From 3fdf3e734a934b9454b04ae5220a5167bcde0d64 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 12 Apr 2022 11:13:00 +0100 Subject: [PATCH 01/41] Initial soft-state testing --- .../simulator/adversary/drop_packets.go | 4 +- cmd/pineconesim/simulator/commands.go | 8 +- router/manhole.go | 7 +- router/peer.go | 11 +- router/state.go | 37 +- router/state_forward.go | 56 +- router/state_snek.go | 569 ++---------------- router/version.go | 3 +- types/frame.go | 146 +---- types/frame_test.go | 171 ------ 10 files changed, 83 insertions(+), 929 deletions(-) diff --git a/cmd/pineconesim/simulator/adversary/drop_packets.go b/cmd/pineconesim/simulator/adversary/drop_packets.go index e8a0772a..c295bcd6 100644 --- a/cmd/pineconesim/simulator/adversary/drop_packets.go +++ b/cmd/pineconesim/simulator/adversary/drop_packets.go @@ -88,10 +88,10 @@ func defaultFrameCount() PeerFrameCount { frameCount[types.TypeKeepalive] = atomic.NewUint64(0) frameCount[types.TypeTreeAnnouncement] = atomic.NewUint64(0) frameCount[types.TypeVirtualSnakeBootstrap] = atomic.NewUint64(0) - frameCount[types.TypeVirtualSnakeBootstrapACK] = atomic.NewUint64(0) + /*frameCount[types.TypeVirtualSnakeBootstrapACK] = atomic.NewUint64(0) frameCount[types.TypeVirtualSnakeSetup] = atomic.NewUint64(0) frameCount[types.TypeVirtualSnakeSetupACK] = atomic.NewUint64(0) - frameCount[types.TypeVirtualSnakeTeardown] = atomic.NewUint64(0) + frameCount[types.TypeVirtualSnakeTeardown] = atomic.NewUint64(0)*/ frameCount[types.TypeTreeRouted] = atomic.NewUint64(0) frameCount[types.TypeVirtualSnakeRouted] = atomic.NewUint64(0) diff --git a/cmd/pineconesim/simulator/commands.go b/cmd/pineconesim/simulator/commands.go index 69295ac5..bbce263a 100644 --- a/cmd/pineconesim/simulator/commands.go +++ b/cmd/pineconesim/simulator/commands.go @@ -137,7 +137,7 @@ func UnmarshalCommandJSON(command *SimCommandMsg) (SimCommand, error) { } else { err = fmt.Errorf("%sConfigureAdversaryDefaults.DropRates.VirtualSnakeBootstrap field doesn't exist", FAILURE_PREAMBLE) } - if subVal, subOk := val.(map[string]interface{})["VirtualSnakeBootstrapACK"]; subOk { + /*if subVal, subOk := val.(map[string]interface{})["VirtualSnakeBootstrapACK"]; subOk { intVal, _ := strconv.Atoi(subVal.(string)) dropRates.Frames[types.TypeVirtualSnakeBootstrapACK] = uint64(intVal) } else { @@ -160,7 +160,7 @@ func UnmarshalCommandJSON(command *SimCommandMsg) (SimCommand, error) { dropRates.Frames[types.TypeVirtualSnakeTeardown] = uint64(intVal) } else { err = fmt.Errorf("%sConfigureAdversaryDefaults.DropRates.VirtualSnakeTeardown field doesn't exist", FAILURE_PREAMBLE) - } + }*/ if subVal, subOk := val.(map[string]interface{})["VirtualSnakeRouted"]; subOk { intVal, _ := strconv.Atoi(subVal.(string)) dropRates.Frames[types.TypeVirtualSnakeRouted] = uint64(intVal) @@ -217,7 +217,7 @@ func UnmarshalCommandJSON(command *SimCommandMsg) (SimCommand, error) { } else { err = fmt.Errorf("%sConfigureAdversaryPeer.DropRates.VirtualSnakeBootstrap field doesn't exist", FAILURE_PREAMBLE) } - if subVal, subOk := val.(map[string]interface{})["VirtualSnakeBootstrapACK"]; subOk { + /*if subVal, subOk := val.(map[string]interface{})["VirtualSnakeBootstrapACK"]; subOk { intVal, _ := strconv.Atoi(subVal.(string)) dropRates.Frames[types.TypeVirtualSnakeBootstrapACK] = uint64(intVal) } else { @@ -240,7 +240,7 @@ func UnmarshalCommandJSON(command *SimCommandMsg) (SimCommand, error) { dropRates.Frames[types.TypeVirtualSnakeTeardown] = uint64(intVal) } else { err = fmt.Errorf("%sConfigureAdversaryPeer.DropRates.VirtualSnakeTeardown field doesn't exist", FAILURE_PREAMBLE) - } + }*/ if subVal, subOk := val.(map[string]interface{})["VirtualSnakeRouted"]; subOk { intVal, _ := strconv.Atoi(subVal.(string)) dropRates.Frames[types.TypeVirtualSnakeRouted] = uint64(intVal) diff --git a/router/manhole.go b/router/manhole.go index 6d3aac94..bff73901 100644 --- a/router/manhole.go +++ b/router/manhole.go @@ -78,7 +78,6 @@ func (r *Router) ManholeHandler(w http.ResponseWriter, req *http.Request) { public := p.public.String() response.Peers[public] = append(response.Peers[public], info) } - response.SNEK.Ascending = r.state._ascending response.SNEK.Descending = r.state._descending for _, p := range r.state._table { response.SNEK.Paths = append(response.SNEK.Paths, p) @@ -90,11 +89,7 @@ func (r *Router) ManholeHandler(w http.ResponseWriter, req *http.Request) { }) } sort.Slice(response.SNEK.Paths, func(i, j int) bool { - c := response.SNEK.Paths[i].PublicKey.CompareTo(response.SNEK.Paths[j].PublicKey) - if c == 0 { - return response.SNEK.Paths[i].PathID.CompareTo(response.SNEK.Paths[j].PathID) < 0 - } - return c < 0 + return response.SNEK.Paths[i].PublicKey.CompareTo(response.SNEK.Paths[j].PublicKey) < 0 }) encoder := json.NewEncoder(w) encoder.SetIndent("", " ") diff --git a/router/peer.go b/router/peer.go index 63a6640c..3e19fbe7 100644 --- a/router/peer.go +++ b/router/peer.go @@ -86,13 +86,6 @@ func (p *peer) String() string { // to make sim less ugly return fmt.Sprintf("%d", p.port) } -// local returns true if the peer refers to the local router peer, or -// false if the peer is an actual connected peer. It is safe to be called from -// other actors. -func (p *peer) local() bool { - return p == p.router.local -} - // send queues a frame to be sent to this peer. It is safe to be called from // other actors. The frame will be allocated to the correct queue automatically // depending on whether it is a protocol frame or a traffic frame. This function @@ -103,9 +96,7 @@ func (p *peer) send(f *types.Frame) bool { // Protocol messages case types.TypeTreeAnnouncement, types.TypeKeepalive: fallthrough - case types.TypeVirtualSnakeBootstrap, types.TypeVirtualSnakeBootstrapACK: - fallthrough - case types.TypeVirtualSnakeSetup, types.TypeVirtualSnakeSetupACK, types.TypeVirtualSnakeTeardown: + case types.TypeVirtualSnakeBootstrap: if p.proto == nil { // The local peer doesn't have a protocol queue so we should check // for nils to prevent panics. diff --git a/router/state.go b/router/state.go index 3f8b6197..777dbd01 100644 --- a/router/state.go +++ b/router/state.go @@ -37,7 +37,6 @@ type state struct { phony.Inbox r *Router _peers []*peer // All switch ports, connected and disconnected - _ascending *virtualSnakeEntry // Next ascending node in keyspace _descending *virtualSnakeEntry // Next descending node in keyspace _candidate *virtualSnakeEntry // Candidate to replace the ascending node _parent *peer // Our chosen parent in the tree @@ -54,7 +53,6 @@ type state struct { // _start resets the state and starts tree and virtual snake maintenance. func (s *state) _start() { s._setParent(nil) - s._setAscendingNode(nil) s._setDescendingNode(nil) s._candidate = nil @@ -172,23 +170,6 @@ func (s *state) _setParent(peer *peer) { }) } -func (s *state) _setAscendingNode(node *virtualSnakeEntry) { - s._ascending = node - - s.r.Act(nil, func() { - peerID := "" - pathID := []byte{} - if node != nil { - peerID = node.Origin.String() - if node.virtualSnakeIndex != nil { - pathID, _ = node.PathID.MarshalJSON() - } - } - - s.r._publish(events.SnakeAscUpdate{PeerID: peerID, PathID: string(pathID)}) - }) -} - func (s *state) _setDescendingNode(node *virtualSnakeEntry) { s._descending = node @@ -197,9 +178,9 @@ func (s *state) _setDescendingNode(node *virtualSnakeEntry) { pathID := []byte{} if node != nil { peerID = node.PublicKey.String() - if node.virtualSnakeIndex != nil { + /*if node.virtualSnakeIndex != nil { pathID, _ = node.PathID.MarshalJSON() - } + }*/ } s.r._publish(events.SnakeDescUpdate{PeerID: peerID, PathID: string(pathID)}) @@ -235,24 +216,16 @@ func (s *state) _portDisconnected(peer *peer) { // direction, so that nodes further along the path will learn that the // path was broken. for k, v := range s._table { - if v.Destination == peer || v.Source == peer { - s._sendTeardownForExistingPath(peer, k.PublicKey, k.PathID) + if v.Source == peer { + delete(s._table, k) } } - // If the ascending path was also lost because it went via the now-dead - // peering then clear that path (although we can't send a teardown) and - // then bootstrap again. - if asc := s._ascending; asc != nil && asc.Destination == peer { - s._teardownPath(s.r.local, asc.PublicKey, asc.PathID) - bootstrap = true - } - // If the descending path was lost because it went via the now-dead // peering then clear that path (although we can't send a teardown) and // wait for another incoming setup. if desc := s._descending; desc != nil && desc.Source == peer { - s._teardownPath(s.r.local, desc.PublicKey, desc.PathID) + s._descending = nil } // If the peer that died was our chosen tree parent, then we will need to diff --git a/router/state_forward.go b/router/state_forward.go index 4998bfee..b563ab43 100644 --- a/router/state_forward.go +++ b/router/state_forward.go @@ -27,20 +27,12 @@ import ( func (s *state) _nextHopsFor(from *peer, frame *types.Frame) *peer { var nexthop *peer switch frame.Type { - case types.TypeVirtualSnakeTeardown: - // Teardowns have their own logic so we do nothing with them - return nil - - case types.TypeVirtualSnakeSetupACK: - // Setup ACKs have their own logic so we do nothing with them - return nil - // SNEK routing case types.TypeVirtualSnakeRouted, types.TypeVirtualSnakeBootstrap, types.TypeSNEKPing, types.TypeSNEKPong: nexthop = s._nextHopsSNEK(frame, frame.Type == types.TypeVirtualSnakeBootstrap) // Tree routing - case types.TypeTreeRouted, types.TypeVirtualSnakeBootstrapACK, types.TypeVirtualSnakeSetup, types.TypeTreePing, types.TypeTreePong: + case types.TypeTreeRouted, types.TypeTreePing, types.TypeTreePong: nexthop = s._nextHopsTree(from, frame) } return nexthop @@ -75,55 +67,13 @@ func (s *state) _forward(p *peer, f *types.Frame) error { case types.TypeVirtualSnakeBootstrap: // Bootstrap messages are only handled specially when they reach a dead end. // Otherwise they are forwarded normally by falling through. - if deadend { - if err := s._handleBootstrap(p, f); err != nil { - return fmt.Errorf("s._handleBootstrap (port %d): %w", p.port, err) - } - return nil + if err := s._handleBootstrap(p, f); err != nil { + return fmt.Errorf("s._handleBootstrap (port %d): %w", p.port, err) } - - case types.TypeVirtualSnakeBootstrapACK: - // Bootstrap ACK messages are only handled specially when they reach a dead end. - // Otherwise they are forwarded normally by falling through. if deadend { - if err := s._handleBootstrapACK(p, f); err != nil { - return fmt.Errorf("s._handleBootstrapACK (port %d): %w", p.port, err) - } return nil } - case types.TypeVirtualSnakeSetup: - // Setup messages are handled at each node on the path. Since the _handleSetup - // function needs to be sure that the setup message was queued to the next-hop - // before installing the route, we do not need to forward the packet here. - if err := s._handleSetup(p, f, nexthop); err != nil { - return fmt.Errorf("s._handleSetup (port %d): %w", p.port, err) - } - return nil - - case types.TypeVirtualSnakeSetupACK: - // Setup ACK messages are handled at each node on the path. Since the _handleSetupACK - // function needs to be sure that the setup ACK message was queued to the next-hop - // before activating the route, we do not need to forward the packet here. - if err := s._handleSetupACK(p, f, nexthop); err != nil { - return fmt.Errorf("s._handleSetupACK (port %d): %w", p.port, err) - } - return nil - - case types.TypeVirtualSnakeTeardown: - // Teardown messages are a special case where there might be more than one - // next-hop, so this is handled specifically. - if nexthops, err := s._handleTeardown(p, f); err != nil { - return fmt.Errorf("s._handleTeardown (port %d): %w", p.port, err) - } else { - for _, nexthop := range nexthops { - if nexthop != nil && nexthop.proto != nil { - nexthop.proto.push(f) - } - } - } - return nil - case types.TypeVirtualSnakeRouted, types.TypeTreeRouted: // Traffic type packets are forwarded normally by falling through. There // are no special rules to apply to these packets, regardless of whether diff --git a/router/state_snek.go b/router/state_snek.go index 1e58e98b..4dc9073d 100644 --- a/router/state_snek.go +++ b/router/state_snek.go @@ -27,25 +27,23 @@ import ( // NOTE: Functions prefixed with an underscore (_) are only safe to be called // from the actor that owns them, in order to prevent data races. -const virtualSnakeMaintainInterval = time.Second -const virtualSnakeNeighExpiryPeriod = time.Hour +const virtualSnakeMaintainInterval = time.Second * 2 +const virtualSnakeNeighExpiryPeriod = time.Second * 5 type virtualSnakeTable map[virtualSnakeIndex]*virtualSnakeEntry type virtualSnakeIndex struct { - PublicKey types.PublicKey `json:"public_key"` - PathID types.VirtualSnakePathID `json:"path_id"` + PublicKey types.PublicKey `json:"public_key"` } type virtualSnakeEntry struct { *virtualSnakeIndex - Origin types.PublicKey `json:"origin"` - Target types.PublicKey `json:"target"` - Source *peer `json:"source"` - Destination *peer `json:"destination"` - LastSeen time.Time `json:"last_seen"` - Root types.Root `json:"root"` - Active bool `json:"active"` + Origin types.PublicKey `json:"origin"` + Target types.PublicKey `json:"target"` + Source *peer `json:"source"` + LastSeen time.Time `json:"last_seen"` + Root types.Root `json:"root"` + Active bool `json:"active"` } // valid returns true if the update hasn't expired, or false if it has. It is @@ -55,6 +53,10 @@ func (e *virtualSnakeEntry) valid() bool { return time.Since(e.LastSeen) < virtualSnakeNeighExpiryPeriod } +func (e *virtualSnakeEntry) validForForwarding() bool { + return time.Since(e.LastSeen) < (virtualSnakeNeighExpiryPeriod * 2) +} + // _maintainSnake is responsible for working out if we need to send bootstraps // or to clean up any old paths. func (s *state) _maintainSnake() { @@ -70,53 +72,28 @@ func (s *state) _maintainSnake() { // bootstraps are sent up to the next ascending node, but as the root, // we already have the highest key on the network. rootAnn := s._rootAnnouncement() - canBootstrap := s._parent != nil && rootAnn.RootPublicKey != s.r.public - willBootstrap := false - // The ascending node is the node with the next highest key. - if asc := s._ascending; asc != nil { + // The descending node is the node with the next lowest key. + if desc := s._descending; desc != nil { switch { - case !asc.valid(): - // The ascending path entry has expired, so tear it down and then - // see if we can bootstrap again. - s._sendTeardownForExistingPath(s.r.local, asc.PublicKey, asc.PathID) + case !desc.valid(): fallthrough - case !asc.Root.EqualTo(&rootAnn.Root): - // The ascending node was set up with a different root key or sequence - // number. In this case, we will send another bootstrap to the remote - // side in order to hopefully replace the path with a new one. - willBootstrap = canBootstrap + case !desc.Root.EqualTo(&rootAnn.Root): + s._setDescendingNode(nil) } - } else { - // We don't have an ascending node at all, so if we can, we'll try - // bootstrapping to locate it. - willBootstrap = canBootstrap - } - - // The descending node is the node with the next lowest key. - if desc := s._descending; desc != nil && !desc.valid() { - // The descending path has expired, so tear it down and then that should - // prompt the remote side into sending a new bootstrap to set up a new - // path, if they are still alive. - s._sendTeardownForExistingPath(s.r.local, desc.PublicKey, desc.PathID) } // Clean up any paths that were installed more than 5 seconds ago but haven't // been activated by a setup ACK. for k, v := range s._table { - if !v.Active && time.Since(v.LastSeen) > time.Second*5 { - s._sendTeardownForExistingPath(s.r.local, k.PublicKey, k.PathID) - if s._candidate == v { - s._candidate = nil - } + if !v.valid() { + delete(s._table, k) } } // If one of the previous conditions means that we need to bootstrap, then // send the actual bootstrap message into the network. - if willBootstrap { - s._bootstrapNow() - } + s._bootstrapNow() } // _bootstrapNow is responsible for sending a bootstrap message to the network. @@ -127,20 +104,10 @@ func (s *state) _bootstrapNow() { if s._parent == nil { return } - // If we already have a relationship with an ascending node and that has the - // same root key and sequence number (i.e. nothing has changed in the tree since - // the path was set up) then we don't need to send another bootstrap message just - // yet. We'll either wait for the path to be torn down, expire or for the tree to - // change. - ann := s._rootAnnouncement() - if asc := s._ascending; asc != nil && asc.Source.started.Load() { - if asc.Root.EqualTo(&ann.Root) { - return - } - } // Construct the bootstrap packet. We will include our root key and sequence // number in the update so that the remote side can determine if we are both using // the same root node when processing the update. + ann := s._rootAnnouncement() b := frameBufferPool.Get().(*[types.MaxFrameSize]byte) defer frameBufferPool.Put(b) bootstrap := types.VirtualSnakeBootstrap{ @@ -294,7 +261,7 @@ func getNextHopSNEK(params virtualSnakeNextHopParams) *peer { // higher one, this is effectively looking for paths that descend through // keyspace toward lower keys rather than ascend toward higher ones. for _, entry := range params.snakeRoutes { - if !entry.Source.started.Load() || !entry.valid() || entry.Source == params.selfPeer { + if !entry.Source.started.Load() || !entry.validForForwarding() || entry.Source == params.selfPeer { continue } if !params.isBootstrap && !entry.Active { @@ -350,480 +317,56 @@ func (s *state) _handleBootstrap(from *peer, rx *types.Frame) error { if !root.Root.EqualTo(&bootstrap.Root) { return nil } - // In response to a bootstrap, we'll send back a bootstrap ACK packet to - // the sender. We'll include our own root details in the ACK. - bootstrapACK := types.VirtualSnakeBootstrapACK{ - PathID: bootstrap.PathID, - Root: root.Root, - SourceSig: bootstrap.SourceSig, - } - if s.r.secure { - // Since we're the "destination" of the bootstrap, we'll add a new - // "destination signature", in which we sign the source signature, - // the path key and the path ID. This allows anyone else to verify - // that we accepted this specific bootstrap. - copy( - bootstrapACK.DestinationSig[:], - ed25519.Sign( - s.r.private[:], - append(bootstrap.SourceSig[:], append(rx.DestinationKey[:], bootstrap.PathID[:]...)...), - ), - ) - } - b := frameBufferPool.Get().(*[types.MaxFrameSize]byte) - defer frameBufferPool.Put(b) - n, err := bootstrapACK.MarshalBinary(b[:]) - if err != nil { - return fmt.Errorf("bootstrapACK.MarshalBinary: %w", err) - } - // Bootstrap ACKs are routed using tree routing, so we need to take the - // coordinates from the source field of the received packet and set the - // destination of the ACK packet to that. - send := getFrame() - send.Type = types.TypeVirtualSnakeBootstrapACK - send.Destination = rx.Source - send.DestinationKey = rx.DestinationKey - send.Source = s._coords() - send.SourceKey = s.r.public - send.Payload = append(send.Payload[:0], b[:n]...) - if p := s._nextHopsTree(s.r.local, send); p != nil && p.proto != nil { - p.proto.push(send) - } - return nil -} -// _handleBootstrapACK is called in response to receiving a bootstrap ACK -// packet. This function will work out whether the remote node is a suitable -// candidate to set up an outbound path to, and if so, will send path setup -// packets to the network. -func (s *state) _handleBootstrapACK(from *peer, rx *types.Frame) error { - // Unmarshal the bootstrap ACK. - var bootstrapACK types.VirtualSnakeBootstrapACK - _, err := bootstrapACK.UnmarshalBinary(rx.Payload) - if err != nil { - return fmt.Errorf("bootstrapACK.UnmarshalBinary: %w", err) + // Create a routing table entry. + index := virtualSnakeIndex{ + PublicKey: rx.DestinationKey, } - if s.r.secure { - // Verify that the source signature hasn't been changed by the remote - // side. If it has then it won't validate using our own public key. - if !ed25519.Verify( - s.r.public[:], - append(s.r.public[:], bootstrapACK.PathID[:]...), - bootstrapACK.SourceSig[:], - ) { - return nil - } - // Verify that the destination signature is OK, which allows us to confirm - // that the remote node accepted our bootstrap and that the remote node is - // who they claim to be. - if !ed25519.Verify( - rx.SourceKey[:], - append(bootstrapACK.SourceSig[:], append(rx.DestinationKey[:], bootstrapACK.PathID[:]...)...), - bootstrapACK.DestinationSig[:], - ) { - return nil - } + s._table[index] = &virtualSnakeEntry{ + virtualSnakeIndex: &index, + Origin: rx.DestinationKey, + Target: rx.DestinationKey, + Source: from, + LastSeen: time.Now(), + Root: bootstrap.Root, + Active: true, } - root := s._rootAnnouncement() + + // Now let's see if this is a suitable ascending entry. update := false - asc := s._ascending + desc := s._descending switch { - case rx.SourceKey == s.r.public: - // We received a bootstrap ACK from ourselves. This shouldn't happen, - // so either another node has forwarded it to us incorrectly, or - // a routing loop has occurred somewhere. Don't act on the bootstrap - // in that case. - case !bootstrapACK.Root.EqualTo(&root.Root): + case !root.Root.EqualTo(&bootstrap.Root): // The root key in the bootstrap ACK doesn't match our own key, or the // sequence doesn't match, so it is quite possible that routing setup packets // using tree routing would fail. - case asc != nil && asc.valid(): - // We already have an ascending entry and it hasn't expired yet. + case !util.LessThan(rx.DestinationKey, s.r.public): + // The bootstrapping key should be less than ours but it isn't. + case desc != nil && desc.valid(): + // We already have a descending entry and it hasn't expired. switch { - case asc.Origin == rx.SourceKey && bootstrapACK.PathID != asc.PathID: - // We've received another bootstrap ACK from our direct ascending node. - // Just refresh the record and then send a new path setup message to - // that node. + case desc.PublicKey == rx.DestinationKey: + // We've received another bootstrap from our direct descending node. + // Send back an acknowledgement as this is OK. update = true - case util.DHTOrdered(s.r.public, rx.SourceKey, asc.Origin): - // We know about an ascending node already but it turns out that this - // new node that we've received a bootstrap from is actually closer to - // us than the previous node. We'll update our record to use the new - // node instead and then send a new path setup message to it. + case util.DHTOrdered(desc.PublicKey, rx.DestinationKey, s.r.public): + // The bootstrapping node is closer to us than our previous descending + // node was. update = true } - case asc == nil || !asc.valid(): - // We don't have an ascending entry, or we did but it expired. - if util.LessThan(s.r.public, rx.SourceKey) { - // We don't know about an ascending node and at the moment we don't know - // any better candidates, so we'll accept a bootstrap ACK from a node with a - // key higher than ours (so that it matches descending order). + case desc == nil || !desc.valid(): + // We don't have a descending entry, or we did but it expired. + if util.LessThan(rx.DestinationKey, s.r.public) { + // The bootstrapping key is less than ours so we'll acknowledge it. update = true } default: - // The bootstrap ACK conditions weren't met. This might just be because + // The bootstrap conditions weren't met. This might just be because // there's a node out there that hasn't converged to a closer node - // yet, so we'll just ignore the acknowledgement. - } - // If we haven't decided we like the update then we won't do anything at this - // point so give up. - if !update { - return nil - } - // Include our own root information in the update. - setup := types.VirtualSnakeSetup{ // nolint:gosimple - PathID: bootstrapACK.PathID, - Root: root.Root, - SourceSig: bootstrapACK.SourceSig, - DestinationSig: bootstrapACK.DestinationSig, - } - b := frameBufferPool.Get().(*[types.MaxFrameSize]byte) - defer frameBufferPool.Put(b) - n, err := setup.MarshalBinary(b[:]) - if err != nil { - return fmt.Errorf("setup.MarshalBinary: %w", err) - } - // Setup messages routed using tree routing. The destination key is set in the - // header so that a node can determine if the setup message arrived at the - // intended destination instead of forwarding it. The source key is set to our - // public key, since this is the lower of the two keys that intermediate nodes - // will populate into their routing tables. - send := getFrame() - send.Type = types.TypeVirtualSnakeSetup - send.Destination = rx.Source - send.DestinationKey = rx.SourceKey - send.SourceKey = s.r.public - send.Payload = append(send.Payload[:0], b[:n]...) - nexthop := s.r.state._nextHopsTree(s.r.local, send) - // Importantly, we will only create a DHT entry if it appears as though our next - // hop has actually accepted the packet. Otherwise we'll create a path entry and - // the setup message won't go anywhere. - switch { - case nexthop == nil: - fallthrough // No peer was identified, which shouldn't happen. - case nexthop.local(): - fallthrough // The peer is local, which shouldn't happen. - case !nexthop.started.Load(): - fallthrough // The peer has shut down or errored. - case nexthop.proto == nil: - fallthrough // The peer doesn't have a protocol queue for some reason. - case !nexthop.proto.push(send): - return nil // We failed to push the message into the peer queue. - } - index := virtualSnakeIndex{ - PublicKey: s.r.public, - PathID: bootstrapACK.PathID, + // yet, so we'll just ignore the bootstrap. } - entry := &virtualSnakeEntry{ - virtualSnakeIndex: &index, - Origin: rx.SourceKey, - Target: rx.SourceKey, - Source: s.r.local, - Destination: nexthop, - LastSeen: time.Now(), - Root: bootstrapACK.Root, - } - // The remote side is responsible for clearing up the replaced path, but - // we do want to make sure we don't have any old paths to other nodes - // that *aren't* the new ascending node lying around. This helps to avoid - // routing loops. - for dhtKey, entry := range s._table { - if entry.Source == s.r.local && entry.PublicKey != rx.SourceKey { - s._sendTeardownForExistingPath(s.r.local, dhtKey.PublicKey, dhtKey.PathID) - } - } - // Install the new route into the DHT. - s._table[index] = entry - s._candidate = entry - return nil -} - -// _handleSetup is called in response to receiving setup packets. Note that -// these packets are handled even as we forward them, as setup packets should be -// processed by each node on the path. -func (s *state) _handleSetup(from *peer, rx *types.Frame, nexthop *peer) error { - root := s._rootAnnouncement() - // Unmarshal the setup. - var setup types.VirtualSnakeSetup - if _, err := setup.UnmarshalBinary(rx.Payload); err != nil { - return fmt.Errorf("setup.UnmarshalBinary: %w", err) - } - if s.r.secure { - // Verify the source signature using the source key. A valid signature proves - // that the node that sent the setup is actually who they say they are. - if !ed25519.Verify( - rx.SourceKey[:], - append(rx.SourceKey[:], setup.PathID[:]...), - setup.SourceSig[:], - ) { - s._sendTeardownForRejectedPath(rx.SourceKey, setup.PathID, from) - return nil - } - // Verify the destination signature using the destination key. A valid signature - // proves that the node that the setup message is being sent to actually accepted - // the bootstrap and therefore this path is legitimate and not spoofed. - if !ed25519.Verify( - rx.DestinationKey[:], - append(setup.SourceSig[:], append(rx.SourceKey[:], setup.PathID[:]...)...), - setup.DestinationSig[:], - ) { - s._sendTeardownForRejectedPath(rx.SourceKey, setup.PathID, from) - return nil - } - } - if !root.Root.EqualTo(&setup.Root) { - s._sendTeardownForRejectedPath(rx.SourceKey, setup.PathID, from) - return nil - } - index := virtualSnakeIndex{ - PublicKey: rx.SourceKey, - PathID: setup.PathID, - } - // If we already have a path for this public key and path ID combo, which - // *shouldn't* happen, then we need to tear down both the existing path and - // then send back a teardown to the sender notifying them that there was a - // problem. This will probably trigger a new setup, but that's OK, it should - // have a new path ID. - if _, ok := s._table[index]; ok { - s._sendTeardownForExistingPath(s.r.local, rx.SourceKey, setup.PathID) // first call fixes routing table - s._sendTeardownForRejectedPath(rx.SourceKey, setup.PathID, from) // second call sends back to origin - return nil - } - // If we're at the destination of the setup then update our predecessor - // with information from the bootstrap. - if rx.DestinationKey == s.r.public { - update := false - desc := s._descending - switch { - case !root.Root.EqualTo(&setup.Root): - // The root key in the bootstrap ACK doesn't match our own key, or the - // sequence doesn't match, so it is quite possible that routing setup packets - // using tree routing would fail. - case !util.LessThan(rx.SourceKey, s.r.public): - // The bootstrapping key should be less than ours but it isn't. - case desc != nil && desc.valid(): - // We already have a descending entry and it hasn't expired. - switch { - case desc.PublicKey == rx.SourceKey && setup.PathID != desc.PathID: - // We've received another bootstrap from our direct descending node. - // Send back an acknowledgement as this is OK. - update = true - case util.DHTOrdered(desc.PublicKey, rx.SourceKey, s.r.public): - // The bootstrapping node is closer to us than our previous descending - // node was. - update = true - } - case desc == nil || !desc.valid(): - // We don't have a descending entry, or we did but it expired. - if util.LessThan(rx.SourceKey, s.r.public) { - // The bootstrapping key is less than ours so we'll acknowledge it. - update = true - } - default: - // The bootstrap conditions weren't met. This might just be because - // there's a node out there that hasn't converged to a closer node - // yet, so we'll just ignore the bootstrap. - } - if !update { - s._sendTeardownForRejectedPath(rx.SourceKey, setup.PathID, from) - return nil - } - if desc != nil { - // Tear down the previous path, if there was one. - s._sendTeardownForExistingPath(s.r.local, desc.PublicKey, desc.PathID) - } - entry := &virtualSnakeEntry{ - virtualSnakeIndex: &index, - Origin: rx.SourceKey, - Target: rx.DestinationKey, - Source: from, - Destination: s.r.local, - LastSeen: time.Now(), - Root: setup.Root, - } - s._table[index] = entry - s._setDescendingNode(entry) - // Send back a setup ACK to the remote side. - setupACK := types.VirtualSnakeSetupACK{ - PathID: setup.PathID, - Root: setup.Root, - } - if s.r.secure { - // Sign the path key and path ID with our own key. This forms the "target - // signature", which anyone can use to verify that we sent the setup ACK. - copy( - setupACK.TargetSig[:], - ed25519.Sign(s.r.private[:], append(index.PublicKey[:], index.PathID[:]...)), - ) - } - b := frameBufferPool.Get().(*[types.MaxFrameSize]byte) - defer frameBufferPool.Put(b) - n, err := setupACK.MarshalBinary(b[:]) - if err != nil { - return fmt.Errorf("setupACK.MarshalBinary: %w", err) - } - send := getFrame() - send.Type = types.TypeVirtualSnakeSetupACK - send.DestinationKey = rx.SourceKey - send.Payload = append(send.Payload[:0], b[:n]...) - if entry.Source.send(send) { - entry.Active = true - } - return nil - } - // Try to forward the setup onto the next node first. If we - // can't do that then there's no point in keeping the path. - switch { - case nexthop == nil: - fallthrough // No peer was identified, which shouldn't happen. - case nexthop.local(): - fallthrough // The peer is local, which shouldn't happen. - case !nexthop.started.Load(): - fallthrough // The peer has shut down or errored. - case nexthop.proto == nil: - fallthrough // The peer doesn't have a protocol queue for some reason. - case !nexthop.proto.push(rx): - s._sendTeardownForRejectedPath(rx.SourceKey, setup.PathID, from) - return nil // We failed to push the message into the peer queue. - } - // Add a new routing table entry as we are intermediate to - // the path. - s._table[index] = &virtualSnakeEntry{ - virtualSnakeIndex: &index, - Origin: rx.SourceKey, - Target: rx.DestinationKey, - LastSeen: time.Now(), - Root: setup.Root, - Source: from, // node with lower of the two keys - Destination: nexthop, // node with higher of the two keys - } - return nil -} - -// _handleSetupACK is called in response to receiving a setup ACK packet from the -// network. -func (s *state) _handleSetupACK(from *peer, rx *types.Frame, nexthop *peer) error { - // Unmarshal the setup. - var setup types.VirtualSnakeSetupACK - if _, err := setup.UnmarshalBinary(rx.Payload); err != nil { - return fmt.Errorf("setup.UnmarshalBinary: %w", err) - } - // Look up to see if we have a matching route. The route must be not active - // (i.e. we haven't received a setup ACK for it yet) and must have arrived - // from the port that the entry was populated with. - for k, v := range s._table { - if v.Active || k.PublicKey != rx.DestinationKey || k.PathID != setup.PathID { - continue - } - switch { - case from.local(): - fallthrough - case from == v.Destination: - if s.r.secure && !ed25519.Verify(v.Target[:], append(k.PublicKey[:], k.PathID[:]...), setup.TargetSig[:]) { - continue - } - if v.Source.local() || v.Source.send(rx) { - v.Active = true - if v == s._candidate { - s._setAscendingNode(v) - s._candidate = nil - } - } - } - } - return nil -} - -// _handleTeardown is called in response to receiving a teardown packet from the -// network. -func (s *state) _handleTeardown(from *peer, rx *types.Frame) ([]*peer, error) { - if len(rx.Payload) < types.VirtualSnakePathIDLength { - return nil, fmt.Errorf("payload too short") - } - var teardown types.VirtualSnakeTeardown - if _, err := teardown.UnmarshalBinary(rx.Payload); err != nil { - return nil, fmt.Errorf("teardown.UnmarshalBinary: %w", err) - } - return s._teardownPath(from, rx.DestinationKey, teardown.PathID), nil -} - -// _sendTeardownForRejectedPath sends a teardown into the network for a path -// that was received but not accepted. -func (s *state) _sendTeardownForRejectedPath(pathKey types.PublicKey, pathID types.VirtualSnakePathID, via *peer) { - if via != nil { - via.proto.push(s._getTeardown(pathKey, pathID)) - } -} - -// _sendTeardownForExistingPath sends a teardown into the network for a path -// that was already accepted into the routing table but is being replaced or -// removed. -func (s *state) _sendTeardownForExistingPath(from *peer, pathKey types.PublicKey, pathID types.VirtualSnakePathID) { - frame := s._getTeardown(pathKey, pathID) - for _, nexthop := range s._teardownPath(from, pathKey, pathID) { - if nexthop != nil && nexthop.proto != nil { - nexthop.proto.push(frame) - } - } -} - -// _getTeardown generates a frame containing a teardown message for the given -// path key and path ID. -func (s *state) _getTeardown(pathKey types.PublicKey, pathID types.VirtualSnakePathID) *types.Frame { - payload := frameBufferPool.Get().(*[types.MaxFrameSize]byte) - defer frameBufferPool.Put(payload) - teardown := types.VirtualSnakeTeardown{ - PathID: pathID, - } - n, err := teardown.MarshalBinary(payload[:]) - if err != nil { - return nil - } - frame := getFrame() - frame.Type = types.TypeVirtualSnakeTeardown - frame.DestinationKey = pathKey - frame.Payload = append(frame.Payload[:0], payload[:n]...) - return frame -} - -// _teardownPath processes a teardown message by tearing down any -// related routes, returning a slice of next-hop candidates that the -// teardown must be forwarded to. -func (s *state) _teardownPath(from *peer, pathKey types.PublicKey, pathID types.VirtualSnakePathID) []*peer { - if asc := s._ascending; asc != nil && asc.PublicKey == pathKey && asc.PathID == pathID { - switch { - case from.local(): // originated locally - fallthrough - case from == asc.Destination: // from network - s._setAscendingNode(nil) - delete(s._table, virtualSnakeIndex{asc.PublicKey, asc.PathID}) - return []*peer{asc.Destination} - } - } - if desc := s._descending; desc != nil && desc.PublicKey == pathKey && desc.PathID == pathID { - switch { - case from == desc.Source: // from network - fallthrough - case from.local(): // originated locally - s._setDescendingNode(nil) - delete(s._table, virtualSnakeIndex{desc.PublicKey, desc.PathID}) - return []*peer{desc.Source} - } - } - for k, v := range s._table { - if k.PublicKey == pathKey && k.PathID == pathID { - switch { - case from.local(): // happens when we're tearing down an existing duplicate path - delete(s._table, k) - return []*peer{v.Destination, v.Source} - case from == v.Source: // from network, return the opposite direction - delete(s._table, k) - return []*peer{v.Destination} - case from == v.Destination: // from network, return the opposite direction - delete(s._table, k) - return []*peer{v.Source} - } - } + if update { + s._setDescendingNode(s._table[index]) } return nil } diff --git a/router/version.go b/router/version.go index 838f4769..4743e45f 100644 --- a/router/version.go +++ b/router/version.go @@ -18,7 +18,8 @@ const ( capabilityLengthenedRootInterval = iota + 1 capabilityCryptographicSetups capabilitySetupACKs + capabilitySoftState ) const ourVersion uint8 = 1 -const ourCapabilities uint32 = capabilityLengthenedRootInterval | capabilityCryptographicSetups | capabilitySetupACKs +const ourCapabilities uint32 = capabilityLengthenedRootInterval | capabilityCryptographicSetups | capabilitySetupACKs | capabilitySoftState diff --git a/types/frame.go b/types/frame.go index 52d7008b..e908f396 100644 --- a/types/frame.go +++ b/types/frame.go @@ -34,19 +34,15 @@ type FrameVersion uint8 type FrameType uint8 const ( - TypeKeepalive FrameType = iota // protocol frame, direct to peers only - TypeTreeAnnouncement // protocol frame, bypasses queues - TypeTreeRouted // traffic frame, forwarded using tree routing - TypeVirtualSnakeBootstrap // protocol frame, forwarded using SNEK - TypeVirtualSnakeBootstrapACK // protocol frame, forwarded using tree routing - TypeVirtualSnakeSetup // protocol frame, forwarded using tree routing - TypeVirtualSnakeSetupACK // protocol frame, forwarded using special rules - TypeVirtualSnakeTeardown // protocol frame, forwarded using special rules - TypeVirtualSnakeRouted // traffic frame, forwarded using SNEK - TypeSNEKPing FrameType = iota + 200 // traffic frame, forwarded using SNEK - TypeSNEKPong // traffic frame, forwarded using SNEK - TypeTreePing // traffic frame, forwarded using tree - TypeTreePong // traffic frame, forwarded using tree + TypeKeepalive FrameType = iota // protocol frame, direct to peers only + TypeTreeAnnouncement // protocol frame, bypasses queues + TypeTreeRouted // traffic frame, forwarded using tree routing + TypeVirtualSnakeBootstrap // protocol frame, forwarded using SNEK + TypeVirtualSnakeRouted // traffic frame, forwarded using SNEK + TypeSNEKPing FrameType = iota + 200 // traffic frame, forwarded using SNEK + TypeSNEKPong // traffic frame, forwarded using SNEK + TypeTreePing // traffic frame, forwarded using tree + TypeTreePong // traffic frame, forwarded using tree ) const ( @@ -102,62 +98,6 @@ func (f *Frame) MarshalBinary(buffer []byte) (int, error) { offset += copy(buffer[offset:], f.Payload[:payloadLen]) } - case TypeVirtualSnakeBootstrapACK: - payloadLen := len(f.Payload) - binary.BigEndian.PutUint16(buffer[offset+0:offset+2], uint16(payloadLen)) - dn, err := f.Destination.MarshalBinary(buffer[offset+6:]) - if err != nil { - return 0, fmt.Errorf("f.Destination.MarshalBinary: %w", err) - } - sn, err := f.Source.MarshalBinary(buffer[offset+6+dn:]) - if err != nil { - return 0, fmt.Errorf("f.Source.MarshalBinary: %w", err) - } - binary.BigEndian.PutUint16(buffer[offset+2:offset+4], uint16(dn)) - binary.BigEndian.PutUint16(buffer[offset+4:offset+6], uint16(sn)) - offset += 6 + dn + sn - offset += copy(buffer[offset:], f.DestinationKey[:ed25519.PublicKeySize]) - offset += copy(buffer[offset:], f.SourceKey[:ed25519.PublicKeySize]) - if f.Payload != nil { - f.Payload = f.Payload[:payloadLen] - offset += copy(buffer[offset:], f.Payload[:payloadLen]) - } - - case TypeVirtualSnakeSetup: // destination = coords & key, source = key - payloadLen := len(f.Payload) - binary.BigEndian.PutUint16(buffer[offset+0:offset+2], uint16(payloadLen)) - dn, err := f.Destination.MarshalBinary(buffer[offset+2:]) - if err != nil { - return 0, fmt.Errorf("f.Destination.MarshalBinary: %w", err) - } - offset += 2 + dn - offset += copy(buffer[offset:], f.SourceKey[:ed25519.PublicKeySize]) - offset += copy(buffer[offset:], f.DestinationKey[:ed25519.PublicKeySize]) - if f.Payload != nil { - f.Payload = f.Payload[:payloadLen] - offset += copy(buffer[offset:], f.Payload[:payloadLen]) - } - - case TypeVirtualSnakeSetupACK: // detination = key - payloadLen := len(f.Payload) - binary.BigEndian.PutUint16(buffer[offset+0:offset+2], uint16(payloadLen)) - offset += 2 - offset += copy(buffer[offset:], f.DestinationKey[:ed25519.PublicKeySize]) - if f.Payload != nil { - f.Payload = f.Payload[:payloadLen] - offset += copy(buffer[offset:], f.Payload[:payloadLen]) - } - - case TypeVirtualSnakeTeardown: // destination = key - payloadLen := len(f.Payload) - binary.BigEndian.PutUint16(buffer[offset+0:offset+2], uint16(payloadLen)) - offset += 2 - offset += copy(buffer[offset:], f.DestinationKey[:ed25519.PublicKeySize]) - if f.Payload != nil { - f.Payload = f.Payload[:payloadLen] - offset += copy(buffer[offset:], f.Payload[:payloadLen]) - } - case TypeVirtualSnakeRouted, TypeSNEKPing, TypeSNEKPong: // destination = key, source = key payloadLen := len(f.Payload) binary.BigEndian.PutUint16(buffer[offset+0:offset+2], uint16(payloadLen)) @@ -230,66 +170,6 @@ func (f *Frame) UnmarshalBinary(data []byte) (int, error) { offset += copy(f.Payload, data[offset:]) return offset, nil - case TypeVirtualSnakeBootstrapACK: - payloadLen := int(binary.BigEndian.Uint16(data[offset+0 : offset+2])) - if payloadLen > cap(f.Payload) { - return 0, fmt.Errorf("payload length exceeds frame capacity") - } - dstLen := int(binary.BigEndian.Uint16(data[offset+2 : offset+4])) - srcLen := int(binary.BigEndian.Uint16(data[offset+4 : offset+6])) - offset += 6 - if _, err := f.Destination.UnmarshalBinary(data[offset:]); err != nil { - return 0, fmt.Errorf("f.Destination.UnmarshalBinary: %w", err) - } - offset += dstLen - if _, err := f.Source.UnmarshalBinary(data[offset:]); err != nil { - return 0, fmt.Errorf("f.Destination.UnmarshalBinary: %w", err) - } - offset += srcLen - offset += copy(f.DestinationKey[:], data[offset:]) - offset += copy(f.SourceKey[:], data[offset:]) - f.Payload = f.Payload[:payloadLen] - offset += copy(f.Payload, data[offset:]) - return offset, nil - - case TypeVirtualSnakeSetup: // destination = coords & key, source = key - payloadLen := int(binary.BigEndian.Uint16(data[offset+0 : offset+2])) - if payloadLen > cap(f.Payload) { - return 0, fmt.Errorf("payload length exceeds frame capacity") - } - dstLen := int(binary.BigEndian.Uint16(data[offset+2 : offset+4])) - if _, err := f.Destination.UnmarshalBinary(data[offset+2:]); err != nil { - return 0, fmt.Errorf("f.Destination.UnmarshalBinary: %w", err) - } - offset += 4 + dstLen - offset += copy(f.SourceKey[:], data[offset:]) - offset += copy(f.DestinationKey[:], data[offset:]) - f.Payload = f.Payload[:payloadLen] - offset += copy(f.Payload, data[offset:]) - return offset, nil - - case TypeVirtualSnakeSetupACK: // destination = key - payloadLen := int(binary.BigEndian.Uint16(data[offset+0 : offset+2])) - if payloadLen > cap(f.Payload) { - return 0, fmt.Errorf("payload length exceeds frame capacity") - } - offset += 2 - offset += copy(f.DestinationKey[:], data[offset:]) - f.Payload = f.Payload[:payloadLen] - offset += copy(f.Payload, data[offset:]) - return offset, nil - - case TypeVirtualSnakeTeardown: // destination = key - payloadLen := int(binary.BigEndian.Uint16(data[offset+0 : offset+2])) - if payloadLen > cap(f.Payload) { - return 0, fmt.Errorf("payload length exceeds frame capacity") - } - offset += 2 - offset += copy(f.DestinationKey[:], data[offset:]) - f.Payload = f.Payload[:payloadLen] - offset += copy(f.Payload, data[offset:]) - return offset, nil - case TypeVirtualSnakeRouted, TypeSNEKPing, TypeSNEKPong: // destination = key, source = key payloadLen := int(binary.BigEndian.Uint16(data[offset+0 : offset+2])) if payloadLen > cap(f.Payload) { @@ -338,16 +218,8 @@ func (t FrameType) String() string { return "TreeRouted" case TypeVirtualSnakeBootstrap: return "VirtualSnakeBootstrap" - case TypeVirtualSnakeBootstrapACK: - return "VirtualSnakeBootstrapACK" - case TypeVirtualSnakeSetup: - return "VirtualSnakeSetup" - case TypeVirtualSnakeSetupACK: - return "VirtualSnakeSetupACK" case TypeVirtualSnakeRouted: return "VirtualSnakeRouted" - case TypeVirtualSnakeTeardown: - return "VirtualSnakeTeardown" case TypeKeepalive: return "Keepalive" case TypeSNEKPing: diff --git a/types/frame_test.go b/types/frame_test.go index 44fd6b1d..81ce9c59 100644 --- a/types/frame_test.go +++ b/types/frame_test.go @@ -138,177 +138,6 @@ func TestMarshalUnmarshalSNEKBootstrapFrame(t *testing.T) { } } -func TestMarshalUnmarshalSNEKBootstrapACKFrame(t *testing.T) { - pk1, _, _ := ed25519.GenerateKey(nil) - pk2, _, _ := ed25519.GenerateKey(nil) - input := Frame{ - Version: Version0, - Type: TypeVirtualSnakeBootstrapACK, - Source: Coordinates{1, 2, 3, 4, 5}, - Destination: Coordinates{5, 4, 3, 2, 1}, - Payload: []byte{9, 9, 9, 9, 9}, - } - copy(input.DestinationKey[:], pk1) - copy(input.SourceKey[:], pk2) - expected := []byte{ - 0x70, 0x69, 0x6e, 0x65, // magic bytes - 0, // version 0 - byte(TypeVirtualSnakeBootstrapACK), // type greedy - 0, 0, // extra - 0, 99, // frame length - 0, 5, // payload length - 0, 7, // destination length - 0, 7, // source length - 0, 5, 5, 4, 3, 2, 1, // destination coordinates - 0, 5, 1, 2, 3, 4, 5, // source coordinates - } - expected = append(expected, pk1...) - expected = append(expected, pk2...) - expected = append(expected, input.Payload...) - buf := make([]byte, 65535) - n, err := input.MarshalBinary(buf) - if err != nil { - t.Fatal(err) - } - if n != len(expected) { - t.Fatalf("wrong marshalled length, got %d, expected %d", n, len(expected)) - } - if !bytes.Equal(buf[:n], expected) { - t.Fatalf("wrong marshalled output, got %v, expected %v", buf[:n], expected) - } - - t.Log("Got: ", buf[:n]) - t.Log("Want:", expected) - - output := Frame{ - Payload: make([]byte, 0, MaxPayloadSize), - } - if _, err := output.UnmarshalBinary(buf[:n]); err != nil { - t.Fatal(err) - } - if output.Version != input.Version { - t.Fatal("wrong version") - } - if output.Type != input.Type { - t.Fatal("wrong version") - } - if !output.Destination.EqualTo(input.Destination) { - t.Fatal("wrong path") - } - if !bytes.Equal(input.Payload, output.Payload) { - t.Fatal("wrong payload") - } -} - -func TestMarshalUnmarshalSNEKSetupFrame(t *testing.T) { - pk1, _, _ := ed25519.GenerateKey(nil) - pk2, _, _ := ed25519.GenerateKey(nil) - input := Frame{ - Version: Version0, - Type: TypeVirtualSnakeSetup, - Destination: Coordinates{5, 4, 3, 2, 1}, - Payload: []byte{9, 9, 9, 9, 9, 9, 9, 9, 9, 9}, - } - copy(input.DestinationKey[:], pk1) - copy(input.SourceKey[:], pk2) - expected := []byte{ - 0x70, 0x69, 0x6e, 0x65, // magic bytes - 0, // version 0 - byte(TypeVirtualSnakeSetup), // type greedy - 0, 0, // extra - 0, 93, // frame length - 0, 10, // payload length - 0, 5, 5, 4, 3, 2, 1, // destination coordinates - } - expected = append(expected, pk2...) - expected = append(expected, pk1...) - expected = append(expected, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9) // payload - buf := make([]byte, 65535) - n, err := input.MarshalBinary(buf) - if err != nil { - t.Fatal(err) - } - if n != len(expected) { - t.Fatalf("wrong marshalled length, \ngot %d, \nexpected %d", n, len(expected)) - } - if !bytes.Equal(buf[:n], expected) { - t.Fatalf("wrong marshalled output, \ngot %v, \nexpected %v", buf[:n], expected) - } - - t.Log("Got: ", buf[:n]) - t.Log("Want:", expected) - - output := Frame{ - Payload: make([]byte, 0, MaxPayloadSize), - } - if _, err := output.UnmarshalBinary(buf[:n]); err != nil { - t.Fatal(err) - } - if output.Version != input.Version { - t.Fatal("wrong version") - } - if output.Type != input.Type { - t.Fatal("wrong version") - } - if !output.Destination.EqualTo(input.Destination) { - t.Fatal("wrong path") - } - if !bytes.Equal(input.Payload, output.Payload) { - t.Fatal("wrong payload") - } -} - -func TestMarshalUnmarshalSNEKTeardownFrame(t *testing.T) { - pk1, _, _ := ed25519.GenerateKey(nil) - input := Frame{ - Version: Version0, - Type: TypeVirtualSnakeTeardown, - } - copy(input.DestinationKey[:], pk1) - expected := []byte{ - 0x70, 0x69, 0x6e, 0x65, // magic bytes - 0, // version 0 - byte(TypeVirtualSnakeTeardown), // type greedy - 0, 0, // extra - 0, 44, // frame length - 0, 0, // payload length - } - expected = append(expected, pk1...) - buf := make([]byte, 65535) - n, err := input.MarshalBinary(buf) - if err != nil { - t.Fatal(err) - } - if n != len(expected) { - t.Fatalf("wrong marshalled length, \ngot %d, \nexpected %d", n, len(expected)) - } - if !bytes.Equal(buf[:n], expected) { - t.Fatalf("wrong marshalled output, \ngot %v, \nexpected %v", buf[:n], expected) - } - - t.Log("Got: ", buf[:n]) - t.Log("Want:", expected) - - output := Frame{ - Payload: make([]byte, 0, MaxPayloadSize), - } - if _, err := output.UnmarshalBinary(buf[:n]); err != nil { - t.Fatal(err) - } - if output.Version != input.Version { - t.Fatal("wrong version") - } - if output.Type != input.Type { - t.Fatal("wrong version") - } - if !output.Destination.EqualTo(input.Destination) { - t.Fatal("wrong path") - } - if !bytes.Equal(input.Payload, output.Payload) { - t.Fatal("wrong payload") - } -} - func TestMarshalUnmarshalSNEKFrame(t *testing.T) { pk1, _, _ := ed25519.GenerateKey(nil) pk2, _, _ := ed25519.GenerateKey(nil) From 9cacda9b56f442b28fbc4b116c11c016e6551d65 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 12 Apr 2022 12:03:48 +0100 Subject: [PATCH 02/41] Fix `pineconeip` --- router/state_snek.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/router/state_snek.go b/router/state_snek.go index 4dc9073d..af5f5e89 100644 --- a/router/state_snek.go +++ b/router/state_snek.go @@ -184,7 +184,7 @@ func getNextHopSNEK(params virtualSnakeNextHopParams) *peer { // candidate has to improve on our own key in order to forward the frame. var bestPeer *peer var bestAnn *rootAnnouncementWithTime - if !params.isTraffic { + if params.isTraffic { bestPeer = params.selfPeer } bestKey := params.publicKey From 6c5325671503205a220d1fcad43efae84455dcf1 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 12 Apr 2022 13:19:35 +0100 Subject: [PATCH 03/41] Try raising the intervals a bit --- router/state_snek.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/router/state_snek.go b/router/state_snek.go index af5f5e89..4cce189b 100644 --- a/router/state_snek.go +++ b/router/state_snek.go @@ -27,8 +27,8 @@ import ( // NOTE: Functions prefixed with an underscore (_) are only safe to be called // from the actor that owns them, in order to prevent data races. -const virtualSnakeMaintainInterval = time.Second * 2 -const virtualSnakeNeighExpiryPeriod = time.Second * 5 +const virtualSnakeMaintainInterval = time.Second * 5 +const virtualSnakeNeighExpiryPeriod = virtualSnakeMaintainInterval * 2 type virtualSnakeTable map[virtualSnakeIndex]*virtualSnakeEntry From a6f5f7126be88672751260a95591a8ab1b21dacc Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 12 Apr 2022 13:44:22 +0100 Subject: [PATCH 04/41] Tighter expiry --- router/state_snek.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/router/state_snek.go b/router/state_snek.go index 4cce189b..01723b50 100644 --- a/router/state_snek.go +++ b/router/state_snek.go @@ -53,10 +53,6 @@ func (e *virtualSnakeEntry) valid() bool { return time.Since(e.LastSeen) < virtualSnakeNeighExpiryPeriod } -func (e *virtualSnakeEntry) validForForwarding() bool { - return time.Since(e.LastSeen) < (virtualSnakeNeighExpiryPeriod * 2) -} - // _maintainSnake is responsible for working out if we need to send bootstraps // or to clean up any old paths. func (s *state) _maintainSnake() { @@ -261,7 +257,7 @@ func getNextHopSNEK(params virtualSnakeNextHopParams) *peer { // higher one, this is effectively looking for paths that descend through // keyspace toward lower keys rather than ascend toward higher ones. for _, entry := range params.snakeRoutes { - if !entry.Source.started.Load() || !entry.validForForwarding() || entry.Source == params.selfPeer { + if !entry.Source.started.Load() || !entry.valid() || entry.Source == params.selfPeer { continue } if !params.isBootstrap && !entry.Active { From 9e0e66acb867f27d24792af71943368fbea52ec1 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 12 Apr 2022 14:19:36 +0100 Subject: [PATCH 05/41] Tweaks --- router/state_forward.go | 2 +- router/state_snek.go | 7 ------- router/state_snek_test.go | 12 ++++++------ 3 files changed, 7 insertions(+), 14 deletions(-) diff --git a/router/state_forward.go b/router/state_forward.go index b563ab43..7ff71ee4 100644 --- a/router/state_forward.go +++ b/router/state_forward.go @@ -49,7 +49,7 @@ func (s *state) _forward(p *peer, f *types.Frame) error { } nexthop := s._nextHopsFor(p, f) - deadend := nexthop == p.router.local + deadend := nexthop == nil || nexthop == p.router.local switch f.Type { case types.TypeTreeAnnouncement: diff --git a/router/state_snek.go b/router/state_snek.go index 01723b50..70231515 100644 --- a/router/state_snek.go +++ b/router/state_snek.go @@ -39,11 +39,9 @@ type virtualSnakeIndex struct { type virtualSnakeEntry struct { *virtualSnakeIndex Origin types.PublicKey `json:"origin"` - Target types.PublicKey `json:"target"` Source *peer `json:"source"` LastSeen time.Time `json:"last_seen"` Root types.Root `json:"root"` - Active bool `json:"active"` } // valid returns true if the update hasn't expired, or false if it has. It is @@ -260,9 +258,6 @@ func getNextHopSNEK(params virtualSnakeNextHopParams) *peer { if !entry.Source.started.Load() || !entry.valid() || entry.Source == params.selfPeer { continue } - if !params.isBootstrap && !entry.Active { - continue - } newCheckedCandidate(entry.PublicKey, entry.Source) } @@ -321,11 +316,9 @@ func (s *state) _handleBootstrap(from *peer, rx *types.Frame) error { s._table[index] = &virtualSnakeEntry{ virtualSnakeIndex: &index, Origin: rx.DestinationKey, - Target: rx.DestinationKey, Source: from, LastSeen: time.Now(), Root: bootstrap.Root, - Active: true, } // Now let's see if this is a suitable ascending entry. diff --git a/router/state_snek_test.go b/router/state_snek_test.go index 74b69b03..5820b95e 100644 --- a/router/state_snek_test.go +++ b/router/state_snek_test.go @@ -245,9 +245,9 @@ func TestSNEKNextHopSelection(t *testing.T) { }, virtualSnakeTable{ virtualSnakeIndex{}: &virtualSnakeEntry{ - Source: peers[3], - LastSeen: time.Now(), - Active: true, + Source: peers[3], + LastSeen: time.Now(), + // Active: true, virtualSnakeIndex: &virtualSnakeIndex{PublicKey: destDownKey}, }}, }, peers[3]}, @@ -264,9 +264,9 @@ func TestSNEKNextHopSelection(t *testing.T) { }, virtualSnakeTable{ virtualSnakeIndex{}: &virtualSnakeEntry{ - Source: peers[3], - LastSeen: time.Now(), - Active: true, + Source: peers[3], + LastSeen: time.Now(), + // Active: true, virtualSnakeIndex: &virtualSnakeIndex{PublicKey: destDownKey}, }}, }, peers[0]}, // handle a bootstrap received from a lower key node From 954442130d9268d1c9b4469cf6411f1363a49b5a Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 12 Apr 2022 14:46:00 +0100 Subject: [PATCH 06/41] Try to speed up convergence but reduce idle noise --- router/state.go | 5 +++++ router/state_snek.go | 21 +++++++++++---------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/router/state.go b/router/state.go index 777dbd01..a21d1dfd 100644 --- a/router/state.go +++ b/router/state.go @@ -46,6 +46,7 @@ type state struct { _sequence uint64 // Used to sequence our root tree announcements _treetimer *time.Timer // Tree maintenance timer _snaketimer *time.Timer // Virtual snake maintenance timer + _lastbootstrap time.Time // When did we last bootstrap? _waiting bool // Is the tree waiting to reparent? _filterPacket FilterFn // Function called when forwarding packets } @@ -171,6 +172,10 @@ func (s *state) _setParent(peer *peer) { } func (s *state) _setDescendingNode(node *virtualSnakeEntry) { + if s._descending != node { + defer s._bootstrapNow() + } + s._descending = node s.r.Act(nil, func() { diff --git a/router/state_snek.go b/router/state_snek.go index 70231515..f270e6d0 100644 --- a/router/state_snek.go +++ b/router/state_snek.go @@ -27,8 +27,9 @@ import ( // NOTE: Functions prefixed with an underscore (_) are only safe to be called // from the actor that owns them, in order to prevent data races. -const virtualSnakeMaintainInterval = time.Second * 5 -const virtualSnakeNeighExpiryPeriod = virtualSnakeMaintainInterval * 2 +const virtualSnakeMaintainInterval = time.Second +const virtualSnakeBootstrapInterval = time.Second * 5 +const virtualSnakeNeighExpiryPeriod = virtualSnakeBootstrapInterval * 2 type virtualSnakeTable map[virtualSnakeIndex]*virtualSnakeEntry @@ -38,10 +39,9 @@ type virtualSnakeIndex struct { type virtualSnakeEntry struct { *virtualSnakeIndex - Origin types.PublicKey `json:"origin"` - Source *peer `json:"source"` - LastSeen time.Time `json:"last_seen"` - Root types.Root `json:"root"` + Source *peer `json:"source"` + LastSeen time.Time `json:"last_seen"` + Root types.Root `json:"root"` } // valid returns true if the update hasn't expired, or false if it has. It is @@ -85,9 +85,10 @@ func (s *state) _maintainSnake() { } } - // If one of the previous conditions means that we need to bootstrap, then - // send the actual bootstrap message into the network. - s._bootstrapNow() + // Send a new bootstrap. + if time.Since(s._lastbootstrap) >= virtualSnakeBootstrapInterval { + s._bootstrapNow() + } } // _bootstrapNow is responsible for sending a bootstrap message to the network. @@ -135,6 +136,7 @@ func (s *state) _bootstrapNow() { // bootstrap packets. if p := s._nextHopsSNEK(send, true); p != nil && p.proto != nil { p.proto.push(send) + s._lastbootstrap = time.Now() } } @@ -315,7 +317,6 @@ func (s *state) _handleBootstrap(from *peer, rx *types.Frame) error { } s._table[index] = &virtualSnakeEntry{ virtualSnakeIndex: &index, - Origin: rx.DestinationKey, Source: from, LastSeen: time.Now(), Root: bootstrap.Root, From 1eafa35efcf655a831e3d24e46c926fe19070014 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 12 Apr 2022 14:57:09 +0100 Subject: [PATCH 07/41] More tweaks --- router/state.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/router/state.go b/router/state.go index a21d1dfd..cdbb267b 100644 --- a/router/state.go +++ b/router/state.go @@ -172,7 +172,10 @@ func (s *state) _setParent(peer *peer) { } func (s *state) _setDescendingNode(node *virtualSnakeEntry) { - if s._descending != node { + switch { + case node == nil: + fallthrough + case s._descending != nil && s._descending.PublicKey != node.PublicKey: defer s._bootstrapNow() } @@ -195,7 +198,6 @@ func (s *state) _setDescendingNode(node *virtualSnakeEntry) { // _portDisconnected is called when a peer disconnects. func (s *state) _portDisconnected(peer *peer) { peercount := 0 - bootstrap := false // Work out how many peers are connected now that this peer has // disconnected. @@ -230,18 +232,14 @@ func (s *state) _portDisconnected(peer *peer) { // peering then clear that path (although we can't send a teardown) and // wait for another incoming setup. if desc := s._descending; desc != nil && desc.Source == peer { - s._descending = nil + s._setDescendingNode(nil) } // If the peer that died was our chosen tree parent, then we will need to // select a new parent. If we successfully choose a new parent (as in, we // don't end up promoting ourselves to a root) then we will also need to // send a new bootstrap into the network. - if s._parent == peer { - bootstrap = bootstrap || s._selectNewParent() - } - - if bootstrap { + if s._parent == peer && s._selectNewParent() { s._bootstrapNow() } } From e980db4ba3be047ff2d2ee801057c0c12471e6cc Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 12 Apr 2022 15:11:42 +0100 Subject: [PATCH 08/41] Handle disconnections on destination ports, tweak next-hops --- router/state.go | 4 ++-- router/state_forward.go | 2 +- router/state_snek.go | 12 +++++++----- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/router/state.go b/router/state.go index cdbb267b..0aaa37b4 100644 --- a/router/state.go +++ b/router/state.go @@ -173,7 +173,7 @@ func (s *state) _setParent(peer *peer) { func (s *state) _setDescendingNode(node *virtualSnakeEntry) { switch { - case node == nil: + case s._descending == nil || node == nil: fallthrough case s._descending != nil && s._descending.PublicKey != node.PublicKey: defer s._bootstrapNow() @@ -223,7 +223,7 @@ func (s *state) _portDisconnected(peer *peer) { // direction, so that nodes further along the path will learn that the // path was broken. for k, v := range s._table { - if v.Source == peer { + if v.Source == peer || v.Destination == peer { delete(s._table, k) } } diff --git a/router/state_forward.go b/router/state_forward.go index 7ff71ee4..485db42d 100644 --- a/router/state_forward.go +++ b/router/state_forward.go @@ -67,7 +67,7 @@ func (s *state) _forward(p *peer, f *types.Frame) error { case types.TypeVirtualSnakeBootstrap: // Bootstrap messages are only handled specially when they reach a dead end. // Otherwise they are forwarded normally by falling through. - if err := s._handleBootstrap(p, f); err != nil { + if err := s._handleBootstrap(p, nexthop, f); err != nil { return fmt.Errorf("s._handleBootstrap (port %d): %w", p.port, err) } if deadend { diff --git a/router/state_snek.go b/router/state_snek.go index f270e6d0..694c8e8e 100644 --- a/router/state_snek.go +++ b/router/state_snek.go @@ -39,9 +39,10 @@ type virtualSnakeIndex struct { type virtualSnakeEntry struct { *virtualSnakeIndex - Source *peer `json:"source"` - LastSeen time.Time `json:"last_seen"` - Root types.Root `json:"root"` + Source *peer `json:"source"` + Destination *peer `json:"destination"` + LastSeen time.Time `json:"last_seen"` + Root types.Root `json:"root"` } // valid returns true if the update hasn't expired, or false if it has. It is @@ -257,7 +258,7 @@ func getNextHopSNEK(params virtualSnakeNextHopParams) *peer { // higher one, this is effectively looking for paths that descend through // keyspace toward lower keys rather than ascend toward higher ones. for _, entry := range params.snakeRoutes { - if !entry.Source.started.Load() || !entry.valid() || entry.Source == params.selfPeer { + if !entry.Source.started.Load() || !entry.valid() { continue } newCheckedCandidate(entry.PublicKey, entry.Source) @@ -286,7 +287,7 @@ func getNextHopSNEK(params virtualSnakeNextHopParams) *peer { // _handleBootstrap is called in response to receiving a bootstrap packet. // This function will send a bootstrap ACK back to the sender. -func (s *state) _handleBootstrap(from *peer, rx *types.Frame) error { +func (s *state) _handleBootstrap(from, to *peer, rx *types.Frame) error { // Unmarshal the bootstrap. var bootstrap types.VirtualSnakeBootstrap if _, err := bootstrap.UnmarshalBinary(rx.Payload); err != nil { @@ -318,6 +319,7 @@ func (s *state) _handleBootstrap(from *peer, rx *types.Frame) error { s._table[index] = &virtualSnakeEntry{ virtualSnakeIndex: &index, Source: from, + Destination: to, LastSeen: time.Now(), Root: bootstrap.Root, } From 3153da525b852e7c26188920d9981c7564bb9b0a Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 12 Apr 2022 15:35:12 +0100 Subject: [PATCH 09/41] All traffic is either bootstrap or not, more tweaking --- router/state_snek.go | 8 +++----- router/state_snek_test.go | 26 +++++++++++++------------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/router/state_snek.go b/router/state_snek.go index 694c8e8e..1dae3357 100644 --- a/router/state_snek.go +++ b/router/state_snek.go @@ -137,13 +137,12 @@ func (s *state) _bootstrapNow() { // bootstrap packets. if p := s._nextHopsSNEK(send, true); p != nil && p.proto != nil { p.proto.push(send) - s._lastbootstrap = time.Now() } + s._lastbootstrap = time.Now() } type virtualSnakeNextHopParams struct { isBootstrap bool - isTraffic bool destinationKey types.PublicKey publicKey types.PublicKey parentPeer *peer @@ -158,8 +157,7 @@ type virtualSnakeNextHopParams struct { // specific rules — this should only be used for VirtualSnakeBootstrap frames. func (s *state) _nextHopsSNEK(rx *types.Frame, bootstrap bool) *peer { return getNextHopSNEK(virtualSnakeNextHopParams{ - bootstrap, - rx.Type == types.TypeVirtualSnakeRouted || rx.Type == types.TypeTreeRouted, + rx.Type == types.TypeVirtualSnakeBootstrap, rx.DestinationKey, s.r.public, s._parent, @@ -181,7 +179,7 @@ func getNextHopSNEK(params virtualSnakeNextHopParams) *peer { // candidate has to improve on our own key in order to forward the frame. var bestPeer *peer var bestAnn *rootAnnouncementWithTime - if params.isTraffic { + if !params.isBootstrap { bestPeer = params.selfPeer } bestKey := params.publicKey diff --git a/router/state_snek_test.go b/router/state_snek_test.go index 5820b95e..25314d0c 100644 --- a/router/state_snek_test.go +++ b/router/state_snek_test.go @@ -86,7 +86,7 @@ func TestSNEKNextHopSelection(t *testing.T) { }{ {"TestNotBootstrapNoValidNextHop", virtualSnakeNextHopParams{ false, - false, + //false, destUpKey, selfKey, peers[1], @@ -99,7 +99,7 @@ func TestSNEKNextHopSelection(t *testing.T) { }, peers[1]}, // default peer with no next hop is parent {"TestBootstrapNoValidNextHop", virtualSnakeNextHopParams{ true, - false, + //false, destUpKey, selfKey, peers[1], @@ -112,7 +112,7 @@ func TestSNEKNextHopSelection(t *testing.T) { }, peers[1]}, // default bootstrap peer with no next hop is parent {"TestNotBootstrapDestIsSelf", virtualSnakeNextHopParams{ false, - false, + //false, destUpKey, destUpKey, peers[1], @@ -126,7 +126,7 @@ func TestSNEKNextHopSelection(t *testing.T) { }, peers[0]}, {"TestBootstrapDestIsSelf", virtualSnakeNextHopParams{ true, - false, + //false, destUpKey, destUpKey, peers[1], @@ -139,7 +139,7 @@ func TestSNEKNextHopSelection(t *testing.T) { }, peers[1]}, // bootstraps always start working towards root via parent {"TestNotBootstrapPeerIsDestination", virtualSnakeNextHopParams{ false, - false, + //false, destUpKey, selfKey, peers[1], @@ -153,7 +153,7 @@ func TestSNEKNextHopSelection(t *testing.T) { }, peers[2]}, {"TestBootstrapPeerIsDestination", virtualSnakeNextHopParams{ true, - false, + //false, destUpKey, selfKey, peers[1], @@ -167,7 +167,7 @@ func TestSNEKNextHopSelection(t *testing.T) { }, peers[1]}, // bootstraps work their way toward the root {"TestNotBootstrapParentKnowsDestination", virtualSnakeNextHopParams{ false, - false, + //false, destUpKey, selfKey, peers[1], @@ -180,7 +180,7 @@ func TestSNEKNextHopSelection(t *testing.T) { }, peers[1]}, {"TestNotBootstrapPeerKnowsDestination", virtualSnakeNextHopParams{ false, - false, + //false, destUpKey, selfKey, peers[1], @@ -194,7 +194,7 @@ func TestSNEKNextHopSelection(t *testing.T) { }, peers[2]}, {"TestBootstrapPeerKnowsDestination", virtualSnakeNextHopParams{ true, - false, + //false, destUpKey, selfKey, peers[1], @@ -208,7 +208,7 @@ func TestSNEKNextHopSelection(t *testing.T) { }, peers[1]}, // bootstraps work their way toward the root {"TestNotBootstrapParentKnowsCloser", virtualSnakeNextHopParams{ false, - false, + //false, destUpKey, selfKey, peers[1], @@ -221,7 +221,7 @@ func TestSNEKNextHopSelection(t *testing.T) { }, peers[1]}, {"TestBootstrapParentKnowsCloser", virtualSnakeNextHopParams{ true, - false, + //false, destUpKey, selfKey, peers[1], @@ -234,7 +234,7 @@ func TestSNEKNextHopSelection(t *testing.T) { }, peers[1]}, {"TestNotBootstrapSnakeEntryIsDest", virtualSnakeNextHopParams{ false, - false, + //false, destDownKey, selfKey, peers[1], @@ -253,7 +253,7 @@ func TestSNEKNextHopSelection(t *testing.T) { }, peers[3]}, {"TestBootstrapSnakeEntryIsDest", virtualSnakeNextHopParams{ true, - false, + //false, destDownKey, selfKey, peers[1], From db28a008ac80f65e276358cb917b88edb28ecc87 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 14 Apr 2022 15:18:37 +0100 Subject: [PATCH 10/41] Protocol calming, loop detection, bootstrap signatures --- router/api.go | 4 + router/packetconn.go | 5 ++ router/state.go | 6 +- router/state_forward.go | 31 ++++--- router/state_snek.go | 132 +++++++++++++++++++---------- router/state_snek_test.go | 43 ++++++---- router/state_tree.go | 4 +- types/ed25519.go | 5 ++ types/frame.go | 39 ++++++--- types/frame_test.go | 38 +++++++-- types/varu64.go | 4 + types/varu64_test.go | 58 +++++++++---- types/virtualsnake.go | 172 +++++++------------------------------- 13 files changed, 292 insertions(+), 249 deletions(-) diff --git a/router/api.go b/router/api.go index 1b3f6c35..b1ec9892 100644 --- a/router/api.go +++ b/router/api.go @@ -84,6 +84,10 @@ func (r *Router) Ping(ctx context.Context, a net.Addr) (uint16, time.Duration, e frame.Type = types.TypeSNEKPing frame.DestinationKey = dst frame.SourceKey = r.public + frame.Watermark = types.VirtualSnakeWatermark{ + PublicKey: types.FullMask, + Sequence: 0, + } _ = r.state._forward(r.local, frame) }) diff --git a/router/packetconn.go b/router/packetconn.go index b4bbbbd1..62bc048f 100644 --- a/router/packetconn.go +++ b/router/packetconn.go @@ -100,8 +100,13 @@ func (r *Router) WriteTo(p []byte, addr net.Addr) (n int, err error) { frame := getFrame() frame.Type = types.TypeVirtualSnakeRouted frame.DestinationKey = ga + frame.Watermark.PublicKey = types.FullMask frame.SourceKey = r.public frame.Payload = append(frame.Payload[:0], p...) + frame.Watermark = types.VirtualSnakeWatermark{ + PublicKey: types.FullMask, + Sequence: 0, + } _ = r.state._forward(r.local, frame) }) return len(p), nil diff --git a/router/state.go b/router/state.go index 0aaa37b4..e25a6410 100644 --- a/router/state.go +++ b/router/state.go @@ -175,8 +175,8 @@ func (s *state) _setDescendingNode(node *virtualSnakeEntry) { switch { case s._descending == nil || node == nil: fallthrough - case s._descending != nil && s._descending.PublicKey != node.PublicKey: - defer s._bootstrapNow() + case s._descending != nil && node != nil && s._descending.PublicKey != node.PublicKey: + s._bootstrapSoon() } s._descending = node @@ -240,6 +240,6 @@ func (s *state) _portDisconnected(peer *peer) { // don't end up promoting ourselves to a root) then we will also need to // send a new bootstrap into the network. if s._parent == peer && s._selectNewParent() { - s._bootstrapNow() + s._bootstrapSoon() } } diff --git a/router/state_forward.go b/router/state_forward.go index 485db42d..da1caf39 100644 --- a/router/state_forward.go +++ b/router/state_forward.go @@ -24,18 +24,19 @@ import ( // _nextHopsFor returns the next-hop for the given frame. It will examine the packet // type and use the correct routing algorithm to determine the next-hop. It is possible // for this function to return `nil` if there is no suitable candidate. -func (s *state) _nextHopsFor(from *peer, frame *types.Frame) *peer { +func (s *state) _nextHopsFor(from *peer, frame *types.Frame) (*peer, types.VirtualSnakeWatermark) { var nexthop *peer + var watermark types.VirtualSnakeWatermark switch frame.Type { // SNEK routing case types.TypeVirtualSnakeRouted, types.TypeVirtualSnakeBootstrap, types.TypeSNEKPing, types.TypeSNEKPong: - nexthop = s._nextHopsSNEK(frame, frame.Type == types.TypeVirtualSnakeBootstrap) + nexthop, watermark = s._nextHopsSNEK(from, frame, frame.Type == types.TypeVirtualSnakeBootstrap) // Tree routing case types.TypeTreeRouted, types.TypeTreePing, types.TypeTreePong: nexthop = s._nextHopsTree(from, frame) } - return nexthop + return nexthop, watermark } // _forward handles frames received from a given peer. In most cases, this function will @@ -48,7 +49,7 @@ func (s *state) _forward(p *peer, f *types.Frame) error { return nil } - nexthop := s._nextHopsFor(p, f) + nexthop, watermark := s._nextHopsFor(p, f) deadend := nexthop == nil || nexthop == p.router.local switch f.Type { @@ -67,10 +68,7 @@ func (s *state) _forward(p *peer, f *types.Frame) error { case types.TypeVirtualSnakeBootstrap: // Bootstrap messages are only handled specially when they reach a dead end. // Otherwise they are forwarded normally by falling through. - if err := s._handleBootstrap(p, nexthop, f); err != nil { - return fmt.Errorf("s._handleBootstrap (port %d): %w", p.port, err) - } - if deadend { + if !s._handleBootstrap(p, nexthop, f) || deadend { return nil } @@ -88,7 +86,11 @@ func (s *state) _forward(p *peer, f *types.Frame) error { f.DestinationKey = of.SourceKey f.SourceKey = s.r.public f.Extra = of.Extra - nexthop = s._nextHopsFor(s.r.local, f) + f.Watermark = types.VirtualSnakeWatermark{ + PublicKey: types.FullMask, + Sequence: 0, + } + nexthop, watermark = s._nextHopsFor(s.r.local, f) } else { hops := binary.BigEndian.Uint16(f.Extra[:]) hops++ @@ -118,7 +120,7 @@ func (s *state) _forward(p *peer, f *types.Frame) error { f.Destination = append(f.Destination[:0], of.Source...) f.Source = append(f.Source[:0], s._coords()...) f.Extra = of.Extra - nexthop = s._nextHopsFor(s.r.local, f) + nexthop, watermark = s._nextHopsFor(s.r.local, f) } else { hops := binary.BigEndian.Uint16(f.Extra[:]) hops++ @@ -140,9 +142,18 @@ func (s *state) _forward(p *peer, f *types.Frame) error { } } + // If the packet's watermark is higher than the previous one or we are + // obviously looping, drop the packet. + if nexthop == p || watermark.WorseThan(f.Watermark) { + return nil + } + // If there's a suitable next-hop then try sending the packet. If we fail // to queue up the packet then we will log it but there isn't an awful lot // we can do at this point. + if watermark.Sequence > 0 { + f.Watermark = watermark + } if nexthop != nil && !nexthop.send(f) { s.r.log.Println("Dropping forwarded packet of type", f.Type) } diff --git a/router/state_snek.go b/router/state_snek.go index 1dae3357..43d074d2 100644 --- a/router/state_snek.go +++ b/router/state_snek.go @@ -16,8 +16,6 @@ package router import ( "crypto/ed25519" - "crypto/rand" - "fmt" "time" "github.com/matrix-org/pinecone/types" @@ -39,10 +37,11 @@ type virtualSnakeIndex struct { type virtualSnakeEntry struct { *virtualSnakeIndex - Source *peer `json:"source"` - Destination *peer `json:"destination"` - LastSeen time.Time `json:"last_seen"` - Root types.Root `json:"root"` + Source *peer `json:"source"` + Destination *peer `json:"destination"` + Watermark types.VirtualSnakeWatermark `json:"watermark"` + LastSeen time.Time `json:"last_seen"` + Root types.Root `json:"root"` } // valid returns true if the update hasn't expired, or false if it has. It is @@ -92,6 +91,13 @@ func (s *state) _maintainSnake() { } } +// _bootstrapSoon will reset the bootstrap timer so that we will bootstrap on +// the next maintenance interval. This is better than calling _bootstrapNow +// directly which might cause more protocol traffic than necessary. +func (s *state) _bootstrapSoon() { + s._lastbootstrap = time.Now().Add(-virtualSnakeBootstrapInterval) +} + // _bootstrapNow is responsible for sending a bootstrap message to the network. func (s *state) _bootstrapNow() { // If we are the root node then there's no point in trying to bootstrap. We @@ -107,18 +113,17 @@ func (s *state) _bootstrapNow() { b := frameBufferPool.Get().(*[types.MaxFrameSize]byte) defer frameBufferPool.Put(b) bootstrap := types.VirtualSnakeBootstrap{ - Root: ann.Root, - } - // Generate a random path ID. - if _, err := rand.Read(bootstrap.PathID[:]); err != nil { - return + Root: ann.Root, + Sequence: types.Varu64(time.Now().UnixMilli()), } if s.r.secure { - // Sign the path key and path ID with our own key. This forms the "source - // signature", which anyone can use to verify that we sent the bootstrap. + protected, err := bootstrap.ProtectedPayload() + if err != nil { + return + } copy( - bootstrap.SourceSig[:], - ed25519.Sign(s.r.private[:], append(s.r.public[:], bootstrap.PathID[:]...)), + bootstrap.Signature[:], + ed25519.Sign(s.r.private[:], protected), ) } n, err := bootstrap.MarshalBinary(b[:]) @@ -133,18 +138,25 @@ func (s *state) _bootstrapNow() { send.DestinationKey = s.r.public send.Source = s._coords() send.Payload = append(send.Payload[:0], b[:n]...) + send.Watermark = types.VirtualSnakeWatermark{ + PublicKey: types.FullMask, + Sequence: 0, + } // Bootstrap messages are routed using SNEK routing with special rules for // bootstrap packets. - if p := s._nextHopsSNEK(send, true); p != nil && p.proto != nil { + if p, w := s._nextHopsSNEK(s.r.local, send, true); p != nil && p.proto != nil { + send.Watermark = w p.proto.push(send) } s._lastbootstrap = time.Now() } type virtualSnakeNextHopParams struct { + from *peer isBootstrap bool destinationKey types.PublicKey publicKey types.PublicKey + watermark types.VirtualSnakeWatermark parentPeer *peer selfPeer *peer lastAnnouncement *rootAnnouncementWithTime @@ -155,11 +167,13 @@ type virtualSnakeNextHopParams struct { // _nextHopsSNEK locates the best next-hop for a given SNEK-routed frame. The // bootstrap flag determines whether the frame should be routed using bootstrap // specific rules — this should only be used for VirtualSnakeBootstrap frames. -func (s *state) _nextHopsSNEK(rx *types.Frame, bootstrap bool) *peer { +func (s *state) _nextHopsSNEK(from *peer, rx *types.Frame, bootstrap bool) (*peer, types.VirtualSnakeWatermark) { return getNextHopSNEK(virtualSnakeNextHopParams{ + from, rx.Type == types.TypeVirtualSnakeBootstrap, rx.DestinationKey, s.r.public, + rx.Watermark, s._parent, s.r.local, s._rootAnnouncement(), @@ -168,35 +182,37 @@ func (s *state) _nextHopsSNEK(rx *types.Frame, bootstrap bool) *peer { }) } -func getNextHopSNEK(params virtualSnakeNextHopParams) *peer { +func getNextHopSNEK(params virtualSnakeNextHopParams) (*peer, types.VirtualSnakeWatermark) { // If the message isn't a bootstrap message and the destination is for our // own public key, handle the frame locally — it's basically loopback. if !params.isBootstrap && params.publicKey == params.destinationKey { - return params.selfPeer + return params.selfPeer, params.watermark } // We start off with our own key as the best key. Any suitable next-hop // candidate has to improve on our own key in order to forward the frame. var bestPeer *peer var bestAnn *rootAnnouncementWithTime + var bestSeq types.Varu64 if !params.isBootstrap { bestPeer = params.selfPeer } bestKey := params.publicKey destKey := params.destinationKey + watermark := params.watermark // newCandidate updates the best key and best peer with new candidates. - newCandidate := func(key types.PublicKey, p *peer) { - bestKey, bestPeer, bestAnn = key, p, params.peerAnnouncements[p] + newCandidate := func(key types.PublicKey, seq types.Varu64, p *peer) { + bestKey, bestSeq, bestPeer, bestAnn = key, seq, p, params.peerAnnouncements[p] } // newCheckedCandidate performs some sanity checks on the candidate before // passing it to newCandidate. - newCheckedCandidate := func(candidate types.PublicKey, p *peer) { + newCheckedCandidate := func(candidate types.PublicKey, seq types.Varu64, p *peer) { switch { case !params.isBootstrap && candidate == destKey && bestKey != destKey: - newCandidate(candidate, p) + newCandidate(candidate, seq, p) case util.DHTOrdered(destKey, candidate, bestKey): - newCandidate(candidate, p) + newCandidate(candidate, seq, p) } } @@ -212,14 +228,14 @@ func getNextHopSNEK(params virtualSnakeNextHopParams) *peer { case util.DHTOrdered(bestKey, destKey, params.lastAnnouncement.RootPublicKey): // The destination key is higher than our own key, so start using // the path to the root as the first candidate. - newCandidate(params.lastAnnouncement.RootPublicKey, params.parentPeer) + newCandidate(params.lastAnnouncement.RootPublicKey, 0, params.parentPeer) } // Check our direct ancestors in the tree, that is, all nodes between // ourselves and the root node via the parent port. if ann := params.peerAnnouncements[params.parentPeer]; ann != nil { for _, ancestor := range ann.Signatures { - newCheckedCandidate(ancestor.PublicKey, params.parentPeer) + newCheckedCandidate(ancestor.PublicKey, 0, params.parentPeer) } } } @@ -231,7 +247,7 @@ func getNextHopSNEK(params virtualSnakeNextHopParams) *peer { continue } for _, hop := range ann.Signatures { - newCheckedCandidate(hop.PublicKey, p) + newCheckedCandidate(hop.PublicKey, 0, p) } } @@ -247,7 +263,7 @@ func getNextHopSNEK(params virtualSnakeNextHopParams) *peer { if peerKey := p.public; bestKey == peerKey { // We've seen this key already and we are directly peered, so use // the peering instead of the previous selected port. - newCandidate(peerKey, p) + newCandidate(bestKey, 0, p) } } @@ -259,7 +275,10 @@ func getNextHopSNEK(params virtualSnakeNextHopParams) *peer { if !entry.Source.started.Load() || !entry.valid() { continue } - newCheckedCandidate(entry.PublicKey, entry.Source) + if entry.Watermark.WorseThan(watermark) { + continue + } + newCheckedCandidate(entry.PublicKey, entry.Watermark.Sequence, entry.Source) } // Finally, be sure that we're using the best-looking path to our next-hop. @@ -272,34 +291,50 @@ func getNextHopSNEK(params virtualSnakeNextHopParams) *peer { continue case p.peertype < bestPeer.peertype: // Prefer faster classes of links if possible. - newCandidate(peerKey, p) + newCandidate(bestKey, bestSeq, p) case p.peertype == bestPeer.peertype && ann.receiveOrder < bestAnn.receiveOrder: // Prefer links that have the lowest latency to the root. - newCandidate(peerKey, p) + newCandidate(bestKey, bestSeq, p) } } } - return bestPeer + // Only SNEK paths will have a sequence number higher than 0, so + // it's a safe bet that if it's greater than 0, we have hit upon + // a newly watermarkable path. + if bestSeq > 0 { + watermark = types.VirtualSnakeWatermark{ + PublicKey: bestKey, + Sequence: bestSeq, + } + } + + return bestPeer, watermark } // _handleBootstrap is called in response to receiving a bootstrap packet. -// This function will send a bootstrap ACK back to the sender. -func (s *state) _handleBootstrap(from, to *peer, rx *types.Frame) error { +// This function will send a bootstrap ACK back to the sender. Returns true +// if the bootstrap was handled and false otherwise. +func (s *state) _handleBootstrap(from, to *peer, rx *types.Frame) bool { // Unmarshal the bootstrap. var bootstrap types.VirtualSnakeBootstrap - if _, err := bootstrap.UnmarshalBinary(rx.Payload); err != nil { - return fmt.Errorf("bootstrap.UnmarshalBinary: %w", err) + _, err := bootstrap.UnmarshalBinary(rx.Payload) + if err != nil { + return false } if s.r.secure { - // Check that the bootstrap message was signed by the node that claims + // Check that the bootstrap message was protected by the node that claims // to have sent it. Silently drop it if there's a signature problem. + protected, err := bootstrap.ProtectedPayload() + if err != nil { + return false + } if !ed25519.Verify( rx.DestinationKey[:], - append(rx.DestinationKey[:], bootstrap.PathID[:]...), - bootstrap.SourceSig[:], + protected, + bootstrap.Signature[:], ) { - return nil + return false } } // Check that the root key and sequence number in the update match our @@ -307,19 +342,32 @@ func (s *state) _handleBootstrap(from, to *peer, rx *types.Frame) error { // tree routing anyway. If they don't match, silently drop the bootstrap. root := s._rootAnnouncement() if !root.Root.EqualTo(&bootstrap.Root) { - return nil + return false } // Create a routing table entry. index := virtualSnakeIndex{ PublicKey: rx.DestinationKey, } + if existing, ok := s._table[index]; ok { + switch { + case !existing.Root.EqualTo(&bootstrap.Root): + break // the root is different + case bootstrap.Sequence <= existing.Watermark.Sequence: + // TODO: less than-equal to might not be the right thing to do + return false + } + } s._table[index] = &virtualSnakeEntry{ virtualSnakeIndex: &index, Source: from, Destination: to, LastSeen: time.Now(), Root: bootstrap.Root, + Watermark: types.VirtualSnakeWatermark{ + PublicKey: index.PublicKey, + Sequence: bootstrap.Sequence, + }, } // Now let's see if this is a suitable ascending entry. @@ -358,5 +406,5 @@ func (s *state) _handleBootstrap(from, to *peer, rx *types.Frame) error { if update { s._setDescendingNode(s._table[index]) } - return nil + return true } diff --git a/router/state_snek_test.go b/router/state_snek_test.go index 25314d0c..5aed265a 100644 --- a/router/state_snek_test.go +++ b/router/state_snek_test.go @@ -85,10 +85,11 @@ func TestSNEKNextHopSelection(t *testing.T) { expected *peer // index into peer list }{ {"TestNotBootstrapNoValidNextHop", virtualSnakeNextHopParams{ + nil, false, - //false, destUpKey, selfKey, + types.VirtualSnakeWatermark{PublicKey: types.FullMask, Sequence: 0}, peers[1], peers[0], &selfAnn, @@ -98,10 +99,11 @@ func TestSNEKNextHopSelection(t *testing.T) { virtualSnakeTable{}, }, peers[1]}, // default peer with no next hop is parent {"TestBootstrapNoValidNextHop", virtualSnakeNextHopParams{ - true, - //false, + nil, + false, destUpKey, selfKey, + types.VirtualSnakeWatermark{PublicKey: types.FullMask, Sequence: 0}, peers[1], peers[0], &selfAnn, @@ -111,10 +113,11 @@ func TestSNEKNextHopSelection(t *testing.T) { virtualSnakeTable{}, }, peers[1]}, // default bootstrap peer with no next hop is parent {"TestNotBootstrapDestIsSelf", virtualSnakeNextHopParams{ + nil, false, - //false, destUpKey, destUpKey, + types.VirtualSnakeWatermark{PublicKey: types.FullMask, Sequence: 0}, peers[1], peers[0], &selfAnn, @@ -125,10 +128,11 @@ func TestSNEKNextHopSelection(t *testing.T) { virtualSnakeTable{}, }, peers[0]}, {"TestBootstrapDestIsSelf", virtualSnakeNextHopParams{ + nil, true, - //false, destUpKey, destUpKey, + types.VirtualSnakeWatermark{PublicKey: types.FullMask, Sequence: 0}, peers[1], peers[0], &selfAnn, @@ -138,10 +142,11 @@ func TestSNEKNextHopSelection(t *testing.T) { virtualSnakeTable{}, }, peers[1]}, // bootstraps always start working towards root via parent {"TestNotBootstrapPeerIsDestination", virtualSnakeNextHopParams{ + nil, false, - //false, destUpKey, selfKey, + types.VirtualSnakeWatermark{PublicKey: types.FullMask, Sequence: 0}, peers[1], peers[0], &selfAnn, @@ -152,10 +157,11 @@ func TestSNEKNextHopSelection(t *testing.T) { virtualSnakeTable{}, }, peers[2]}, {"TestBootstrapPeerIsDestination", virtualSnakeNextHopParams{ + nil, true, - //false, destUpKey, selfKey, + types.VirtualSnakeWatermark{PublicKey: types.FullMask, Sequence: 0}, peers[1], peers[0], &selfAnn, @@ -166,10 +172,11 @@ func TestSNEKNextHopSelection(t *testing.T) { virtualSnakeTable{}, }, peers[1]}, // bootstraps work their way toward the root {"TestNotBootstrapParentKnowsDestination", virtualSnakeNextHopParams{ + nil, false, - //false, destUpKey, selfKey, + types.VirtualSnakeWatermark{PublicKey: types.FullMask, Sequence: 0}, peers[1], peers[0], &selfAnn, @@ -179,10 +186,11 @@ func TestSNEKNextHopSelection(t *testing.T) { virtualSnakeTable{}, }, peers[1]}, {"TestNotBootstrapPeerKnowsDestination", virtualSnakeNextHopParams{ + nil, false, - //false, destUpKey, selfKey, + types.VirtualSnakeWatermark{PublicKey: types.FullMask, Sequence: 0}, peers[1], peers[0], &selfAnn, @@ -193,10 +201,11 @@ func TestSNEKNextHopSelection(t *testing.T) { virtualSnakeTable{}, }, peers[2]}, {"TestBootstrapPeerKnowsDestination", virtualSnakeNextHopParams{ + nil, true, - //false, destUpKey, selfKey, + types.VirtualSnakeWatermark{PublicKey: types.FullMask, Sequence: 0}, peers[1], peers[0], &selfAnn, @@ -207,10 +216,11 @@ func TestSNEKNextHopSelection(t *testing.T) { virtualSnakeTable{}, }, peers[1]}, // bootstraps work their way toward the root {"TestNotBootstrapParentKnowsCloser", virtualSnakeNextHopParams{ + nil, false, - //false, destUpKey, selfKey, + types.VirtualSnakeWatermark{PublicKey: types.FullMask, Sequence: 0}, peers[1], peers[0], &selfAnn, @@ -220,10 +230,11 @@ func TestSNEKNextHopSelection(t *testing.T) { virtualSnakeTable{}, }, peers[1]}, {"TestBootstrapParentKnowsCloser", virtualSnakeNextHopParams{ + nil, true, - //false, destUpKey, selfKey, + types.VirtualSnakeWatermark{PublicKey: types.FullMask, Sequence: 0}, peers[1], peers[0], &selfAnn, @@ -233,10 +244,11 @@ func TestSNEKNextHopSelection(t *testing.T) { virtualSnakeTable{}, }, peers[1]}, {"TestNotBootstrapSnakeEntryIsDest", virtualSnakeNextHopParams{ + nil, false, - //false, destDownKey, selfKey, + types.VirtualSnakeWatermark{PublicKey: types.FullMask, Sequence: 0}, peers[1], peers[0], &selfAnn, @@ -252,10 +264,11 @@ func TestSNEKNextHopSelection(t *testing.T) { }}, }, peers[3]}, {"TestBootstrapSnakeEntryIsDest", virtualSnakeNextHopParams{ + nil, true, - //false, destDownKey, selfKey, + types.VirtualSnakeWatermark{PublicKey: types.FullMask, Sequence: 0}, peers[1], peers[0], &selfAnn, @@ -274,7 +287,7 @@ func TestSNEKNextHopSelection(t *testing.T) { for _, tc := range cases { t.Run(tc.desc, func(t *testing.T) { - actual := getNextHopSNEK(tc.input) + actual, _ := getNextHopSNEK(tc.input) actualString, expectedString := convertToString(actual, tc.expected, peers) if actual != tc.expected { diff --git a/router/state_tree.go b/router/state_tree.go index 30edb39c..21dd7580 100644 --- a/router/state_tree.go +++ b/router/state_tree.go @@ -343,7 +343,7 @@ func (s *state) _handleTreeAnnouncement(p *peer, f *types.Frame) error { s._sendTreeAnnouncements() case SelectNewParent: if s._selectNewParent() { - s._bootstrapNow() + s._bootstrapSoon() } case SelectNewParentWithWait: s._waiting = true @@ -353,7 +353,7 @@ func (s *state) _handleTreeAnnouncement(p *peer, f *types.Frame) error { s.Act(nil, func() { s._waiting = false if s._selectNewParent() { - s._bootstrapNow() + s._bootstrapSoon() } }) }) diff --git a/types/ed25519.go b/types/ed25519.go index 37664536..79f3e553 100644 --- a/types/ed25519.go +++ b/types/ed25519.go @@ -40,6 +40,11 @@ var FullMask = PublicKey{ 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, } +func (a PublicKey) IsEmpty() bool { + empty := PublicKey{} + return a == empty +} + func (a PublicKey) EqualMaskTo(b, m PublicKey) bool { for i := range a { if (a[i] & m[i]) != (b[i] & m[i]) { diff --git a/types/frame.go b/types/frame.go index e908f396..5f1a49ae 100644 --- a/types/frame.go +++ b/types/frame.go @@ -62,6 +62,7 @@ type Frame struct { DestinationKey PublicKey Source Coordinates SourceKey PublicKey + Watermark VirtualSnakeWatermark Payload []byte } @@ -74,6 +75,7 @@ func (f *Frame) Reset() { f.DestinationKey = PublicKey{} f.Source = Coordinates{} f.SourceKey = PublicKey{} + f.Watermark = VirtualSnakeWatermark{} f.Payload = f.Payload[:0] } @@ -87,12 +89,13 @@ func (f *Frame) MarshalBinary(buffer []byte) (int, error) { payloadLen := len(f.Payload) binary.BigEndian.PutUint16(buffer[offset+0:offset+2], uint16(payloadLen)) offset += 2 - n, err := f.Source.MarshalBinary(buffer[offset:]) + offset += copy(buffer[offset:], f.DestinationKey[:ed25519.PublicKeySize]) + offset += copy(buffer[offset:], f.Watermark.PublicKey[:ed25519.PublicKeySize]) + n, err := f.Watermark.Sequence.MarshalBinary(buffer[offset:]) if err != nil { - return 0, fmt.Errorf("f.Source.MarshalBinary: %w", err) + return 0, fmt.Errorf("f.WatermarkSeq.MarshalBinary: %w", err) } offset += n - offset += copy(buffer[offset:], f.DestinationKey[:ed25519.PublicKeySize]) if f.Payload != nil { f.Payload = f.Payload[:payloadLen] offset += copy(buffer[offset:], f.Payload[:payloadLen]) @@ -104,6 +107,12 @@ func (f *Frame) MarshalBinary(buffer []byte) (int, error) { offset += 2 offset += copy(buffer[offset:], f.DestinationKey[:ed25519.PublicKeySize]) offset += copy(buffer[offset:], f.SourceKey[:ed25519.PublicKeySize]) + offset += copy(buffer[offset:], f.Watermark.PublicKey[:ed25519.PublicKeySize]) + n, err := f.Watermark.Sequence.MarshalBinary(buffer[offset:]) + if err != nil { + return 0, fmt.Errorf("f.WatermarkSeq.MarshalBinary: %w", err) + } + offset += n if f.Payload != nil { f.Payload = f.Payload[:payloadLen] offset += copy(buffer[offset:], f.Payload[:payloadLen]) @@ -160,14 +169,16 @@ func (f *Frame) UnmarshalBinary(data []byte) (int, error) { if payloadLen > cap(f.Payload) { return 0, fmt.Errorf("payload length exceeds frame capacity") } - srcLen := int(binary.BigEndian.Uint16(data[offset+2 : offset+4])) - if _, err := f.Source.UnmarshalBinary(data[offset+2:]); err != nil { - return 0, fmt.Errorf("f.Source.UnmarshalBinary: %w", err) - } - offset += 4 + srcLen + offset += 2 offset += copy(f.DestinationKey[:], data[offset:]) + offset += copy(f.Watermark.PublicKey[:], data[offset:]) + n, err := f.Watermark.Sequence.UnmarshalBinary(data[offset:]) + if err != nil { + return 0, fmt.Errorf("f.WatermarkSeq.UnmarshalBinary: %w", err) + } + offset += n f.Payload = f.Payload[:payloadLen] - offset += copy(f.Payload, data[offset:]) + offset += copy(f.Payload[:payloadLen], data[offset:]) return offset, nil case TypeVirtualSnakeRouted, TypeSNEKPing, TypeSNEKPong: // destination = key, source = key @@ -178,9 +189,15 @@ func (f *Frame) UnmarshalBinary(data []byte) (int, error) { offset += 2 offset += copy(f.DestinationKey[:], data[offset:]) offset += copy(f.SourceKey[:], data[offset:]) + offset += copy(f.Watermark.PublicKey[:], data[offset:]) + n, err := f.Watermark.Sequence.UnmarshalBinary(data[offset:]) + if err != nil { + return 0, fmt.Errorf("f.WatermarkSeq.UnmarshalBinary: %w", err) + } + offset += n f.Payload = f.Payload[:payloadLen] - offset += copy(f.Payload, data[offset:]) - return offset + payloadLen, nil + offset += copy(f.Payload[:payloadLen], data[offset:]) + return offset, nil case TypeKeepalive: return offset, nil diff --git a/types/frame_test.go b/types/frame_test.go index 81ce9c59..3c0c214a 100644 --- a/types/frame_test.go +++ b/types/frame_test.go @@ -84,30 +84,42 @@ func TestMarshalUnmarshalFrame(t *testing.T) { func TestMarshalUnmarshalSNEKBootstrapFrame(t *testing.T) { pk, _, _ := ed25519.GenerateKey(nil) + wpk, _, _ := ed25519.GenerateKey(nil) input := Frame{ Version: Version0, Type: TypeVirtualSnakeBootstrap, - Source: Coordinates{1, 2, 3, 4, 5}, Payload: []byte{9, 9, 9, 9, 9}, + Watermark: VirtualSnakeWatermark{ + Sequence: 100, + }, } copy(input.DestinationKey[:], pk) + copy(input.Watermark.PublicKey[:], wpk) + input.Watermark.Sequence = 100 expected := []byte{ 0x70, 0x69, 0x6e, 0x65, // magic bytes 0, // version 0 byte(TypeVirtualSnakeBootstrap), // type greedy 0, 0, // extra - 0, 56, // frame length + 0, 82, // frame length 0, 5, // payload length - 0, 5, // source length - 1, 2, 3, 4, 5, // source coordinates } expected = append(expected, pk...) + expected = append(expected, wpk...) + var seq [4]byte + n, err := input.Watermark.Sequence.MarshalBinary(seq[:]) + if err != nil { + t.Fatal(err) + } + expected = append(expected, seq[:n]...) expected = append(expected, input.Payload...) buf := make([]byte, 65535) - n, err := input.MarshalBinary(buf) + n, err = input.MarshalBinary(buf) if err != nil { t.Fatal(err) } + fmt.Println("Got", buf[:n]) + fmt.Println("Want", expected) if n != len(expected) { t.Fatalf("wrong marshalled length, got %d, expected %d", n, len(expected)) } @@ -141,26 +153,38 @@ func TestMarshalUnmarshalSNEKBootstrapFrame(t *testing.T) { func TestMarshalUnmarshalSNEKFrame(t *testing.T) { pk1, _, _ := ed25519.GenerateKey(nil) pk2, _, _ := ed25519.GenerateKey(nil) + wpk, _, _ := ed25519.GenerateKey(nil) input := Frame{ Version: Version0, Type: TypeVirtualSnakeRouted, Payload: []byte("HELLO!"), + Watermark: VirtualSnakeWatermark{ + Sequence: 100, + }, } copy(input.SourceKey[:], pk1) copy(input.DestinationKey[:], pk2) + copy(input.Watermark.PublicKey[:], wpk) expected := []byte{ 0x70, 0x69, 0x6e, 0x65, // magic bytes 0, // version 0 byte(TypeVirtualSnakeRouted), // type greedy 0, 0, // extra - 0, 82, // frame length + 0, 115, // frame length 0, 6, // payload length } expected = append(expected, pk2...) expected = append(expected, pk1...) + expected = append(expected, wpk...) + var seq [4]byte + n, err := input.Watermark.Sequence.MarshalBinary(seq[:]) + if err != nil { + t.Fatal(err) + } + expected = append(expected, seq[:n]...) expected = append(expected, input.Payload...) buf := make([]byte, 65535) - n, err := input.MarshalBinary(buf) + n, err = input.MarshalBinary(buf) if err != nil { t.Fatal(err) } diff --git a/types/varu64.go b/types/varu64.go index 22f8e8e3..12440faa 100644 --- a/types/varu64.go +++ b/types/varu64.go @@ -53,3 +53,7 @@ func (n Varu64) Length() int { } return l } + +func (n Varu64) MinLength() int { + return 1 +} diff --git a/types/varu64_test.go b/types/varu64_test.go index 8d5cc5f7..26b3a76d 100644 --- a/types/varu64_test.go +++ b/types/varu64_test.go @@ -21,27 +21,49 @@ import ( func TestMarshalBinaryVaru64(t *testing.T) { var bin [4]byte - input := Varu64(12345678) - expected := []byte{133, 241, 194, 78} - if _, err := input.MarshalBinary(bin[:]); err != nil { - t.Fatal(err) - } - if !bytes.Equal(bin[:], expected) { - t.Fatalf("expected %v, got %v", expected, bin) - } - if length := input.Length(); length != len(expected) { - t.Fatalf("expected length %d, got %d", length, len(expected)) + for input, expected := range map[Varu64][]byte{ + 0: {0}, + 1: {1}, + 12: {12}, + 123: {123}, + 1234: {137, 82}, + 12345: {224, 57}, + 123456: {135, 196, 64}, + 1234567: {203, 173, 7}, + 12345678: {133, 241, 194, 78}, + } { + n, err := input.MarshalBinary(bin[:]) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(bin[:n], expected) { + t.Fatalf("for %d expected %v, got %v", input, expected, bin[:n]) + } + if length := input.Length(); length != len(expected) { + t.Fatalf("for %d expected length %d, got %d", input, length, len(expected)) + } } } func TestUnmarshalBinaryVaru64(t *testing.T) { - input := []byte{133, 241, 194, 78, 0, 1, 2, 3, 4, 5, 6} - expected := Varu64(12345678) - var num Varu64 - if _, err := num.UnmarshalBinary(input); err != nil { - t.Fatal(err) - } - if num != expected { - t.Fatalf("expected %v, got %v", expected, num) + for input, expected := range map[[7]byte]Varu64{ + {0}: 0, + {1}: 1, + {12}: 12, + {123}: 123, + {137, 82}: 1234, + {224, 57}: 12345, + {135, 196, 64}: 123456, + {203, 173, 7}: 1234567, + {133, 241, 194, 78}: 12345678, + {133, 241, 194, 78, 1, 2, 3}: 12345678, + } { + var num Varu64 + if _, err := num.UnmarshalBinary(input[:]); err != nil { + t.Fatal(err) + } + if num != expected { + t.Fatalf("expected %v, got %v", expected, num) + } } } diff --git a/types/virtualsnake.go b/types/virtualsnake.go index 8b74480a..29e8cdcb 100644 --- a/types/virtualsnake.go +++ b/types/virtualsnake.go @@ -21,181 +21,71 @@ func (a VirtualSnakePathID) CompareTo(b VirtualSnakePathID) int { } type VirtualSnakeBootstrap struct { - PathID VirtualSnakePathID - SourceSig VirtualSnakePathSig + Sequence Varu64 Root + Signature [ed25519.SignatureSize]byte } -func (v *VirtualSnakeBootstrap) MarshalBinary(buf []byte) (int, error) { - if len(buf) < VirtualSnakePathIDLength+v.Root.Length()+ed25519.SignatureSize { - return 0, fmt.Errorf("buffer too small") - } - offset := 0 - offset += copy(buf[offset:], v.PathID[:]) - offset += copy(buf[offset:], v.RootPublicKey[:]) - n, err := v.RootSequence.MarshalBinary(buf[offset:]) - if err != nil { - return 0, fmt.Errorf("v.RootSequence.MarshalBinary: %w", err) - } - offset += n - offset += copy(buf[offset:], v.SourceSig[:]) - return offset, nil -} - -func (v *VirtualSnakeBootstrap) UnmarshalBinary(buf []byte) (int, error) { - if len(buf) < VirtualSnakePathIDLength+v.Root.MinLength()+ed25519.SignatureSize { - return 0, fmt.Errorf("buffer too small") - } - offset := 0 - offset += copy(v.PathID[:], buf[offset:]) - offset += copy(v.RootPublicKey[:], buf[offset:]) - l, err := v.RootSequence.UnmarshalBinary(buf[offset:]) - if err != nil { - return 0, fmt.Errorf("v.RootSequence.UnmarshalBinary: %w", err) - } - offset += l - offset += copy(v.SourceSig[:], buf[offset:]) - return offset, nil +type VirtualSnakeWatermark struct { + PublicKey PublicKey `json:"public_key"` + Sequence Varu64 `json:"sequence"` } -type VirtualSnakeBootstrapACK struct { - PathID VirtualSnakePathID - SourceSig VirtualSnakePathSig - DestinationSig VirtualSnakePathSig - Root +func (a VirtualSnakeWatermark) WorseThan(b VirtualSnakeWatermark) bool { + diff := a.PublicKey.CompareTo(b.PublicKey) + return diff > 0 || (diff == 0 && a.Sequence < b.Sequence) } -func (v *VirtualSnakeBootstrapACK) MarshalBinary(buf []byte) (int, error) { - if len(buf) < VirtualSnakePathIDLength+v.Root.Length()+(ed25519.SignatureSize*2) { - return 0, fmt.Errorf("buffer too small") +func (v *VirtualSnakeBootstrap) ProtectedPayload() ([]byte, error) { + buffer := make([]byte, ed25519.SignatureSize+v.Sequence.Length()) + sn, err := v.Sequence.MarshalBinary(buffer[:]) + if err != nil { + return nil, fmt.Errorf("v.Sequence.MarshalBinary: %w", err) } - offset := 0 - offset += copy(buf[offset:], v.PathID[:]) - offset += copy(buf[offset:], v.RootPublicKey[:]) - n, err := v.RootSequence.MarshalBinary(buf[offset:]) + rn := copy(buffer[:sn], v.RootPublicKey[:]) + rsn, err := v.RootSequence.MarshalBinary(buffer[sn+rn:]) if err != nil { - return 0, fmt.Errorf("v.RootSequence.MarshalBinary: %w", err) + return nil, fmt.Errorf("v.Sequence.MarshalBinary: %w", err) } - offset += n - offset += copy(buf[offset:], v.SourceSig[:]) - offset += copy(buf[offset:], v.DestinationSig[:]) - return offset, nil + return buffer[:sn+rn+rsn], nil } -func (v *VirtualSnakeBootstrapACK) UnmarshalBinary(buf []byte) (int, error) { - if len(buf) < VirtualSnakePathIDLength+v.Root.MinLength()+(ed25519.SignatureSize*2) { +func (v *VirtualSnakeBootstrap) MarshalBinary(buf []byte) (int, error) { + if len(buf) < v.Sequence.Length()+v.Root.Length()+ed25519.SignatureSize { return 0, fmt.Errorf("buffer too small") } offset := 0 - offset += copy(v.PathID[:], buf[offset:]) - offset += copy(v.RootPublicKey[:], buf[offset:]) - l, err := v.RootSequence.UnmarshalBinary(buf[offset:]) + n, err := v.Sequence.MarshalBinary(buf[offset:]) if err != nil { - return 0, fmt.Errorf("v.RootSequence.UnmarshalBinary: %w", err) - } - offset += l - offset += copy(v.SourceSig[:], buf[offset:]) - offset += copy(v.DestinationSig[:], buf[offset:]) - return offset, nil -} - -type VirtualSnakeSetup struct { - PathID VirtualSnakePathID - SourceSig VirtualSnakePathSig - DestinationSig VirtualSnakePathSig - Root -} - -func (v *VirtualSnakeSetup) MarshalBinary(buf []byte) (int, error) { - if len(buf) < VirtualSnakePathIDLength+v.Root.Length()+(ed25519.SignatureSize*2) { - return 0, fmt.Errorf("buffer too small") + return 0, fmt.Errorf("v.Sequence.MarshalBinary: %w", err) } - offset := 0 - offset += copy(buf[offset:], v.PathID[:]) + offset += n offset += copy(buf[offset:], v.RootPublicKey[:]) - n, err := v.RootSequence.MarshalBinary(buf[offset:]) + n, err = v.RootSequence.MarshalBinary(buf[offset:]) if err != nil { return 0, fmt.Errorf("v.RootSequence.MarshalBinary: %w", err) } offset += n - offset += copy(buf[offset:], v.SourceSig[:]) - offset += copy(buf[offset:], v.DestinationSig[:]) - return offset, nil -} - -func (v *VirtualSnakeSetup) UnmarshalBinary(buf []byte) (int, error) { - if len(buf) < VirtualSnakePathIDLength+v.Root.MinLength()+(ed25519.SignatureSize*2) { - return 0, fmt.Errorf("buffer too small") - } - offset := 0 - offset += copy(v.PathID[:], buf[offset:]) - offset += copy(v.RootPublicKey[:], buf[offset:]) - l, err := v.RootSequence.UnmarshalBinary(buf[offset:]) - if err != nil { - return 0, fmt.Errorf("v.RootSequence.UnmarshalBinary: %w", err) - } - offset += l - offset += copy(v.SourceSig[:], buf[offset:]) - offset += copy(v.DestinationSig[:], buf[offset:]) + offset += copy(buf[offset:], v.Signature[:]) return offset, nil } -type VirtualSnakeSetupACK struct { - PathID VirtualSnakePathID - TargetSig VirtualSnakePathSig - Root -} - -func (v *VirtualSnakeSetupACK) MarshalBinary(buf []byte) (int, error) { - if len(buf) < VirtualSnakePathIDLength+v.Root.Length()+ed25519.SignatureSize { +func (v *VirtualSnakeBootstrap) UnmarshalBinary(buf []byte) (int, error) { + if len(buf) < v.Sequence.MinLength()+v.Root.MinLength()+ed25519.SignatureSize { return 0, fmt.Errorf("buffer too small") } offset := 0 - offset += copy(buf[offset:], v.PathID[:]) - offset += copy(buf[offset:], v.RootPublicKey[:]) - n, err := v.RootSequence.MarshalBinary(buf[offset:]) + n, err := v.Sequence.UnmarshalBinary(buf[offset:]) if err != nil { - return 0, fmt.Errorf("v.RootSequence.MarshalBinary: %w", err) + return 0, fmt.Errorf("v.Sequence.UnmarshalBinary: %w", err) } offset += n - offset += copy(buf[offset:], v.TargetSig[:]) - return offset, nil -} - -func (v *VirtualSnakeSetupACK) UnmarshalBinary(buf []byte) (int, error) { - if len(buf) < VirtualSnakePathIDLength+v.Root.MinLength()+ed25519.SignatureSize { - return 0, fmt.Errorf("buffer too small") - } - offset := 0 - offset += copy(v.PathID[:], buf[offset:]) offset += copy(v.RootPublicKey[:], buf[offset:]) - l, err := v.RootSequence.UnmarshalBinary(buf[offset:]) + n, err = v.RootSequence.UnmarshalBinary(buf[offset:]) if err != nil { return 0, fmt.Errorf("v.RootSequence.UnmarshalBinary: %w", err) } - offset += l - offset += copy(v.TargetSig[:], buf[offset:]) - return offset, nil -} - -type VirtualSnakeTeardown struct { - PathID VirtualSnakePathID -} - -func (v *VirtualSnakeTeardown) MarshalBinary(buf []byte) (int, error) { - if len(buf) < VirtualSnakePathIDLength { - return 0, fmt.Errorf("buffer too small") - } - offset := 0 - offset += copy(buf[offset:], v.PathID[:]) - return offset, nil -} - -func (v *VirtualSnakeTeardown) UnmarshalBinary(buf []byte) (int, error) { - if len(buf) < VirtualSnakePathIDLength { - return 0, fmt.Errorf("buffer too small") - } - offset := 0 - offset += copy(v.PathID[:], buf[offset:]) + offset += n + offset += copy(v.Signature[:], buf[offset:]) return offset, nil } From ca3f61af1b94421865808e69cc68a3c3854011e9 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 14 Apr 2022 15:20:47 +0100 Subject: [PATCH 11/41] Tidy up sim --- .../simulator/adversary/drop_packets.go | 4 -- cmd/pineconesim/simulator/commands.go | 48 ------------------- 2 files changed, 52 deletions(-) diff --git a/cmd/pineconesim/simulator/adversary/drop_packets.go b/cmd/pineconesim/simulator/adversary/drop_packets.go index c295bcd6..3fad42d8 100644 --- a/cmd/pineconesim/simulator/adversary/drop_packets.go +++ b/cmd/pineconesim/simulator/adversary/drop_packets.go @@ -88,10 +88,6 @@ func defaultFrameCount() PeerFrameCount { frameCount[types.TypeKeepalive] = atomic.NewUint64(0) frameCount[types.TypeTreeAnnouncement] = atomic.NewUint64(0) frameCount[types.TypeVirtualSnakeBootstrap] = atomic.NewUint64(0) - /*frameCount[types.TypeVirtualSnakeBootstrapACK] = atomic.NewUint64(0) - frameCount[types.TypeVirtualSnakeSetup] = atomic.NewUint64(0) - frameCount[types.TypeVirtualSnakeSetupACK] = atomic.NewUint64(0) - frameCount[types.TypeVirtualSnakeTeardown] = atomic.NewUint64(0)*/ frameCount[types.TypeTreeRouted] = atomic.NewUint64(0) frameCount[types.TypeVirtualSnakeRouted] = atomic.NewUint64(0) diff --git a/cmd/pineconesim/simulator/commands.go b/cmd/pineconesim/simulator/commands.go index bbce263a..da533c61 100644 --- a/cmd/pineconesim/simulator/commands.go +++ b/cmd/pineconesim/simulator/commands.go @@ -137,30 +137,6 @@ func UnmarshalCommandJSON(command *SimCommandMsg) (SimCommand, error) { } else { err = fmt.Errorf("%sConfigureAdversaryDefaults.DropRates.VirtualSnakeBootstrap field doesn't exist", FAILURE_PREAMBLE) } - /*if subVal, subOk := val.(map[string]interface{})["VirtualSnakeBootstrapACK"]; subOk { - intVal, _ := strconv.Atoi(subVal.(string)) - dropRates.Frames[types.TypeVirtualSnakeBootstrapACK] = uint64(intVal) - } else { - err = fmt.Errorf("%sConfigureAdversaryDefaults.DropRates.VirtualSnakeBootstrapACK field doesn't exist", FAILURE_PREAMBLE) - } - if subVal, subOk := val.(map[string]interface{})["VirtualSnakeSetup"]; subOk { - intVal, _ := strconv.Atoi(subVal.(string)) - dropRates.Frames[types.TypeVirtualSnakeSetup] = uint64(intVal) - } else { - err = fmt.Errorf("%sConfigureAdversaryDefaults.DropRates.VirtualSnakeSetup field doesn't exist", FAILURE_PREAMBLE) - } - if subVal, subOk := val.(map[string]interface{})["VirtualSnakeSetupACK"]; subOk { - intVal, _ := strconv.Atoi(subVal.(string)) - dropRates.Frames[types.TypeVirtualSnakeSetupACK] = uint64(intVal) - } else { - err = fmt.Errorf("%sConfigureAdversaryDefaults.DropRates.VirtualSnakeSetupACK field doesn't exist", FAILURE_PREAMBLE) - } - if subVal, subOk := val.(map[string]interface{})["VirtualSnakeTeardown"]; subOk { - intVal, _ := strconv.Atoi(subVal.(string)) - dropRates.Frames[types.TypeVirtualSnakeTeardown] = uint64(intVal) - } else { - err = fmt.Errorf("%sConfigureAdversaryDefaults.DropRates.VirtualSnakeTeardown field doesn't exist", FAILURE_PREAMBLE) - }*/ if subVal, subOk := val.(map[string]interface{})["VirtualSnakeRouted"]; subOk { intVal, _ := strconv.Atoi(subVal.(string)) dropRates.Frames[types.TypeVirtualSnakeRouted] = uint64(intVal) @@ -217,30 +193,6 @@ func UnmarshalCommandJSON(command *SimCommandMsg) (SimCommand, error) { } else { err = fmt.Errorf("%sConfigureAdversaryPeer.DropRates.VirtualSnakeBootstrap field doesn't exist", FAILURE_PREAMBLE) } - /*if subVal, subOk := val.(map[string]interface{})["VirtualSnakeBootstrapACK"]; subOk { - intVal, _ := strconv.Atoi(subVal.(string)) - dropRates.Frames[types.TypeVirtualSnakeBootstrapACK] = uint64(intVal) - } else { - err = fmt.Errorf("%sConfigureAdversaryPeer.DropRates.VirtualSnakeBootstrapACK field doesn't exist", FAILURE_PREAMBLE) - } - if subVal, subOk := val.(map[string]interface{})["VirtualSnakeSetup"]; subOk { - intVal, _ := strconv.Atoi(subVal.(string)) - dropRates.Frames[types.TypeVirtualSnakeSetup] = uint64(intVal) - } else { - err = fmt.Errorf("%sConfigureAdversaryPeer.DropRates.VirtualSnakeSetup field doesn't exist", FAILURE_PREAMBLE) - } - if subVal, subOk := val.(map[string]interface{})["VirtualSnakeSetupACK"]; subOk { - intVal, _ := strconv.Atoi(subVal.(string)) - dropRates.Frames[types.TypeVirtualSnakeSetupACK] = uint64(intVal) - } else { - err = fmt.Errorf("%sConfigureAdversaryPeer.DropRates.VirtualSnakeSetupACK field doesn't exist", FAILURE_PREAMBLE) - } - if subVal, subOk := val.(map[string]interface{})["VirtualSnakeTeardown"]; subOk { - intVal, _ := strconv.Atoi(subVal.(string)) - dropRates.Frames[types.TypeVirtualSnakeTeardown] = uint64(intVal) - } else { - err = fmt.Errorf("%sConfigureAdversaryPeer.DropRates.VirtualSnakeTeardown field doesn't exist", FAILURE_PREAMBLE) - }*/ if subVal, subOk := val.(map[string]interface{})["VirtualSnakeRouted"]; subOk { intVal, _ := strconv.Atoi(subVal.(string)) dropRates.Frames[types.TypeVirtualSnakeRouted] = uint64(intVal) From 2e65ba29580e8a6031ca096eba93f91145ca8958 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 26 Apr 2022 11:48:20 -0500 Subject: [PATCH 12/41] Fix unit test --- router/state_snek_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/router/state_snek_test.go b/router/state_snek_test.go index 5aed265a..46288c81 100644 --- a/router/state_snek_test.go +++ b/router/state_snek_test.go @@ -282,7 +282,7 @@ func TestSNEKNextHopSelection(t *testing.T) { // Active: true, virtualSnakeIndex: &virtualSnakeIndex{PublicKey: destDownKey}, }}, - }, peers[0]}, // handle a bootstrap received from a lower key node + }, nil}, // handle a bootstrap received from a lower key node } for _, tc := range cases { From f8aa052d730720d5d0cdd38cd3043adf5156b2a8 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Wed, 27 Apr 2022 14:07:25 -0500 Subject: [PATCH 13/41] Don't drop pong responses as looping traffic --- router/state_forward.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/router/state_forward.go b/router/state_forward.go index da1caf39..9d736ec2 100644 --- a/router/state_forward.go +++ b/router/state_forward.go @@ -52,6 +52,8 @@ func (s *state) _forward(p *peer, f *types.Frame) error { nexthop, watermark := s._nextHopsFor(p, f) deadend := nexthop == nil || nexthop == p.router.local + isInitialPongResponse := false + switch f.Type { case types.TypeTreeAnnouncement: // Tree announcements are a special case. The _handleTreeAnnouncement function @@ -79,6 +81,7 @@ func (s *state) _forward(p *peer, f *types.Frame) error { case types.TypeSNEKPing: if f.DestinationKey == s.r.public { + isInitialPongResponse = true of := f defer framePool.Put(of) f = getFrame() @@ -113,6 +116,7 @@ func (s *state) _forward(p *peer, f *types.Frame) error { case types.TypeTreePing: if deadend { + isInitialPongResponse = true of := f defer framePool.Put(of) f = getFrame() @@ -144,7 +148,9 @@ func (s *state) _forward(p *peer, f *types.Frame) error { // If the packet's watermark is higher than the previous one or we are // obviously looping, drop the packet. - if nexthop == p || watermark.WorseThan(f.Watermark) { + // In the case of initial pong response frames, they are routed back to + // the peer we received the ping from so the "loop" is desired. + if (nexthop == p && !isInitialPongResponse) || watermark.WorseThan(f.Watermark) { return nil } From 9bd82ae0ff583a1bdd7d14df42ba902a0ea2db51 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Thu, 28 Apr 2022 12:25:10 -0500 Subject: [PATCH 14/41] Remove unused snake path id --- router/api.go | 1 - types/virtualsnake.go | 15 --------------- 2 files changed, 16 deletions(-) diff --git a/router/api.go b/router/api.go index b1ec9892..ef0c7bdc 100644 --- a/router/api.go +++ b/router/api.go @@ -31,7 +31,6 @@ import ( type NeighbourInfo struct { PublicKey types.PublicKey - PathID types.VirtualSnakePathID } type PeerInfo struct { diff --git a/types/virtualsnake.go b/types/virtualsnake.go index 29e8cdcb..2479ec37 100644 --- a/types/virtualsnake.go +++ b/types/virtualsnake.go @@ -1,25 +1,10 @@ package types import ( - "bytes" "crypto/ed25519" - "encoding/hex" "fmt" ) -const VirtualSnakePathIDLength = 8 - -type VirtualSnakePathID [VirtualSnakePathIDLength]byte -type VirtualSnakePathSig [ed25519.SignatureSize]byte - -func (p VirtualSnakePathID) MarshalJSON() ([]byte, error) { - return []byte("\"" + hex.EncodeToString(p[:]) + "\""), nil -} - -func (a VirtualSnakePathID) CompareTo(b VirtualSnakePathID) int { - return bytes.Compare(a[:], b[:]) -} - type VirtualSnakeBootstrap struct { Sequence Varu64 Root From d6fb3d113cd6baa3620b1d2487222d13f7a8eee0 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Thu, 28 Apr 2022 12:25:58 -0500 Subject: [PATCH 15/41] Remove unused variable from nextHopsSNEK --- router/state_forward.go | 2 +- router/state_snek.go | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/router/state_forward.go b/router/state_forward.go index 9d736ec2..ca39c4d9 100644 --- a/router/state_forward.go +++ b/router/state_forward.go @@ -30,7 +30,7 @@ func (s *state) _nextHopsFor(from *peer, frame *types.Frame) (*peer, types.Virtu switch frame.Type { // SNEK routing case types.TypeVirtualSnakeRouted, types.TypeVirtualSnakeBootstrap, types.TypeSNEKPing, types.TypeSNEKPong: - nexthop, watermark = s._nextHopsSNEK(from, frame, frame.Type == types.TypeVirtualSnakeBootstrap) + nexthop, watermark = s._nextHopsSNEK(from, frame) // Tree routing case types.TypeTreeRouted, types.TypeTreePing, types.TypeTreePong: diff --git a/router/state_snek.go b/router/state_snek.go index 43d074d2..9a3224c5 100644 --- a/router/state_snek.go +++ b/router/state_snek.go @@ -142,9 +142,10 @@ func (s *state) _bootstrapNow() { PublicKey: types.FullMask, Sequence: 0, } + // Bootstrap messages are routed using SNEK routing with special rules for // bootstrap packets. - if p, w := s._nextHopsSNEK(s.r.local, send, true); p != nil && p.proto != nil { + if p, w := s._nextHopsSNEK(s.r.local, send); p != nil && p.proto != nil { send.Watermark = w p.proto.push(send) } @@ -164,10 +165,8 @@ type virtualSnakeNextHopParams struct { snakeRoutes virtualSnakeTable } -// _nextHopsSNEK locates the best next-hop for a given SNEK-routed frame. The -// bootstrap flag determines whether the frame should be routed using bootstrap -// specific rules — this should only be used for VirtualSnakeBootstrap frames. -func (s *state) _nextHopsSNEK(from *peer, rx *types.Frame, bootstrap bool) (*peer, types.VirtualSnakeWatermark) { +// _nextHopsSNEK locates the best next-hop for a given SNEK-routed frame. +func (s *state) _nextHopsSNEK(from *peer, rx *types.Frame) (*peer, types.VirtualSnakeWatermark) { return getNextHopSNEK(virtualSnakeNextHopParams{ from, rx.Type == types.TypeVirtualSnakeBootstrap, From f76b5d8c5f976fafa01c6aff0758cc983d7ebeb5 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Thu, 28 Apr 2022 12:26:16 -0500 Subject: [PATCH 16/41] Unassign setup acks capability from soft state capabilites --- router/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/router/version.go b/router/version.go index 2b718795..c2a464fc 100644 --- a/router/version.go +++ b/router/version.go @@ -23,4 +23,4 @@ const ( ) const ourVersion uint8 = 1 -const ourCapabilities uint32 = capabilityLengthenedRootInterval | capabilityCryptographicSetups | capabilitySetupACKs | capabilityDedupedCoordinateInfo | capabilitySoftState +const ourCapabilities uint32 = capabilityLengthenedRootInterval | capabilityCryptographicSetups | capabilityDedupedCoordinateInfo | capabilitySoftState From 2616e826948db720239f61c431add34b55197633 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Thu, 28 Apr 2022 12:27:00 -0500 Subject: [PATCH 17/41] Cleanup comments to reflect new softstate logic --- router/state.go | 9 +++------ router/state_forward.go | 4 ++-- router/state_snek.go | 17 ++++++++--------- 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/router/state.go b/router/state.go index e25a6410..3aeb9719 100644 --- a/router/state.go +++ b/router/state.go @@ -218,10 +218,8 @@ func (s *state) _portDisconnected(peer *peer) { // Delete the last tree announcement that we received from this peer. delete(s._announcements, peer) - // Scan the local DHT table for any routes that transited this now-dead - // peering. If we find any then we need to send teardowns in the opposite - // direction, so that nodes further along the path will learn that the - // path was broken. + // Scan the local routing table for any routes that transited this now-dead + // peering and remove them from the routing table. for k, v := range s._table { if v.Source == peer || v.Destination == peer { delete(s._table, k) @@ -229,8 +227,7 @@ func (s *state) _portDisconnected(peer *peer) { } // If the descending path was lost because it went via the now-dead - // peering then clear that path (although we can't send a teardown) and - // wait for another incoming setup. + // peering then clear that path and wait for another incoming setup. if desc := s._descending; desc != nil && desc.Source == peer { s._setDescendingNode(nil) } diff --git a/router/state_forward.go b/router/state_forward.go index ca39c4d9..94a9d789 100644 --- a/router/state_forward.go +++ b/router/state_forward.go @@ -41,8 +41,8 @@ func (s *state) _nextHopsFor(from *peer, frame *types.Frame) (*peer, types.Virtu // _forward handles frames received from a given peer. In most cases, this function will // look up the best next-hop for a given frame and forward it to the appropriate peer -// queue if possible. In some special cases, like tree announcements, path setups and -// teardowns, special handling will be done before forwarding if needed. +// queue if possible. In some special cases, like tree announcements, +// special handling will be done before forwarding if needed. func (s *state) _forward(p *peer, f *types.Frame) error { if s._filterPacket != nil && s._filterPacket(p.public, f) { s.r.log.Printf("Packet of type %s destined for port %d [%s] was dropped due to filter rules", f.Type.String(), p.port, p.public.String()[:8]) diff --git a/router/state_snek.go b/router/state_snek.go index 9a3224c5..1ed875ff 100644 --- a/router/state_snek.go +++ b/router/state_snek.go @@ -77,8 +77,7 @@ func (s *state) _maintainSnake() { } } - // Clean up any paths that were installed more than 5 seconds ago but haven't - // been activated by a setup ACK. + // Clean up any paths that are older than the expiry period. for k, v := range s._table { if !v.valid() { delete(s._table, k) @@ -130,6 +129,7 @@ func (s *state) _bootstrapNow() { if err != nil { return } + // Construct the frame. We set the destination key to be our own public key. As // the bootstrap routing defaults to routing towards higher keys, this should // mean that the message gets forwarded up to the next highest key from ours. @@ -312,8 +312,7 @@ func getNextHopSNEK(params virtualSnakeNextHopParams) (*peer, types.VirtualSnake } // _handleBootstrap is called in response to receiving a bootstrap packet. -// This function will send a bootstrap ACK back to the sender. Returns true -// if the bootstrap was handled and false otherwise. +// Returns true if the bootstrap was handled and false otherwise. func (s *state) _handleBootstrap(from, to *peer, rx *types.Frame) bool { // Unmarshal the bootstrap. var bootstrap types.VirtualSnakeBootstrap @@ -336,6 +335,7 @@ func (s *state) _handleBootstrap(from, to *peer, rx *types.Frame) bool { return false } } + // Check that the root key and sequence number in the update match our // current root, otherwise we won't be able to route back to them using // tree routing anyway. If they don't match, silently drop the bootstrap. @@ -369,14 +369,13 @@ func (s *state) _handleBootstrap(from, to *peer, rx *types.Frame) bool { }, } - // Now let's see if this is a suitable ascending entry. + // Now let's see if this is a suitable descending entry. update := false desc := s._descending switch { case !root.Root.EqualTo(&bootstrap.Root): - // The root key in the bootstrap ACK doesn't match our own key, or the - // sequence doesn't match, so it is quite possible that routing setup packets - // using tree routing would fail. + // The root key in the bootstrap doesn't match our own key + // so it is quite possible that tree routing would fail. case !util.LessThan(rx.DestinationKey, s.r.public): // The bootstrapping key should be less than ours but it isn't. case desc != nil && desc.valid(): @@ -384,7 +383,7 @@ func (s *state) _handleBootstrap(from, to *peer, rx *types.Frame) bool { switch { case desc.PublicKey == rx.DestinationKey: // We've received another bootstrap from our direct descending node. - // Send back an acknowledgement as this is OK. + // Accept the update as this is OK. update = true case util.DHTOrdered(desc.PublicKey, rx.DestinationKey, s.r.public): // The bootstrapping node is closer to us than our previous descending From 189af67f4cdad3a97aa7c17209ec18b6919e37f4 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Thu, 28 Apr 2022 13:12:54 -0500 Subject: [PATCH 18/41] Remove unused candidate variable from state --- router/state.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/router/state.go b/router/state.go index 3aeb9719..d3dbbcc6 100644 --- a/router/state.go +++ b/router/state.go @@ -38,7 +38,6 @@ type state struct { r *Router _peers []*peer // All switch ports, connected and disconnected _descending *virtualSnakeEntry // Next descending node in keyspace - _candidate *virtualSnakeEntry // Candidate to replace the ascending node _parent *peer // Our chosen parent in the tree _announcements announcementTable // Announcements received from our peers _table virtualSnakeTable // Virtual snake DHT entries @@ -56,7 +55,6 @@ func (s *state) _start() { s._setParent(nil) s._setDescendingNode(nil) - s._candidate = nil s._ordering = 0 s._waiting = false From a6b5c11a557799f492e6b610c6cd48af62074c8c Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Thu, 28 Apr 2022 13:13:37 -0500 Subject: [PATCH 19/41] Update sim and events to reflect softstate changes --- cmd/pineconesim/main.go | 11 ++---- cmd/pineconesim/simulator/api.go | 16 +++----- cmd/pineconesim/simulator/events.go | 21 ++-------- cmd/pineconesim/simulator/simulator.go | 13 +------ cmd/pineconesim/simulator/state.go | 49 ++++++++---------------- cmd/pineconesim/ui/main.js | 12 ++---- cmd/pineconesim/ui/modules/graph.js | 33 +++------------- cmd/pineconesim/ui/modules/server-api.js | 5 +-- router/events/events.go | 26 ------------- router/manhole.go | 1 - router/state.go | 6 +-- 11 files changed, 42 insertions(+), 151 deletions(-) diff --git a/cmd/pineconesim/main.go b/cmd/pineconesim/main.go index 37e42182..dc62373b 100644 --- a/cmd/pineconesim/main.go +++ b/cmd/pineconesim/main.go @@ -305,12 +305,9 @@ func userProxyReporter(conn *websocket.Conn, connID uint64, sim *simulator.Simul AnnTime: node.Announcement.Time, Coords: node.Coords, }, - Peers: peerConns, - TreeParent: node.Parent, - SnakeAsc: node.AscendingPeer, - SnakeAscPath: node.AscendingPathID, - SnakeDesc: node.DescendingPeer, - SnakeDescPath: node.DescendingPathID, + Peers: peerConns, + TreeParent: node.Parent, + SnakeDesc: node.DescendingPeer, } if batchSize == int(maxBatchSize) || end { @@ -363,8 +360,6 @@ func handleSimEvents(log *log.Logger, conn *websocket.Conn, ch <-chan simulator. eventType = simulator.SimPeerRemoved case simulator.TreeParentUpdate: eventType = simulator.SimTreeParentUpdated - case simulator.SnakeAscUpdate: - eventType = simulator.SimSnakeAscUpdated case simulator.SnakeDescUpdate: eventType = simulator.SimSnakeDescUpdated case simulator.TreeRootAnnUpdate: diff --git a/cmd/pineconesim/simulator/api.go b/cmd/pineconesim/simulator/api.go index e4d79602..173a6b7d 100644 --- a/cmd/pineconesim/simulator/api.go +++ b/cmd/pineconesim/simulator/api.go @@ -38,7 +38,6 @@ const ( SimPeerAdded SimPeerRemoved SimTreeParentUpdated - SimSnakeAscUpdated SimSnakeDescUpdated SimTreeRootAnnUpdated ) @@ -64,15 +63,12 @@ const ( ) type InitialNodeState struct { - PublicKey string - NodeType APINodeType - RootState RootState - Peers []PeerInfo - TreeParent string - SnakeAsc string - SnakeAscPath string - SnakeDesc string - SnakeDescPath string + PublicKey string + NodeType APINodeType + RootState RootState + Peers []PeerInfo + TreeParent string + SnakeDesc string } type RootState struct { diff --git a/cmd/pineconesim/simulator/events.go b/cmd/pineconesim/simulator/events.go index 08e0fe7f..b5b4af58 100644 --- a/cmd/pineconesim/simulator/events.go +++ b/cmd/pineconesim/simulator/events.go @@ -62,21 +62,10 @@ type TreeParentUpdate struct { // Tag TreeParentUpdate as an Event func (e TreeParentUpdate) isEvent() {} -type SnakeAscUpdate struct { - Node string - Peer string - Prev string - PathID string -} - -// Tag SnakeAscUpdate as an Event -func (e SnakeAscUpdate) isEvent() {} - type SnakeDescUpdate struct { - Node string - Peer string - Prev string - PathID string + Node string + Peer string + Prev string } // Tag SnakeDescUpdate as an Event @@ -111,10 +100,8 @@ func (h eventHandler) Run(quit <-chan bool, sim *Simulator) { sim.handlePeerRemoved(h.node, e.PeerID, int(e.Port)) case events.TreeParentUpdate: sim.handleTreeParentUpdate(h.node, e.PeerID) - case events.SnakeAscUpdate: - sim.handleSnakeAscUpdate(h.node, e.PeerID, e.PathID) case events.SnakeDescUpdate: - sim.handleSnakeDescUpdate(h.node, e.PeerID, e.PathID) + sim.handleSnakeDescUpdate(h.node, e.PeerID) case events.TreeRootAnnUpdate: sim.handleTreeRootAnnUpdate(h.node, e.Root, e.Sequence, e.Time, e.Coords) default: diff --git a/cmd/pineconesim/simulator/simulator.go b/cmd/pineconesim/simulator/simulator.go index e201b073..f8f9695d 100644 --- a/cmd/pineconesim/simulator/simulator.go +++ b/cmd/pineconesim/simulator/simulator.go @@ -196,21 +196,12 @@ func (sim *Simulator) handleTreeParentUpdate(node string, peerID string) { sim.State.Act(nil, func() { sim.State._updateParent(node, peerName) }) } -func (sim *Simulator) handleSnakeAscUpdate(node string, peerID string, pathID string) { +func (sim *Simulator) handleSnakeDescUpdate(node string, peerID string) { peerName := "" if peerNode, err := sim.State.GetNodeName(peerID); err == nil { peerName = peerNode } - - sim.State.Act(nil, func() { sim.State._updateAscendingPeer(node, peerName, pathID) }) -} - -func (sim *Simulator) handleSnakeDescUpdate(node string, peerID string, pathID string) { - peerName := "" - if peerNode, err := sim.State.GetNodeName(peerID); err == nil { - peerName = peerNode - } - sim.State.Act(nil, func() { sim.State._updateDescendingPeer(node, peerName, pathID) }) + sim.State.Act(nil, func() { sim.State._updateDescendingPeer(node, peerName) }) } func (sim *Simulator) handleTreeRootAnnUpdate(node string, root string, sequence uint64, time uint64, coords []uint64) { diff --git a/cmd/pineconesim/simulator/state.go b/cmd/pineconesim/simulator/state.go index f628a305..423f3cb4 100644 --- a/cmd/pineconesim/simulator/state.go +++ b/cmd/pineconesim/simulator/state.go @@ -28,30 +28,24 @@ type RootAnnouncement struct { } type NodeState struct { - PeerID string - NodeType APINodeType - Connections map[int]string - Parent string - Coords []uint64 - Announcement RootAnnouncement - AscendingPeer string - AscendingPathID string - DescendingPeer string - DescendingPathID string + PeerID string + NodeType APINodeType + Connections map[int]string + Parent string + Coords []uint64 + Announcement RootAnnouncement + DescendingPeer string } func NewNodeState(peerID string, nodeType APINodeType) *NodeState { node := &NodeState{ - PeerID: peerID, - NodeType: nodeType, - Connections: make(map[int]string), - Parent: "", - Announcement: RootAnnouncement{}, - Coords: []uint64{}, - AscendingPeer: "", - AscendingPathID: "", - DescendingPeer: "", - DescendingPathID: "", + PeerID: peerID, + NodeType: nodeType, + Connections: make(map[int]string), + Parent: "", + Announcement: RootAnnouncement{}, + Coords: []uint64{}, + DescendingPeer: "", } return node } @@ -202,23 +196,12 @@ func (s *StateAccessor) _updateParent(node string, peerID string) { } } -func (s *StateAccessor) _updateAscendingPeer(node string, peerID string, pathID string) { - if _, ok := s._state.Nodes[node]; ok { - prev := s._state.Nodes[node].AscendingPeer - s._state.Nodes[node].AscendingPeer = peerID - s._state.Nodes[node].AscendingPathID = pathID - - s._publish(SnakeAscUpdate{Node: node, Peer: peerID, Prev: prev, PathID: pathID}) - } -} - -func (s *StateAccessor) _updateDescendingPeer(node string, peerID string, pathID string) { +func (s *StateAccessor) _updateDescendingPeer(node string, peerID string) { if _, ok := s._state.Nodes[node]; ok { prev := s._state.Nodes[node].DescendingPeer s._state.Nodes[node].DescendingPeer = peerID - s._state.Nodes[node].DescendingPathID = pathID - s._publish(SnakeDescUpdate{Node: node, Peer: peerID, Prev: prev, PathID: pathID}) + s._publish(SnakeDescUpdate{Node: node, Peer: peerID, Prev: prev}) } } diff --git a/cmd/pineconesim/ui/main.js b/cmd/pineconesim/ui/main.js index 0e4c1122..873b5492 100644 --- a/cmd/pineconesim/ui/main.js +++ b/cmd/pineconesim/ui/main.js @@ -17,11 +17,8 @@ function handleSimMessage(msg) { } } - if (value.SnakeAsc && value.SnakeAscPath) { - graph.setSnekAsc(key, value.SnakeAsc, "", value.SnakeAscPath); - } - if (value.SnakeDesc && value.SnakeDescPath) { - graph.setSnekDesc(key, value.SnakeDesc, "", value.SnakeDescPath); + if (value.SnakeDesc) { + graph.setSnekDesc(key, value.SnakeDesc, ""); } if (value.TreeParent) { @@ -52,11 +49,8 @@ function handleSimMessage(msg) { case APIUpdateID.TreeParentUpdated: graph.setTreeParent(event.Node, event.Peer, event.Prev); break; - case APIUpdateID.SnakeAscUpdated: - graph.setSnekAsc(event.Node, event.Peer, event.Prev, event.PathID); - break; case APIUpdateID.SnakeDescUpdated: - graph.setSnekDesc(event.Node, event.Peer, event.Prev, event.PathID); + graph.setSnekDesc(event.Node, event.Peer, event.Prev); break; case APIUpdateID.TreeRootAnnUpdated: graph.updateRootAnnouncement(event.Node, event.Root, event.Sequence, event.Time, event.Coords); diff --git a/cmd/pineconesim/ui/modules/graph.js b/cmd/pineconesim/ui/modules/graph.js index ed5988c0..447d32c2 100644 --- a/cmd/pineconesim/ui/modules/graph.js +++ b/cmd/pineconesim/ui/modules/graph.js @@ -307,22 +307,7 @@ class Graph { } } - setSnekAsc(id, asc, prev, path) { - this.removeEdge("snake", id, prev); - if (asc != "") { - this.addEdge("snake", id, asc); - } - - if (Nodes.has(id)) { - let node = Nodes.get(id); - node.snekAsc = asc; - node.snekAscPath = path.replace(/\"/g, "").toUpperCase(); - - this.updateUI(id); - } - } - - setSnekDesc(id, desc, prev, path) { + setSnekDesc(id, desc, prev) { this.removeEdge("snake", id, prev); if (desc != "") { this.addEdge("snake", id, desc); @@ -331,7 +316,6 @@ class Graph { if (Nodes.has(id)) { let node = Nodes.get(id); node.snekDesc = desc; - node.snekDescPath = path.replace(/\"/g, "").toUpperCase(); this.updateUI(id); } @@ -622,10 +606,7 @@ function newNode(key, type) { peers: [], key: key, treeParent: "", - snekAsc: "", - snekAscPath: "", snekDesc: "", - snekDescPath: "", }; } @@ -659,7 +640,6 @@ function handleNodeHoverUpdate() { "
Coords: [" + node.coords + "]" + "
Tree Parent: " + node.treeParent + "
SNEK Desc: " + node.snekDesc + - "
SNEK Asc: " + node.snekAsc + "

Announcement" + "
Root: Node " + node.announcement.root + "
Sequence: " + node.announcement.sequence + @@ -722,9 +702,6 @@ function handleNodePanelUpdate() { "Root Key:" + getNodeKey(node.announcement.root).slice(0, 16).replace(/\"/g, "").toUpperCase() + "" + "Tree Parent:" + node.treeParent + "" + "Descending Node:" + node.snekDesc + "" + - "Descending Path:" + node.snekDescPath + "" + - "Ascending Node:" + node.snekAsc + "" + - "Ascending Path:" + node.snekAscPath + "" + "" + "

Peers

" + "" + @@ -733,8 +710,8 @@ function handleNodePanelUpdate() { "
" + "

SNEK Routes

" + "" + - "" + - "" + + "" + + "" + "
Public KeyPath IDSrcDstSeq
N/AN/AN/AN/AN/A
Public KeySrcDstSeq
N/AN/AN/AN/A

"; } } @@ -750,7 +727,7 @@ function handleStatsPanelUpdate() { if (graph && graph.isStarted()) { for (const [key, value] of Nodes.entries()) { - nodeTable += "" + key + "[" + value.coords + "]" + value.announcement.root + "" + getNodeKey(value.snekDesc).slice(0, 4).replace(/\"/g, "").toUpperCase() + "" + value.key.slice(0, 4).replace(/\"/g, "").toUpperCase() + "" + getNodeKey(value.snekAsc).slice(0, 4).replace(/\"/g, "").toUpperCase() + ""; + nodeTable += "" + key + "[" + value.coords + "]" + value.announcement.root + "" + getNodeKey(value.snekDesc).slice(0, 4).replace(/\"/g, "").toUpperCase() + "" + value.key.slice(0, 4).replace(/\"/g, "").toUpperCase(); peerLinks += value.peers.length; if (rootConvergence.has(value.announcement.root)) { @@ -777,7 +754,7 @@ function handleStatsPanelUpdate() { "" + "

Node Summary

" + "" + - "" + + "" + nodeTable + "
NameCoordsRoot↓Key↑
NameCoordsRoot↓Key
" + "

Tree Building

" + diff --git a/cmd/pineconesim/ui/modules/server-api.js b/cmd/pineconesim/ui/modules/server-api.js index 8f4d75de..e587338b 100644 --- a/cmd/pineconesim/ui/modules/server-api.js +++ b/cmd/pineconesim/ui/modules/server-api.js @@ -16,9 +16,8 @@ export const APIUpdateID = { PeerAdded: 3, PeerRemoved: 4, TreeParentUpdated: 5, - SnakeAscUpdated: 6, - SnakeDescUpdated: 7, - TreeRootAnnUpdated: 8, + SnakeDescUpdated: 6, + TreeRootAnnUpdated: 7, }; export const APICommandID = { diff --git a/router/events/events.go b/router/events/events.go index 2ab308e7..642b64b5 100644 --- a/router/events/events.go +++ b/router/events/events.go @@ -18,23 +18,6 @@ import ( "github.com/matrix-org/pinecone/types" ) -/* API Events: -DONE: - Peer Added - Peer Removed - Tree Parent Changed - Snake Descending Node Changed - Snake Ascending Node Changed - Tree Root Announcement Changed - -TODO: - Snake Table Entry Added - Snake Table Entry Removed - -NOTE: - Events need to be processed in FIFO order. -*/ - type Event interface { isEvent() } @@ -62,17 +45,8 @@ type TreeParentUpdate struct { // Tag TreeParentUpdate as an Event func (e TreeParentUpdate) isEvent() {} -type SnakeAscUpdate struct { - PeerID string - PathID string -} - -// Tag SnakeAscUpdate as an Event -func (e SnakeAscUpdate) isEvent() {} - type SnakeDescUpdate struct { PeerID string - PathID string } // Tag SnakeDescUpdate as an Event diff --git a/router/manhole.go b/router/manhole.go index bff73901..31b6a931 100644 --- a/router/manhole.go +++ b/router/manhole.go @@ -30,7 +30,6 @@ type manholeResponse struct { Parent *peer `json:"parent"` Peers map[string][]manholePeer `json:"peers"` SNEK struct { - Ascending *virtualSnakeEntry `json:"ascending"` Descending *virtualSnakeEntry `json:"descending"` Paths []*virtualSnakeEntry `json:"paths"` } `json:"snek"` diff --git a/router/state.go b/router/state.go index d3dbbcc6..bf016035 100644 --- a/router/state.go +++ b/router/state.go @@ -181,15 +181,11 @@ func (s *state) _setDescendingNode(node *virtualSnakeEntry) { s.r.Act(nil, func() { peerID := "" - pathID := []byte{} if node != nil { peerID = node.PublicKey.String() - /*if node.virtualSnakeIndex != nil { - pathID, _ = node.PathID.MarshalJSON() - }*/ } - s.r._publish(events.SnakeDescUpdate{PeerID: peerID, PathID: string(pathID)}) + s.r._publish(events.SnakeDescUpdate{PeerID: peerID}) }) } From bcba8b3875912d97dec4ad93f720bc248492c948 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Thu, 28 Apr 2022 14:23:35 -0500 Subject: [PATCH 20/41] Update wireshark dissector to match softstate frames --- pinecone.lua | 180 +++++++++++++-------------------------------------- 1 file changed, 45 insertions(+), 135 deletions(-) diff --git a/pinecone.lua b/pinecone.lua index cd3e3bb5..48b68ac7 100644 --- a/pinecone.lua +++ b/pinecone.lua @@ -21,15 +21,11 @@ local frame_types = { [1] = "Tree Announcement", [2] = "Tree Routed", [3] = "Bootstrap", - [4] = "Bootstrap ACK", - [5] = "Setup", - [6] = "Setup ACK", - [7] = "Teardown", - [8] = "SNEK Routed", - [9] = "SNEK Ping", - [10] = "SNEK Pong", - [11] = "Tree Ping", - [12] = "Tree Pong", + [4] = "SNEK Routed", + [5] = "SNEK Ping", + [6] = "SNEK Pong", + [7] = "Tree Ping", + [8] = "Tree Pong", } header_size = 10 @@ -58,8 +54,6 @@ source = ProtoField.string("pinecone.src", "Source Coords") source_key = ProtoField.bytes("pinecone.srckey", "Source Key") source_sig = ProtoField.bytes("pinecone.srcsig", "Source Signature") -path_sig = ProtoField.bytes("pinecone.pathsig", "Path Signature") - hop_count = ProtoField.uint16("pinecone.hops", "Hop Count") payload = ProtoField.bytes("pinecone.payload", "Payload", base.SPACE) @@ -70,13 +64,16 @@ sigport = ProtoField.uint8("pinecone.sigport", "Port") sigkey = ProtoField.bytes("pinecone.sigkey", "Public key") sigsig = ProtoField.bytes("pinecone.sigsig", "Signature") -pathid = ProtoField.bytes("pinecone.pathid", "Path ID") -failing = ProtoField.uint8("pinecone.failing", "Failing Bootstrap") +bootstrap_seq = ProtoField.uint32("pinecone.bootstrapseq", "Bootstrap sequence number") + +watermark_key = ProtoField.bytes("pinecone.wmarkkey", "Watermark public key") +watermark_seq = ProtoField.uint32("pinecone.wmarkseq", "Watermark sequence number") pinecone_protocol.fields = { magic_bytes, frame_version, frame_type, extra_bytes, frame_len, destination_len, source_len, payload_len, destination, source, destination_key, source_key, destination_sig, source_sig, - path_sig, payload, rootkey, rootseq, sigkey, sigport, sigsig, roottgt, pathid, failing + payload, rootkey, rootseq, sigkey, sigport, sigsig, roottgt, bootstrap_seq, watermark_key, + watermark_seq } function short_pk(key) @@ -129,122 +126,30 @@ local function do_pinecone_dissect(buffer, pinfo, tree) elseif ftype == 3 then -- Bootstrap local plen = buffer(f_payload_idx, 2):uint() - local slen = buffer(f_payload_idx + 2, 2):uint() - local srccoords = coords(buffer(f_payload_idx + 4, slen)) - local dstkey = buffer(f_payload_idx + 4 + slen, 32) - local pload = buffer(f_payload_idx + 4 + slen + 32, plen) - subtree:add(payload_len, buffer(f_payload_idx, 2), plen) - - local srcsubtree = subtree:add(subtree, buffer(f_payload_idx + 2, slen + 2), "Source") - srcsubtree:set_text("Source " .. srccoords) - srcsubtree:add(source_len, buffer(f_payload_idx + 2, 2), slen) - srcsubtree:add(source, buffer(f_payload_idx + 4, slen), srccoords) - - subtree:add(destination_key, dstkey) - - local psubtree = subtree:add(subtree, pload, "Payload") - psubtree:set_text("Payload") - psubtree:add(pathid, pload(0, 8)) - psubtree:add(rootkey, pload(8, 32)) - local seq, offset = varu64(pload(40):bytes()) - psubtree:add(rootseq, pload(40, offset), seq) - psubtree:add(sigsig, pload(40 + offset, 64)) - -- psubtree:add(failing, pload(104 + offset, 1)) - -- TODO : Add sigs if failing - - -- Info column - pinfo.cols.info:set(frame_types[3]) - pinfo.cols.info:append(" " .. srccoords .. " → [" .. - short_pk(dstkey:bytes():raw()) .. "]") - elseif ftype == 4 then - -- Bootstrap ACK - local plen = buffer(f_payload_idx, 2):uint() - subtree:add(payload_len, buffer(f_payload_idx, 2), plen) - - local dlen = buffer(f_payload_idx + 2, 2):uint() - local slen = buffer(f_payload_idx + 4 + dlen, 2):uint() - local dstcoords = coords(buffer(f_payload_idx + 2 + 2, dlen)) - local srccoords = coords(buffer(f_payload_idx + 4 + dlen + 2, slen)) - subtree:add(destination_len, buffer(f_payload_idx + 2, 2), dlen) - subtree:add(destination, buffer(f_payload_idx + 4, dlen), dstcoords) - subtree:add(source_len, buffer(f_payload_idx + 4 + dlen, 2), slen) - subtree:add(source, buffer(f_payload_idx + 4 + dlen + 2, slen), srccoords) - subtree:add(destination_key, buffer(f_payload_idx + 6 + dlen + slen, 32)) - subtree:add(source_key, buffer(f_payload_idx + 6 + dlen + slen + 32, 32)) - local pload = buffer(f_payload_idx + 6 + dlen + slen + 64, plen) - local psubtree = subtree:add(subtree, pload, "Payload") - psubtree:set_text("Payload") - psubtree:add(pathid, pload(0, 8)) - psubtree:add(rootkey, pload(8, 32)) - local seq, offset = varu64(pload(40):bytes()) - psubtree:add(rootseq, pload(40, offset), seq) - psubtree:add(source_sig, pload(40 + offset, 64)) - psubtree:add(destination_sig, pload(104 + offset, 64)) - - -- Info column - pinfo.cols.info:set(frame_types[4]) - pinfo.cols.info:append(" " .. srccoords .. " → " .. dstcoords) - elseif ftype == 5 then - -- Setup - local plen = buffer(f_payload_idx, 2):uint() - subtree:add(payload_len, buffer(f_payload_idx, 2), plen) - local dlen = buffer(f_payload_idx + 2, 2):uint() - local dstcoords = coords(buffer(f_payload_idx + 4, dlen)) - subtree:add(destination_len, buffer(f_payload_idx + 2, 2), dlen) - subtree:add(destination, buffer(f_payload_idx + 4, dlen), dstcoords) - local srckey = buffer(f_payload_idx + 4 + dlen, 32) - subtree:add(source_key, buffer(f_payload_idx + 4 + dlen, 32)) - subtree:add(destination_key, buffer(f_payload_idx + 4 + dlen + 32, 32)) - - local pload = buffer(f_payload_idx + 4 + dlen + 64, plen) - local psubtree = subtree:add(subtree, pload, "Payload") - psubtree:set_text("Payload") - psubtree:add(pathid, pload(0, 8)) - psubtree:add(rootkey, pload(8, 32)) - local seq, offset = varu64(pload(40):bytes()) - psubtree:add(rootseq, pload(40, offset), seq) - psubtree:add(source_sig, pload(40 + offset, 64)) - psubtree:add(destination_sig, pload(104 + offset, 64)) - - -- Info column - pinfo.cols.info:set(frame_types[5]) - pinfo.cols.info:append(" [" .. short_pk(srckey:bytes():raw()) .. "] → " .. - dstcoords) - elseif ftype == 6 then - -- Setup ACK - local plen = buffer(f_payload_idx, 2):uint() - subtree:add(payload_len, buffer(f_payload_idx, 2), plen) local dstkey = buffer(f_payload_idx + 2, 32) - subtree:add(destination_key, buffer(f_payload_idx + 2, 32)) - local pload = buffer(f_payload_idx + 2 + 32, plen) - local psubtree = subtree:add(subtree, pload, "Payload") - psubtree:set_text("Payload") - psubtree:add(pathid, pload(0, 8)) - psubtree:add(rootkey, pload(8, 32)) - local seq, offset = varu64(pload(40):bytes()) - psubtree:add(rootseq, pload(40, offset), seq) - psubtree:add(path_sig, pload(40 + offset, 64)) + local wmarkkey = buffer(f_payload_idx + 2 + 32, 32) + subtree:add(watermark_key, buffer(f_payload_idx + 2 + 32, 32)) + local wmarkseq, offset = varu64(buffer(f_payload_idx + 2 + 64):bytes()) + subtree:add(watermark_seq, buffer(f_payload_idx + 2 + 64, offset), wmarkseq) - -- Info column - pinfo.cols.info:set(frame_types[6]) - pinfo.cols.info:append(" → [" .. short_pk(dstkey:bytes():raw()) .. "]") - elseif ftype == 7 then - -- Teardown - local plen = buffer(f_payload_idx, 2):uint() + local pload = buffer(f_payload_idx + 2 + 32 + 32 + offset, plen) subtree:add(payload_len, buffer(f_payload_idx, 2), plen) - local dstkey = buffer(f_payload_idx + 2, 32) - subtree:add(destination_key, buffer(f_payload_idx + 2, 32)) + subtree:add(destination_key, dstkey) - local pload = buffer(f_payload_idx + 2 + 32, plen) local psubtree = subtree:add(subtree, pload, "Payload") psubtree:set_text("Payload") - psubtree:add(pathid, pload(0, 8)) + local seq, offset = varu64(pload(0):bytes()) + psubtree:add(bootstrap_seq, pload(0, offset), seq) + psubtree:add(rootkey, pload(offset, 32)) + local root_seq, root_offset = varu64(pload(offset + 32):bytes()) + psubtree:add(rootseq, pload(offset + 32, root_offset), root_seq) + psubtree:add(sigsig, pload(offset + 32 + root_offset, 64)) -- Info column - pinfo.cols.info:set(frame_types[7]) - pinfo.cols.info:append(" → [" .. short_pk(dstkey:bytes():raw()) .. "]") - elseif (ftype == 8 or ftype == 9 or ftype == 10) then + pinfo.cols.info:set(frame_types[3]) + pinfo.cols.info:append(" " .. short_pk(dstkey:bytes():raw()) .. " → ") + elseif (ftype == 4 or ftype == 5 or ftype == 6) then -- SNEK Routed -- SNEK Ping -- SNEK Pong @@ -255,25 +160,30 @@ local function do_pinecone_dissect(buffer, pinfo, tree) local srckey = buffer(f_payload_idx + 2 + 32, 32) subtree:add(source_key, buffer(f_payload_idx + 2 + 32, 32)) - local pload = buffer(f_payload_idx + 2 + 64, plen) + local wmarkkey = buffer(f_payload_idx + 2 + 32 + 32, 32) + subtree:add(watermark_key, buffer(f_payload_idx + 2 + 32 + 32, 32)) + local wmarkseq, offset = varu64(buffer(f_payload_idx + 2 + 64 + 32):bytes()) + subtree:add(watermark_seq, buffer(f_payload_idx + 2 + 64 + 32, offset), wmarkseq) + + local pload = buffer(f_payload_idx + 2 + 64 + 32 + offset, plen) local psubtree = subtree:add(subtree, pload, "Payload") psubtree:set_text("Payload") - if plen > 0 and ftype == 8 then + if plen > 0 and ftype == 4 then -- SNEK Routed quic_dissector = Dissector.get("quic") quic_dissector:call(pload:tvb(), pinfo, tree) if pinfo.cols.protocol ~= pinecone_protocol.name then pinfo.cols.protocol:prepend(pinecone_protocol.name .. "-") end - pinfo.cols.info:set(frame_types[8]) - elseif (ftype == 9 or ftype == 10) then - if ftype == 9 then + pinfo.cols.info:set(frame_types[4]) + elseif (ftype == 5 or ftype == 6) then + if ftype == 5 then -- SNEK Ping - pinfo.cols.info:set(frame_types[9]) - elseif ftype == 10 then + pinfo.cols.info:set(frame_types[5]) + elseif ftype == 6 then -- SNEK Pong - pinfo.cols.info:set(frame_types[10]) + pinfo.cols.info:set(frame_types[6]) end subtree:add(hop_count, buffer(f_extra_idx, 2), buffer(f_extra_idx, 2):uint()) end @@ -332,7 +242,7 @@ local function do_pinecone_dissect(buffer, pinfo, tree) short_pk(payload(0, 32):bytes():raw()) .. "]") pinfo.cols.info:append(" Coords=[" .. table.concat(ports, " ") .. "]") - elseif (ftype == 2 or ftype == 11 or ftype == 12) then + elseif (ftype == 2 or ftype == 7 or ftype == 8) then if plen > 0 and ftype == 2 then -- Tree Routed quic_dissector = Dissector.get("quic") @@ -341,13 +251,13 @@ local function do_pinecone_dissect(buffer, pinfo, tree) pinfo.cols.protocol:prepend(pinecone_protocol.name .. "-") end pinfo.cols.info:set(frame_types[2]) - elseif (ftype == 11 or ftype == 12) then - if ftype == 11 then + elseif (ftype == 7 or ftype == 8) then + if ftype == 7 then -- Tree Ping - pinfo.cols.info:set(frame_types[11]) - elseif ftype == 12 then + pinfo.cols.info:set(frame_types[7]) + elseif ftype == 8 then -- Tree Pong - pinfo.cols.info:set(frame_types[12]) + pinfo.cols.info:set(frame_types[8]) end subtree:add(hop_count, buffer(f_extra_idx, 2), buffer(f_extra_idx, 2):uint()) end From efe17108ae135f837d2de61ebe91f350f21a95da Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Thu, 28 Apr 2022 16:18:09 -0500 Subject: [PATCH 21/41] Add unit test for snake bootstrap --- types/virtualsnake_test.go | 74 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 types/virtualsnake_test.go diff --git a/types/virtualsnake_test.go b/types/virtualsnake_test.go new file mode 100644 index 00000000..5c31b22a --- /dev/null +++ b/types/virtualsnake_test.go @@ -0,0 +1,74 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +import ( + "bytes" + "crypto/ed25519" + "fmt" + "testing" +) + +func TestMarshalUnmarshalBootstrap(t *testing.T) { + pkr, _, _ := ed25519.GenerateKey(nil) + _, sk1, _ := ed25519.GenerateKey(nil) + input := &VirtualSnakeBootstrap{ + Sequence: 7, + Root: Root{ + RootSequence: 1, + }, + } + copy(input.RootPublicKey[:], pkr) + var err error + protected, err := input.ProtectedPayload() + if err != nil { + t.Fatal(err) + } + copy( + input.Signature[:], + ed25519.Sign(sk1[:], protected), + ) + var buffer [65535]byte + n, err := input.MarshalBinary(buffer[:]) + if err != nil { + t.Fatal(err) + } + + var output VirtualSnakeBootstrap + if _, err = output.UnmarshalBinary(buffer[:n]); err != nil { + t.Fatal(err) + } + + if output.Sequence != input.Sequence { + fmt.Println("expected:", input.Sequence) + fmt.Println("got:", output.Sequence) + t.Fatalf("bootstrap sequence doesn't match") + } + if !bytes.Equal(pkr, output.RootPublicKey[:]) { + fmt.Println("expected:", pkr) + fmt.Println("got:", output.RootPublicKey) + t.Fatalf("root public key doesn't match") + } + if output.RootSequence != input.RootSequence { + fmt.Println("expected:", input.RootSequence) + fmt.Println("got:", output.RootSequence) + t.Fatalf("root sequence doesn't match") + } + if !bytes.Equal(input.Signature[:], output.Signature[:]) { + fmt.Println("expected:", input.Signature) + fmt.Println("got:", output.Signature) + t.Fatalf("root public key doesn't match") + } +} From 06f254784875c92a2a9a5e1edd88406c72ee3448 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Thu, 28 Apr 2022 17:17:07 -0500 Subject: [PATCH 22/41] Fix incorrect comment about bootstrap forwarding conditions --- router/state_forward.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/router/state_forward.go b/router/state_forward.go index 94a9d789..f438ae55 100644 --- a/router/state_forward.go +++ b/router/state_forward.go @@ -68,8 +68,7 @@ func (s *state) _forward(p *peer, f *types.Frame) error { return nil case types.TypeVirtualSnakeBootstrap: - // Bootstrap messages are only handled specially when they reach a dead end. - // Otherwise they are forwarded normally by falling through. + // Bootstrap messages are handled at each node along the path. if !s._handleBootstrap(p, nexthop, f) || deadend { return nil } From a8a5b35060bc5d3822e780c65b92617acff497b3 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 29 Apr 2022 14:52:32 +0100 Subject: [PATCH 23/41] Add `queue` interface --- router/manhole.go | 4 ++-- router/peer.go | 4 ++-- router/queue.go | 26 ++++++++++++++++++++++++++ router/queuelifo.go | 4 ++++ router/version.go | 2 +- 5 files changed, 35 insertions(+), 5 deletions(-) create mode 100644 router/queue.go diff --git a/router/manhole.go b/router/manhole.go index 31b6a931..b5cf4d2e 100644 --- a/router/manhole.go +++ b/router/manhole.go @@ -42,8 +42,8 @@ type manholePeer struct { PeerType ConnectionPeerType `json:"type,omitempty"` PeerZone ConnectionZone `json:"zone,omitempty"` PeerURI ConnectionURI `json:"uri,omitempty"` - ProtoQueue *fifoQueue `json:"proto_queue"` - TrafficQueue *fairFIFOQueue `json:"traffic_queue"` + ProtoQueue queue `json:"proto_queue"` + TrafficQueue queue `json:"traffic_queue"` } func (r *Router) ManholeHandler(w http.ResponseWriter, req *http.Request) { diff --git a/router/peer.go b/router/peer.go index 3e19fbe7..e3601bdc 100644 --- a/router/peer.go +++ b/router/peer.go @@ -62,8 +62,8 @@ type peer struct { public types.PublicKey // Not mutated after peer setup. keepalives bool // Not mutated after peer setup. started atomic.Bool // Thread-safe toggle for marking a peer as down. - proto *fifoQueue // Thread-safe queue for outbound protocol messages. - traffic *fairFIFOQueue // Thread-safe queue for outbound traffic messages. + proto queue // Thread-safe queue for outbound protocol messages. + traffic queue // Thread-safe queue for outbound traffic messages. } func (p *peer) MarshalJSON() ([]byte, error) { diff --git a/router/queue.go b/router/queue.go new file mode 100644 index 00000000..d1a209e3 --- /dev/null +++ b/router/queue.go @@ -0,0 +1,26 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package router + +import "github.com/matrix-org/pinecone/types" + +type queue interface { + queuecount() int + queuesize() int + push(frame *types.Frame) bool + pop() <-chan *types.Frame + ack() + reset() +} diff --git a/router/queuelifo.go b/router/queuelifo.go index 3b2f312b..af8cbe84 100644 --- a/router/queuelifo.go +++ b/router/queuelifo.go @@ -79,6 +79,10 @@ func (q *lifoQueue) pop() (*types.Frame, bool) { // nolint:unused return frame, true } +func (q *lifoQueue) ack() { // nolint:unused + // no-op on this queue type +} + func (q *lifoQueue) reset() { // nolint:unused q.mutex.Lock() defer q.mutex.Unlock() diff --git a/router/version.go b/router/version.go index c2a464fc..44f19128 100644 --- a/router/version.go +++ b/router/version.go @@ -17,7 +17,7 @@ package router const ( capabilityLengthenedRootInterval = 1 << iota capabilityCryptographicSetups - capabilitySetupACKs + capabilitySetupACKs // nolint:deadcode,varcheck capabilityDedupedCoordinateInfo capabilitySoftState ) From 1ee3761efdb084664d14bb67641e04229a0d5218 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 13 May 2022 10:29:29 -0500 Subject: [PATCH 24/41] Initial docs update for softstate algorithm --- docs/introduction.md | 2 +- docs/introduction/2_node_anatomy.md | 2 +- docs/peer_management/2_peer_disconnects.md | 8 +- docs/virtual_snake/1_neighbours.md | 18 ++--- docs/virtual_snake/2_bootstrapping.md | 23 ++---- docs/virtual_snake/3_bootstraps.md | 63 ++++------------ docs/virtual_snake/4_bootstrap_acks.md | 87 ---------------------- docs/virtual_snake/5_path_setups.md | 68 ----------------- docs/virtual_snake/6_teardowns.md | 34 --------- docs/virtual_snake/7_next_hop.md | 50 ------------- docs/virtual_snake/8_maintenance.md | 14 ---- 11 files changed, 34 insertions(+), 335 deletions(-) delete mode 100644 docs/virtual_snake/4_bootstrap_acks.md delete mode 100644 docs/virtual_snake/5_path_setups.md delete mode 100644 docs/virtual_snake/6_teardowns.md delete mode 100644 docs/virtual_snake/7_next_hop.md delete mode 100644 docs/virtual_snake/8_maintenance.md diff --git a/docs/introduction.md b/docs/introduction.md index 71671c79..e52fe63f 100644 --- a/docs/introduction.md +++ b/docs/introduction.md @@ -48,5 +48,5 @@ The Pinecone routing scheme effectively is built up of two major components: * Provides efficient, low-stretch and strictly loop-free routing; * Effective means of exchanging path setup messages, even before SNEK bootstrap has taken place; -* A **virtual snake** (or **SNEK**) — a double-linked linear routing topology: +* A **virtual snake** (or **SNEK**) — a single-linked linear routing topology: * Provides resilient public key-based routing across the overlay network. diff --git a/docs/introduction/2_node_anatomy.md b/docs/introduction/2_node_anatomy.md index c663ac98..774bfce5 100644 --- a/docs/introduction/2_node_anatomy.md +++ b/docs/introduction/2_node_anatomy.md @@ -16,6 +16,6 @@ The router maintains state that allows it to participate in the network, includi * Information about all directly connected peers, including their public key, last tree announcement received, how they are connected to us etc; * Which of the node’s directly connected peers is our chosen parent in the tree, if any; * A routing table, containing information about SNEK paths that have been set up through this node by other nodes; -* Information about our ascending and descending keyspace neighbours — that is, which node has the next highest (ascending) and next lowest (descending) key to the node’s own; +* Information about our descending keyspace neighbour — that is, which node has the next lowest (descending) key to the node’s own; * A sequence number, used when operating as a root node and sending the nodes own root announcements into the network; * Maintenance timers for tree and SNEK maintenance. diff --git a/docs/peer_management/2_peer_disconnects.md b/docs/peer_management/2_peer_disconnects.md index 6097691f..79d07812 100644 --- a/docs/peer_management/2_peer_disconnects.md +++ b/docs/peer_management/2_peer_disconnects.md @@ -11,10 +11,6 @@ A Pinecone node must immediately remove any state related to a peer that disconn If the chosen parent is the disconnected peer, the node must re-run the parent selection algorithm immediately, either selecting a new parent (with the equivalent **Root public key** and **Root sequence**) or by becoming a root node and waiting for a stronger update from a peer. -If the disconnected peer appears in any entry in the routing table, as either the **Source port** or **Destination port**, a teardown must be sent to the remaining port. For example, if the **Source port** is now disconnected, a teardown for the path must be sent to the **Destination port**. The entry should then be removed from the routing table. +If the disconnected peer appears in any entry in the routing table, as either the **Source port** or **Destination port**, the entry should be removed from the routing table. -If the disconnected port is the **Destination port** of the ascending node entry, the ascending entry should be cleared. There is no **Source port** for an ascending node entry, therefore it is not possible to send a teardown for this path. The node on the other side of the failed connection is responsible for sending a teardown in this case. - -If the disconnected port is the **Source port** of the descending node entry, the descending entry should be cleared. There is no **Destination port** for a descending node entry, therefore it is not possible to send a teardown for this path. The node on the other side of the failed connection is responsible for sending a teardown in this case. - -The next iteration of the routine maintenance should send a bootstrap message into the network if the ascending node entry was cleared. +If the disconnected port is the **Source port** of the descending node entry, the descending entry should be cleared. diff --git a/docs/virtual_snake/1_neighbours.md b/docs/virtual_snake/1_neighbours.md index c525a275..5408f7b5 100644 --- a/docs/virtual_snake/1_neighbours.md +++ b/docs/virtual_snake/1_neighbours.md @@ -7,15 +7,15 @@ permalink: /virtual_snake/neighbours # Snake Neighbours -Each node in the topology has a reference to an ascending and a descending path. The ascending path is the path on the network that leads to the closest public key to our own in the ascending direction (the next highest key), and the descending path is the path on the network that leads to the next closest public key to our own in the descending direction (the next lowest key). +Each node in the topology has a reference to a descending path. The descending path is the path on the network that leads to the next closest public key to our own in the descending direction (the next lowest key). -There are only two exceptions to this rule: the node with the highest key on the network will have only a descending path (as there is no higher key to build an ascending path to) and the node with the lowest key on the network will have only an ascending path (as there is no lower key to build a descending path). +There is only one exception to this rule: the node with the lowest key on the network will have only an ascending path (as there is no lower key to build a descending path). -For the ascending and descending paths, a node should store the following information: +For the descending path, a node should store the following information: -1. The **Path public key** and **Path ID**, as exchanged during bootstrap/path setup; -2. The **Origin public key**, noting which node initiated the path creation; -3. The **Source port**, where the Bootstrap ACK message arrived from (for descending path entries); -4. The **Destination port**, where the Path Setup message was forwarded to next (for ascending path entries); -5. The **Last seen** time, noting when the entry was populated; -6. The **Root public key** and **Root sequence** that the path was set up with. +1. The **Origin public key**, noting which node initiated the path creation; +2. The **Source port**, where the Bootstrap message arrived from; +3. The **Destination port**, where the Bootstrap message was forwarded to next (if applicable); +4. The **Last seen** time, noting when the entry was populated; +5. The **Root public key** and **Root sequence** that the path was set up with. +6. The **Watermark public key** and **Watermark sequence** that the path was set up with. diff --git a/docs/virtual_snake/2_bootstrapping.md b/docs/virtual_snake/2_bootstrapping.md index 17e5912f..f3c98ae4 100644 --- a/docs/virtual_snake/2_bootstrapping.md +++ b/docs/virtual_snake/2_bootstrapping.md @@ -7,25 +7,16 @@ permalink: /virtual_snake/bootstrapping # Bootstrapping -Bootstrapping is the process of joining the snake topology. Bootstrapping takes place in three steps: +Bootstrapping is the process of joining the snake topology. Bootstrapping takes place every 5 seconds and takes place in two steps: -1. The bootstrapping node sends a bootstrap message into the network, with their own public key as the “target” key, which will be routed to the nearest keyspace neighbour; -2. The nearest keyspace neighbour will respond to the bootstrap message by sending back a Bootstrap ACK; -3. The bootstrapping node will respond to the Bootstrap ACK by sending a Path Setup message to the nearest keyspace neighbour. - -Nodes bootstrap when they do not have an ascending path — that is, they do not know who the next highest public key belongs to. The descending path is populated passively by another node bootstrapping and building a path to that node. +1. The bootstrapping node sends a bootstrap message into the network, with their own public key as the “destination” key, which will be routed to the nearest keyspace neighbour; +2. The nearest keyspace neighbour will add the bootstrapping node as their descending neighbour. The bootstrap message contains the following fields: - - - - - @@ -40,8 +31,6 @@ The bootstrap message contains the following fields:
Source coordinates -
Path public key - Path ID + Bootstrap Sequence
-The combination of the **Path public key** and the **Path ID** uniquely identify a path. While the **Path public key** is predetermined by the public key of the bootstrapping node, the **Path ID** must be generated randomly by the bootstrapping node and should not be reused. - -The **Source signature** is an ed25519 signature covering both the **Path public key** and **Path ID** by concatenating them together and signing the result. This enables the remote side to verify that the bootstrap was genuinely initiated by the sending node and has not been forged. +The **Source signature** is an ed25519 signature covering the **Bootstrap Sequence**, **Root public key** and **Root Sequence** by concatenating them together and signing the result. This enables the remote side to verify that the bootstrap was genuinely initiated by the sending node and has not been forged. -Bootstraps will travel through the network, forwarded **using SNEK routing with bootstrap rules**, towards the target key, until they arrive at the node that is closest to the target key. The forwarding logic specifically will not deliver bootstrap messages to the actual target key, so the bootstrap message will eventually arrive at a “dead end” at the next closest key. +Bootstraps will travel through the network, forwarded **using SNEK routing with bootstrap rules**, towards the destination key, until they arrive at the node that is closest to the destination key. The forwarding logic specifically will not deliver bootstrap messages to the actual destination key, so the bootstrap message will eventually arrive at a “dead end” at the next closest key. diff --git a/docs/virtual_snake/3_bootstraps.md b/docs/virtual_snake/3_bootstraps.md index b58888e7..4910f1b2 100644 --- a/docs/virtual_snake/3_bootstraps.md +++ b/docs/virtual_snake/3_bootstraps.md @@ -7,56 +7,23 @@ permalink: /virtual_snake/bootstraps # Handling Bootstrap Messages -Once the bootstrap message arrives at a dead end, the node will respond using **tree routing** back to the node with a Bootstrap ACK message. +Once the bootstrap message arrives at a dead end, the node will update it's descending node entry if it makes sense to do so (ie. same **Root public key** and **Root sequence** and a closer key than the previous descending entry or an update from our existing descending node). Before doing anything, the node must ensure that the signature in the **Source signature** field is valid by checking against the **Destination public key**. If the signature is invalid, the bootstrap message should be silently dropped and not processed any further. Additionally, the node should ensure that the **Root public key** and **Root sequence** of the bootstrap message match those of the most recent root announcement from our chosen parent, if any, or the node’s own public key and sequence number if the node is currently acting as a root node. If this is not true, the bootstrap message should be silently dropped and not processed any further. -A Bootstrap ACK message contains the following fields: - - - - - - - - - - - - - - - - - - - - - -
Destination coordinates - Source coordinates -
Destination public key - Source public key -
Path ID -
Root public key - Root sequence -
Source signature - Destination signature -
- -The responding node should: - -1. Copy the bootstrap **Source coordinates** into the **Destination coordinates** field; -2. Copy the bootstrap **Path public key** into the **Destination public key** field; -3. Copy the bootstrap **Path ID** into the **Path ID** field; -4. Copy the bootstrap **Source signature** into the **Source signature** field; -5. Populate the node’s own current coordinates into the **Source coordinates** field; -6. Populate the node’s own public key into the **Source public key** field; -7. Copy their own parent’s last root announcement **Root public key** and **Root sequence** fields into the corresponding fields; -8. Add the **Destination signature**, as below. - -The **Destination signature** is a signature that covers the **Source signature**, **Path public key** and **Path ID** fields by concatenating all three values together and then signing the result. It enables any node on the network to verify that the acknowledging node accepted a specific bootstrap for a specific path. - -Note that the **Source signature** is copied and preserved from the original packet without modification. This is so that the bootstrap ACK contains signatures from both ends of the bootstrap. +Each node along the bootstrapping path should install the bootstrapping node into their routing table. + +## Install route into routing table + +Regardless of whether the bootstrap message is considered to have arrived at its intended destination (there is no closer node to route to) or not, a bootstrap message should result in the route being installed into the routing table of each node handling the message. + +To install the route into the routing table, the node should either create a new entry or overwrite the existing entry and: + +1. Copy the **Origin public key** into the virtual snake index; +2. Copy the bootstrap **Root public key** and **Root sequence** into the appropriate fields; +3. Populate the **Last seen time** with the current time; +4. Populate the **Source port** with a reference to the port that the setup message was received from; +5. Populate the **Destination port** with a reference to the chosen next-hop port, unless the bootstrap message has reached its intended destination, in which case this field should remain empty; +6. Copy the setup **Watermark public key** and **Watermark sequence** into the appropriate fields. diff --git a/docs/virtual_snake/4_bootstrap_acks.md b/docs/virtual_snake/4_bootstrap_acks.md deleted file mode 100644 index 458f2fe0..00000000 --- a/docs/virtual_snake/4_bootstrap_acks.md +++ /dev/null @@ -1,87 +0,0 @@ ---- -title: Handling Bootstrap ACKs -parent: Virtual Snake -nav_order: 4 -permalink: /virtual_snake/bootstrap_acks ---- - -# Handling Bootstrap ACK Messages - -Upon receipt of a Bootstrap ACK packet, the bootstrapping node now is considered to have “found” a potential ascending node candidate. - -The Bootstrap ACK packet will contain two signatures: the **Source signature** and the **Destination signature**. Both of these signatures should be verified to ensure authenticity. The **Source signature** must be verified using the node’s own public key and the **Destination signature** must be verified using the **Source public key**. If either signature is invalid, the packet should be dropped and not processed any further. - -If the signature is valid, the following checks should be made to see if it is suitable: - -1. Drop the update and do not process further if any of the following are true: - 1. If the **Source public key** is the same as the node’s own public key, implying that the node somehow received a Bootstrap ACK from itself; - 2. If the **Root public key** does not match that of our chosen parent’s last announcement; - 3. If the **Root sequence** does not match that of our chosen parent’s last announcement; -2. If the node already has an ascending entry, and it has not expired: - 1. If the **Source public key** is the same as the existing ascending entry’s public key, and the **Path ID** is different to the existing ascending entry’s path ID, accept the update; - 2. If the ordering **Node public key ＜ Source public key ＜ Ascending origin public key** is true, that is that the public key that the Bootstrap ACK came from is closer to us in keyspace than our previous ascending node, accept the update; -3. If the node does not have an ascending entry, or the node has an expired ascending entry: - 1. If the **Source public key** is greater than the node’s own public key, accept the update. - -If the update has not been accepted, it should be dropped. There is no new path to tear down and it is not necessary to respond to the bootstrapping node. - -If the update has been accepted, a Path Setup message is constructed. A Path Setup message contains the following fields: - - - - - - - - - - - - - - - - - - - - -
Destination public key - Destination coordinates -
Source public key -
Path ID -
Root public key - Root sequence -
Source signature - Destination signature -
- -The responding node should: - -1. Copy the bootstrap ACK **Source public key** into the **Destination public key** field; -2. Copy the bootstrap ACK **Source coordinates** into the **Destination coordinates** field; -3. Populate the node’s own public key into the **Source public key** field; -4. Copy the bootstrap ACK’s **Path ID** into the **Path ID** field; -5. Copy their own parent’s last root announcement **Root public key** and **Root sequence** fields into the corresponding fields; -6. Copy the bootstrap ACK’s **Source signature** into the **Source signature** field; -7. Copy the bootstrap ACK’s **Destination signature** into the **Destination signature** field. - -Path setup messages are routed using **tree routing** — that is, the **Destination coordinates** are the prominent field when making routing decisions. The **Destination public key** field is used to confirm that the setup has reached its intended destination. - -Each path setup message will contain both a **Source signature** and a **Destination signature**. The **Source signature** must be verified using the **Source public key** and the **Destination signature** must be verified using the **Destination public key**. If either of the signatures is invalid, the setup message should be dropped and a teardown must be sent back for the new path to the port that the setup message was received on. - -The node should attempt to look up the next hop for the message and then attempt to forward the Path Setup onto the first hop. If this fails, either because there is no suitable next-hop identified or because the packet was not successfully deliverable to the first hop, the setup message should be dropped and a teardown must be sent back for the new path to the port that the setup message was received on. - -The path is only useful if we can assert that it arrived at the intended destination and was set up correctly at all intermediate nodes without being torn down at any point. Since no routing information has been installed yet, there is nothing to tear down, so dropping is sufficient to abort the path setup altogether. - -If the Path Setup was successfully forwarded, the node’s ascending reference should be populated to point to the node from which the Bootstrap ACK arrived from: - -1. Copy the bootstrap ACK **Destination public key** into the **Path public key** field; -2. Copy the bootstrap ACK **Path ID** into the **Path ID** field; -3. Copy the bootstrap ACK **Source public key** into the **Origin public key** field; -4. Copy the bootstrap ACK **Root public key** and **Root sequence** into the appropriate fields; -5. Populate the **Last seen** time with the current time; -6. Populate the **Source** port with a reference to the port that the bootstrap ACK was received from; -7. Populate the **Destination** port with a reference to the chosen next-hop port, from above. - -Finally, the node must then iterate through the routing table and search for all entries where the **Source** port refers to nowhere/the local node, and the **Path public key** does not match the bootstrap ACK **Source public key**, sending teardowns for each of these paths and removing them from the routing table. This ensures that any stale paths from other nodes are torn down, but it will not remove paths from the newly bootstrapping node until it has received the Path Setup and torn down the path itself, avoiding a race condition. Teardowns are described in a later section. diff --git a/docs/virtual_snake/5_path_setups.md b/docs/virtual_snake/5_path_setups.md deleted file mode 100644 index 7094ed1e..00000000 --- a/docs/virtual_snake/5_path_setups.md +++ /dev/null @@ -1,68 +0,0 @@ ---- -title: Handling Path Setups -parent: Virtual Snake -nav_order: 5 -permalink: /virtual_snake/setups ---- - -# Handling Path Setup Messages - -Path Setup messages are responsible for populating the routing table, in addition to updating the reference to the descending node when the message reaches its final destination. They are different to Bootstrap and Bootstrap ACK messages in that they must be processed by all intermediate nodes on the path before being forwarded. - -Importantly, if a node decides that it wants to reject a setup message for any reason, it **must** send a teardown for the new path to ensure that it is cleaned up, as it will have possibly been installed into the routing table of other nodes on the path. This includes if the setup message cannot be forwarded to its destination due to reaching a dead end. - -If the setup message has reached its intended destination, that is that the **Destination public key** matches the node’s public key, then the responding node should decide whether or not to update their descending reference to the new path. - -If an entry in the routing table already exists with the **Destination public key** and the **Path ID**, the routing entry is a duplicate: - -1. The new path must be torn down; -2. The old path must be torn down; -3. The update must be rejected and not processed any further. - -## Arrived at intended destination - -If the **Destination public key** is equal to the node’s public key, the update is considered to have arrived at its intended destination, therefore the following checks should be performed as to whether or not to update the descending node reference: - -1. Drop the update and do not process further if any of the following are true: - 1. If the **Root public key** does not match that of our chosen parent’s last announcement; - 2. If the **Root sequence** does not match that of our chosen parent’s last announcement; - 3. The **Source public key** is not less than the node’s own public key; -2. If the node already has a descending entry, and it has not expired: - 1. If the **Source public key** is the same as the existing descending entry’s public key, and the **Path ID** is different to the existing descending entry’s **Path ID**, accept the update; - 2. If the ordering **Descending public key ＜ Source public key ＜ Node public key** is true, that is that the public key that the setup came from is closer to us in keyspace than our previous descending node, accept the update; -3. If the node does not have a descending entry, or the node has an expired descending entry: - 1. If the **Source public key** is less than the node’s own public key, accept the update. - -If the update has not been accepted, a teardown of the new path must be sent back via the receiving port and the update should be dropped. - -If the update has been accepted, the node’s descending reference should be populated to point to the node from which the Setup message arrived from: - -1. Copy the setup **Source public key** into both the **Path public key** and **Origin public key** fields; -2. Copy the setup **Path ID** into the **Path ID** field; -3. Copy the setup **Root public key** and **Root sequence** into the appropriate fields; -4. Populate the **Last seen time** with the current time; -5. Populate the **Source port** with a reference to the port that the setup message was received from; -6. Leave the **Destination port** empty, as there should be no next-hop once the setup message has been processed at its intended destination. - -Then proceed into the next section to install the route into the routing table. - -## Install route into routing table - -Regardless of whether the setup message is considered to have arrived at its intended destination (the node’s public key matches the **Destination public key**) or not, a setup message should result in the route being installed into the routing table of each node handling the message. - -In the event that the setup message is due to be forwarded (i.e. the setup message has not yet reached its intended destination), installing the routing table entry should be done **after** the message has been forwarded. - -By doing this, all transitive nodes on a given setup path will contain routing information for the newly built path. There is one of three possible outcomes: - -1. The intended destination for the setup message will accept the route, therefore the route will remain up; -2. The intended destination will reject the route, sending back a teardown along the path, causing the routing table entry to be deleted; -3. The setup message will never arrive at the intended destination, instead hitting a dead end, with the node at the dead end sending back a teardown along the path, causing the routing table entry to be deleted. - -To install the route into the routing table, the node should create a new entry and: - -1. Copy the setup **Source public key** into both the **Path public key** and **Origin public key** fields; -2. Copy the setup **Path ID** into the **Path ID** field; -3. Copy the setup **Root public key** and **Root sequence** into the appropriate fields; -4. Populate the **Last seen time** with the current time; -5. Populate the **Source port** with a reference to the port that the setup message was received from; -6. Populate the **Destination port** with a reference to the chosen next-hop port, unless the setup message has reached its intended destination, in which case this field should remain empty. diff --git a/docs/virtual_snake/6_teardowns.md b/docs/virtual_snake/6_teardowns.md deleted file mode 100644 index 9bb9f297..00000000 --- a/docs/virtual_snake/6_teardowns.md +++ /dev/null @@ -1,34 +0,0 @@ ---- -title: Path Teardown -parent: Virtual Snake -nav_order: 6 -permalink: /virtual_snake/teardown ---- - -# Path Teardown - -A teardown is a special type of protocol message that signals to a node that a path is no longer valid and should be removed. A teardown message contains the following fields: - - - - - - -
Path public key - Path ID -
- -If the teardown message arrived from any port that was not the **Source port** or the **Destination port**, the teardown must be dropped and ignored. A valid teardown will only ever arrive from the same ports as the original path was built on. - -Upon receipt of a path teardown message that matches a routing table entry and arrives from either the **Source port** or the **Destination port**, the node should clear any routing table, ascending or descending path entries that match the **Path public key** and **Path ID**. - -Once the teardown has been actioned, it must be forwarded based on the following rules: - -1. If the teardown arrived via the **Source port**, and the **Destination port** is not empty, forward it to the **Destination port**; -2. If the teardown arrived via the **Destination port**, and the **Source port** is not empty, forward it to the **Source port**. - -A teardown that originates locally must be forwarded to all related ports — that is, if both the **Destination port** and **Source port** are known, a teardown must be sent to each port. - -If a received teardown message does not match any routing table entries and/or the ascending or descending entry, the teardown should be ignored and dropped, and must not be forwarded. - -In the case that the teardown message results in the ascending path being torn down, the node should then re-bootstrap by sending a new bootstrap message into the network. diff --git a/docs/virtual_snake/7_next_hop.md b/docs/virtual_snake/7_next_hop.md deleted file mode 100644 index d0ed31d1..00000000 --- a/docs/virtual_snake/7_next_hop.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: Next Hop Calculation -parent: Virtual Snake -nav_order: 7 -permalink: /virtual_snake/nexthop ---- - -# Next Hop Calculation - -When using SNEK routing to route towards a certain public key, a number of rules apply in order to calculate the next-hop. - -These rules slightly differ based on whether the frame is considered to be a “bootstrap” message. Only “Bootstrap” frames follow the bootstrap rules (but **not** “Bootstrap ACK”, “Path setup” etc frames, which are routed using tree routing instead). - -1. Start with a best key set to the node’s public key, and a best candidate set to the node’s own router port; -2. If the **Destination public key** is equal to the node’s own public key and the frame is not a bootstrap message, handle the packet locally without forwarding; -3. If the node has a chosen parent (i.e. is not a root node) and an announcement has been received from that parent: - 1. If the frame is a bootstrap message and the best key still equals the node’s public key, which should always be the case to begin with, ensure that a worst-case route up to the root is chosen: - - Set the best key to the chosen parent’s root public key; - - Set the best candidate to the port through which the parent is reachable; - 2. If the ordering **Best key ＜ Destination public key ＜ Root public key** is true, implying that the target is higher in keyspace than our own key, ensure that a worst-case route up to the root is chosen: - - Set the best key to the chosen parent’s root public key; - - Set the best candidate to the port through which the parent is reachable; - 3. For each of the node’s ancestors — that is, public keys that appear in the **Signatures** section of the last received root update from the chosen parent: - 1. If the frame is not a bootstrap message, the candidate ancestor key equals the **Destination public key** and the best key does not equal the **Destination public key**, meaning that we know that the target is one of our ancestors: - - Set the best key to the ancestor key; - - Set the best candidate to the port through which the parent is reachable; - 2. If the ordering **Destination public key ＜ Ancestor key ＜ Best key** is true, meaning that we believe one of our ancestors takes us closer to the target: - - Set the best key to the ancestor key; - - Set the best candidate to the port through which the parent is reachable; -4. For each of the node’s directly connected peers (first loop): - 1. For each of the connected peer’s ancestors — that is, public keys that appear in the **Signatures** section of the last received root update from this peer: - 1. If the frame is not a bootstrap message, the candidate peer ancestor key equals the **Destination public key** and the best key does not equal the **Destination public key**, meaning that we believe that the target is one of our direct peer’s ancestors: - - Set the best key to the ancestor key; - - Set the best candidate to the port through which the peer is reachable; -5. For each of the node’s directly connected peers (second loop): - 1. If the best key equals the connected peer’s public key, i.e. we have previously found the peer’s key as an ancestor of another node but not using the most direct port, we can now refine the path to use the direct connection to that peer instead: - - Set the best key to the peer’s public key; - - Set the best candidate to the port through which the peer is reachable; -6. For each of our routing table entries, to look for any transitive paths that may take the packet closer to the target than any of our direct peering knowledge has provided us: - 1. Skip the routing table entry if either of the following conditions are true: - - The routing table entry has expired; - - The **Source port** of the routing table entry refers to the local router; - 2. If the frame is not a bootstrap message, the **Path public key** is equal to the **Destination public key** and the best key is not equal to the **Destination public key**: - - Set the best key to the **Path public key**; - - Set the best candidate to the **Source port** from the entry; - 3. If the ordering **Destination public key ＜ Path public key ＜ Best key** is true: - - Set the best key to the **Path public key**; - - Set the best candidate to the **Source port** from the entry. - -If none of the above conditions have matched for the given **Destination public key**, then it is expected that the best candidate will still refer to the local router port, in which case the node is expected to handle the traffic as if it was destined for the local node. diff --git a/docs/virtual_snake/8_maintenance.md b/docs/virtual_snake/8_maintenance.md deleted file mode 100644 index dfadda4b..00000000 --- a/docs/virtual_snake/8_maintenance.md +++ /dev/null @@ -1,14 +0,0 @@ ---- -title: Routine Maintenance -parent: Virtual Snake -nav_order: 8 -permalink: /virtual_snake/maintenance ---- - -# Routine Maintenance - -At a specified interval, typically every 1 second, the node should run the following checks: - -1. If the descending node entry has expired, that is, the time since the **Last seen** entry has passed 1 hour, tear down the path; -2. If the ascending node entry has expired, that is, the time since the **Last seen** entry has passed 1 hour, tear down the path; -3. If the ascending node entry is empty, either before or after step 2, send a bootstrap message into the network. From 3740c7ffea33fd511c16a746aaba737c140dbebf Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 13 May 2022 16:03:39 -0500 Subject: [PATCH 25/41] Softstate doc updates --- .gitignore | 1 + docs/introduction/3_frame_forwarding.md | 2 + docs/virtual_snake/4_next_hop.md | 52 +++++++++++++++++++++++++ docs/virtual_snake/5_maintenance.md | 14 +++++++ 4 files changed, 69 insertions(+) create mode 100644 docs/virtual_snake/4_next_hop.md create mode 100644 docs/virtual_snake/5_maintenance.md diff --git a/.gitignore b/.gitignore index 063785e7..4995cc71 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ # Stuff from GitHub Pages docs/_site +.jekyll-metadata diff --git a/docs/introduction/3_frame_forwarding.md b/docs/introduction/3_frame_forwarding.md index 10200367..8a785b14 100644 --- a/docs/introduction/3_frame_forwarding.md +++ b/docs/introduction/3_frame_forwarding.md @@ -5,6 +5,8 @@ nav_order: 3 permalink: /introduction/frame_forwarding --- +// TODO : Watermark! + # Frame Forwarding When receiving an incoming data frame on a port, the switch then uses the correct lookup method for the frame type to determine which port to forward the frame to. diff --git a/docs/virtual_snake/4_next_hop.md b/docs/virtual_snake/4_next_hop.md new file mode 100644 index 00000000..e363207a --- /dev/null +++ b/docs/virtual_snake/4_next_hop.md @@ -0,0 +1,52 @@ +--- +title: Next Hop Calculation +parent: Virtual Snake +nav_order: 4 +permalink: /virtual_snake/nexthop +--- + +**TODO: Watermark Info** + +# Next Hop Calculation + +When using SNEK routing to route towards a certain public key, a number of rules apply in order to calculate the next-hop. + +These rules slightly differ based on whether the frame is considered to be a “bootstrap” message. Only “Bootstrap” frames follow the bootstrap rules. + +1. Start with a best key set to the node’s public key, and a best candidate set to the node’s own router port; +2. If the **Destination public key** is equal to the node’s own public key and the frame is not a bootstrap message, handle the packet locally without forwarding; +3. If the node has a chosen parent (i.e. is not a root node) and an announcement has been received from that parent: + 1. If the frame is a bootstrap message and the best key still equals the node’s public key, which should always be the case to begin with, ensure that a worst-case route up to the root is chosen: + - Set the best key to the chosen parent’s root public key; + - Set the best candidate to the port through which the parent is reachable; + 2. If the ordering **Best key ＜ Destination public key ＜ Root public key** is true, implying that the target is higher in keyspace than our own key, ensure that a worst-case route up to the root is chosen: + - Set the best key to the chosen parent’s root public key; + - Set the best candidate to the port through which the parent is reachable; + 3. For each of the node’s ancestors — that is, public keys that appear in the **Signatures** section of the last received root update from the chosen parent: + 1. If the frame is not a bootstrap message, the candidate ancestor key equals the **Destination public key** and the best key does not equal the **Destination public key**, meaning that we know that the target is one of our ancestors: + - Set the best key to the ancestor key; + - Set the best candidate to the port through which the parent is reachable; + 2. If the ordering **Destination public key ＜ Ancestor key ＜ Best key** is true, meaning that we believe one of our ancestors takes us closer to the target: + - Set the best key to the ancestor key; + - Set the best candidate to the port through which the parent is reachable; +4. For each of the node’s directly connected peers (first loop): + 1. For each of the connected peer’s ancestors — that is, public keys that appear in the **Signatures** section of the last received root update from this peer: + 1. If the frame is not a bootstrap message, the candidate peer ancestor key equals the **Destination public key** and the best key does not equal the **Destination public key**, meaning that we believe that the target is one of our direct peer’s ancestors: + - Set the best key to the ancestor key; + - Set the best candidate to the port through which the peer is reachable; +5. For each of the node’s directly connected peers (second loop): + 1. If the best key equals the connected peer’s public key, i.e. we have previously found the peer’s key as an ancestor of another node but not using the most direct port, we can now refine the path to use the direct connection to that peer instead: + - Set the best key to the peer’s public key; + - Set the best candidate to the port through which the peer is reachable; +6. For each of our routing table entries, to look for any transitive paths that may take the packet closer to the target than any of our direct peering knowledge has provided us: + 1. Skip the routing table entry if either of the following conditions are true: + - The routing table entry has expired; + - The **Source port** of the routing table entry refers to the local router; + 2. If the frame is not a bootstrap message, the **Path public key** is equal to the **Destination public key** and the best key is not equal to the **Destination public key**: + - Set the best key to the **Path public key**; + - Set the best candidate to the **Source port** from the entry; + 3. If the ordering **Destination public key ＜ Path public key ＜ Best key** is true: + - Set the best key to the **Path public key**; + - Set the best candidate to the **Source port** from the entry. + +If none of the above conditions have matched for the given **Destination public key**, then it is expected that the best candidate will still refer to the local router port, in which case the node is expected to handle the traffic as if it was destined for the local node. diff --git a/docs/virtual_snake/5_maintenance.md b/docs/virtual_snake/5_maintenance.md new file mode 100644 index 00000000..26a730de --- /dev/null +++ b/docs/virtual_snake/5_maintenance.md @@ -0,0 +1,14 @@ +--- +title: Routine Maintenance +parent: Virtual Snake +nav_order: 5 +permalink: /virtual_snake/maintenance +--- + +# Routine Maintenance + +At a specified interval, typically every 1 second, the node should run the following checks: + +1. If the descending node entry has expired, that is, the time since the **Last seen** entry has passed 10 seconds, remove the entry; +2. If the descending node entry has different root information, remove the entry; +3. Remove any routing table entries that are older than 10 seconds. From fcb5164fcd8b891dbc1ab36f2388477c097779ac Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Wed, 25 May 2022 10:17:32 -0400 Subject: [PATCH 26/41] Fix logic for selecting lowest latency links --- router/state_snek.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/router/state_snek.go b/router/state_snek.go index 6097d9a8..8d2b95c5 100644 --- a/router/state_snek.go +++ b/router/state_snek.go @@ -289,7 +289,9 @@ func getNextHopSNEK(params virtualSnakeNextHopParams) (*peer, types.VirtualSnake case p.peertype < bestPeer.peertype: // Prefer faster classes of links if possible. newCandidate(bestKey, bestSeq, p) - case p.peertype == bestPeer.peertype && ann.receiveOrder < bestAnn.receiveOrder: + case p.peertype == bestPeer.peertype && + ann.RootSequence == bestAnn.RootSequence && + ann.receiveOrder < bestAnn.receiveOrder: // Prefer links that have the lowest latency to the root. newCandidate(bestKey, bestSeq, p) } From 39f8332e30c7e845a7021278632eaa68149386bb Mon Sep 17 00:00:00 2001 From: devonh Date: Wed, 25 May 2022 17:18:56 +0000 Subject: [PATCH 27/41] Update docs with watermark information (#52) * Update docs with watermark information * Further clarifications to watermark docs --- docs/introduction/3_frame_forwarding.md | 12 +++++-- .../4_handling_root_announcements.md | 9 +++++ docs/virtual_snake/3_bootstraps.md | 4 ++- docs/virtual_snake/4_next_hop.md | 35 ++++++++++++++----- 4 files changed, 49 insertions(+), 11 deletions(-) diff --git a/docs/introduction/3_frame_forwarding.md b/docs/introduction/3_frame_forwarding.md index 8a785b14..aa28f72e 100644 --- a/docs/introduction/3_frame_forwarding.md +++ b/docs/introduction/3_frame_forwarding.md @@ -5,8 +5,6 @@ nav_order: 3 permalink: /introduction/frame_forwarding --- -// TODO : Watermark! - # Frame Forwarding When receiving an incoming data frame on a port, the switch then uses the correct lookup method for the frame type to determine which port to forward the frame to. @@ -18,3 +16,13 @@ Protocol frames often have specific rules governing their behaviour, including i Traffic frames, on the other hand, are always forwarded using SNEK routing or tree routing (typically the former) and are not required to be otherwise inspected by an intermediate node. If a suitable next-hop is identified, the frame will be forwarded to the chosen next-hop peer. + +## Watermarks + +In the case of **traffic frames** there is also a watermark used to help detect routing loops. Watermarks only factor into the equation when next-hops are selected from the SNEK routing table as these routes can become stale much more quickly than route information obtained through the global spanning tree. The watermark ensures that forward progress towards a given key is being made and that a packet will never be forwarded onto a path that is worse than the path selected at the last hop. If any node along the path does not know of either the same best destination node or a closer destination node, then the packet has reached a location where further forwarding could result in routing loops. + +Frames should be dropped if the path watermark (derived from the path key and bootstrap sequence) of the chosen next-hop is worse than the watermark on the received frame. A watermark is defined as being worse if either of the following conditions is met: +- The new watermark has a higher public key than the existing watermark; +- The new watermark has the same public key but a lower sequence number than the existing watermark; + +Before forwarding the frame to the next-hop, if the chosen next-hop was selected using the snake routing table then the watermark on the frame should be updated with the path watermark of the next-hop. diff --git a/docs/spanning_tree/4_handling_root_announcements.md b/docs/spanning_tree/4_handling_root_announcements.md index cc68f772..420330c5 100644 --- a/docs/spanning_tree/4_handling_root_announcements.md +++ b/docs/spanning_tree/4_handling_root_announcements.md @@ -29,6 +29,15 @@ The following sanity checks must be performed on all incoming root announcement If any of these conditions fail, the update is considered to be invalid, the update should be dropped and the peer that sent us this announcement should be disconnected as a result of the error. +## Storing root announcements + +When storing a root announcement for a given peer, the following information should be kept: + +- The announcement itself; +- The time the announcement was received; +- The order in which the announcement was received; + - The order of received root announcements must be global across all peers; + ## Deciding to re-parent Once the sanity checks have passed, if the update came from the currently selected parent, perform the following checks in order: diff --git a/docs/virtual_snake/3_bootstraps.md b/docs/virtual_snake/3_bootstraps.md index 4910f1b2..050d4d22 100644 --- a/docs/virtual_snake/3_bootstraps.md +++ b/docs/virtual_snake/3_bootstraps.md @@ -11,7 +11,7 @@ Once the bootstrap message arrives at a dead end, the node will update it's desc Before doing anything, the node must ensure that the signature in the **Source signature** field is valid by checking against the **Destination public key**. If the signature is invalid, the bootstrap message should be silently dropped and not processed any further. -Additionally, the node should ensure that the **Root public key** and **Root sequence** of the bootstrap message match those of the most recent root announcement from our chosen parent, if any, or the node’s own public key and sequence number if the node is currently acting as a root node. If this is not true, the bootstrap message should be silently dropped and not processed any further. +The node should ensure that the **Root public key** and **Root sequence** of the bootstrap message match those of the most recent root announcement from our chosen parent, if any, or the node’s own public key and sequence number if the node is currently acting as a root node. If this is not true, the bootstrap message should be silently dropped and not processed any further. Each node along the bootstrapping path should install the bootstrapping node into their routing table. @@ -19,6 +19,8 @@ Each node along the bootstrapping path should install the bootstrapping node int Regardless of whether the bootstrap message is considered to have arrived at its intended destination (there is no closer node to route to) or not, a bootstrap message should result in the route being installed into the routing table of each node handling the message. +Before installing the bootstrapping node into the routing table, each node should compare the bootstrap sequence against any existing entry for the bootstrapping node. If an entry exists and the new bootstrap sequence number isn't higher than the current entry, then the bootstrap should be dropped and not processed any further. The only reason to see a bootstrap with a lower or equal sequence number to a bootstrap the node has seen before is if there is a routing loop present. + To install the route into the routing table, the node should either create a new entry or overwrite the existing entry and: 1. Copy the **Origin public key** into the virtual snake index; diff --git a/docs/virtual_snake/4_next_hop.md b/docs/virtual_snake/4_next_hop.md index e363207a..f5c9373e 100644 --- a/docs/virtual_snake/4_next_hop.md +++ b/docs/virtual_snake/4_next_hop.md @@ -5,48 +5,67 @@ nav_order: 4 permalink: /virtual_snake/nexthop --- -**TODO: Watermark Info** - # Next Hop Calculation When using SNEK routing to route towards a certain public key, a number of rules apply in order to calculate the next-hop. These rules slightly differ based on whether the frame is considered to be a “bootstrap” message. Only “Bootstrap” frames follow the bootstrap rules. -1. Start with a best key set to the node’s public key, and a best candidate set to the node’s own router port; +1. Start with a best key set to the node’s public key, a best candidate set to the node’s own router port, and a best sequence set to 0; 2. If the **Destination public key** is equal to the node’s own public key and the frame is not a bootstrap message, handle the packet locally without forwarding; 3. If the node has a chosen parent (i.e. is not a root node) and an announcement has been received from that parent: 1. If the frame is a bootstrap message and the best key still equals the node’s public key, which should always be the case to begin with, ensure that a worst-case route up to the root is chosen: - Set the best key to the chosen parent’s root public key; - Set the best candidate to the port through which the parent is reachable; + - Set the best sequence to **0**; 2. If the ordering **Best key ＜ Destination public key ＜ Root public key** is true, implying that the target is higher in keyspace than our own key, ensure that a worst-case route up to the root is chosen: - Set the best key to the chosen parent’s root public key; - Set the best candidate to the port through which the parent is reachable; + - Set the best sequence to **0**; 3. For each of the node’s ancestors — that is, public keys that appear in the **Signatures** section of the last received root update from the chosen parent: 1. If the frame is not a bootstrap message, the candidate ancestor key equals the **Destination public key** and the best key does not equal the **Destination public key**, meaning that we know that the target is one of our ancestors: - Set the best key to the ancestor key; - Set the best candidate to the port through which the parent is reachable; + - Set the best sequence to **0**; 2. If the ordering **Destination public key ＜ Ancestor key ＜ Best key** is true, meaning that we believe one of our ancestors takes us closer to the target: - Set the best key to the ancestor key; - Set the best candidate to the port through which the parent is reachable; + - Set the best sequence to **0**; 4. For each of the node’s directly connected peers (first loop): 1. For each of the connected peer’s ancestors — that is, public keys that appear in the **Signatures** section of the last received root update from this peer: 1. If the frame is not a bootstrap message, the candidate peer ancestor key equals the **Destination public key** and the best key does not equal the **Destination public key**, meaning that we believe that the target is one of our direct peer’s ancestors: - Set the best key to the ancestor key; - Set the best candidate to the port through which the peer is reachable; + - Set the best sequence to **0**; 5. For each of the node’s directly connected peers (second loop): 1. If the best key equals the connected peer’s public key, i.e. we have previously found the peer’s key as an ancestor of another node but not using the most direct port, we can now refine the path to use the direct connection to that peer instead: - Set the best key to the peer’s public key; - Set the best candidate to the port through which the peer is reachable; + - Set the best sequence to **0**; 6. For each of our routing table entries, to look for any transitive paths that may take the packet closer to the target than any of our direct peering knowledge has provided us: 1. Skip the routing table entry if either of the following conditions are true: - The routing table entry has expired; - The **Source port** of the routing table entry refers to the local router; - 2. If the frame is not a bootstrap message, the **Path public key** is equal to the **Destination public key** and the best key is not equal to the **Destination public key**: - - Set the best key to the **Path public key**; + - The **Watermark** of the entry has a higher public key than the watermark of the packet; + - The **Watermark** of the entry has the same public key but a lower sequence number than the watermark of the packet; + 2. If the frame is not a bootstrap message, the **Entry public key** is equal to the **Destination public key** and the best key is not equal to the **Destination public key**: + - Set the best key to the **Entry public key**; + - Set the best candidate to the **Source port** from the entry; + - Set the best sequence to the **Sequence number** from the entry; + 3. If the ordering **Destination public key ＜ Entry public key ＜ Best key** is true: + - Set the best key to the **Entry public key**; - Set the best candidate to the **Source port** from the entry; - 3. If the ordering **Destination public key ＜ Path public key ＜ Best key** is true: - - Set the best key to the **Path public key**; - - Set the best candidate to the **Source port** from the entry. + - Set the best sequence to the **Sequence number** from the entry; +7. For each of the node's directly connected peers (third loop): + 1. If the best key equals the connected peer’s public key, i.e. we have previously found the peer’s key to be the best key, we can further refine the path to use either the faster or lower latency link type to route to that peer: + 1. If the **Best candidate** has a slower peer connection type (**Multicast > Remote > Bluetooth**) than the connected peer: + - Set the best candidate to the connected peer from the entry; + 2. If the **Best candidate** has the same peer connection type as the connected peer, the same root sequence number and a higher receive order number for it's tree root announcement than the connected peer: + - Set the best candidate to the connected peer from the entry; + +A **Watermark** should be returned with the **Best candidate** from the next-hop algorithm as it is used to update the packet before forwarding. There are two cases to consider for what the watermark should be: + +1. If the **Best sequence number** is 0, then the **Watermark** returned should be the same as the existing watermark on the packet; +2. If the **Best sequence number** is higher than 0, then the **Watermark** returned should contain the **Best key** and **Best sequence number** information; If none of the above conditions have matched for the given **Destination public key**, then it is expected that the best candidate will still refer to the local router port, in which case the node is expected to handle the traffic as if it was destined for the local node. From b2736318244a2398803d2be793e65304ddd0737f Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 26 May 2022 14:30:36 +0100 Subject: [PATCH 28/41] Remove spanning tree --- .../simulator/adversary/drop_packets.go | 6 - cmd/pineconesim/simulator/commands.go | 52 +- cmd/pineconesim/simulator/pathfind.go | 27 - cmd/pineconesim/simulator/router.go | 40 +- cmd/pineconesim/simulator/simulator.go | 4 - router/api.go | 17 - router/manhole.go | 18 - router/packetconn.go | 14 - router/peer.go | 21 +- router/queuefairfifo.go | 7 - router/state.go | 59 +- router/state_forward.go | 23 +- router/state_snek.go | 117 +-- router/state_snek_test.go | 2 + router/state_tree.go | 545 ------------- router/state_tree_test.go | 750 ------------------ types/frame.go | 56 +- types/frame_test.go | 65 -- types/virtualsnake.go | 26 +- types/virtualsnake_test.go | 15 - 20 files changed, 60 insertions(+), 1804 deletions(-) delete mode 100644 router/state_tree.go delete mode 100644 router/state_tree_test.go diff --git a/cmd/pineconesim/simulator/adversary/drop_packets.go b/cmd/pineconesim/simulator/adversary/drop_packets.go index 09350b07..af54880d 100644 --- a/cmd/pineconesim/simulator/adversary/drop_packets.go +++ b/cmd/pineconesim/simulator/adversary/drop_packets.go @@ -86,9 +86,7 @@ func NewPacketsReceived() PacketsReceived { func defaultFrameCount() PeerFrameCount { frameCount := make(FrameCounts, 9) frameCount[types.TypeKeepalive] = atomic.NewUint64(0) - frameCount[types.TypeTreeAnnouncement] = atomic.NewUint64(0) frameCount[types.TypeVirtualSnakeBootstrap] = atomic.NewUint64(0) - frameCount[types.TypeTreeRouted] = atomic.NewUint64(0) frameCount[types.TypeVirtualSnakeRouted] = atomic.NewUint64(0) peerFrameCount := PeerFrameCount{ @@ -135,10 +133,6 @@ func (a *AdversaryRouter) Ping(ctx context.Context, addr net.Addr) (uint16, time return 0, 0, nil } -func (a *AdversaryRouter) Coords() types.Coordinates { - return a.rtr.Coords() -} - func (a *AdversaryRouter) ConfigureFilterDefaults(rates DropRates) { a.dropSettings.overall = rates } diff --git a/cmd/pineconesim/simulator/commands.go b/cmd/pineconesim/simulator/commands.go index c444f3a7..3a67023f 100644 --- a/cmd/pineconesim/simulator/commands.go +++ b/cmd/pineconesim/simulator/commands.go @@ -119,18 +119,20 @@ func UnmarshalCommandJSON(command *SimCommandMsg) (SimCommand, error) { } else { err = fmt.Errorf("%sConfigureAdversaryDefaults.DropRates.Keepalive field doesn't exist", FAILURE_PREAMBLE) } - if subVal, subOk := val.(map[string]interface{})["TreeAnnouncement"]; subOk { - intVal, _ := strconv.Atoi(subVal.(string)) - dropRates.Frames[types.TypeTreeAnnouncement] = uint64(intVal) - } else { - err = fmt.Errorf("%sConfigureAdversaryDefaults.DropRates.TreeAnnouncement field doesn't exist", FAILURE_PREAMBLE) - } - if subVal, subOk := val.(map[string]interface{})["TreeRouted"]; subOk { - intVal, _ := strconv.Atoi(subVal.(string)) - dropRates.Frames[types.TypeTreeRouted] = uint64(intVal) - } else { - err = fmt.Errorf("%sConfigureAdversaryDefaults.DropRates.TreeRouted field doesn't exist", FAILURE_PREAMBLE) - } + /* + if subVal, subOk := val.(map[string]interface{})["TreeAnnouncement"]; subOk { + intVal, _ := strconv.Atoi(subVal.(string)) + dropRates.Frames[types.TypeTreeAnnouncement] = uint64(intVal) + } else { + err = fmt.Errorf("%sConfigureAdversaryDefaults.DropRates.TreeAnnouncement field doesn't exist", FAILURE_PREAMBLE) + } + if subVal, subOk := val.(map[string]interface{})["TreeRouted"]; subOk { + intVal, _ := strconv.Atoi(subVal.(string)) + dropRates.Frames[types.TypeTreeRouted] = uint64(intVal) + } else { + err = fmt.Errorf("%sConfigureAdversaryDefaults.DropRates.TreeRouted field doesn't exist", FAILURE_PREAMBLE) + } + */ if subVal, subOk := val.(map[string]interface{})["VirtualSnakeBootstrap"]; subOk { intVal, _ := strconv.Atoi(subVal.(string)) dropRates.Frames[types.TypeVirtualSnakeBootstrap] = uint64(intVal) @@ -175,18 +177,20 @@ func UnmarshalCommandJSON(command *SimCommandMsg) (SimCommand, error) { } else { err = fmt.Errorf("%sConfigureAdversaryPeer.DropRates.Keepalive field doesn't exist", FAILURE_PREAMBLE) } - if subVal, subOk := val.(map[string]interface{})["TreeAnnouncement"]; subOk { - intVal, _ := strconv.Atoi(subVal.(string)) - dropRates.Frames[types.TypeTreeAnnouncement] = uint64(intVal) - } else { - err = fmt.Errorf("%sConfigureAdversaryPeer.DropRates.TreeAnnouncement field doesn't exist", FAILURE_PREAMBLE) - } - if subVal, subOk := val.(map[string]interface{})["TreeRouted"]; subOk { - intVal, _ := strconv.Atoi(subVal.(string)) - dropRates.Frames[types.TypeTreeRouted] = uint64(intVal) - } else { - err = fmt.Errorf("%sConfigureAdversaryPeer.DropRates.TreeRouted field doesn't exist", FAILURE_PREAMBLE) - } + /* + if subVal, subOk := val.(map[string]interface{})["TreeAnnouncement"]; subOk { + intVal, _ := strconv.Atoi(subVal.(string)) + dropRates.Frames[types.TypeTreeAnnouncement] = uint64(intVal) + } else { + err = fmt.Errorf("%sConfigureAdversaryPeer.DropRates.TreeAnnouncement field doesn't exist", FAILURE_PREAMBLE) + } + if subVal, subOk := val.(map[string]interface{})["TreeRouted"]; subOk { + intVal, _ := strconv.Atoi(subVal.(string)) + dropRates.Frames[types.TypeTreeRouted] = uint64(intVal) + } else { + err = fmt.Errorf("%sConfigureAdversaryPeer.DropRates.TreeRouted field doesn't exist", FAILURE_PREAMBLE) + } + */ if subVal, subOk := val.(map[string]interface{})["VirtualSnakeBootstrap"]; subOk { intVal, _ := strconv.Atoi(subVal.(string)) dropRates.Frames[types.TypeVirtualSnakeBootstrap] = uint64(intVal) diff --git a/cmd/pineconesim/simulator/pathfind.go b/cmd/pineconesim/simulator/pathfind.go index 19add920..7a12b166 100644 --- a/cmd/pineconesim/simulator/pathfind.go +++ b/cmd/pineconesim/simulator/pathfind.go @@ -20,33 +20,6 @@ import ( "time" ) -func (sim *Simulator) PingTree(from, to string) (uint16, time.Duration, error) { - fromnode := sim.nodes[from] - tonode := sim.nodes[to] - success := false - - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - - defer func() { - sim.treePathConvergenceMutex.Lock() - if _, ok := sim.treePathConvergence[from]; !ok { - sim.treePathConvergence[from] = map[string]bool{} - } - sim.treePathConvergence[from][to] = success - sim.treePathConvergenceMutex.Unlock() - }() - - hops, rtt, err := fromnode.Ping(ctx, tonode.Coords()) - if err != nil { - return 0, 0, fmt.Errorf("fromnode.TreePing: %w", err) - } - - success = true - sim.ReportDistance(from, to, int64(hops), false) - return hops, rtt, nil -} - func (sim *Simulator) PingSNEK(from, to string) (uint16, time.Duration, error) { fromnode := sim.nodes[from] tonode := sim.nodes[to] diff --git a/cmd/pineconesim/simulator/router.go b/cmd/pineconesim/simulator/router.go index 9fa3e75e..8e08d912 100644 --- a/cmd/pineconesim/simulator/router.go +++ b/cmd/pineconesim/simulator/router.go @@ -33,7 +33,6 @@ type SimRouter interface { Connect(conn net.Conn, options ...router.ConnectionOption) (types.SwitchPortID, error) Subscribe(ch chan events.Event) Ping(ctx context.Context, a net.Addr) (uint16, time.Duration, error) - Coords() types.Coordinates ConfigureFilterDefaults(rates adversary.DropRates) ConfigureFilterPeer(peer types.PublicKey, rates adversary.DropRates) ManholeHandler(w http.ResponseWriter, req *http.Request) @@ -56,10 +55,6 @@ func (r *DefaultRouter) Connect(conn net.Conn, options ...router.ConnectionOptio return r.rtr.Connect(conn, options...) } -func (r *DefaultRouter) Coords() types.Coordinates { - return r.rtr.Coords() -} - func (r *DefaultRouter) ConfigureFilterDefaults(rates adversary.DropRates) {} func (r *DefaultRouter) ConfigureFilterPeer(peer types.PublicKey, rates adversary.DropRates) {} @@ -76,10 +71,6 @@ func (r *DefaultRouter) Ping(ctx context.Context, a net.Addr) (uint16, time.Dura var pingType PingType switch a.(type) { - case types.Coordinates: - origin = r.Coords() - frameType = types.TypeTreeRouted - pingType = TreePing case types.PublicKey: origin = r.PublicKey() frameType = types.TypeVirtualSnakeRouted @@ -150,31 +141,6 @@ func (r *DefaultRouter) OverlayReadHandler(quit <-chan bool) { pingAtDest := false var frameType types.FrameType switch payload.pingType { - case TreePing: - switch dest := (payload.destination).(type) { - case types.Coordinates: - frameType = types.TypeTreeRouted - if dest.EqualTo(r.Coords()) { - pingAtDest = true - } - } - case TreePong: - switch orig := (payload.origin).(type) { - case types.Coordinates: - frameType = types.TypeTreeRouted - if orig.EqualTo(r.Coords()) { - id := payload.destination.String() - v, ok := r.pings.Load(id) - if !ok { - continue - } - ch := v.(chan uint16) - ch <- payload.hops - close(ch) - r.pings.Delete(id) - continue - } - } case SNEKPing: switch dest := (payload.destination).(type) { case types.PublicKey: @@ -211,11 +177,7 @@ func (r *DefaultRouter) OverlayReadHandler(quit <-chan bool) { payload.hops++ } else { fromAddr = nil - if frameType == types.TypeTreeRouted { - payload.pingType = TreePong - } else { - payload.pingType = SNEKPong - } + payload.pingType = SNEKPong } } diff --git a/cmd/pineconesim/simulator/simulator.go b/cmd/pineconesim/simulator/simulator.go index bc3ce756..eaf365ce 100644 --- a/cmd/pineconesim/simulator/simulator.go +++ b/cmd/pineconesim/simulator/simulator.go @@ -150,10 +150,6 @@ func (sim *Simulator) StartPinging(ping_period time.Duration) { for i := 0; i < numWorkers; i++ { go func() { for pair := range tasks { - sim.log.Println("Tree ping from", pair.from, "to", pair.to) - if _, _, err := sim.PingTree(pair.from, pair.to); err != nil { - sim.log.Println("Tree ping from", pair.from, "to", pair.to, "failed:", err) - } sim.log.Println("SNEK ping from", pair.from, "to", pair.to) if _, _, err := sim.PingSNEK(pair.from, pair.to); err != nil { sim.log.Println("SNEK ping from", pair.from, "to", pair.to, "failed:", err) diff --git a/router/api.go b/router/api.go index 039e40d3..9ea1d1e6 100644 --- a/router/api.go +++ b/router/api.go @@ -45,10 +45,6 @@ func (r *Router) Subscribe(ch chan<- events.Event) { }) } -func (r *Router) Coords() types.Coordinates { - return r.state.coords() -} - func (r *Router) Peers() []PeerInfo { var infos []PeerInfo phony.Block(r.state, func() { @@ -89,19 +85,6 @@ func (r *Router) NextHop(from net.Addr, frameType types.FrameType, dest net.Addr if nextPeer != nil { switch (dest).(type) { - case types.Coordinates: - var err error - coords := types.Coordinates{} - phony.Block(r.state, func() { - coords, err = nextPeer._coords() - }) - - if err != nil { - r.log.Println("failed retrieving coords for nexthop: %w") - return nil - } - - nexthop = coords case types.PublicKey: nexthop = nextPeer.public } diff --git a/router/manhole.go b/router/manhole.go index b5cf4d2e..a5ccc06d 100644 --- a/router/manhole.go +++ b/router/manhole.go @@ -26,8 +26,6 @@ import ( type manholeResponse struct { Public types.PublicKey `json:"public_key"` Coords types.Coordinates `json:"coords"` - Root *types.Root `json:"root"` - Parent *peer `json:"parent"` Peers map[string][]manholePeer `json:"peers"` SNEK struct { Descending *virtualSnakeEntry `json:"descending"` @@ -36,8 +34,6 @@ type manholeResponse struct { } type manholePeer struct { - Coords types.Coordinates `json:"coords,omitempty"` - Order uint64 `json:"order,omitempty"` Port types.SwitchPortID `json:"port"` PeerType ConnectionPeerType `json:"type,omitempty"` PeerZone ConnectionZone `json:"zone,omitempty"` @@ -53,11 +49,6 @@ func (r *Router) ManholeHandler(w http.ResponseWriter, req *http.Request) { } phony.Block(r.state, func() { response.Public = r.public - response.Coords = r.state._coords() - response.Parent = r.state._parent - if rootAnn := r.state._rootAnnouncement(); rootAnn != nil { - response.Root = &rootAnn.Root - } for _, p := range r.state._peers { if p == nil || !p.started.Load() { continue @@ -70,10 +61,6 @@ func (r *Router) ManholeHandler(w http.ResponseWriter, req *http.Request) { ProtoQueue: p.proto, TrafficQueue: p.traffic, } - if ann := r.state._announcements[p]; ann != nil { - info.Coords = ann.Coords() - info.Order = ann.receiveOrder - } public := p.public.String() response.Peers[public] = append(response.Peers[public], info) } @@ -82,11 +69,6 @@ func (r *Router) ManholeHandler(w http.ResponseWriter, req *http.Request) { response.SNEK.Paths = append(response.SNEK.Paths, p) } }) - for _, p := range response.Peers { - sort.Slice(p, func(i, j int) bool { - return p[i].Order < p[j].Order - }) - } sort.Slice(response.SNEK.Paths, func(i, j int) bool { return response.SNEK.Paths[i].PublicKey.CompareTo(response.SNEK.Paths[j].PublicKey) < 0 }) diff --git a/router/packetconn.go b/router/packetconn.go index 57a5ea33..fd386d54 100644 --- a/router/packetconn.go +++ b/router/packetconn.go @@ -57,9 +57,6 @@ func (r *Router) ReadFrom(p []byte) (n int, addr net.Addr, err error) { r.local.traffic.ack() } switch frame.Type { - case types.TypeTreeRouted: - addr = frame.Source - case types.TypeVirtualSnakeRouted: addr = frame.SourceKey @@ -86,17 +83,6 @@ func (r *Router) WriteTo(p []byte, addr net.Addr) (n int, err error) { }() switch ga := addr.(type) { - case types.Coordinates: - frame := getFrame() - frame.Type = types.TypeTreeRouted - frame.Destination = ga - frame.Source = r.state.coords() - frame.Payload = append(frame.Payload[:0], p...) - phony.Block(r.state, func() { - _ = r.state._forward(r.local, frame) - }) - return len(p), nil - case types.PublicKey: frame := getFrame() frame.Type = types.TypeVirtualSnakeRouted diff --git a/router/peer.go b/router/peer.go index 9eb6734e..e1735281 100644 --- a/router/peer.go +++ b/router/peer.go @@ -94,7 +94,7 @@ func (p *peer) String() string { // to make sim less ugly func (p *peer) send(f *types.Frame) bool { switch f.Type { // Protocol messages - case types.TypeTreeAnnouncement, types.TypeKeepalive: + case types.TypeKeepalive: fallthrough case types.TypeVirtualSnakeBootstrap: if p.proto == nil { @@ -105,7 +105,7 @@ func (p *peer) send(f *types.Frame) bool { return p.proto.push(f) // Traffic messages - case types.TypeVirtualSnakeRouted, types.TypeTreeRouted: + case types.TypeVirtualSnakeRouted: return p.traffic.push(f) } @@ -351,20 +351,3 @@ func (p *peer) _read() { // the actor inbox. p.reader.Act(nil, p._read) } - -func (p *peer) _coords() (types.Coordinates, error) { - var err error - var coords types.Coordinates - - if p == p.router.local { - coords = p.router.state._coords() - } else { - if announcement, ok := p.router.state._announcements[p]; ok { - coords = announcement.PeerCoords() - } else { - err = fmt.Errorf("no root announcement found for peer") - } - } - - return coords, err -} diff --git a/router/queuefairfifo.go b/router/queuefairfifo.go index b9d6e0ef..a65eb259 100644 --- a/router/queuefairfifo.go +++ b/router/queuefairfifo.go @@ -59,13 +59,6 @@ func (q *fairFIFOQueue) queuesize() int { // nolint:unused func (q *fairFIFOQueue) hash(frame *types.Frame) uint16 { h := q.offset switch frame.Type { - case types.TypeTreeRouted: - for _, v := range frame.Source { - h += uint64(v) - } - for _, v := range frame.Destination { - h += uint64(v) - } case types.TypeVirtualSnakeRouted: for _, v := range frame.SourceKey { h += uint64(v) diff --git a/router/state.go b/router/state.go index 358487c4..9cb4bba4 100644 --- a/router/state.go +++ b/router/state.go @@ -37,13 +37,9 @@ type state struct { phony.Inbox r *Router _peers []*peer // All switch ports, connected and disconnected + _highest *virtualSnakeEntry // The highest entry we've seen recently _descending *virtualSnakeEntry // Next descending node in keyspace - _parent *peer // Our chosen parent in the tree - _announcements announcementTable // Announcements received from our peers _table virtualSnakeTable // Virtual snake DHT entries - _ordering uint64 // Used to order incoming tree announcements - _sequence uint64 // Used to sequence our root tree announcements - _treetimer *time.Timer // Tree maintenance timer _snaketimer *time.Timer // Virtual snake maintenance timer _lastbootstrap time.Time // When did we last bootstrap? _waiting bool // Is the tree waiting to reparent? @@ -52,43 +48,21 @@ type state struct { // _start resets the state and starts tree and virtual snake maintenance. func (s *state) _start() { - s._setParent(nil) s._setDescendingNode(nil) - s._ordering = 0 s._waiting = false - s._announcements = make(announcementTable, portCount) s._table = virtualSnakeTable{} - if s._treetimer == nil { - s._treetimer = time.AfterFunc(announcementInterval, func() { - s.Act(nil, s._maintainTree) - }) - } - if s._snaketimer == nil { s._snaketimer = time.AfterFunc(time.Second, func() { s.Act(nil, s._maintainSnake) }) } - s._maintainTreeIn(0) s._maintainSnakeIn(0) } -// _maintainTreeIn resets the tree maintenance timer to the specified -// duration. -func (s *state) _maintainTreeIn(d time.Duration) { - if !s._treetimer.Stop() { - select { - case <-s._treetimer.C: - default: - } - } - s._treetimer.Reset(d) -} - // _maintainSnakeIn resets the virtual snake maintenance timer to the // specified duration. func (s *state) _maintainSnakeIn(d time.Duration) { @@ -133,7 +107,6 @@ func (s *state) _addPeer(conn net.Conn, public types.PublicKey, uri ConnectionUR s.r.log.Println("Connected to peer", new.public.String(), "on port", new.port) v, _ := s.r.active.LoadOrStore(hex.EncodeToString(new.public[:])+string(zone), atomic.NewUint64(0)) v.(*atomic.Uint64).Inc() - new.proto.push(s.r.state._rootAnnouncement().forPeer(new)) new.started.Store(true) new.reader.Act(nil, new._read) new.writer.Act(nil, new._write) @@ -156,19 +129,6 @@ func (s *state) _removePeer(port types.SwitchPortID) { }) } -func (s *state) _setParent(peer *peer) { - s._parent = peer - - s.r.Act(nil, func() { - peerID := "" - if peer != nil { - peerID = peer.public.String() - } - - s.r._publish(events.TreeParentUpdate{PeerID: peerID}) - }) -} - func (s *state) _setDescendingNode(node *virtualSnakeEntry) { switch { case s._descending == nil || node == nil: @@ -209,9 +169,6 @@ func (s *state) _portDisconnected(peer *peer) { return } - // Delete the last tree announcement that we received from this peer. - delete(s._announcements, peer) - // Scan the local routing table for any routes that transited this now-dead // peering and remove them from the routing table. for k, v := range s._table { @@ -225,14 +182,6 @@ func (s *state) _portDisconnected(peer *peer) { if desc := s._descending; desc != nil && desc.Source == peer { s._setDescendingNode(nil) } - - // If the peer that died was our chosen tree parent, then we will need to - // select a new parent. If we successfully choose a new parent (as in, we - // don't end up promoting ourselves to a root) then we will also need to - // send a new bootstrap into the network. - if s._parent == peer && s._selectNewParent() { - s._bootstrapSoon() - } } // _lookupPeerForAddr finds and returns the peer corresponding to the provided @@ -246,12 +195,6 @@ func (s *state) _lookupPeerForAddr(addr net.Addr) *peer { } switch fromAddr := addr.(type) { - case types.Coordinates: - coords, err := p._coords() - if err == nil && fromAddr.EqualTo(coords) { - result = p - break - } case types.PublicKey: if fromAddr == p.public { result = p diff --git a/router/state_forward.go b/router/state_forward.go index 5c0f14b0..4e88fbe9 100644 --- a/router/state_forward.go +++ b/router/state_forward.go @@ -15,7 +15,6 @@ package router import ( - "fmt" "net" "github.com/matrix-org/pinecone/types" @@ -34,13 +33,6 @@ func (s *state) _nextHopsFor(from *peer, frameType types.FrameType, dest net.Add case types.PublicKey: nexthop, newWatermark = s._nextHopsSNEK(dest, frameType, watermark) } - - // Tree routing - case types.TypeTreeRouted: - switch dest := (dest).(type) { - case types.Coordinates: - nexthop = s._nextHopsTree(from, dest) - } } return nexthop, newWatermark } @@ -56,9 +48,8 @@ func (s *state) _forward(p *peer, f *types.Frame) error { } // Allow overlay loopback traffic by directly forwarding it to the local router. - isTreeLoopback := f.Type == types.TypeTreeRouted && f.Destination.EqualTo(s._coords()) isSnakeLoopback := f.Type == types.TypeVirtualSnakeRouted && f.DestinationKey == s.r.public - if isTreeLoopback || isSnakeLoopback { + if isSnakeLoopback { s.r.local.send(f) return nil } @@ -66,22 +57,12 @@ func (s *state) _forward(p *peer, f *types.Frame) error { var nexthop *peer var watermark types.VirtualSnakeWatermark switch f.Type { - case types.TypeTreeRouted: - nexthop, watermark = s._nextHopsFor(p, f.Type, f.Destination, f.Watermark) case types.TypeVirtualSnakeBootstrap, types.TypeVirtualSnakeRouted: nexthop, watermark = s._nextHopsFor(p, f.Type, f.DestinationKey, f.Watermark) } deadend := nexthop == nil || nexthop == p.router.local switch f.Type { - case types.TypeTreeAnnouncement: - // Tree announcements are a special case. The _handleTreeAnnouncement function - // will generate new tree announcements and send them to peers if needed. - if err := s._handleTreeAnnouncement(p, f); err != nil { - return fmt.Errorf("s._handleTreeAnnouncement (port %d): %w", p.port, err) - } - return nil - case types.TypeKeepalive: // Keepalives are sent on a peering and are never forwarded. return nil @@ -92,7 +73,7 @@ func (s *state) _forward(p *peer, f *types.Frame) error { return nil } - case types.TypeVirtualSnakeRouted, types.TypeTreeRouted: + case types.TypeVirtualSnakeRouted: // Traffic type packets are forwarded normally by falling through. There // are no special rules to apply to these packets, regardless of whether // they are SNEK-routed or tree-routed. diff --git a/router/state_snek.go b/router/state_snek.go index 8d2b95c5..45b962fb 100644 --- a/router/state_snek.go +++ b/router/state_snek.go @@ -61,20 +61,9 @@ func (s *state) _maintainSnake() { defer s._maintainSnakeIn(virtualSnakeMaintainInterval) } - // Work out if we are able to bootstrap. If we are the root node then - // we don't send bootstraps, since there's nowhere for them to go — - // bootstraps are sent up to the next ascending node, but as the root, - // we already have the highest key on the network. - rootAnn := s._rootAnnouncement() - // The descending node is the node with the next lowest key. - if desc := s._descending; desc != nil { - switch { - case !desc.valid(): - fallthrough - case !desc.Root.EqualTo(&rootAnn.Root): - s._setDescendingNode(nil) - } + if desc := s._descending; desc != nil && !desc.valid() { + s._setDescendingNode(nil) } // Clean up any paths that are older than the expiry period. @@ -99,20 +88,12 @@ func (s *state) _bootstrapSoon() { // _bootstrapNow is responsible for sending a bootstrap message to the network. func (s *state) _bootstrapNow() { - // If we are the root node then there's no point in trying to bootstrap. We - // already have the highest public key on the network so a bootstrap won't be - // able to go anywhere in ascending order. - if s._parent == nil { - return - } // Construct the bootstrap packet. We will include our root key and sequence // number in the update so that the remote side can determine if we are both using // the same root node when processing the update. - ann := s._rootAnnouncement() b := frameBufferPool.Get().(*[types.MaxFrameSize]byte) defer frameBufferPool.Put(b) bootstrap := types.VirtualSnakeBootstrap{ - Root: ann.Root, Sequence: types.Varu64(time.Now().UnixMilli()), } if s.r.secure { @@ -136,7 +117,6 @@ func (s *state) _bootstrapNow() { send := getFrame() send.Type = types.TypeVirtualSnakeBootstrap send.DestinationKey = s.r.public - send.Source = s._coords() send.Payload = append(send.Payload[:0], b[:n]...) send.Watermark = types.VirtualSnakeWatermark{ PublicKey: types.FullMask, @@ -153,28 +133,24 @@ func (s *state) _bootstrapNow() { } type virtualSnakeNextHopParams struct { - isBootstrap bool - destinationKey types.PublicKey - publicKey types.PublicKey - watermark types.VirtualSnakeWatermark - parentPeer *peer - selfPeer *peer - lastAnnouncement *rootAnnouncementWithTime - peerAnnouncements announcementTable - snakeRoutes virtualSnakeTable + isBootstrap bool + peers []*peer + destinationKey types.PublicKey + publicKey types.PublicKey + watermark types.VirtualSnakeWatermark + selfPeer *peer + snakeRoutes virtualSnakeTable } // _nextHopsSNEK locates the best next-hop for a given SNEK-routed frame. func (s *state) _nextHopsSNEK(dest types.PublicKey, frameType types.FrameType, watermark types.VirtualSnakeWatermark) (*peer, types.VirtualSnakeWatermark) { return getNextHopSNEK(virtualSnakeNextHopParams{ frameType == types.TypeVirtualSnakeBootstrap, + s._peers, dest, s.r.public, watermark, - s._parent, s.r.local, - s._rootAnnouncement(), - s._announcements, s._table, }) } @@ -189,7 +165,6 @@ func getNextHopSNEK(params virtualSnakeNextHopParams) (*peer, types.VirtualSnake // We start off with our own key as the best key. Any suitable next-hop // candidate has to improve on our own key in order to forward the frame. var bestPeer *peer - var bestAnn *rootAnnouncementWithTime var bestSeq types.Varu64 if !params.isBootstrap { bestPeer = params.selfPeer @@ -200,7 +175,7 @@ func getNextHopSNEK(params virtualSnakeNextHopParams) (*peer, types.VirtualSnake // newCandidate updates the best key and best peer with new candidates. newCandidate := func(key types.PublicKey, seq types.Varu64, p *peer) { - bestKey, bestSeq, bestPeer, bestAnn = key, seq, p, params.peerAnnouncements[p] + bestKey, bestSeq, bestPeer = key, seq, p } // newCheckedCandidate performs some sanity checks on the candidate before // passing it to newCandidate. @@ -213,39 +188,13 @@ func getNextHopSNEK(params virtualSnakeNextHopParams) (*peer, types.VirtualSnake } } - // Check if we can use the path to the root via our parent as a starting - // point. We can't do this if we are the root node as there would be no - // parent or ascending paths. - if params.parentPeer != nil && params.parentPeer.started.Load() { - switch { - case params.isBootstrap && bestKey == destKey: - // Bootstraps always start working towards thear root so that they - // go somewhere rather than getting stuck. - fallthrough - case util.DHTOrdered(bestKey, destKey, params.lastAnnouncement.RootPublicKey): - // The destination key is higher than our own key, so start using - // the path to the root as the first candidate. - newCandidate(params.lastAnnouncement.RootPublicKey, 0, params.parentPeer) - } - - // Check our direct ancestors in the tree, that is, all nodes between - // ourselves and the root node via the parent port. - if ann := params.peerAnnouncements[params.parentPeer]; ann != nil { - for _, ancestor := range ann.Signatures { - newCheckedCandidate(ancestor.PublicKey, 0, params.parentPeer) - } - } - } - // Check all of the ancestors of our direct peers too, that is, all nodes // between our direct peer and the root node. - for p, ann := range params.peerAnnouncements { - if !p.started.Load() { + for _, p := range params.peers { + if p == nil || !p.started.Load() { continue } - for _, hop := range ann.Signatures { - newCheckedCandidate(hop.PublicKey, 0, p) - } + newCheckedCandidate(p.public, 0, p) } // Check whether our current best candidate is actually a direct peer. @@ -253,8 +202,8 @@ func getNextHopSNEK(params virtualSnakeNextHopParams) (*peer, types.VirtualSnake // example, only in this case it would make more sense to route directly // to the peer via our peering with them as opposed to routing via our // parent port. - for p := range params.peerAnnouncements { - if !p.started.Load() { + for _, p := range params.peers { + if p == nil || !p.started.Load() { continue } if peerKey := p.public; bestKey == peerKey { @@ -278,26 +227,6 @@ func getNextHopSNEK(params virtualSnakeNextHopParams) (*peer, types.VirtualSnake newCheckedCandidate(entry.PublicKey, entry.Watermark.Sequence, entry.Source) } - // Finally, be sure that we're using the best-looking path to our next-hop. - // Prefer faster link types and, if not, lower latencies to the root. - if bestPeer != nil && bestAnn != nil { - for p, ann := range params.peerAnnouncements { - peerKey := p.public - switch { - case bestKey != peerKey: - continue - case p.peertype < bestPeer.peertype: - // Prefer faster classes of links if possible. - newCandidate(bestKey, bestSeq, p) - case p.peertype == bestPeer.peertype && - ann.RootSequence == bestAnn.RootSequence && - ann.receiveOrder < bestAnn.receiveOrder: - // Prefer links that have the lowest latency to the root. - newCandidate(bestKey, bestSeq, p) - } - } - } - // Only SNEK paths will have a sequence number higher than 0, so // it's a safe bet that if it's greater than 0, we have hit upon // a newly watermarkable path. @@ -336,22 +265,12 @@ func (s *state) _handleBootstrap(from, to *peer, rx *types.Frame) bool { } } - // Check that the root key and sequence number in the update match our - // current root, otherwise we won't be able to route back to them using - // tree routing anyway. If they don't match, silently drop the bootstrap. - root := s._rootAnnouncement() - if !root.Root.EqualTo(&bootstrap.Root) { - return false - } - // Create a routing table entry. index := virtualSnakeIndex{ PublicKey: rx.DestinationKey, } if existing, ok := s._table[index]; ok { switch { - case !existing.Root.EqualTo(&bootstrap.Root): - break // the root is different case bootstrap.Sequence <= existing.Watermark.Sequence: // TODO: less than-equal to might not be the right thing to do return false @@ -362,7 +281,6 @@ func (s *state) _handleBootstrap(from, to *peer, rx *types.Frame) bool { Source: from, Destination: to, LastSeen: time.Now(), - Root: bootstrap.Root, Watermark: types.VirtualSnakeWatermark{ PublicKey: index.PublicKey, Sequence: bootstrap.Sequence, @@ -373,9 +291,6 @@ func (s *state) _handleBootstrap(from, to *peer, rx *types.Frame) bool { update := false desc := s._descending switch { - case !root.Root.EqualTo(&bootstrap.Root): - // The root key in the bootstrap doesn't match our own key - // so it is quite possible that tree routing would fail. case !util.LessThan(rx.DestinationKey, s.r.public): // The bootstrapping key should be less than ours but it isn't. case desc != nil && desc.valid(): diff --git a/router/state_snek_test.go b/router/state_snek_test.go index e78ef499..ac4bee2c 100644 --- a/router/state_snek_test.go +++ b/router/state_snek_test.go @@ -1,5 +1,6 @@ package router +/* import ( "testing" "time" @@ -283,3 +284,4 @@ func TestSNEKNextHopSelection(t *testing.T) { }) } } +*/ diff --git a/router/state_tree.go b/router/state_tree.go deleted file mode 100644 index d509a9c0..00000000 --- a/router/state_tree.go +++ /dev/null @@ -1,545 +0,0 @@ -// Copyright 2021 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package router - -import ( - "fmt" - "math" - "time" - - "github.com/Arceliar/phony" - "github.com/matrix-org/pinecone/router/events" - "github.com/matrix-org/pinecone/types" -) - -// NOTE: Functions prefixed with an underscore (_) are only safe to be called -// from the actor that owns them, in order to prevent data races. - -// announcementInterval is the frequency at which this -// node will send root announcements to other peers. -const announcementInterval = time.Minute * 30 - -// announcementTimeout is the amount of time that must -// pass without receiving a root announcement before we -// will assume that the peer is dead. -const announcementTimeout = time.Minute * 45 - -type announcementTable map[*peer]*rootAnnouncementWithTime - -// _maintainTree sends out root announcements if we are -// considering ourselves to be a root node. -func (s *state) _maintainTree() { - select { - case <-s.r.context.Done(): - return - default: - defer s._maintainTreeIn(announcementInterval) - } - - // If we don't have a parent then we are acting as if we are a root node, - // so we need to send tree announcements to our peers. In each instance, - // we will update the sequence number so that downstream nodes know that - // it's a new update. - if s._parent == nil { - s._sequence++ - s._sendTreeAnnouncements() - } -} - -type rootAnnouncementWithTime struct { - types.SwitchAnnouncement - receiveTime time.Time // when did we receive the update? - receiveOrder uint64 // the relative order that the update was received -} - -// forPeer generates a frame with a signed root announcement for the given -// peer. -func (a *rootAnnouncementWithTime) forPeer(p *peer) *types.Frame { - if p == nil || p.port == 0 { - panic("trying to send announcement to nil port or port 0") - } - announcement := a.SwitchAnnouncement - announcement.Signatures = append([]types.SignatureWithHop{}, a.Signatures...) - for _, sig := range announcement.Signatures { - if p.router.public == sig.PublicKey { - // For some reason the announcement that we want to send already - // includes our signature. This shouldn't really happen but if we - // did send it, other nodes would end up ignoring the announcement - // anyway since it would appear to be a routing loop. - panic("trying to send announcement with loop") - } - } - // Sign the announcement. - if err := announcement.Sign(p.router.private[:], p.port); err != nil { - panic("failed to sign switch announcement: " + err.Error()) - } - frame := getFrame() - frame.Type = types.TypeTreeAnnouncement - n, err := announcement.MarshalBinary(frame.Payload[:cap(frame.Payload)]) - if err != nil { - panic("failed to marshal switch announcement: " + err.Error()) - } - frame.Payload = frame.Payload[:n] - return frame -} - -// _rootAnnouncement returns the latest root announcement from our parent. -// If we are the root, or the announcement from the parent has expired, we -// will instead return a root update with ourselves as the root. -func (s *state) _rootAnnouncement() *rootAnnouncementWithTime { - if s._parent == nil || s._announcements[s._parent] == nil { - return &rootAnnouncementWithTime{ - SwitchAnnouncement: types.SwitchAnnouncement{ - Root: types.Root{ - RootPublicKey: s.r.public, - RootSequence: types.Varu64(s._sequence), - }, - }, - } - } - return s._announcements[s._parent] -} - -// coords returns our tree coordinates, or an empty array if we are the -// root. This function is safe to be called from other actors. -func (s *state) coords() types.Coordinates { - var coords types.Coordinates - phony.Block(s, func() { - coords = s._coords() - }) - return coords -} - -// _coords returns our tree coordinates, or an empty array if we are the -// root. -func (s *state) _coords() types.Coordinates { - if ann := s._rootAnnouncement(); ann != nil { - return ann.Coords() - } - return types.Coordinates{} -} - -// _becomeRoot removes our current parent, effectively making us a root -// node. It then kicks off tree maintenance, which will result in a tree -// announcement being sent to our peers. -func (s *state) _becomeRoot() { - if s._parent == nil { - return - } - s._setParent(nil) - s._maintainTree() -} - -// sendTreeAnnouncementToPeer signs and sends the given root announcement -// to a given peer. -func (s *state) sendTreeAnnouncementToPeer(ann *rootAnnouncementWithTime, p *peer) { - p.proto.push(ann.forPeer(p)) -} - -// _sendTreeAnnouncements signs and sends the current root announcement to -// all of our active peers. -func (s *state) _sendTreeAnnouncements() { - ann := s._rootAnnouncement() - for _, p := range s._peers { - if p == nil || p.port == 0 || !p.started.Load() { - continue - } - s.sendTreeAnnouncementToPeer(ann, p) - } - - s.r.Act(nil, func() { - coords := []uint64{} - for _, val := range ann.Coords() { - coords = append(coords, uint64(val)) - } - - var announcementTime int64 - if ann.RootPublicKey == s.r.public { - announcementTime = time.Now().UnixNano() - } else { - announcementTime = ann.receiveTime.UnixNano() - } - - s.r._publish(events.TreeRootAnnUpdate{ - Root: ann.RootPublicKey.String(), - Sequence: uint64(ann.RootSequence), - Time: uint64(announcementTime), - Coords: coords, - }) - }) -} - -type treeNextHopParams struct { - destinationCoords types.Coordinates - ourCoords types.Coordinates - fromPeer *peer - selfPeer *peer - lastAnnouncement *rootAnnouncementWithTime - peerAnnouncements *announcementTable -} - -// _nextHopsTree returns the best next-hop candidate for a given frame. The -// "from" peer must be supplied in order to prevent routing loops. It is -// possible for this function to return nil if no next best-hop is available. -func (s *state) _nextHopsTree(from *peer, dest types.Coordinates) *peer { - nextHopParams := treeNextHopParams{ - dest, - s._coords(), - from, - s.r.local, - s._rootAnnouncement(), - &s._announcements, - } - - return getNextHopTree(nextHopParams) -} - -func getNextHopTree(params treeNextHopParams) *peer { - // If it's loopback then don't bother doing anything else. - if params.destinationCoords.EqualTo(params.ourCoords) { - return params.selfPeer - } - - // Work out how close our own coordinates are to the destination - // message. This is important because we'll only forward a frame - // to a peer that takes the message closer to the destination than - // we are. - ourDist := int64(params.ourCoords.DistanceTo(params.destinationCoords)) - if ourDist == 0 { - // It's impossible to get closer so there's a pretty good - // chance at this point that the traffic is destined for us. - // Pass it up to the router. - return params.selfPeer - } - - // Now work out which of our peers takes the message closer. - var bestPeer *peer - bestDist := ourDist - bestOrdering := uint64(math.MaxUint64) - ourRoot := params.lastAnnouncement - for p, ann := range *params.peerAnnouncements { - switch { - case !p.started.Load(): - continue // ignore peers that have stopped - case ann == nil: - continue // ignore peers that haven't sent us announcements - case p == params.fromPeer: - continue // don't route back where the packet came from - case !ourRoot.Root.EqualTo(&ann.Root): - continue // ignore peers that are following a different root or seq - } - - // Look up the coordinates of the peer, and the distance - // across the tree to those coordinates. - peerCoords := ann.PeerCoords() - peerDist := int64(peerCoords.DistanceTo(params.destinationCoords)) - if isBetterNextHopCandidate(peerDist, bestDist, ann.receiveOrder, bestOrdering, - bestPeer != nil) { - bestPeer, bestDist, bestOrdering = p, peerDist, ann.receiveOrder - } - } - - return bestPeer -} - -func isBetterNextHopCandidate(peerDistance int64, bestDistance int64, - peerOrder uint64, bestOrder uint64, candidateExists bool) bool { - betterCandidate := false - - switch { - case peerDistance < bestDistance: - // The peer is closer to the destination. - betterCandidate = true - case peerDistance > bestDistance: - // The peer is further away from the destination. - case candidateExists && peerOrder < bestOrder: - // The peer has a lower latency path to the root as a - // last-resort tiebreak. - betterCandidate = true - } - - return betterCandidate -} - -type TreeAnnouncementAction int64 - -const ( - DropFrame TreeAnnouncementAction = iota // Default value - AcceptUpdate - AcceptNewParent - SelectNewParent - SelectNewParentWithWait - InformPeerOfStrongerRoot -) - -// _handleTreeAnnouncement is called whenever a tree announcement is -// received from a direct peer. It stores the update and then works out -// if that update is good news or bad news. -func (s *state) _handleTreeAnnouncement(p *peer, f *types.Frame) error { - // Unmarshal the frame and check that it is sane. The sanity checks - // do things like ensure that all updates are signed, the first - // signature is from the root, the last signature is from our direct - // peer etc. - var newUpdate types.SwitchAnnouncement - if _, err := newUpdate.UnmarshalBinary(f.Payload); err != nil { - return fmt.Errorf("update unmarshal failed: %w", err) - } - if err := newUpdate.SanityCheck(p.public); err != nil { - return fmt.Errorf("update sanity checks failed: %w", err) - } - - // If the peer is replaying an old sequence number to us then we - // assume that they are up to no good. - if ann := s._announcements[p]; ann != nil { - if newUpdate.RootPublicKey == ann.RootPublicKey && newUpdate.RootSequence < ann.RootSequence { - return fmt.Errorf("update replays old sequence number") - } - } - - // Get the key of our current root and then work out if the root - // key in the new update is stronger, weaker or the same key. - lastParentUpdate := s._rootAnnouncement() - lastRootKey := s.r.public - if lastParentUpdate != nil { - lastRootKey = lastParentUpdate.RootPublicKey - } - rootDelta := newUpdate.RootPublicKey.CompareTo(lastRootKey) - - // Save the root announcement for the peer. If the update is not - // obviously bad then it isn't safe to "skip" storing updates. - s._ordering++ - s._announcements[p] = &rootAnnouncementWithTime{ - SwitchAnnouncement: newUpdate, - receiveTime: time.Now(), - receiveOrder: s._ordering, - } - - // If we're currently waiting to re-parent then there is no - // further action - if !s._waiting { - announcementAction := determineAnnouncementAction(p == s._parent, - newUpdate.IsLoopOrChildOf(s.r.public), rootDelta, - newUpdate.RootSequence, lastParentUpdate.RootSequence) - - switch announcementAction { - case DropFrame: - // Do nothing - case AcceptUpdate: - s._sendTreeAnnouncements() - case AcceptNewParent: - s._setParent(p) - s._sendTreeAnnouncements() - case SelectNewParent: - if s._selectNewParent() { - s._bootstrapSoon() - } - case SelectNewParentWithWait: - s._waiting = true - s._becomeRoot() - // Start the 1 second timer to re-run parent selection. - time.AfterFunc(time.Second, func() { - s.Act(nil, func() { - s._waiting = false - if s._selectNewParent() { - s._bootstrapSoon() - } - }) - }) - case InformPeerOfStrongerRoot: - s.sendTreeAnnouncementToPeer(lastParentUpdate, p) - } - } - - return nil -} - -// determineAnnouncementAction performs the algorithm used to decide how to react -// when a new tree announcement is received. -func determineAnnouncementAction(senderIsParent bool, updateContainsLoop bool, - rootDelta int, newRootSequence types.Varu64, lastRootSequence types.Varu64) TreeAnnouncementAction { - action := DropFrame - if senderIsParent { // The update came from our current parent. - switch { - case updateContainsLoop: - // The update seems to contain our own key already, so it - // would appear that our chosen parent has suddenly decided - // to start replaying our own updates back to us. This is - // bad news. - action = SelectNewParentWithWait - case rootDelta < 0: - // The update contains a weaker root key, which is also bad - // news. - action = SelectNewParentWithWait - case rootDelta == 0 && newRootSequence == lastRootSequence: - // The update contains the same root key, but the sequence - // number is being replayed. This usually happens when the - // parent has chosen a new parent and is re-signing the last - // update to notify their peers of their new coordinates. - // In this case, we consider this also to be bad news. We - // will switch to being the root, which notifies our peers - // of the bad news (since the update will be coming from a - // weaker key) and then we start a 1 second timer, after - // which we will re-run the parent selection. During that 1 - // second period, we will not act on root updates apart from - // saving them. - action = SelectNewParentWithWait - case rootDelta > 0: - // The root update contains a stronger key than before. - // Since this node is already our parent, we can just send out - // the update as normal. - action = AcceptUpdate - case rootDelta == 0 && newRootSequence > lastRootSequence: - // The root update contains the same key as before but it has - // a new sequence number, so the parent is repeating a new - // update to us. We will repeat that update to our peers. - action = AcceptUpdate - } - } else { // Update came from another peer - switch { - case updateContainsLoop: - // The update seems to be signed to us already. This happens - // because one of our peers has chosen us as their parent, but - // they still have to send an update back to us so that we know - // their coordinates. In this case, we will not do anything more - // with the update since it would create a loop otherwise. - action = DropFrame - case rootDelta > 0: - // The update seems to contain a stronger root than our existing - // root. In that case, we will switch to this node as our parent - // and then send out tree announcements to our peers, notifying - // them of the change. - action = AcceptNewParent - case rootDelta < 0: - // The update seems to contain a weaker root key than our existing - // root. In this case the best thing to do is to send an update - // back to this specific peer containing our stronger key in the - // hope that they will accept the update and re-parent. - action = InformPeerOfStrongerRoot - default: - // The update contains the same root key so we will check if it - // still makes sense to keep our current parent. We will reparent - // if not, sending out a new SNEK bootstrap into the network. - action = SelectNewParent - } - } - - return action -} - -// _selectNewParent will examine the root updates from all of our peers -// and decide if we should re-parent. If a new peer is selected, this -// function will return true. If no change is made, or we become the root -// as a result, this function will return false. -func (s *state) _selectNewParent() bool { - // Start with our current root key as the strongest candidate. If we - // don't have any peers that also have this root update then this will - // cause us to fail parent selection, marking ourselves as the root. - root := s._rootAnnouncement() - bestRoot := root.Root - - // If our own key happens to be stronger than our current root for some - // reason then we will just compare against our own key instead. - if bestRoot.RootPublicKey.CompareTo(s.r.public) < 0 { - bestRoot = types.Root{ - RootPublicKey: s.r.public, - RootSequence: 0, - } - } - bestOrder := uint64(math.MaxUint64) - var bestPeer *peer - - // Iterate through all of the announcements received from our peers. - // This will exclude any peers that haven't sent us updates yet. - for peer, ann := range s._announcements { - if !peer.started.Load() { - // The peer has been stopped for some reason, possibly due to a - // timeout or other protocol handling error. - continue - } - - if ann != nil { - if isBetterParentCandidate(*ann, bestRoot, bestOrder, ann.IsLoopOrChildOf(s.r.public)) { - bestRoot = ann.Root - bestPeer = peer - bestOrder = ann.receiveOrder - } - } - } - - // If we found a suitable candidate then we should see if a change needs - // to be made. - if bestPeer != nil { - if bestPeer != s._parent { - // The chosen candidate is different to our current parent, so we - // will update to our new parent and then send tree announcements - // to our peers to notify them of the change. - s._setParent(bestPeer) - s._sendTreeAnnouncements() - return true - } - // The chosen candidate is the same as our current parent, so there is - // nothing to do. - return false - } - - // No suitable other peer was found, so we'll just become the root and wait - // for one of our peers corrects us with future updates. - s._becomeRoot() - return false -} - -func isBetterParentCandidate(ann rootAnnouncementWithTime, bestRoot types.Root, - bestOrder uint64, containsLoop bool) bool { - isBetterCandidate := false - - if time.Since(ann.receiveTime) >= announcementTimeout { - // If the announcement has expired then don't consider this peer - // as a possible candidate. - return false - } - - // Work out if the parent's announcement contains a stronger root - // key than our current best candidate. - keyDelta := ann.RootPublicKey.CompareTo(bestRoot.RootPublicKey) - switch { - case containsLoop: - // The announcement from this peer contains our own public key in - // the signatures, which implies they are a child of ours in the - // tree. We therefore can't use this peer as a parent as this would - // create a loop in the tree. - case keyDelta > 0: - // The peer has a stronger root key, so they are a better candidate. - isBetterCandidate = true - case keyDelta < 0: - // The peer has a weaker root key than our current best candidate, - // so ignore this peer. - case ann.RootSequence > bestRoot.RootSequence: - // The peer has the same root key as our current candidate but the - // sequence number is higher, so they have sent us a newer tree - // announcement. They are a better candidate as a result. - isBetterCandidate = true - case ann.RootSequence < bestRoot.RootSequence: - // The peer has the same root key as our current candidate but a - // worse sequence number, so their announcement is out of date. - case ann.receiveOrder < bestOrder: - // The peer has the same root key and update sequence number as our - // current best candidate, but the update from this peer was received - // first. This condition is a tie-break that helps us to pick a parent - // which will have the lowest latency path to the root, all else equal. - isBetterCandidate = true - } - - return isBetterCandidate -} diff --git a/router/state_tree_test.go b/router/state_tree_test.go deleted file mode 100644 index de2a66ae..00000000 --- a/router/state_tree_test.go +++ /dev/null @@ -1,750 +0,0 @@ -package router - -import ( - "strconv" - "testing" - "time" - - "github.com/matrix-org/pinecone/types" - "go.uber.org/atomic" -) - -func TestHandleTreeAnnouncement(t *testing.T) { - cases := []struct { - desc string - senderIsParent bool - updateContainsLoop bool - rootDelta int - newRootSequence types.Varu64 - lastRootSequence types.Varu64 - expected TreeAnnouncementAction - }{ - {"TestParentLoop1", true, true, -1, 1, 1, SelectNewParentWithWait}, - {"TestParentLoop2", true, true, -1, 1, 2, SelectNewParentWithWait}, - {"TestParentLoop3", true, true, -1, 2, 1, SelectNewParentWithWait}, - {"TestParentLoop4", true, true, 1, 1, 1, SelectNewParentWithWait}, - {"TestParentLoop5", true, true, 1, 1, 2, SelectNewParentWithWait}, - {"TestParentLoop6", true, true, 1, 2, 1, SelectNewParentWithWait}, - {"TestParentLoop7", true, true, 0, 1, 1, SelectNewParentWithWait}, - {"TestParentLoop8", true, true, 0, 1, 2, SelectNewParentWithWait}, - {"TestParentLoop9", true, true, 0, 2, 1, SelectNewParentWithWait}, - {"TestParentLowerRoot1", true, false, -1, 1, 1, SelectNewParentWithWait}, - {"TestParentLowerRoot2", true, false, -1, 1, 2, SelectNewParentWithWait}, - {"TestParentLowerRoot3", true, false, -1, 2, 1, SelectNewParentWithWait}, - {"TestParentHigherRoot1", true, false, 1, 1, 1, AcceptUpdate}, - {"TestParentHigherRoot2", true, false, 1, 1, 2, AcceptUpdate}, - {"TestParentHigherRoot3", true, false, 1, 2, 1, AcceptUpdate}, - {"TestParentSameRootSameSeq", true, false, 0, 1, 1, SelectNewParentWithWait}, - {"TestParentSameRootLowerSeq", true, false, 0, 1, 2, DropFrame}, - {"TestParentSameRootHigherSeq", true, false, 0, 2, 1, AcceptUpdate}, - - {"TestNonParentLoop1", false, true, -1, 1, 1, DropFrame}, - {"TestNonParentLoop2", false, true, -1, 1, 2, DropFrame}, - {"TestNonParentLoop3", false, true, -1, 2, 1, DropFrame}, - {"TestNonParentLoop4", false, true, 1, 1, 1, DropFrame}, - {"TestNonParentLoop5", false, true, 1, 1, 2, DropFrame}, - {"TestNonParentLoop6", false, true, 1, 2, 1, DropFrame}, - {"TestNonParentLoop7", false, true, 0, 1, 1, DropFrame}, - {"TestNonParentLoop8", false, true, 0, 1, 2, DropFrame}, - {"TestNonParentLoop9", false, true, 0, 2, 1, DropFrame}, - {"TestNonParentLowerRoot1", false, false, -1, 1, 1, InformPeerOfStrongerRoot}, - {"TestNonParentLowerRoot2", false, false, -1, 1, 2, InformPeerOfStrongerRoot}, - {"TestNonParentLowerRoot3", false, false, -1, 2, 1, InformPeerOfStrongerRoot}, - {"TestNonParentHigherRoot1", false, false, 1, 1, 1, AcceptNewParent}, - {"TestNonParentHigherRoot2", false, false, 1, 1, 2, AcceptNewParent}, - {"TestNonParentHigherRoot3", false, false, 1, 2, 1, AcceptNewParent}, - {"TestNonParentSameRoot1", false, false, 0, 1, 1, SelectNewParent}, - {"TestNonParentSameRoot2", false, false, 0, 1, 2, SelectNewParent}, - {"TestNonParentSameRoot3", false, false, 0, 2, 1, SelectNewParent}, - } - - for _, tc := range cases { - t.Run(tc.desc, func(t *testing.T) { - actual := determineAnnouncementAction(tc.senderIsParent, tc.updateContainsLoop, - tc.rootDelta, tc.newRootSequence, tc.lastRootSequence) - if actual != tc.expected { - t.Fatalf("expected: %d got: %d", tc.expected, actual) - } - }) - } -} - -func TestTreeParentSelection(t *testing.T) { - cases := []struct { - desc string - announcement rootAnnouncementWithTime - bestRoot types.Root - bestOrder uint64 - containsLoop bool - expected bool - }{ - {desc: "TestAnnouncementTooOld", - announcement: rootAnnouncementWithTime{ - receiveTime: time.Now().Add(-announcementTimeout * 2), - receiveOrder: 0, - SwitchAnnouncement: types.SwitchAnnouncement{ - Root: types.Root{ - RootPublicKey: types.PublicKey{6}, RootSequence: 1, - }}}, - bestRoot: types.Root{ - RootPublicKey: types.PublicKey{5}, RootSequence: 1, - }, - bestOrder: 0, - containsLoop: false, - expected: false, - }, - {desc: "TestContainsLoop", - announcement: rootAnnouncementWithTime{ - receiveTime: time.Now(), - receiveOrder: 0, - SwitchAnnouncement: types.SwitchAnnouncement{ - Root: types.Root{ - RootPublicKey: types.PublicKey{6}, - }}}, - bestRoot: types.Root{ - RootPublicKey: types.PublicKey{5}, RootSequence: 1, - }, - bestOrder: 0, - containsLoop: true, - expected: false, - }, - {desc: "TestLowerRoot1", - announcement: rootAnnouncementWithTime{ - receiveTime: time.Now(), - receiveOrder: 0, - SwitchAnnouncement: types.SwitchAnnouncement{ - Root: types.Root{ - RootPublicKey: types.PublicKey{4}, - }}}, - bestRoot: types.Root{ - RootPublicKey: types.PublicKey{5}, RootSequence: 1, - }, - bestOrder: 0, - containsLoop: false, - expected: false, - }, - {desc: "TestLowerRoot2", - announcement: rootAnnouncementWithTime{ - receiveTime: time.Now(), - receiveOrder: 0, - SwitchAnnouncement: types.SwitchAnnouncement{ - Root: types.Root{ - RootPublicKey: types.PublicKey{4}, RootSequence: 2, - }}}, - bestRoot: types.Root{ - RootPublicKey: types.PublicKey{5}, RootSequence: 1, - }, - bestOrder: 0, - containsLoop: false, - expected: false, - }, - {desc: "TestLowerRoot3", - announcement: rootAnnouncementWithTime{ - receiveTime: time.Now(), - receiveOrder: 0, - SwitchAnnouncement: types.SwitchAnnouncement{ - Root: types.Root{ - RootPublicKey: types.PublicKey{4}, RootSequence: 1, - }}}, - bestRoot: types.Root{ - RootPublicKey: types.PublicKey{5}, RootSequence: 2, - }, - bestOrder: 0, - containsLoop: false, - expected: false, - }, - {desc: "TestLowerRoot4", - announcement: rootAnnouncementWithTime{ - receiveTime: time.Now(), - receiveOrder: 1, - SwitchAnnouncement: types.SwitchAnnouncement{ - Root: types.Root{ - RootPublicKey: types.PublicKey{4}, RootSequence: 1, - }}}, - bestRoot: types.Root{ - RootPublicKey: types.PublicKey{5}, RootSequence: 1, - }, - bestOrder: 0, - containsLoop: false, - expected: false, - }, - {desc: "TestLowerRoot5", - announcement: rootAnnouncementWithTime{ - receiveTime: time.Now(), - receiveOrder: 1, - SwitchAnnouncement: types.SwitchAnnouncement{ - Root: types.Root{ - RootPublicKey: types.PublicKey{4}, RootSequence: 2, - }}}, - bestRoot: types.Root{ - RootPublicKey: types.PublicKey{5}, RootSequence: 1, - }, - bestOrder: 0, - containsLoop: false, - expected: false, - }, - {desc: "TestLowerRoot6", - announcement: rootAnnouncementWithTime{ - receiveTime: time.Now(), - receiveOrder: 1, - SwitchAnnouncement: types.SwitchAnnouncement{ - Root: types.Root{ - RootPublicKey: types.PublicKey{4}, RootSequence: 1, - }}}, - bestRoot: types.Root{ - RootPublicKey: types.PublicKey{5}, RootSequence: 2, - }, - bestOrder: 0, - containsLoop: false, - expected: false, - }, - {desc: "TestLowerRoot7", - announcement: rootAnnouncementWithTime{ - receiveTime: time.Now(), - receiveOrder: 0, - SwitchAnnouncement: types.SwitchAnnouncement{ - Root: types.Root{ - RootPublicKey: types.PublicKey{4}, RootSequence: 1, - }}}, - bestRoot: types.Root{ - RootPublicKey: types.PublicKey{5}, RootSequence: 1, - }, - bestOrder: 1, - containsLoop: false, - expected: false, - }, - {desc: "TestLowerRoot8", - announcement: rootAnnouncementWithTime{ - receiveTime: time.Now(), - receiveOrder: 0, - SwitchAnnouncement: types.SwitchAnnouncement{ - Root: types.Root{ - RootPublicKey: types.PublicKey{4}, RootSequence: 2, - }}}, - bestRoot: types.Root{ - RootPublicKey: types.PublicKey{5}, RootSequence: 1, - }, - bestOrder: 1, - containsLoop: false, - expected: false, - }, - {desc: "TestLowerRoot9", - announcement: rootAnnouncementWithTime{ - receiveTime: time.Now(), - receiveOrder: 0, - SwitchAnnouncement: types.SwitchAnnouncement{ - Root: types.Root{ - RootPublicKey: types.PublicKey{4}, RootSequence: 1, - }}}, - bestRoot: types.Root{ - RootPublicKey: types.PublicKey{5}, RootSequence: 2, - }, - bestOrder: 1, - containsLoop: false, - expected: false, - }, - {desc: "TestHigherRoot1", - announcement: rootAnnouncementWithTime{ - receiveTime: time.Now(), - receiveOrder: 0, - SwitchAnnouncement: types.SwitchAnnouncement{ - Root: types.Root{ - RootPublicKey: types.PublicKey{6}, - }}}, - bestRoot: types.Root{ - RootPublicKey: types.PublicKey{5}, RootSequence: 1, - }, - bestOrder: 0, - containsLoop: false, - expected: true, - }, - {desc: "TestHigherRoot2", - announcement: rootAnnouncementWithTime{ - receiveTime: time.Now(), - receiveOrder: 0, - SwitchAnnouncement: types.SwitchAnnouncement{ - Root: types.Root{ - RootPublicKey: types.PublicKey{6}, RootSequence: 2, - }}}, - bestRoot: types.Root{ - RootPublicKey: types.PublicKey{5}, RootSequence: 1, - }, - bestOrder: 0, - containsLoop: false, - expected: true, - }, - {desc: "TestHigherRoot3", - announcement: rootAnnouncementWithTime{ - receiveTime: time.Now(), - receiveOrder: 0, - SwitchAnnouncement: types.SwitchAnnouncement{ - Root: types.Root{ - RootPublicKey: types.PublicKey{6}, RootSequence: 1, - }}}, - bestRoot: types.Root{ - RootPublicKey: types.PublicKey{5}, RootSequence: 2, - }, - bestOrder: 0, - containsLoop: false, - expected: true, - }, - {desc: "TestHigherRoot4", - announcement: rootAnnouncementWithTime{ - receiveTime: time.Now(), - receiveOrder: 1, - SwitchAnnouncement: types.SwitchAnnouncement{ - Root: types.Root{ - RootPublicKey: types.PublicKey{6}, RootSequence: 1, - }}}, - bestRoot: types.Root{ - RootPublicKey: types.PublicKey{5}, RootSequence: 1, - }, - bestOrder: 0, - containsLoop: false, - expected: true, - }, - {desc: "TestHigherRoot5", - announcement: rootAnnouncementWithTime{ - receiveTime: time.Now(), - receiveOrder: 1, - SwitchAnnouncement: types.SwitchAnnouncement{ - Root: types.Root{ - RootPublicKey: types.PublicKey{6}, RootSequence: 2, - }}}, - bestRoot: types.Root{ - RootPublicKey: types.PublicKey{5}, RootSequence: 1, - }, - bestOrder: 0, - containsLoop: false, - expected: true, - }, - {desc: "TestHigherRoot6", - announcement: rootAnnouncementWithTime{ - receiveTime: time.Now(), - receiveOrder: 1, - SwitchAnnouncement: types.SwitchAnnouncement{ - Root: types.Root{ - RootPublicKey: types.PublicKey{6}, RootSequence: 1, - }}}, - bestRoot: types.Root{ - RootPublicKey: types.PublicKey{5}, RootSequence: 2, - }, - bestOrder: 0, - containsLoop: false, - expected: true, - }, - {desc: "TestHigherRoot7", - announcement: rootAnnouncementWithTime{ - receiveTime: time.Now(), - receiveOrder: 0, - SwitchAnnouncement: types.SwitchAnnouncement{ - Root: types.Root{ - RootPublicKey: types.PublicKey{6}, RootSequence: 1, - }}}, - bestRoot: types.Root{ - RootPublicKey: types.PublicKey{5}, RootSequence: 1, - }, - bestOrder: 1, - containsLoop: false, - expected: true, - }, - {desc: "TestHigherRoot8", - announcement: rootAnnouncementWithTime{ - receiveTime: time.Now(), - receiveOrder: 0, - SwitchAnnouncement: types.SwitchAnnouncement{ - Root: types.Root{ - RootPublicKey: types.PublicKey{6}, RootSequence: 2, - }}}, - bestRoot: types.Root{ - RootPublicKey: types.PublicKey{5}, RootSequence: 1, - }, - bestOrder: 1, - containsLoop: false, - expected: true, - }, - {desc: "TestHigherRoot9", - announcement: rootAnnouncementWithTime{ - receiveTime: time.Now(), - receiveOrder: 0, - SwitchAnnouncement: types.SwitchAnnouncement{ - Root: types.Root{ - RootPublicKey: types.PublicKey{6}, RootSequence: 1, - }}}, - bestRoot: types.Root{ - RootPublicKey: types.PublicKey{5}, RootSequence: 2, - }, - bestOrder: 1, - containsLoop: false, - expected: true, - }, - {desc: "TestSameRootHigherSequenceHigherOrder", - announcement: rootAnnouncementWithTime{ - receiveTime: time.Now(), - receiveOrder: 1, - SwitchAnnouncement: types.SwitchAnnouncement{ - Root: types.Root{ - RootPublicKey: types.PublicKey{5}, RootSequence: 2, - }}}, - bestRoot: types.Root{ - RootPublicKey: types.PublicKey{5}, RootSequence: 1, - }, - bestOrder: 0, - containsLoop: false, - expected: true, - }, - {desc: "TestSameRootHigherSequenceLowerOrder", - announcement: rootAnnouncementWithTime{ - receiveTime: time.Now(), - receiveOrder: 0, - SwitchAnnouncement: types.SwitchAnnouncement{ - Root: types.Root{ - RootPublicKey: types.PublicKey{5}, RootSequence: 2, - }}}, - bestRoot: types.Root{ - RootPublicKey: types.PublicKey{5}, RootSequence: 1, - }, - bestOrder: 1, - containsLoop: false, - expected: true, - }, - {desc: "TestSameRootHigherSequenceSameOrder", - announcement: rootAnnouncementWithTime{ - receiveTime: time.Now(), - receiveOrder: 0, - SwitchAnnouncement: types.SwitchAnnouncement{ - Root: types.Root{ - RootPublicKey: types.PublicKey{5}, RootSequence: 2, - }}}, - bestRoot: types.Root{ - RootPublicKey: types.PublicKey{5}, RootSequence: 1, - }, - bestOrder: 0, - containsLoop: false, - expected: true, - }, - {desc: "TestSameRootLowerSequenceHigherOrder", - announcement: rootAnnouncementWithTime{ - receiveTime: time.Now(), - receiveOrder: 1, - SwitchAnnouncement: types.SwitchAnnouncement{ - Root: types.Root{ - RootPublicKey: types.PublicKey{5}, RootSequence: 1, - }}}, - bestRoot: types.Root{ - RootPublicKey: types.PublicKey{5}, RootSequence: 2, - }, - bestOrder: 0, - containsLoop: false, - expected: false, - }, - {desc: "TestSameRootLowerSequenceLowerOrder", - announcement: rootAnnouncementWithTime{ - receiveTime: time.Now(), - receiveOrder: 0, - SwitchAnnouncement: types.SwitchAnnouncement{ - Root: types.Root{ - RootPublicKey: types.PublicKey{5}, RootSequence: 1, - }}}, - bestRoot: types.Root{ - RootPublicKey: types.PublicKey{5}, RootSequence: 2, - }, - bestOrder: 1, - containsLoop: false, - expected: false, - }, - {desc: "TestSameRootLowerSequenceSameOrder", - announcement: rootAnnouncementWithTime{ - receiveTime: time.Now(), - receiveOrder: 0, - SwitchAnnouncement: types.SwitchAnnouncement{ - Root: types.Root{ - RootPublicKey: types.PublicKey{5}, RootSequence: 1, - }}}, - bestRoot: types.Root{ - RootPublicKey: types.PublicKey{5}, RootSequence: 2, - }, - bestOrder: 0, - containsLoop: false, - expected: false, - }, - {desc: "TestSameRootSameSequenceHigherOrder", - announcement: rootAnnouncementWithTime{ - receiveTime: time.Now(), - receiveOrder: 1, - SwitchAnnouncement: types.SwitchAnnouncement{ - Root: types.Root{ - RootPublicKey: types.PublicKey{5}, RootSequence: 1, - }}}, - bestRoot: types.Root{ - RootPublicKey: types.PublicKey{5}, RootSequence: 1, - }, - bestOrder: 0, - containsLoop: false, - expected: false, - }, - {desc: "TestSameRootSameSequenceLowerOrder", - announcement: rootAnnouncementWithTime{ - receiveTime: time.Now(), - receiveOrder: 0, - SwitchAnnouncement: types.SwitchAnnouncement{ - Root: types.Root{ - RootPublicKey: types.PublicKey{5}, RootSequence: 1, - }}}, - bestRoot: types.Root{ - RootPublicKey: types.PublicKey{5}, RootSequence: 1, - }, - bestOrder: 1, - containsLoop: false, - expected: true, - }, - {desc: "TestSameRootSameSequenceSameOrder", - announcement: rootAnnouncementWithTime{ - receiveTime: time.Now(), - receiveOrder: 0, - SwitchAnnouncement: types.SwitchAnnouncement{ - Root: types.Root{ - RootPublicKey: types.PublicKey{5}, RootSequence: 1, - }}}, - bestRoot: types.Root{ - RootPublicKey: types.PublicKey{5}, RootSequence: 1, - }, - bestOrder: 0, - containsLoop: false, - expected: false, - }, - } - - for _, tc := range cases { - t.Run(tc.desc, func(t *testing.T) { - actual := isBetterParentCandidate(tc.announcement, tc.bestRoot, tc.bestOrder, tc.containsLoop) - if actual != tc.expected { - t.Fatalf("expected: %t got: %t", tc.expected, actual) - } - }) - } -} - -func TestTreeNextHopCandidate(t *testing.T) { - cases := []struct { - desc string - peerDist int64 - bestDist int64 - peerOrder uint64 - bestOrder uint64 - candidateExists bool - expected bool - }{ - {"TestCloserHigherOrderCandidate", 1, 2, 2, 1, true, true}, - {"TestCloserHigherOrderNoCandidate", 1, 2, 2, 1, false, true}, - {"TestCloserLowerOrderCandidate", 1, 2, 1, 2, true, true}, - {"TestCloserLowerOrderNoCandidate", 1, 2, 1, 2, false, true}, - {"TestCloserSameOrderCandidate", 1, 2, 1, 1, true, true}, - {"TestCloserSameOrderNoCandidate", 1, 2, 1, 1, false, true}, - {"TestFurtherHigherOrderCandidate", 2, 1, 2, 1, true, false}, - {"TestFurtherHigherOrderNoCandidate", 2, 1, 2, 1, false, false}, - {"TestFurtherLowerOrderCandidate", 2, 1, 1, 2, true, false}, - {"TestFurtherLowerOrderNoCandidate", 2, 1, 1, 2, false, false}, - {"TestFurtherSameOrderCandidate", 2, 1, 1, 1, true, false}, - {"TestFurtherSameOrderNoCandidate", 2, 1, 1, 1, false, false}, - {"TestEquidistantHigherOrderCandidate", 1, 1, 2, 1, true, false}, - {"TestEquidistantHigherOrderNoCandidate", 1, 1, 2, 1, false, false}, - {"TestEquidistantLowerOrderCandidate", 1, 1, 1, 2, true, true}, - {"TestEquidistantLowerOrderNoCandidate", 1, 1, 1, 2, false, false}, - {"TestEquidistantSameOrderCandidate", 1, 1, 1, 1, true, false}, - {"TestEquidistantSameOrderNoCandidate", 1, 1, 1, 1, false, false}, - } - - for _, tc := range cases { - t.Run(tc.desc, func(t *testing.T) { - actual := isBetterNextHopCandidate(tc.peerDist, tc.bestDist, tc.peerOrder, tc.bestOrder, tc.candidateExists) - if actual != tc.expected { - t.Fatalf("expected: %t got: %t", tc.expected, actual) - } - }) - } -} - -func TestTreeNextHopSelection(t *testing.T) { - peers := []*peer{ - // self - {started: *atomic.NewBool(true)}, - // from - {started: *atomic.NewBool(true)}, - // assorted peers - {started: *atomic.NewBool(true)}, - {started: *atomic.NewBool(true)}, - } - - destCoords := types.Coordinates{1, 1, 1} - - selfRoot := types.Root{ - RootPublicKey: types.PublicKey{5}, RootSequence: 1, - } - otherRoot := types.Root{ - RootPublicKey: types.PublicKey{4}, RootSequence: 1, - } - - destSignatures := []types.SignatureWithHop{ - {Hop: 1}, - {Hop: 1}, - {Hop: 1}, - {Hop: 1}, - } - - selfAnn := rootAnnouncementWithTime{ - receiveTime: time.Now(), - receiveOrder: 1, - SwitchAnnouncement: types.SwitchAnnouncement{ - Root: selfRoot, - Signatures: []types.SignatureWithHop{ - {Hop: 2}, - {Hop: 2}, - }, - }, - } - validAnn := rootAnnouncementWithTime{ - receiveTime: time.Now(), - receiveOrder: 1, - SwitchAnnouncement: types.SwitchAnnouncement{ - Root: selfRoot, - Signatures: []types.SignatureWithHop{ - {Hop: 3}, - {Hop: 3}, - }, - }, - } - destAnn := rootAnnouncementWithTime{ - receiveTime: time.Now(), - receiveOrder: 1, - SwitchAnnouncement: types.SwitchAnnouncement{ - Root: selfRoot, - Signatures: destSignatures, - }, - } - closerAnn := rootAnnouncementWithTime{ - receiveTime: time.Now(), - receiveOrder: 1, - SwitchAnnouncement: types.SwitchAnnouncement{ - Root: selfRoot, - Signatures: []types.SignatureWithHop{ - {Hop: 1}, - {Hop: 1}, - }, - }, - } - differentRootDestAnn := rootAnnouncementWithTime{ - receiveTime: time.Now(), - receiveOrder: 1, - SwitchAnnouncement: types.SwitchAnnouncement{ - Root: otherRoot, - Signatures: destSignatures, - }, - } - - cases := []struct { - desc string - input treeNextHopParams - expected *peer // index into peer list - }{ - {"TestNoValidNextHop", treeNextHopParams{ - destCoords, - types.Coordinates{2}, - peers[1], - peers[0], - &selfAnn, - &announcementTable{peers[1]: &validAnn}, - }, nil}, - {"TestDestIsSelf", treeNextHopParams{ - destCoords, - destCoords, - peers[1], - peers[0], - &selfAnn, - &announcementTable{peers[1]: &validAnn}, - }, peers[0]}, - {"TestPeerIsDestination", treeNextHopParams{ - destCoords, - types.Coordinates{2}, - peers[1], - peers[0], - &selfAnn, - &announcementTable{ - peers[1]: &validAnn, - peers[2]: &destAnn, - peers[3]: &closerAnn, - }, - }, peers[2]}, - {"TestDontCreateLoops", treeNextHopParams{ - destCoords, - types.Coordinates{2}, - peers[1], - peers[0], - &selfAnn, - &announcementTable{ - // Even if from peer is the dest, don't loop back to from peer - peers[1]: &destAnn, - }, - }, nil}, - {"TestDifferentRootIsIgnored", treeNextHopParams{ - destCoords, - types.Coordinates{2}, - peers[1], - peers[0], - &selfAnn, - &announcementTable{ - peers[1]: &validAnn, - peers[2]: &differentRootDestAnn, - }, - }, nil}, - {"TestPeerIsBetterCandidate", treeNextHopParams{ - destCoords, - types.Coordinates{2}, - peers[1], - peers[0], - &selfAnn, - &announcementTable{ - peers[1]: &validAnn, - peers[2]: &validAnn, - peers[3]: &closerAnn, - }, - }, peers[3]}, - } - - for _, tc := range cases { - t.Run(tc.desc, func(t *testing.T) { - actual := getNextHopTree(tc.input) - actualString, expectedString := convertToString(actual, tc.expected, peers) - - if actual != tc.expected { - t.Fatalf("expected: %s got: %s", expectedString, actualString) - } - }) - } -} - -func convertToString(actual *peer, expected *peer, peers []*peer) (string, string) { - actualIndex, expectedIndex := 0, 0 - for i := range peers { - if peers[i] == actual { - actualIndex = i - } - if peers[i] == expected { - expectedIndex = i - } - } - - actualString, expectedString := "", "" - if actual == nil { - actualString = "nil" - } else { - actualString = strconv.Itoa(actualIndex) - } - if expected == nil { - expectedString = "nil" - } else { - expectedString = strconv.Itoa(expectedIndex) - } - - return actualString, expectedString -} diff --git a/types/frame.go b/types/frame.go index 56196d9a..572eb0f1 100644 --- a/types/frame.go +++ b/types/frame.go @@ -19,7 +19,6 @@ import ( "crypto/ed25519" "encoding/binary" "fmt" - "math" ) // MaxPayloadSize is the maximum size that a single frame can contain @@ -35,8 +34,6 @@ type FrameType uint8 const ( TypeKeepalive FrameType = iota // protocol frame, direct to peers only - TypeTreeAnnouncement // protocol frame, bypasses queues - TypeTreeRouted // traffic frame, forwarded using tree routing TypeVirtualSnakeBootstrap // protocol frame, forwarded using SNEK TypeVirtualSnakeRouted // traffic frame, forwarded using SNEK ) @@ -54,9 +51,7 @@ type Frame struct { Version FrameVersion Type FrameType Extra [2]byte - Destination Coordinates DestinationKey PublicKey - Source Coordinates SourceKey PublicKey Watermark VirtualSnakeWatermark Payload []byte @@ -67,9 +62,7 @@ func (f *Frame) Reset() { for i := range f.Extra { f.Extra[i] = 0 } - f.Destination = Coordinates{} f.DestinationKey = PublicKey{} - f.Source = Coordinates{} f.SourceKey = PublicKey{} f.Watermark = VirtualSnakeWatermark{} f.Payload = f.Payload[:0] @@ -116,25 +109,8 @@ func (f *Frame) MarshalBinary(buffer []byte) (int, error) { case TypeKeepalive: - default: // destination = coords, source = coords - payloadLen := len(f.Payload) - binary.BigEndian.PutUint16(buffer[offset+0:offset+2], uint16(payloadLen)) - dn, err := f.Destination.MarshalBinary(buffer[offset+2:]) - if err != nil { - return 0, fmt.Errorf("f.Destination.MarshalBinary: %w", err) - } - sn, err := f.Source.MarshalBinary(buffer[offset+2+dn:]) - if err != nil { - return 0, fmt.Errorf("f.Source.MarshalBinary: %w", err) - } - if dn > math.MaxUint16 || sn > math.MaxUint16 || payloadLen > math.MaxUint16 { - return 0, fmt.Errorf("frame contents too large") - } - offset += 2 + dn + sn - if f.Payload != nil { - f.Payload = f.Payload[:payloadLen] - offset += copy(buffer[offset:], f.Payload[:payloadLen]) - } + default: + return offset, fmt.Errorf("unknown frame type") } binary.BigEndian.PutUint16(buffer[FrameHeaderLength-2:FrameHeaderLength], uint16(offset)) @@ -196,37 +172,13 @@ func (f *Frame) UnmarshalBinary(data []byte) (int, error) { case TypeKeepalive: return offset, nil - default: // destination = coords, source = coords - payloadLen := int(binary.BigEndian.Uint16(data[offset+0 : offset+2])) - if payloadLen > cap(f.Payload) { - return 0, fmt.Errorf("payload length exceeds frame capacity") - } - offset += 2 - dstLen, dstErr := f.Destination.UnmarshalBinary(data[offset:]) - if dstErr != nil { - return 0, fmt.Errorf("f.Destination.UnmarshalBinary: %w", dstErr) - } - offset += dstLen - srcLen, srcErr := f.Source.UnmarshalBinary(data[offset:]) - if srcErr != nil { - return 0, fmt.Errorf("f.Source.UnmarshalBinary: %w", srcErr) - } - offset += srcLen - if size := offset + payloadLen; len(data) != int(size) { - return 0, fmt.Errorf("frame expecting %d total bytes, got %d bytes", size, len(data)) - } - f.Payload = f.Payload[:payloadLen] - offset += copy(f.Payload, data[offset:]) - return offset + payloadLen, nil + default: + return offset, fmt.Errorf("unknown frame type") } } func (t FrameType) String() string { switch t { - case TypeTreeAnnouncement: - return "TreeAnnouncement" - case TypeTreeRouted: - return "TreeRouted" case TypeVirtualSnakeBootstrap: return "VirtualSnakeBootstrap" case TypeVirtualSnakeRouted: diff --git a/types/frame_test.go b/types/frame_test.go index 709deb3d..5081cd5b 100644 --- a/types/frame_test.go +++ b/types/frame_test.go @@ -21,65 +21,6 @@ import ( "testing" ) -func TestMarshalUnmarshalFrame(t *testing.T) { - input := Frame{ - Version: Version0, - Type: TypeTreeRouted, - Destination: Coordinates{1, 2, 3, 4, 5000}, - Source: Coordinates{4, 3, 2, 1}, - Payload: []byte("ABCDEFG"), - } - expected := []byte{ - 0x70, 0x69, 0x6e, 0x65, // magic bytes - 0, // version 0 - byte(TypeTreeRouted), // type greedy - 0, 0, // extra - 0, 33, // frame length - 0, 7, // payload len - 0, 6, 1, 2, 3, 4, 167, 8, // destination (2+6 bytes but 5 ports!) - 0, 4, 4, 3, 2, 1, // source (2+4 bytes) - 65, 66, 67, 68, 69, 70, 71, // payload (7 bytes) - } - buf := make([]byte, 65535) - n, err := input.MarshalBinary(buf) - if err != nil { - t.Fatal(err) - } - if n != len(expected) { - t.Fatalf("wrong marshalled length, got %d, expected %d", n, len(expected)) - } - if !bytes.Equal(buf[:n], expected) { - t.Fatalf("wrong marshalled output, got %v, expected %v", buf[:n], expected) - } - output := Frame{ - Payload: make([]byte, 0, MaxPayloadSize), - } - if _, err := output.UnmarshalBinary(buf[:n]); err != nil { - t.Fatal(err) - } - if output.Version != input.Version { - t.Fatal("wrong version") - } - if output.Type != input.Type { - t.Fatal("wrong version") - } - if l := len(output.Destination); l != 5 { - t.Fatalf("wrong destination length (got %d, expected 5)", l) - } - if l := len(output.Source); l != 4 { - t.Fatalf("wrong source length (got %d, expected 4)", l) - } - if l := len(output.Payload); l != 7 { - t.Fatalf("wrong payload length (got %d, expected 7)", l) - } - if !output.Destination.EqualTo(input.Destination) { - t.Fatal("wrong path") - } - if !bytes.Equal(input.Payload, output.Payload) { - t.Fatal("wrong payload") - } -} - func TestMarshalUnmarshalSNEKBootstrapFrame(t *testing.T) { pk, _, _ := ed25519.GenerateKey(nil) wpk, _, _ := ed25519.GenerateKey(nil) @@ -140,9 +81,6 @@ func TestMarshalUnmarshalSNEKBootstrapFrame(t *testing.T) { if output.Type != input.Type { t.Fatal("wrong version") } - if !output.Destination.EqualTo(input.Destination) { - t.Fatal("wrong path") - } if !bytes.Equal(input.Payload, output.Payload) { t.Fatal("wrong payload") } @@ -207,9 +145,6 @@ func TestMarshalUnmarshalSNEKFrame(t *testing.T) { if output.Type != input.Type { t.Fatal("wrong version") } - if !output.Destination.EqualTo(input.Destination) { - t.Fatal("wrong path") - } if !bytes.Equal(input.Payload, output.Payload) { fmt.Println("want: ", input.Payload) fmt.Println("got: ", output.Payload) diff --git a/types/virtualsnake.go b/types/virtualsnake.go index 2479ec37..9019f969 100644 --- a/types/virtualsnake.go +++ b/types/virtualsnake.go @@ -6,8 +6,7 @@ import ( ) type VirtualSnakeBootstrap struct { - Sequence Varu64 - Root + Sequence Varu64 Signature [ed25519.SignatureSize]byte } @@ -27,16 +26,11 @@ func (v *VirtualSnakeBootstrap) ProtectedPayload() ([]byte, error) { if err != nil { return nil, fmt.Errorf("v.Sequence.MarshalBinary: %w", err) } - rn := copy(buffer[:sn], v.RootPublicKey[:]) - rsn, err := v.RootSequence.MarshalBinary(buffer[sn+rn:]) - if err != nil { - return nil, fmt.Errorf("v.Sequence.MarshalBinary: %w", err) - } - return buffer[:sn+rn+rsn], nil + return buffer[:sn], nil } func (v *VirtualSnakeBootstrap) MarshalBinary(buf []byte) (int, error) { - if len(buf) < v.Sequence.Length()+v.Root.Length()+ed25519.SignatureSize { + if len(buf) < v.Sequence.Length()+ed25519.SignatureSize { return 0, fmt.Errorf("buffer too small") } offset := 0 @@ -45,18 +39,12 @@ func (v *VirtualSnakeBootstrap) MarshalBinary(buf []byte) (int, error) { return 0, fmt.Errorf("v.Sequence.MarshalBinary: %w", err) } offset += n - offset += copy(buf[offset:], v.RootPublicKey[:]) - n, err = v.RootSequence.MarshalBinary(buf[offset:]) - if err != nil { - return 0, fmt.Errorf("v.RootSequence.MarshalBinary: %w", err) - } - offset += n offset += copy(buf[offset:], v.Signature[:]) return offset, nil } func (v *VirtualSnakeBootstrap) UnmarshalBinary(buf []byte) (int, error) { - if len(buf) < v.Sequence.MinLength()+v.Root.MinLength()+ed25519.SignatureSize { + if len(buf) < v.Sequence.MinLength()+ed25519.SignatureSize { return 0, fmt.Errorf("buffer too small") } offset := 0 @@ -65,12 +53,6 @@ func (v *VirtualSnakeBootstrap) UnmarshalBinary(buf []byte) (int, error) { return 0, fmt.Errorf("v.Sequence.UnmarshalBinary: %w", err) } offset += n - offset += copy(v.RootPublicKey[:], buf[offset:]) - n, err = v.RootSequence.UnmarshalBinary(buf[offset:]) - if err != nil { - return 0, fmt.Errorf("v.RootSequence.UnmarshalBinary: %w", err) - } - offset += n offset += copy(v.Signature[:], buf[offset:]) return offset, nil } diff --git a/types/virtualsnake_test.go b/types/virtualsnake_test.go index 5c31b22a..52619572 100644 --- a/types/virtualsnake_test.go +++ b/types/virtualsnake_test.go @@ -22,15 +22,10 @@ import ( ) func TestMarshalUnmarshalBootstrap(t *testing.T) { - pkr, _, _ := ed25519.GenerateKey(nil) _, sk1, _ := ed25519.GenerateKey(nil) input := &VirtualSnakeBootstrap{ Sequence: 7, - Root: Root{ - RootSequence: 1, - }, } - copy(input.RootPublicKey[:], pkr) var err error protected, err := input.ProtectedPayload() if err != nil { @@ -56,16 +51,6 @@ func TestMarshalUnmarshalBootstrap(t *testing.T) { fmt.Println("got:", output.Sequence) t.Fatalf("bootstrap sequence doesn't match") } - if !bytes.Equal(pkr, output.RootPublicKey[:]) { - fmt.Println("expected:", pkr) - fmt.Println("got:", output.RootPublicKey) - t.Fatalf("root public key doesn't match") - } - if output.RootSequence != input.RootSequence { - fmt.Println("expected:", input.RootSequence) - fmt.Println("got:", output.RootSequence) - t.Fatalf("root sequence doesn't match") - } if !bytes.Equal(input.Signature[:], output.Signature[:]) { fmt.Println("expected:", input.Signature) fmt.Println("got:", output.Signature) From a4d9428613607b041d8a0972ff10adc184206f89 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 26 May 2022 15:47:47 +0100 Subject: [PATCH 29/41] Make it work --- router/state.go | 16 +++++++++++ router/state_forward.go | 15 ++++++++++ router/state_snek.go | 63 +++++++++++++++++++++++++---------------- 3 files changed, 69 insertions(+), 25 deletions(-) diff --git a/router/state.go b/router/state.go index 9cb4bba4..fc20cab5 100644 --- a/router/state.go +++ b/router/state.go @@ -51,6 +51,7 @@ func (s *state) _start() { s._setDescendingNode(nil) s._waiting = false + s._highest = s._getHighest() s._table = virtualSnakeTable{} @@ -63,6 +64,21 @@ func (s *state) _start() { s._maintainSnakeIn(0) } +// _getHighest returns the highest key that we know about. If it has +// since expired then we'll return ourselves. +func (s *state) _getHighest() *virtualSnakeEntry { + if s._highest != nil && s._highest.valid() { + return s._highest + } + return &virtualSnakeEntry{ + virtualSnakeIndex: &virtualSnakeIndex{ + PublicKey: s.r.public, + }, + LastSeen: time.Now(), + Source: s.r.local, + } +} + // _maintainSnakeIn resets the virtual snake maintenance timer to the // specified duration. func (s *state) _maintainSnakeIn(d time.Duration) { diff --git a/router/state_forward.go b/router/state_forward.go index 4e88fbe9..a4fa4c61 100644 --- a/router/state_forward.go +++ b/router/state_forward.go @@ -37,6 +37,21 @@ func (s *state) _nextHopsFor(from *peer, frameType types.FrameType, dest net.Add return nexthop, newWatermark } +// _flood sends a frame to all of our connected peers. This is typically used for +// flooding the bootstrap for the highest key to all of our direct peers. +func (s *state) _flood(f *types.Frame) { + for _, p := range s._peers { + if p == s.r.local || p == nil || p.proto == nil || !p.started.Load() { + continue + } + if s._filterPacket != nil && s._filterPacket(p.public, f) { + s.r.log.Printf("Packet of type %s destined for port %d [%s] was dropped due to filter rules", f.Type.String(), p.port, p.public.String()[:8]) + continue + } + p.proto.push(f) + } +} + // _forward handles frames received from a given peer. In most cases, this function will // look up the best next-hop for a given frame and forward it to the appropriate peer // queue if possible. In some special cases, like tree announcements, diff --git a/router/state_snek.go b/router/state_snek.go index 45b962fb..a9f8da13 100644 --- a/router/state_snek.go +++ b/router/state_snek.go @@ -41,7 +41,6 @@ type virtualSnakeEntry struct { Destination *peer `json:"destination"` Watermark types.VirtualSnakeWatermark `json:"watermark"` LastSeen time.Time `json:"last_seen"` - Root types.Root `json:"root"` } // valid returns true if the update hasn't expired, or false if it has. It is @@ -88,9 +87,7 @@ func (s *state) _bootstrapSoon() { // _bootstrapNow is responsible for sending a bootstrap message to the network. func (s *state) _bootstrapNow() { - // Construct the bootstrap packet. We will include our root key and sequence - // number in the update so that the remote side can determine if we are both using - // the same root node when processing the update. + // Construct the bootstrap packet. b := frameBufferPool.Get().(*[types.MaxFrameSize]byte) defer frameBufferPool.Put(b) bootstrap := types.VirtualSnakeBootstrap{ @@ -123,11 +120,20 @@ func (s *state) _bootstrapNow() { Sequence: 0, } - // Bootstrap messages are routed using SNEK routing with special rules for - // bootstrap packets. - if p, w := s._nextHopsSNEK(send.DestinationKey, types.TypeVirtualSnakeBootstrap, send.Watermark); p != nil && p.proto != nil { - send.Watermark = w - p.proto.push(send) + if highest := s._getHighest(); highest.PublicKey == s.r.public { + // If we believe we're the highest key in the network then send our bootstrap + // packet to all peers instead of just to the identified next-hop. This is + // done so that all peers learn about a path that ascends through keyspace. + s._flood(send) + } else { + // Bootstrap messages are routed using SNEK routing with special rules for + // bootstrap packets. + if p, w := s._nextHopsSNEK(send.DestinationKey, types.TypeVirtualSnakeBootstrap, send.Watermark); p != nil && p.proto != nil { + send.Watermark = w + p.proto.push(send) + } else { + s.r.log.Println("No next-hop identified") + } } s._lastbootstrap = time.Now() } @@ -135,6 +141,7 @@ func (s *state) _bootstrapNow() { type virtualSnakeNextHopParams struct { isBootstrap bool peers []*peer + highest *virtualSnakeEntry destinationKey types.PublicKey publicKey types.PublicKey watermark types.VirtualSnakeWatermark @@ -147,6 +154,7 @@ func (s *state) _nextHopsSNEK(dest types.PublicKey, frameType types.FrameType, w return getNextHopSNEK(virtualSnakeNextHopParams{ frameType == types.TypeVirtualSnakeBootstrap, s._peers, + s._getHighest(), dest, s.r.public, watermark, @@ -188,29 +196,26 @@ func getNextHopSNEK(params virtualSnakeNextHopParams) (*peer, types.VirtualSnake } } - // Check all of the ancestors of our direct peers too, that is, all nodes - // between our direct peer and the root node. - for _, p := range params.peers { - if p == nil || !p.started.Load() { - continue + // Start off with a path to the highest key that we know of. + if params.highest != nil { + switch { + case params.isBootstrap && bestKey == destKey: + // Bootstraps always start working towards our highest key so they + // go somewhere instead of getting stuck. + fallthrough + case util.DHTOrdered(bestKey, destKey, params.highest.PublicKey): + // The destination key is higher than our own key, so start using + // the path to the highest key as the first candidate. + newCandidate(params.highest.PublicKey, 0, params.highest.Source) } - newCheckedCandidate(p.public, 0, p) } - // Check whether our current best candidate is actually a direct peer. - // This might happen if we spotted the node in our direct ancestors for - // example, only in this case it would make more sense to route directly - // to the peer via our peering with them as opposed to routing via our - // parent port. + // Check all of our direct peers, in case any of them provide a better path. for _, p := range params.peers { if p == nil || !p.started.Load() { continue } - if peerKey := p.public; bestKey == peerKey { - // We've seen this key already and we are directly peered, so use - // the peering instead of the previous selected port. - newCandidate(bestKey, 0, p) - } + newCheckedCandidate(p.public, 0, p) } // Check our DHT entries. In particular, we are only looking at the source @@ -319,5 +324,13 @@ func (s *state) _handleBootstrap(from, to *peer, rx *types.Frame) bool { if update { s._setDescendingNode(s._table[index]) } + + // If this is a higher key than that which we've seen, update our + // highest entry with it. + if highest := s._getHighest(); index.PublicKey.CompareTo(highest.PublicKey) >= 0 { + s._highest = s._table[index] + s._flood(rx) + } + return true } From df126d9ebc9da2b3ed8e0a711da62b6c22142da8 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 26 May 2022 15:58:29 +0100 Subject: [PATCH 30/41] Clean up some more --- cmd/pineconesim/simulator/ping.go | 36 +----- cmd/pineconesim/simulator/router.go | 4 +- router/manhole.go | 1 - router/state_forward.go | 7 +- router/state_snek.go | 4 +- router/version.go | 3 +- types/announcement.go | 168 ---------------------------- types/announcement_test.go | 80 ------------- types/coordinates.go | 130 --------------------- types/coordinates_test.go | 68 ----------- types/switchport.go | 3 + 11 files changed, 19 insertions(+), 485 deletions(-) delete mode 100644 types/announcement.go delete mode 100644 types/announcement_test.go delete mode 100644 types/coordinates.go delete mode 100644 types/coordinates_test.go create mode 100644 types/switchport.go diff --git a/cmd/pineconesim/simulator/ping.go b/cmd/pineconesim/simulator/ping.go index f2a189ed..8ef066e9 100644 --- a/cmd/pineconesim/simulator/ping.go +++ b/cmd/pineconesim/simulator/ping.go @@ -26,9 +26,7 @@ import ( type PingType uint8 const ( - TreePing PingType = iota - TreePong - SNEKPing + SNEKPing PingType = iota SNEKPong ) @@ -47,25 +45,17 @@ func (p *PingPayload) MarshalBinary(buffer []byte) (int, error) { offset += 2 switch orig := p.origin.(type) { - case types.Coordinates: - on, err := orig.MarshalBinary(buffer[offset:]) - if err != nil { - return 0, fmt.Errorf("f.Destination.MarshalBinary: %w", err) - } - offset += on case types.PublicKey: offset += copy(buffer[offset:], orig[:ed25519.PublicKeySize]) + default: + return 0, fmt.Errorf("unknown address type") } switch dest := p.destination.(type) { - case types.Coordinates: - dn, err := dest.MarshalBinary(buffer[offset:]) - if err != nil { - return 0, fmt.Errorf("f.Destination.MarshalBinary: %w", err) - } - offset += dn case types.PublicKey: offset += copy(buffer[offset:], dest[:ed25519.PublicKeySize]) + default: + return 0, fmt.Errorf("unknown address type") } return offset, nil @@ -78,22 +68,6 @@ func (p *PingPayload) UnmarshalBinary(data []byte) (int, error) { offset += 3 switch p.pingType { - case TreePing, TreePong: - orig := types.Coordinates{} - oriLen, oriErr := orig.UnmarshalBinary(data[offset:]) - if oriErr != nil { - return 0, fmt.Errorf("p.orig.UnmarshalBinary: %w", oriErr) - } - offset += oriLen - p.origin = net.Addr(orig) - - dest := types.Coordinates{} - dstLen, dstErr := dest.UnmarshalBinary(data[offset:]) - if dstErr != nil { - return 0, fmt.Errorf("p.dest.UnmarshalBinary: %w", dstErr) - } - offset += dstLen - p.destination = net.Addr(dest) case SNEKPing, SNEKPong: tempKey := types.PublicKey{} offset += copy(tempKey[:], data[offset:]) diff --git a/cmd/pineconesim/simulator/router.go b/cmd/pineconesim/simulator/router.go index 8e08d912..23197f32 100644 --- a/cmd/pineconesim/simulator/router.go +++ b/cmd/pineconesim/simulator/router.go @@ -172,7 +172,7 @@ func (r *DefaultRouter) OverlayReadHandler(quit <-chan bool) { var fromAddr net.Addr fromAddr = addr - if payload.pingType == TreePing || payload.pingType == SNEKPing { + if payload.pingType == SNEKPing { if !pingAtDest { payload.hops++ } else { @@ -187,7 +187,7 @@ func (r *DefaultRouter) OverlayReadHandler(quit <-chan bool) { } var nexthop net.Addr - if payload.pingType == TreePing || payload.pingType == SNEKPing { + if payload.pingType == SNEKPing { nexthop = r.rtr.NextHop(fromAddr, frameType, payload.destination) } else { nexthop = r.rtr.NextHop(fromAddr, frameType, payload.origin) diff --git a/router/manhole.go b/router/manhole.go index a5ccc06d..343b4dbe 100644 --- a/router/manhole.go +++ b/router/manhole.go @@ -25,7 +25,6 @@ import ( type manholeResponse struct { Public types.PublicKey `json:"public_key"` - Coords types.Coordinates `json:"coords"` Peers map[string][]manholePeer `json:"peers"` SNEK struct { Descending *virtualSnakeEntry `json:"descending"` diff --git a/router/state_forward.go b/router/state_forward.go index a4fa4c61..46df4a41 100644 --- a/router/state_forward.go +++ b/router/state_forward.go @@ -39,9 +39,12 @@ func (s *state) _nextHopsFor(from *peer, frameType types.FrameType, dest net.Add // _flood sends a frame to all of our connected peers. This is typically used for // flooding the bootstrap for the highest key to all of our direct peers. -func (s *state) _flood(f *types.Frame) { +func (s *state) _flood(from *peer, f *types.Frame) { for _, p := range s._peers { - if p == s.r.local || p == nil || p.proto == nil || !p.started.Load() { + if p == nil || p.proto == nil || !p.started.Load() { + continue + } + if p == from || p == s.r.local { continue } if s._filterPacket != nil && s._filterPacket(p.public, f) { diff --git a/router/state_snek.go b/router/state_snek.go index a9f8da13..f0f0e7b1 100644 --- a/router/state_snek.go +++ b/router/state_snek.go @@ -124,7 +124,7 @@ func (s *state) _bootstrapNow() { // If we believe we're the highest key in the network then send our bootstrap // packet to all peers instead of just to the identified next-hop. This is // done so that all peers learn about a path that ascends through keyspace. - s._flood(send) + s._flood(s.r.local, send) } else { // Bootstrap messages are routed using SNEK routing with special rules for // bootstrap packets. @@ -329,7 +329,7 @@ func (s *state) _handleBootstrap(from, to *peer, rx *types.Frame) bool { // highest entry with it. if highest := s._getHighest(); index.PublicKey.CompareTo(highest.PublicKey) >= 0 { s._highest = s._table[index] - s._flood(rx) + s._flood(from, rx) } return true diff --git a/router/version.go b/router/version.go index 44f19128..0ed22364 100644 --- a/router/version.go +++ b/router/version.go @@ -20,7 +20,8 @@ const ( capabilitySetupACKs // nolint:deadcode,varcheck capabilityDedupedCoordinateInfo capabilitySoftState + capabilityNoSpanningTree ) const ourVersion uint8 = 1 -const ourCapabilities uint32 = capabilityLengthenedRootInterval | capabilityCryptographicSetups | capabilityDedupedCoordinateInfo | capabilitySoftState +const ourCapabilities uint32 = capabilityLengthenedRootInterval | capabilityCryptographicSetups | capabilityDedupedCoordinateInfo | capabilitySoftState | capabilityNoSpanningTree diff --git a/types/announcement.go b/types/announcement.go deleted file mode 100644 index c3471725..00000000 --- a/types/announcement.go +++ /dev/null @@ -1,168 +0,0 @@ -// Copyright 2021 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package types - -import ( - "crypto/ed25519" - "fmt" - "os" -) - -type Root struct { - RootPublicKey PublicKey `json:"root_public_key"` - RootSequence Varu64 `json:"root_sequence"` -} - -func (r *Root) Length() int { - return ed25519.PublicKeySize + r.RootSequence.Length() -} - -func (r *Root) MinLength() int { - return ed25519.PublicKeySize + 1 -} - -type SwitchAnnouncement struct { - Root - Signatures []SignatureWithHop -} - -func (a *SwitchAnnouncement) Sign(privKey ed25519.PrivateKey, forPort SwitchPortID) error { - var body [65535]byte - n, err := a.MarshalBinary(body[:]) - if err != nil { - return fmt.Errorf("a.MarshalBinary: %w", err) - } - hop := SignatureWithHop{ - Hop: Varu64(forPort), - } - copy(hop.PublicKey[:], privKey.Public().(ed25519.PublicKey)) - if _, ok := os.LookupEnv("PINECONE_DISABLE_SIGNATURES"); !ok { - copy(hop.Signature[:], ed25519.Sign(privKey, body[:n])) - } - a.Signatures = append(a.Signatures, hop) - return nil -} - -func (a *SwitchAnnouncement) UnmarshalBinary(data []byte) (int, error) { - expected := ed25519.PublicKeySize + 1 - if size := len(data); size < expected { - return 0, fmt.Errorf("expecting at least %d bytes, got %d bytes", expected, size) - } - remaining := data[copy(a.RootPublicKey[:ed25519.PublicKeySize], data):] - if l, err := a.RootSequence.UnmarshalBinary(remaining); err != nil { - return 0, fmt.Errorf("a.Sequence.UnmarshalBinary: %w", err) - } else { - remaining = remaining[l:] - } - for i := Varu64(0); len(remaining) >= ed25519.PublicKeySize+ed25519.SignatureSize+1; i++ { - var signature SignatureWithHop - n, err := signature.UnmarshalBinary(remaining[:]) - if err != nil { - return 0, fmt.Errorf("signature.UnmarshalBinary: %w", err) - } - if _, ok := os.LookupEnv("PINECONE_DISABLE_SIGNATURES"); !ok { - if !ed25519.Verify(signature.PublicKey[:], data[:len(data)-len(remaining)], signature.Signature[:]) { - return 0, fmt.Errorf("signature verification failed for hop %d", signature.Hop) - } - } - a.Signatures = append(a.Signatures, signature) - remaining = remaining[n:] - } - return len(data) - len(remaining), nil -} - -func (a *SwitchAnnouncement) MarshalBinary(buffer []byte) (int, error) { - offset := 0 - offset += copy(buffer[offset:], a.RootPublicKey[:]) // a.RootPublicKey - dn, err := a.RootSequence.MarshalBinary(buffer[offset:]) - if err != nil { - return 0, fmt.Errorf("a.Sequence.MarshalBinary: %w", err) - } - offset += dn - for _, sig := range a.Signatures { - n, err := sig.MarshalBinary(buffer[offset:]) - if err != nil { - return 0, fmt.Errorf("sig.MarshalBinary: %w", err) - } - offset += n - } - return offset, nil -} - -func (a *SwitchAnnouncement) SanityCheck(from PublicKey) error { - if len(a.Signatures) == 0 { - return fmt.Errorf("update has no signatures") - } - sigs := make(map[PublicKey]struct{}, len(a.Signatures)) - for index, sig := range a.Signatures { - if index == 0 && sig.PublicKey != a.RootPublicKey { - return fmt.Errorf("update first signature doesn't match root key") - } - if sig.Hop == 0 { - return fmt.Errorf("update contains invalid 0 hop") - } - if index == len(a.Signatures)-1 && from != sig.PublicKey { - return fmt.Errorf("update last signature is not from direct peer") - } - if _, ok := sigs[sig.PublicKey]; ok { - return fmt.Errorf("update contains routing loop") - } - sigs[sig.PublicKey] = struct{}{} - } - return nil -} - -func (a *SwitchAnnouncement) Coords() Coordinates { - sigs := a.Signatures - coords := make(Coordinates, 0, len(sigs)) - for _, sig := range sigs { - coords = append(coords, SwitchPortID(sig.Hop)) - } - return coords -} - -func (a *SwitchAnnouncement) PeerCoords() Coordinates { - sigs := a.Signatures - coords := make(Coordinates, 0, len(sigs)-1) - for _, sig := range sigs[:len(sigs)-1] { - coords = append(coords, SwitchPortID(sig.Hop)) - } - return coords -} - -func (a *SwitchAnnouncement) AncestorParent() PublicKey { - if len(a.Signatures) < 2 { - return a.RootPublicKey - } - return a.Signatures[len(a.Signatures)-2].PublicKey -} - -func (a *Root) EqualTo(b *Root) bool { - return a.RootPublicKey == b.RootPublicKey && a.RootSequence == b.RootSequence -} - -func (a *SwitchAnnouncement) IsLoopOrChildOf(pk PublicKey) bool { - m := map[PublicKey]struct{}{} - for _, sig := range a.Signatures { - if sig.PublicKey == pk { - return true - } - if _, ok := m[sig.PublicKey]; ok { - return true - } - m[sig.PublicKey] = struct{}{} - } - return false -} diff --git a/types/announcement_test.go b/types/announcement_test.go deleted file mode 100644 index 41c658a7..00000000 --- a/types/announcement_test.go +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright 2021 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package types - -import ( - "bytes" - "crypto/ed25519" - "fmt" - "testing" -) - -func TestMarshalUnmarshalAnnouncement(t *testing.T) { - pkr, _, _ := ed25519.GenerateKey(nil) - pk1, sk1, _ := ed25519.GenerateKey(nil) - pk2, sk2, _ := ed25519.GenerateKey(nil) - pk3, sk3, _ := ed25519.GenerateKey(nil) - input := &SwitchAnnouncement{ - Root: Root{ - RootSequence: 1, - }, - } - copy(input.RootPublicKey[:], pkr) - var err error - err = input.Sign(sk1, 1) - if err != nil { - t.Fatal(err) - } - err = input.Sign(sk2, 2) - if err != nil { - t.Fatal(err) - } - err = input.Sign(sk3, 3) - if err != nil { - t.Fatal(err) - } - var buffer [65535]byte - n, err := input.MarshalBinary(buffer[:]) - if err != nil { - t.Fatal(err) - } - var output SwitchAnnouncement - if _, err = output.UnmarshalBinary(buffer[:n]); err != nil { - t.Fatal(err) - } - if !bytes.Equal(pkr, output.RootPublicKey[:]) { - fmt.Println("expected:", pkr) - fmt.Println("got:", output.RootPublicKey) - t.Fatalf("first public key doesn't match") - } - if len(output.Signatures) < 3 { - t.Fatalf("not enough signatures were found (should be 3)") - } - if !bytes.Equal(pk1, output.Signatures[0].PublicKey[:]) { - fmt.Println("expected:", pk1) - fmt.Println("got:", output.Signatures[0].PublicKey) - t.Fatalf("first public key doesn't match") - } - if !bytes.Equal(pk2, output.Signatures[1].PublicKey[:]) { - fmt.Println("expected:", pk2) - fmt.Println("got:", output.Signatures[1].PublicKey) - t.Fatalf("second public key doesn't match") - } - if !bytes.Equal(pk3, output.Signatures[2].PublicKey[:]) { - fmt.Println("expected:", pk3) - fmt.Println("got:", output.Signatures[2].PublicKey) - t.Fatalf("third public key doesn't match") - } -} diff --git a/types/coordinates.go b/types/coordinates.go deleted file mode 100644 index 67fb82cb..00000000 --- a/types/coordinates.go +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright 2021 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package types - -import ( - "encoding/binary" - "fmt" - "strings" -) - -type SwitchPortID Varu64 -type Coordinates []SwitchPortID - -func (s Coordinates) Network() string { - return "tree" -} - -func (s Coordinates) Len() int { - return len(s) -} - -func (s Coordinates) Swap(i, j int) { - s[i], s[j] = s[j], s[i] -} - -func (s Coordinates) Less(i, j int) bool { - return s[i] < s[j] -} - -func (s Coordinates) String() string { - ports := make([]string, 0, len(s)) - for _, p := range s { - ports = append(ports, fmt.Sprintf("%d", p)) - } - return "[" + strings.Join(ports, " ") + "]" -} - -func (p Coordinates) MarshalBinary(buf []byte) (int, error) { - l := 2 - for _, a := range p { - n, err := Varu64(a).MarshalBinary(buf[l:]) - if err != nil { - return 0, fmt.Errorf("Varu64(a).MarshalBinary: %w", err) - } - l += n - } - binary.BigEndian.PutUint16(buf[:2], uint16(l-2)) - return l, nil -} - -func (p *Coordinates) UnmarshalBinary(b []byte) (int, error) { - l := int(binary.BigEndian.Uint16(b[:2])) - ports := make(Coordinates, 0, 16) - if rl := len(b); rl < 2+l { - return 0, fmt.Errorf("expecting %d bytes but got %d bytes", 2+l, rl) - } - read := 2 - b = b[read : 2+l] - for { - if len(b) < 1 { - break - } - var id Varu64 - l, err := id.UnmarshalBinary(b) - if err != nil { - return 0, fmt.Errorf("id.UnmarshalBinary: %w", err) - } - ports = append(ports, SwitchPortID(id)) - b = b[l:] - read += l - } - *p = ports - return read, nil -} - -func (p Coordinates) MarshalJSON() ([]byte, error) { - s := make([]string, 0, len(p)) - for _, id := range p { - s = append(s, fmt.Sprintf("%d", id)) - } - return []byte(`"[` + strings.Join(s, " ") + `]"`), nil -} - -func (p Coordinates) EqualTo(o Coordinates) bool { - if len(p) != len(o) { - return false - } - for i := range p { - if p[i] != o[i] { - return false - } - } - return true -} - -func (a *Coordinates) Copy() Coordinates { - return append(Coordinates{}, *a...) -} - -func (a Coordinates) DistanceTo(b Coordinates) int { - ancestor := getCommonPrefix(a, b) - return len(a) + len(b) - 2*ancestor -} - -func getCommonPrefix(a, b Coordinates) int { - c := 0 - l := len(a) - if len(b) < l { - l = len(b) - } - for i := 0; i < l; i++ { - if a[i] != b[i] { - break - } - c++ - } - return c -} diff --git a/types/coordinates_test.go b/types/coordinates_test.go deleted file mode 100644 index 11086f55..00000000 --- a/types/coordinates_test.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2021 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package types - -import ( - "bytes" - "testing" -) - -func TestSwitchPorts(t *testing.T) { - var b [7]byte - expected := []byte{0, 5, 1, 2, 3, 159, 32} - input := Coordinates{1, 2, 3, 4000} - _, err := input.MarshalBinary(b[:]) - if err != nil { - t.Fatal(err) - } - if !bytes.Equal(b[:], expected) { - t.Fatalf("MarshalBinary produced %v, expected %v", b, expected) - } - var output Coordinates - if _, err := output.UnmarshalBinary(b[:]); err != nil { - t.Fatal(err) - } - if !input.EqualTo(output) { - t.Fatalf("Expected %v, got %v", input, output) - } -} - -func TestSwitchPortDistances(t *testing.T) { - root := Coordinates{} - parent := Coordinates{1, 2, 3} - us := Coordinates{1, 2, 3, 4} - other := Coordinates{1, 3, 3, 4, 7, 6, 1} - if dist := us.DistanceTo(root); dist != 4 { - t.Fatalf("distance from us to root should be 4, got %d", dist) - } - if dist := parent.DistanceTo(root); dist != 3 { - t.Fatalf("distance from parent to root should be 3, got %d", dist) - } - if dist := root.DistanceTo(us); dist != 4 { - t.Fatalf("distance from root to us should be 4, got %d", dist) - } - if dist := root.DistanceTo(parent); dist != 3 { - t.Fatalf("distance from root to parent should be 3, got %d", dist) - } - if dist := us.DistanceTo(other); dist != 9 { - t.Fatalf("distance from us to other should be 9, got %d", dist) - } - if dist := parent.DistanceTo(other); dist != 8 { - t.Fatalf("distance from parent to other should be 8, got %d", dist) - } - if dist := root.DistanceTo(other); dist != 7 { - t.Fatalf("distance from root to other should be 7, got %d", dist) - } -} diff --git a/types/switchport.go b/types/switchport.go new file mode 100644 index 00000000..ab09a11a --- /dev/null +++ b/types/switchport.go @@ -0,0 +1,3 @@ +package types + +type SwitchPortID int From 3dbaa6cb7b583a27435ef5dff0394b15741f0f7b Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 26 May 2022 16:27:14 +0100 Subject: [PATCH 31/41] Clean up some more --- cmd/pineconesim/simulator/commands.go | 28 --------------------------- router/state_snek.go | 25 ++++++++++++++++-------- 2 files changed, 17 insertions(+), 36 deletions(-) diff --git a/cmd/pineconesim/simulator/commands.go b/cmd/pineconesim/simulator/commands.go index 3a67023f..76c8f04c 100644 --- a/cmd/pineconesim/simulator/commands.go +++ b/cmd/pineconesim/simulator/commands.go @@ -119,20 +119,6 @@ func UnmarshalCommandJSON(command *SimCommandMsg) (SimCommand, error) { } else { err = fmt.Errorf("%sConfigureAdversaryDefaults.DropRates.Keepalive field doesn't exist", FAILURE_PREAMBLE) } - /* - if subVal, subOk := val.(map[string]interface{})["TreeAnnouncement"]; subOk { - intVal, _ := strconv.Atoi(subVal.(string)) - dropRates.Frames[types.TypeTreeAnnouncement] = uint64(intVal) - } else { - err = fmt.Errorf("%sConfigureAdversaryDefaults.DropRates.TreeAnnouncement field doesn't exist", FAILURE_PREAMBLE) - } - if subVal, subOk := val.(map[string]interface{})["TreeRouted"]; subOk { - intVal, _ := strconv.Atoi(subVal.(string)) - dropRates.Frames[types.TypeTreeRouted] = uint64(intVal) - } else { - err = fmt.Errorf("%sConfigureAdversaryDefaults.DropRates.TreeRouted field doesn't exist", FAILURE_PREAMBLE) - } - */ if subVal, subOk := val.(map[string]interface{})["VirtualSnakeBootstrap"]; subOk { intVal, _ := strconv.Atoi(subVal.(string)) dropRates.Frames[types.TypeVirtualSnakeBootstrap] = uint64(intVal) @@ -177,20 +163,6 @@ func UnmarshalCommandJSON(command *SimCommandMsg) (SimCommand, error) { } else { err = fmt.Errorf("%sConfigureAdversaryPeer.DropRates.Keepalive field doesn't exist", FAILURE_PREAMBLE) } - /* - if subVal, subOk := val.(map[string]interface{})["TreeAnnouncement"]; subOk { - intVal, _ := strconv.Atoi(subVal.(string)) - dropRates.Frames[types.TypeTreeAnnouncement] = uint64(intVal) - } else { - err = fmt.Errorf("%sConfigureAdversaryPeer.DropRates.TreeAnnouncement field doesn't exist", FAILURE_PREAMBLE) - } - if subVal, subOk := val.(map[string]interface{})["TreeRouted"]; subOk { - intVal, _ := strconv.Atoi(subVal.(string)) - dropRates.Frames[types.TypeTreeRouted] = uint64(intVal) - } else { - err = fmt.Errorf("%sConfigureAdversaryPeer.DropRates.TreeRouted field doesn't exist", FAILURE_PREAMBLE) - } - */ if subVal, subOk := val.(map[string]interface{})["VirtualSnakeBootstrap"]; subOk { intVal, _ := strconv.Atoi(subVal.(string)) dropRates.Frames[types.TypeVirtualSnakeBootstrap] = uint64(intVal) diff --git a/router/state_snek.go b/router/state_snek.go index f0f0e7b1..c251d512 100644 --- a/router/state_snek.go +++ b/router/state_snek.go @@ -125,15 +125,11 @@ func (s *state) _bootstrapNow() { // packet to all peers instead of just to the identified next-hop. This is // done so that all peers learn about a path that ascends through keyspace. s._flood(s.r.local, send) - } else { + } else if p, w := s._nextHopsSNEK(send.DestinationKey, types.TypeVirtualSnakeBootstrap, send.Watermark); p != nil && p.proto != nil { // Bootstrap messages are routed using SNEK routing with special rules for // bootstrap packets. - if p, w := s._nextHopsSNEK(send.DestinationKey, types.TypeVirtualSnakeBootstrap, send.Watermark); p != nil && p.proto != nil { - send.Watermark = w - p.proto.push(send) - } else { - s.r.log.Println("No next-hop identified") - } + send.Watermark = w + p.proto.push(send) } s._lastbootstrap = time.Now() } @@ -327,7 +323,20 @@ func (s *state) _handleBootstrap(from, to *peer, rx *types.Frame) bool { // If this is a higher key than that which we've seen, update our // highest entry with it. - if highest := s._getHighest(); index.PublicKey.CompareTo(highest.PublicKey) >= 0 { + highest := s._getHighest() + diff := index.PublicKey.CompareTo(highest.PublicKey) + switch { + case diff < 0: + // The bootstrap is for a path with a key lower then our + // currently known highest key. + break + case diff == 0 && bootstrap.Sequence < highest.Watermark.Sequence: + // The bootstrap is for the same path but the bootstrap + // number is out of date. + break + default: + // The bootstrap is for a stronger key, or for the same key + // but a newer sequence number. s._highest = s._table[index] s._flood(from, rx) } From 606aac49c24f3dee1034765b5e3b786b839fcd76 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 26 May 2022 17:39:16 +0100 Subject: [PATCH 32/41] Fix frame reuse --- router/state_forward.go | 4 +++- router/state_snek.go | 20 +++++++++----------- types/frame.go | 20 ++++++++++++++++---- 3 files changed, 28 insertions(+), 16 deletions(-) diff --git a/router/state_forward.go b/router/state_forward.go index 46df4a41..26ed4796 100644 --- a/router/state_forward.go +++ b/router/state_forward.go @@ -51,7 +51,9 @@ func (s *state) _flood(from *peer, f *types.Frame) { s.r.log.Printf("Packet of type %s destined for port %d [%s] was dropped due to filter rules", f.Type.String(), p.port, p.public.String()[:8]) continue } - p.proto.push(f) + frame := framePool.Get().(*types.Frame) + f.CopyInto(frame) + p.proto.push(frame) } } diff --git a/router/state_snek.go b/router/state_snek.go index c251d512..b5b34d51 100644 --- a/router/state_snek.go +++ b/router/state_snek.go @@ -120,14 +120,15 @@ func (s *state) _bootstrapNow() { Sequence: 0, } + // If we think we're the highest key in the network then we'll send our bootstrap + // out to all peers and tell them all of our existence. If they also believe us + // to be the highest key on the network then they will repeat it to their peers. + // Otherwise, we expect to be told from our peers that we're not the highest, in + // which case we'll just bootstrap normally. if highest := s._getHighest(); highest.PublicKey == s.r.public { - // If we believe we're the highest key in the network then send our bootstrap - // packet to all peers instead of just to the identified next-hop. This is - // done so that all peers learn about a path that ascends through keyspace. s._flood(s.r.local, send) + framePool.Put(send) } else if p, w := s._nextHopsSNEK(send.DestinationKey, types.TypeVirtualSnakeBootstrap, send.Watermark); p != nil && p.proto != nil { - // Bootstrap messages are routed using SNEK routing with special rules for - // bootstrap packets. send.Watermark = w p.proto.push(send) } @@ -270,12 +271,9 @@ func (s *state) _handleBootstrap(from, to *peer, rx *types.Frame) bool { index := virtualSnakeIndex{ PublicKey: rx.DestinationKey, } - if existing, ok := s._table[index]; ok { - switch { - case bootstrap.Sequence <= existing.Watermark.Sequence: - // TODO: less than-equal to might not be the right thing to do - return false - } + if existing, ok := s._table[index]; ok && bootstrap.Sequence <= existing.Watermark.Sequence { + // TODO: less than-equal to might not be the right thing to do + return false } s._table[index] = &virtualSnakeEntry{ virtualSnakeIndex: &index, diff --git a/types/frame.go b/types/frame.go index 572eb0f1..b2f5f1b8 100644 --- a/types/frame.go +++ b/types/frame.go @@ -68,6 +68,17 @@ func (f *Frame) Reset() { f.Payload = f.Payload[:0] } +func (f *Frame) CopyInto(t *Frame) { + t.Version = f.Version + t.Type = f.Type + t.Extra = f.Extra + t.DestinationKey = f.DestinationKey + t.SourceKey = f.SourceKey + t.Watermark = f.Watermark + t.Payload = t.Payload[:len(f.Payload)] + copy(t.Payload, f.Payload) +} + func (f *Frame) MarshalBinary(buffer []byte) (int, error) { copy(buffer[:4], FrameMagicBytes) buffer[4], buffer[5] = byte(f.Version), byte(f.Type) @@ -85,8 +96,10 @@ func (f *Frame) MarshalBinary(buffer []byte) (int, error) { return 0, fmt.Errorf("f.WatermarkSeq.MarshalBinary: %w", err) } offset += n - if f.Payload != nil { - f.Payload = f.Payload[:payloadLen] + if payloadLen == 0 { + panic("something is afoot!") + } + if payloadLen > 0 { offset += copy(buffer[offset:], f.Payload[:payloadLen]) } @@ -102,8 +115,7 @@ func (f *Frame) MarshalBinary(buffer []byte) (int, error) { return 0, fmt.Errorf("f.WatermarkSeq.MarshalBinary: %w", err) } offset += n - if f.Payload != nil { - f.Payload = f.Payload[:payloadLen] + if payloadLen > 0 { offset += copy(buffer[offset:], f.Payload[:payloadLen]) } From 17e977c609519151a721d19535c5fda3b67488d8 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 27 May 2022 09:37:35 +0100 Subject: [PATCH 33/41] Only keep first highest key update (should settle on lowest latency path) --- router/state_snek.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/router/state_snek.go b/router/state_snek.go index b5b34d51..6ee54257 100644 --- a/router/state_snek.go +++ b/router/state_snek.go @@ -328,7 +328,7 @@ func (s *state) _handleBootstrap(from, to *peer, rx *types.Frame) bool { // The bootstrap is for a path with a key lower then our // currently known highest key. break - case diff == 0 && bootstrap.Sequence < highest.Watermark.Sequence: + case diff == 0 && bootstrap.Sequence <= highest.Watermark.Sequence: // The bootstrap is for the same path but the bootstrap // number is out of date. break From bb22a130c49de234bff68ade262e69f5cea468bc Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 27 May 2022 09:52:57 +0100 Subject: [PATCH 34/41] Clean up a tree thing --- router/state.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/router/state.go b/router/state.go index fc20cab5..c34baeb0 100644 --- a/router/state.go +++ b/router/state.go @@ -42,7 +42,6 @@ type state struct { _table virtualSnakeTable // Virtual snake DHT entries _snaketimer *time.Timer // Virtual snake maintenance timer _lastbootstrap time.Time // When did we last bootstrap? - _waiting bool // Is the tree waiting to reparent? _filterPacket FilterFn // Function called when forwarding packets } @@ -50,7 +49,6 @@ type state struct { func (s *state) _start() { s._setDescendingNode(nil) - s._waiting = false s._highest = s._getHighest() s._table = virtualSnakeTable{} From 9ccd008b232aa94575a4bee3450a9bf8cf3716c0 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 27 May 2022 10:39:05 +0100 Subject: [PATCH 35/41] Make bootstrapping slightly more aggressive --- router/state.go | 10 ++++++++-- router/state_snek.go | 13 ++++++++++--- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/router/state.go b/router/state.go index c34baeb0..4c047480 100644 --- a/router/state.go +++ b/router/state.go @@ -37,11 +37,13 @@ type state struct { phony.Inbox r *Router _peers []*peer // All switch ports, connected and disconnected + _peercount int // Number of connected peerings in total _highest *virtualSnakeEntry // The highest entry we've seen recently _descending *virtualSnakeEntry // Next descending node in keyspace _table virtualSnakeTable // Virtual snake DHT entries _snaketimer *time.Timer // Virtual snake maintenance timer _lastbootstrap time.Time // When did we last bootstrap? + _interval time.Duration // How often should we send bootstraps? _filterPacket FilterFn // Function called when forwarding packets } @@ -50,7 +52,7 @@ func (s *state) _start() { s._setDescendingNode(nil) s._highest = s._getHighest() - + s._interval = virtualSnakeBootstrapMinInterval s._table = virtualSnakeTable{} if s._snaketimer == nil { @@ -118,13 +120,16 @@ func (s *state) _addPeer(conn net.Conn, public types.PublicKey, uri ConnectionUR traffic: newFairFIFOQueue(queues, s.r.log), } s._peers[i] = new + s._peercount++ s.r.log.Println("Connected to peer", new.public.String(), "on port", new.port) v, _ := s.r.active.LoadOrStore(hex.EncodeToString(new.public[:])+string(zone), atomic.NewUint64(0)) v.(*atomic.Uint64).Inc() new.started.Store(true) new.reader.Act(nil, new._read) new.writer.Act(nil, new._write) - + if s._peercount == 1 { + s._interval = virtualSnakeBootstrapMinInterval + } s.r.Act(nil, func() { s.r._publish(events.PeerAdded{Port: types.SwitchPortID(i), PeerID: new.public.String()}) }) @@ -138,6 +143,7 @@ func (s *state) _addPeer(conn net.Conn, public types.PublicKey, uri ConnectionUR func (s *state) _removePeer(port types.SwitchPortID) { peerID := s._peers[port].public.String() s._peers[port] = nil + s._peercount-- s.r.Act(nil, func() { s.r._publish(events.PeerRemoved{Port: port, PeerID: peerID}) }) diff --git a/router/state_snek.go b/router/state_snek.go index 6ee54257..b8fc9b41 100644 --- a/router/state_snek.go +++ b/router/state_snek.go @@ -25,8 +25,9 @@ import ( // NOTE: Functions prefixed with an underscore (_) are only safe to be called // from the actor that owns them, in order to prevent data races. -const virtualSnakeMaintainInterval = time.Second -const virtualSnakeBootstrapInterval = time.Second * 5 +const virtualSnakeMaintainInterval = time.Second / 2 +const virtualSnakeBootstrapMinInterval = time.Second +const virtualSnakeBootstrapInterval = virtualSnakeBootstrapMinInterval * 5 const virtualSnakeNeighExpiryPeriod = virtualSnakeBootstrapInterval * 2 type virtualSnakeTable map[virtualSnakeIndex]*virtualSnakeEntry @@ -58,6 +59,9 @@ func (s *state) _maintainSnake() { return default: defer s._maintainSnakeIn(virtualSnakeMaintainInterval) + if s._peercount == 0 { + return + } } // The descending node is the node with the next lowest key. @@ -73,9 +77,12 @@ func (s *state) _maintainSnake() { } // Send a new bootstrap. - if time.Since(s._lastbootstrap) >= virtualSnakeBootstrapInterval { + if time.Since(s._lastbootstrap) >= s._interval { s._bootstrapNow() } + if s._interval < virtualSnakeBootstrapInterval { + s._interval += time.Second + } } // _bootstrapSoon will reset the bootstrap timer so that we will bootstrap on From 5e4dcd76c35684b1940d2d9d18318c4c20a87402 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 27 May 2022 16:59:58 +0100 Subject: [PATCH 36/41] Fix some bugs --- router/state_snek.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/router/state_snek.go b/router/state_snek.go index b8fc9b41..8a41136c 100644 --- a/router/state_snek.go +++ b/router/state_snek.go @@ -305,7 +305,7 @@ func (s *state) _handleBootstrap(from, to *peer, rx *types.Frame) bool { case desc.PublicKey == rx.DestinationKey: // We've received another bootstrap from our direct descending node. // Accept the update as this is OK. - update = true + update = bootstrap.Sequence > desc.Watermark.Sequence case util.DHTOrdered(desc.PublicKey, rx.DestinationKey, s.r.public): // The bootstrapping node is closer to us than our previous descending // node was. @@ -324,6 +324,7 @@ func (s *state) _handleBootstrap(from, to *peer, rx *types.Frame) bool { } if update { s._setDescendingNode(s._table[index]) + s._bootstrapSoon() } // If this is a higher key than that which we've seen, update our From ee335f79017cd3378313f86fcbb870fd13ed5615 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 27 May 2022 17:03:17 +0100 Subject: [PATCH 37/41] Update comment --- router/state_snek.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/router/state_snek.go b/router/state_snek.go index 8a41136c..8b5031b7 100644 --- a/router/state_snek.go +++ b/router/state_snek.go @@ -304,7 +304,7 @@ func (s *state) _handleBootstrap(from, to *peer, rx *types.Frame) bool { switch { case desc.PublicKey == rx.DestinationKey: // We've received another bootstrap from our direct descending node. - // Accept the update as this is OK. + // Accept the update if the sequence number is higher only. update = bootstrap.Sequence > desc.Watermark.Sequence case util.DHTOrdered(desc.PublicKey, rx.DestinationKey, s.r.public): // The bootstrapping node is closer to us than our previous descending From 62a5775cf442948e4fd2b480c7e5bbfeea48ba45 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 27 May 2022 17:10:21 +0100 Subject: [PATCH 38/41] Remove bootstrap soon when updating desc node --- router/state_snek.go | 1 - 1 file changed, 1 deletion(-) diff --git a/router/state_snek.go b/router/state_snek.go index 8b5031b7..0d47575d 100644 --- a/router/state_snek.go +++ b/router/state_snek.go @@ -324,7 +324,6 @@ func (s *state) _handleBootstrap(from, to *peer, rx *types.Frame) bool { } if update { s._setDescendingNode(s._table[index]) - s._bootstrapSoon() } // If this is a higher key than that which we've seen, update our From c345c295a3644a6b870fbc94961815089fc7ae93 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 27 May 2022 17:23:00 +0100 Subject: [PATCH 39/41] Speed up updates --- router/state_snek.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/router/state_snek.go b/router/state_snek.go index 0d47575d..c6bf02a3 100644 --- a/router/state_snek.go +++ b/router/state_snek.go @@ -27,7 +27,7 @@ import ( const virtualSnakeMaintainInterval = time.Second / 2 const virtualSnakeBootstrapMinInterval = time.Second -const virtualSnakeBootstrapInterval = virtualSnakeBootstrapMinInterval * 5 +const virtualSnakeBootstrapInterval = virtualSnakeBootstrapMinInterval * 2 const virtualSnakeNeighExpiryPeriod = virtualSnakeBootstrapInterval * 2 type virtualSnakeTable map[virtualSnakeIndex]*virtualSnakeEntry @@ -81,7 +81,7 @@ func (s *state) _maintainSnake() { s._bootstrapNow() } if s._interval < virtualSnakeBootstrapInterval { - s._interval += time.Second + s._interval += time.Second / 2 } } From 615c321f757b795e188b1535c9b97f5189c98e17 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 8 Jun 2022 10:53:15 +0100 Subject: [PATCH 40/41] Try to distribute ascending path information more quickly when peers connect --- router/state.go | 33 +++++++++++++++++++++------------ router/state_snek.go | 11 ++++++++++- 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/router/state.go b/router/state.go index 4c047480..32b3df37 100644 --- a/router/state.go +++ b/router/state.go @@ -36,22 +36,24 @@ type FilterFn func(from types.PublicKey, f *types.Frame) bool type state struct { phony.Inbox r *Router - _peers []*peer // All switch ports, connected and disconnected - _peercount int // Number of connected peerings in total - _highest *virtualSnakeEntry // The highest entry we've seen recently - _descending *virtualSnakeEntry // Next descending node in keyspace - _table virtualSnakeTable // Virtual snake DHT entries - _snaketimer *time.Timer // Virtual snake maintenance timer - _lastbootstrap time.Time // When did we last bootstrap? - _interval time.Duration // How often should we send bootstraps? - _filterPacket FilterFn // Function called when forwarding packets + _peers []*peer // All switch ports, connected and disconnected + _peercount int // Number of connected peerings in total + _highest *virtualSnakeHighest // The highest entry we've seen recently + _descending *virtualSnakeEntry // Next descending node in keyspace + _table virtualSnakeTable // Virtual snake DHT entries + _snaketimer *time.Timer // Virtual snake maintenance timer + _lastbootstrap time.Time // When did we last bootstrap? + _interval time.Duration // How often should we send bootstraps? + _filterPacket FilterFn // Function called when forwarding packets } // _start resets the state and starts tree and virtual snake maintenance. func (s *state) _start() { s._setDescendingNode(nil) - s._highest = s._getHighest() + s._highest = &virtualSnakeHighest{ + virtualSnakeEntry: s._getHighest(), + } s._interval = virtualSnakeBootstrapMinInterval s._table = virtualSnakeTable{} @@ -68,7 +70,7 @@ func (s *state) _start() { // since expired then we'll return ourselves. func (s *state) _getHighest() *virtualSnakeEntry { if s._highest != nil && s._highest.valid() { - return s._highest + return s._highest.virtualSnakeEntry } return &virtualSnakeEntry{ virtualSnakeIndex: &virtualSnakeIndex{ @@ -128,7 +130,11 @@ func (s *state) _addPeer(conn net.Conn, public types.PublicKey, uri ConnectionUR new.reader.Act(nil, new._read) new.writer.Act(nil, new._write) if s._peercount == 1 { - s._interval = virtualSnakeBootstrapMinInterval + s._bootstrapNow() + } else if s._highest != nil && s._highest.valid() { + if tx := s._highest.Frame; tx != nil { + new.proto.push(tx) + } } s.r.Act(nil, func() { s.r._publish(events.PeerAdded{Port: types.SwitchPortID(i), PeerID: new.public.String()}) @@ -144,6 +150,9 @@ func (s *state) _removePeer(port types.SwitchPortID) { peerID := s._peers[port].public.String() s._peers[port] = nil s._peercount-- + if s._peercount == 0 { + s._highest = nil + } s.r.Act(nil, func() { s.r._publish(events.PeerRemoved{Port: port, PeerID: peerID}) }) diff --git a/router/state_snek.go b/router/state_snek.go index c6bf02a3..cf84e5bc 100644 --- a/router/state_snek.go +++ b/router/state_snek.go @@ -44,6 +44,11 @@ type virtualSnakeEntry struct { LastSeen time.Time `json:"last_seen"` } +type virtualSnakeHighest struct { + *virtualSnakeEntry + *types.Frame +} + // valid returns true if the update hasn't expired, or false if it has. It is // required for updates to time out eventually, in the case that paths don't get // torn down properly for some reason. @@ -342,7 +347,11 @@ func (s *state) _handleBootstrap(from, to *peer, rx *types.Frame) bool { default: // The bootstrap is for a stronger key, or for the same key // but a newer sequence number. - s._highest = s._table[index] + s._highest = &virtualSnakeHighest{ + virtualSnakeEntry: s._table[index], + Frame: framePool.Get().(*types.Frame), + } + rx.CopyInto(s._highest.Frame) s._flood(from, rx) } From bb13fd739dcd79ccad8ff162639d59e506fb6a03 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 8 Jun 2022 17:30:06 +0100 Subject: [PATCH 41/41] Fix bugs --- router/peer.go | 4 ++-- router/pools.go | 10 ++++++++++ router/queuefifo.go | 26 +++++++++++--------------- router/queuelifo.go | 1 - router/state.go | 6 ++++-- router/state_forward.go | 4 ++-- router/state_snek.go | 7 +++---- types/frame.go | 3 +++ 8 files changed, 35 insertions(+), 26 deletions(-) diff --git a/router/peer.go b/router/peer.go index e1735281..772a8162 100644 --- a/router/peer.go +++ b/router/peer.go @@ -95,7 +95,7 @@ func (p *peer) send(f *types.Frame) bool { switch f.Type { // Protocol messages case types.TypeKeepalive: - fallthrough + panic("trying to forward keepalive") case types.TypeVirtualSnakeBootstrap: if p.proto == nil { // The local peer doesn't have a protocol queue so we should check @@ -222,7 +222,7 @@ func (p *peer) _write() { p.stop(fmt.Errorf("queue reset")) return } - defer framePool.Put(frame) + defer putFrame(frame) // We might have been waiting for a little while for one of the above // cases to happen, so let's check one more time that the peering wasn't // stopped before we try to marshal and send the frame. diff --git a/router/pools.go b/router/pools.go index 93b8f36e..f2e5af56 100644 --- a/router/pools.go +++ b/router/pools.go @@ -38,6 +38,16 @@ var framePool = &sync.Pool{ func getFrame() *types.Frame { f := framePool.Get().(*types.Frame) + if f.Refs.Inc() > 1 { + panic("frame retrieved from pool has unexpected references") + } f.Reset() return f } + +func putFrame(f *types.Frame, info ...string) { + if f.Refs.Dec() > 0 { + panic("frame still has unexpected references after returning to pool") + } + framePool.Put(f) +} diff --git a/router/queuefifo.go b/router/queuefifo.go index ce1d3ea9..3e3ee40a 100644 --- a/router/queuefifo.go +++ b/router/queuefifo.go @@ -76,24 +76,19 @@ func (q *fifoQueue) push(frame *types.Frame) bool { return false } ch := q.entries[len(q.entries)-1] - ch <- frame - close(ch) - q.entries = append(q.entries, make(chan *types.Frame, 1)) - return true + select { + case ch <- frame: + close(ch) + q.entries = append(q.entries, make(chan *types.Frame, 1)) + return true + default: + panic("queue channel unexpectedly populated already") + } } func (q *fifoQueue) reset() { q.mutex.Lock() defer q.mutex.Unlock() - for _, ch := range q.entries { - select { - case frame := <-ch: - if frame != nil { - framePool.Put(frame) - } - default: - } - } q._initialise() } @@ -106,8 +101,9 @@ func (q *fifoQueue) pop() <-chan *types.Frame { func (q *fifoQueue) ack() { q.mutex.Lock() defer q.mutex.Unlock() - q.entries = q.entries[1:] - if q.max == 0 && len(q.entries) == 0 { + copy(q.entries, q.entries[1:]) + q.entries = q.entries[:len(q.entries)-1] + if q.max == 0 && len(q.entries) == 0 && cap(q.entries) > 16 { q._initialise() } } diff --git a/router/queuelifo.go b/router/queuelifo.go index af8cbe84..e486eb08 100644 --- a/router/queuelifo.go +++ b/router/queuelifo.go @@ -89,7 +89,6 @@ func (q *lifoQueue) reset() { // nolint:unused q.count = 0 for i := range q.frames { if q.frames[i] != nil { - framePool.Put(q.frames[i]) q.frames[i] = nil } } diff --git a/router/state.go b/router/state.go index 32b3df37..206b2c3a 100644 --- a/router/state.go +++ b/router/state.go @@ -132,8 +132,10 @@ func (s *state) _addPeer(conn net.Conn, public types.PublicKey, uri ConnectionUR if s._peercount == 1 { s._bootstrapNow() } else if s._highest != nil && s._highest.valid() { - if tx := s._highest.Frame; tx != nil { - new.proto.push(tx) + if tx := s._highest.Frame; tx != nil && tx.Type == types.TypeVirtualSnakeBootstrap { + frame := getFrame() + tx.CopyInto(frame) + new.send(frame) } } s.r.Act(nil, func() { diff --git a/router/state_forward.go b/router/state_forward.go index 26ed4796..ed300bc4 100644 --- a/router/state_forward.go +++ b/router/state_forward.go @@ -51,9 +51,9 @@ func (s *state) _flood(from *peer, f *types.Frame) { s.r.log.Printf("Packet of type %s destined for port %d [%s] was dropped due to filter rules", f.Type.String(), p.port, p.public.String()[:8]) continue } - frame := framePool.Get().(*types.Frame) + frame := getFrame() f.CopyInto(frame) - p.proto.push(frame) + p.send(frame) } } diff --git a/router/state_snek.go b/router/state_snek.go index cf84e5bc..6d07db40 100644 --- a/router/state_snek.go +++ b/router/state_snek.go @@ -139,10 +139,9 @@ func (s *state) _bootstrapNow() { // which case we'll just bootstrap normally. if highest := s._getHighest(); highest.PublicKey == s.r.public { s._flood(s.r.local, send) - framePool.Put(send) } else if p, w := s._nextHopsSNEK(send.DestinationKey, types.TypeVirtualSnakeBootstrap, send.Watermark); p != nil && p.proto != nil { send.Watermark = w - p.proto.push(send) + p.send(send) } s._lastbootstrap = time.Now() } @@ -349,10 +348,10 @@ func (s *state) _handleBootstrap(from, to *peer, rx *types.Frame) bool { // but a newer sequence number. s._highest = &virtualSnakeHighest{ virtualSnakeEntry: s._table[index], - Frame: framePool.Get().(*types.Frame), + Frame: getFrame(), } rx.CopyInto(s._highest.Frame) - s._flood(from, rx) + s._flood(from, s._highest.Frame) } return true diff --git a/types/frame.go b/types/frame.go index b2f5f1b8..bbbe57bf 100644 --- a/types/frame.go +++ b/types/frame.go @@ -19,6 +19,8 @@ import ( "crypto/ed25519" "encoding/binary" "fmt" + + "go.uber.org/atomic" ) // MaxPayloadSize is the maximum size that a single frame can contain @@ -48,6 +50,7 @@ var FrameMagicBytes = []byte{0x70, 0x69, 0x6e, 0x65} const FrameHeaderLength = 10 type Frame struct { + Refs atomic.Int32 Version FrameVersion Type FrameType Extra [2]byte