Merge pull request #1634 from safchain/debug-weird-node-not-found

handle metadata update error in memory backend

Author: lebauce

Makefile

@@ -162,6 +162,7 @@ endif
 
 ifeq ($(WITH_OPENCONTRAIL), true)
   BUILD_TAGS+=opencontrail
+  EXTRA_ARGS+=-opencontrail
 ifeq ($(OS_RHEL),Y)
   STATIC_LIBS+=libxml2.a
 endif

graffiti/graph/elasticsearch.go

@@ -38,7 +38,7 @@ const graphElementMapping = `
 				"path_match": "*",
 				"path_unmatch": "*.Extra.*",
 				"match_mapping_type": "string",
-                                "mapping": {
+				"mapping": {
 					"type": "keyword"
 				}
 			}

graffiti/graph/memory.go

@@ -39,6 +39,18 @@ type MemoryBackend struct {
 
 // MetadataUpdated return true
 func (m *MemoryBackend) MetadataUpdated(i interface{}) error {
+	switch i := i.(type) {
+	case *Node:
+		if _, ok := m.nodes[i.ID]; !ok {
+
+			return ErrNodeNotFound
+		}
+	case *Edge:
+		if _, ok := m.edges[i.ID]; !ok {
+			return ErrEdgeNotFound
+		}
+	}
+
 	return nil
 }
 

graffiti/hub/pod_endpoint.go

@@ -71,14 +71,14 @@ func (t *TopologyAgentEndpoint) OnStructMessage(c ws.Speaker, msg *ws.StructMess
 		for _, n := range r.Nodes {
 			if t.Graph.GetNode(n.ID) == nil {
 				if err := t.Graph.NodeAdded(n); err != nil {
-					logging.GetLogger().Error(err)
+					logging.GetLogger().Errorf("%s, %+v", err, n)
 				}
 			}
 		}
 		for _, e := range r.Edges {
 			if t.Graph.GetEdge(e.ID) == nil {
 				if err := t.Graph.EdgeAdded(e); err != nil {
-					logging.GetLogger().Error(err)
+					logging.GetLogger().Errorf("%s, %+v", err, e)
 				}
 			}
 		}
@@ -97,7 +97,7 @@ func (t *TopologyAgentEndpoint) OnStructMessage(c ws.Speaker, msg *ws.StructMess
 	}
 
 	if err != nil {
-		logging.GetLogger().Error(err)
+		logging.GetLogger().Errorf("%s, %+v", err, obj)
 	}
 }
 

ovs/ovsdb.go

@@ -211,6 +211,10 @@ func (o *OvsMonitor) bridgeAdded(bridgeUUID string, row *libovsdb.RowUpdate) {
 }
 
 func (o *OvsMonitor) bridgeDeleted(bridgeUUID string, row *libovsdb.RowUpdate) {
+	// for some reason ovs can trigger multiple time this event
+	if _, ok := o.bridgeCache[bridgeUUID]; !ok {
+		return
+	}
 	delete(o.bridgeCache, bridgeUUID)
 
 	logging.GetLogger().Infof("Bridge \"%s(%s)\" got deleted",
@@ -262,6 +266,10 @@ func (o *OvsMonitor) interfaceAdded(interfaceUUID string, row *libovsdb.RowUpdat
 }
 
 func (o *OvsMonitor) interfaceDeleted(interfaceUUID string, row *libovsdb.RowUpdate) {
+	// for some reason ovs can trigger multiple time this event
+	if _, ok := o.interfaceCache[interfaceUUID]; !ok {
+		return
+	}
 	delete(o.interfaceCache, interfaceUUID)
 
 	logging.GetLogger().Infof("Interface \"%s(%s)\" got deleted",
@@ -312,6 +320,10 @@ func (o *OvsMonitor) portAdded(portUUID string, row *libovsdb.RowUpdate) {
 }
 
 func (o *OvsMonitor) portDeleted(portUUID string, row *libovsdb.RowUpdate) {
+	// for some reason ovs can trigger multiple time this event
+	if _, ok := o.portCache[portUUID]; !ok {
+		return
+	}
 	delete(o.portCache, portUUID)
 
 	logging.GetLogger().Infof("Port \"%s(%s)\" got deleted",

scripts/ci/opencontrail-tests.nix

@@ -15,7 +15,7 @@ let
   testScript = ''
     # This is to wait until OpenContrail is ready
     $machine->waitUntilSucceeds("curl -s http://localhost:8083/Snh_ShowBgpNeighborSummaryReq | grep machine | grep -q Established");
-    $machine->succeed("${skydiveTestFunctionals} -test.run TestOpenContrail -standalone");
+    $machine->succeed("${skydiveTestFunctionals} -test.run TestOpenContrail -standalone -opencontrail");
   '';
 
 in 

scripts/ci/run-functional-tests.sh

@@ -7,5 +7,5 @@ DIR="$(dirname "$0")"
 
 . "$DIR/run-tests-utils.sh"
 network_setup
-tests_run
+WITH_OPENCONTRAIL=false tests_run
 exit $RETCODE

tests/tests.go

@@ -82,9 +82,9 @@ agent:
       - ovsdb
       - docker
       - lxd
-      - opencontrail
       - lldp
       - runc
+      {{.OpencontrailProbe}}
     netlink:
       metrics_update: 5
     lldp:
@@ -226,6 +226,7 @@ var (
 	noOFTests         bool
 	standalone        bool
 	topologyBackend   string
+	opencontrailProbe bool
 )
 
 func initConfig(conf string, params ...helperParams) error {
@@ -275,6 +276,9 @@ func initConfig(conf string, params ...helperParams) error {
 	if analyzerProbes != "" {
 		params[0]["AnalyzerProbes"] = strings.Split(analyzerProbes, ",")
 	}
+	if opencontrailProbe {
+		params[0]["OpencontrailProbe"] = "- opencontrail"
+	}
 
 	tmpl, err := template.New("config").Parse(conf)
 	if err != nil {
@@ -792,6 +796,7 @@ func init() {
 	flag.StringVar(&flowBackend, "analyzer.flow.backend", "", "Specify the flow storage backend used")
 	flag.StringVar(&analyzerListen, "analyzer.listen", "0.0.0.0:64500", "Specify the analyzer listen address")
 	flag.StringVar(&analyzerProbes, "analyzer.topology.probes", "", "Specify the analyzer probes to enable")
+	flag.BoolVar(&opencontrailProbe, "opencontrail", false, "Enable opencontrail probe")
 	flag.Parse()
 
 	http.DefaultTransport.(*http.Transport).MaxIdleConnsPerHost = 100

topology/probes/ovsdb/ovs_of.go

@@ -736,7 +736,7 @@ func (probe *BridgeOfProbe) monitorGroup() error {
 	}
 	if _, err = launchOnSwitch(command); err != nil {
 		logging.GetLogger().Warningf("OpenFlow 1.5 not supported on %s", probe.Bridge)
-		return err
+		return nil
 	}
 	randBytes := make([]byte, 16)
 	rand.Read(randBytes)