Add the ability to skip aliased networks

when using Networks and NetworksWithin.

This is probably what most people wanted in the first place. Ideally,
this would be the default behavior, but that is potentially a breaking
change for people.

The code is more complicated than would otherwise be necessary as I am
trying to support the old behavior exactly as it was.

Author: oschwald

example_test.go

@@ -64,7 +64,7 @@ func ExampleReader_Networks() {
 		Domain string `maxminddb:"connection_type"`
 	}{}
 
-	networks := db.Networks()
+	networks := db.Networks(maxminddb.SkipAliasedNetworks)
 	for networks.Next() {
 		subnet, err := networks.Network(&record)
 		if err != nil {
@@ -76,25 +76,6 @@ func ExampleReader_Networks() {
 		log.Fatal(networks.Err())
 	}
 	// Output:
-	// ::100:0/120: Dialup
-	// ::100:100/120: Cable/DSL
-	// ::100:200/119: Dialup
-	// ::100:400/118: Dialup
-	// ::100:800/117: Dialup
-	// ::100:1000/116: Dialup
-	// ::100:2000/115: Dialup
-	// ::100:4000/114: Dialup
-	// ::100:8000/113: Dialup
-	// ::50d6:0/116: Cellular
-	// ::6001:0/112: Cable/DSL
-	// ::600a:0/111: Cable/DSL
-	// ::6045:0/112: Cable/DSL
-	// ::605e:0/111: Cable/DSL
-	// ::6c60:0/107: Cellular
-	// ::af10:c700/120: Dialup
-	// ::bb9c:8a00/120: Cable/DSL
-	// ::c9f3:c800/120: Corporate
-	// ::cfb3:3000/116: Cellular
 	// 1.0.0.0/24: Dialup
 	// 1.0.1.0/24: Cable/DSL
 	// 1.0.2.0/23: Dialup
@@ -114,44 +95,6 @@ func ExampleReader_Networks() {
 	// 187.156.138.0/24: Cable/DSL
 	// 201.243.200.0/24: Corporate
 	// 207.179.48.0/20: Cellular
-	// 2001:0:100::/56: Dialup
-	// 2001:0:100:100::/56: Cable/DSL
-	// 2001:0:100:200::/55: Dialup
-	// 2001:0:100:400::/54: Dialup
-	// 2001:0:100:800::/53: Dialup
-	// 2001:0:100:1000::/52: Dialup
-	// 2001:0:100:2000::/51: Dialup
-	// 2001:0:100:4000::/50: Dialup
-	// 2001:0:100:8000::/49: Dialup
-	// 2001:0:50d6::/52: Cellular
-	// 2001:0:6001::/48: Cable/DSL
-	// 2001:0:600a::/47: Cable/DSL
-	// 2001:0:6045::/48: Cable/DSL
-	// 2001:0:605e::/47: Cable/DSL
-	// 2001:0:6c60::/43: Cellular
-	// 2001:0:af10:c700::/56: Dialup
-	// 2001:0:bb9c:8a00::/56: Cable/DSL
-	// 2001:0:c9f3:c800::/56: Corporate
-	// 2001:0:cfb3:3000::/52: Cellular
-	// 2002:100::/40: Dialup
-	// 2002:100:100::/40: Cable/DSL
-	// 2002:100:200::/39: Dialup
-	// 2002:100:400::/38: Dialup
-	// 2002:100:800::/37: Dialup
-	// 2002:100:1000::/36: Dialup
-	// 2002:100:2000::/35: Dialup
-	// 2002:100:4000::/34: Dialup
-	// 2002:100:8000::/33: Dialup
-	// 2002:50d6::/36: Cellular
-	// 2002:6001::/32: Cable/DSL
-	// 2002:600a::/31: Cable/DSL
-	// 2002:6045::/32: Cable/DSL
-	// 2002:605e::/31: Cable/DSL
-	// 2002:6c60::/27: Cellular
-	// 2002:af10:c700::/40: Dialup
-	// 2002:bb9c:8a00::/40: Cable/DSL
-	// 2002:c9f3:c800::/40: Corporate
-	// 2002:cfb3:3000::/36: Cellular
 	// 2003::/24: Cable/DSL
 }
 
@@ -173,7 +116,7 @@ func ExampleReader_NetworksWithin() {
 		log.Fatal(err)
 	}
 
-	networks := db.NetworksWithin(network)
+	networks := db.NetworksWithin(network, maxminddb.SkipAliasedNetworks)
 	for networks.Next() {
 		subnet, err := networks.Network(&record)
 		if err != nil {

traverse.go

@@ -18,25 +18,40 @@ type Networks struct {
 	nodes    []netNode // Nodes we still have to visit.
 	lastNode netNode
 	err      error
+
+	skipAliasedNetworks bool
 }
 
 var (
 	allIPv4 = &net.IPNet{IP: make(net.IP, 4), Mask: net.CIDRMask(0, 32)}
 	allIPv6 = &net.IPNet{IP: make(net.IP, 16), Mask: net.CIDRMask(0, 128)}
 )
 
+// NetworksOption are options for Networks and NetworksWithin
+type NetworksOption func(*Networks)
+
+// SkipAliasedNetworks is an option for Networks and NetworksWithin that
+// makes them not iterate over aliases of the IPv4 subtree in an IPv6
+// database, e.g., ::ffff:0:0/96, 2001::/32, and 2002::/16.
+//
+// You most likely want to set this. The only reason it isn't the default
+// behavior is to provide backwards compatibility to existing users.
+func SkipAliasedNetworks(networks *Networks) {
+	networks.skipAliasedNetworks = true
+}
+
 // Networks returns an iterator that can be used to traverse all networks in
 // the database.
 //
 // Please note that a MaxMind DB may map IPv4 networks into several locations
 // in an IPv6 database. This iterator will iterate over all of these
 // locations separately.
-func (r *Reader) Networks() *Networks {
+func (r *Reader) Networks(options ...NetworksOption) *Networks {
 	var networks *Networks
 	if r.Metadata.IPVersion == 6 {
-		networks = r.NetworksWithin(allIPv6)
+		networks = r.NetworksWithin(allIPv6, options...)
 	} else {
-		networks = r.NetworksWithin(allIPv4)
+		networks = r.NetworksWithin(allIPv4, options...)
 	}
 
 	return networks
@@ -51,7 +66,7 @@ func (r *Reader) Networks() *Networks {
 //
 // If the provided network is contained within a network in the database, the
 // iterator will iterate over exactly one network, the containing network.
-func (r *Reader) NetworksWithin(network *net.IPNet) *Networks {
+func (r *Reader) NetworksWithin(network *net.IPNet, options ...NetworksOption) *Networks {
 	if r.Metadata.IPVersion == 4 && network.IP.To4() == nil {
 		return &Networks{
 			err: fmt.Errorf(
@@ -61,25 +76,33 @@ func (r *Reader) NetworksWithin(network *net.IPNet) *Networks {
 		}
 	}
 
+	networks := &Networks{reader: r}
+	for _, option := range options {
+		option(networks)
+	}
+
 	ip := network.IP
 	prefixLength, _ := network.Mask.Size()
 
 	if r.Metadata.IPVersion == 6 && len(ip) == net.IPv4len {
-		ip = net.IP.To16(ip)
+		if networks.skipAliasedNetworks {
+			ip = net.IP{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ip[0], ip[1], ip[2], ip[3]}
+		} else {
+			ip = ip.To16()
+		}
 		prefixLength += 96
 	}
 
 	pointer, bit := r.traverseTree(ip, 0, uint(prefixLength))
-	return &Networks{
-		reader: r,
-		nodes: []netNode{
-			{
-				ip:      ip,
-				bit:     uint(bit),
-				pointer: pointer,
-			},
+	networks.nodes = []netNode{
+		{
+			ip:      ip,
+			bit:     uint(bit),
+			pointer: pointer,
 		},
 	}
+
+	return networks
 }
 
 // Next prepares the next network for reading with the Network method. It
@@ -94,6 +117,13 @@ func (n *Networks) Next() bool {
 		n.nodes = n.nodes[:len(n.nodes)-1]
 
 		for node.pointer != n.reader.Metadata.NodeCount {
+			// This skips IPv4 aliases without hardcoding the networks that the writer
+			// currently aliases.
+			if n.skipAliasedNetworks && n.reader.ipv4Start != 0 &&
+				node.pointer == n.reader.ipv4Start && !isInIPv4Subtree(node.ip) {
+				break
+			}
+
 			if node.pointer > n.reader.Metadata.NodeCount {
 				n.lastNode = node
 				return true
@@ -135,13 +165,39 @@ func (n *Networks) Network(result interface{}) (*net.IPNet, error) {
 		return nil, err
 	}
 
+	ip := n.lastNode.ip
+	prefixLength := int(n.lastNode.bit)
+
+	// We do this because uses of SkipAliasedNetworks expect the IPv4 networks
+	// to be returned as IPv4 networks. If we are not skipping aliased
+	// networks, then the user will get IPv4 networks from the ::FFFF:0:0/96
+	// network as Go automatically converts those.
+	if n.skipAliasedNetworks && isInIPv4Subtree(ip) {
+		ip = ip[12:]
+		prefixLength -= 96
+	}
+
 	return &net.IPNet{
-		IP:   n.lastNode.ip,
-		Mask: net.CIDRMask(int(n.lastNode.bit), len(n.lastNode.ip)*8),
+		IP:   ip,
+		Mask: net.CIDRMask(prefixLength, len(ip)*8),
 	}, nil
 }
 
 // Err returns an error, if any, that was encountered during iteration.
 func (n *Networks) Err() error {
 	return n.err
 }
+
+// isInIPv4Subtree returns true if the IP is an IPv6 address in the database's
+// IPv4 subtree.
+func isInIPv4Subtree(ip net.IP) bool {
+	if len(ip) != 16 {
+		return false
+	}
+	for i := 0; i < 12; i++ {
+		if ip[i] != 0 {
+			return false
+		}
+	}
+	return true
+}

traverse_test.go

@@ -53,6 +53,7 @@ type networkTest struct {
 	Network  string
 	Database string
 	Expected []string
+	Options  []NetworksOption
 }
 
 var tests = []networkTest{
@@ -106,6 +107,7 @@ var tests = []networkTest{
 		Expected: []string{
 			"::1:ffff:ffff/128",
 		},
+		Options: []NetworksOption{SkipAliasedNetworks},
 	},
 	{
 		Network:  "::/0",
@@ -117,6 +119,7 @@ var tests = []networkTest{
 			"::2:0:50/125",
 			"::2:0:58/127",
 		},
+		Options: []NetworksOption{SkipAliasedNetworks},
 	},
 	{
 		Network:  "::2:0:40/123",
@@ -126,6 +129,7 @@ var tests = []networkTest{
 			"::2:0:50/125",
 			"::2:0:58/127",
 		},
+		Options: []NetworksOption{SkipAliasedNetworks},
 	},
 	{
 		Network:  "0:0:0:0:0:ffff:ffff:ff00/120",
@@ -145,11 +149,70 @@ var tests = []networkTest{
 		},
 	},
 	{
-		Network:  "1.1.1.16/28",
+		Network:  "0.0.0.0/0",
 		Database: "mixed",
 		Expected: []string{
+			"1.1.1.1/32",
+			"1.1.1.2/31",
+			"1.1.1.4/30",
+			"1.1.1.8/29",
 			"1.1.1.16/28",
+			"1.1.1.32/32",
+		},
+		Options: []NetworksOption{SkipAliasedNetworks},
+	},
+	{
+		Network:  "::/0",
+		Database: "mixed",
+		Expected: []string{
+			"::101:101/128",
+			"::101:102/127",
+			"::101:104/126",
+			"::101:108/125",
+			"::101:110/124",
+			"::101:120/128",
+			"::1:ffff:ffff/128",
+			"::2:0:0/122",
+			"::2:0:40/124",
+			"::2:0:50/125",
+			"::2:0:58/127",
+			"1.1.1.1/32",
+			"1.1.1.2/31",
+			"1.1.1.4/30",
+			"1.1.1.8/29",
+			"1.1.1.16/28",
+			"1.1.1.32/32",
+			"2001:0:101:101::/64",
+			"2001:0:101:102::/63",
+			"2001:0:101:104::/62",
+			"2001:0:101:108::/61",
+			"2001:0:101:110::/60",
+			"2001:0:101:120::/64",
+			"2002:101:101::/48",
+			"2002:101:102::/47",
+			"2002:101:104::/46",
+			"2002:101:108::/45",
+			"2002:101:110::/44",
+			"2002:101:120::/48",
+		},
+	},
+	{
+		Network:  "::/0",
+		Database: "mixed",
+		Expected: []string{
+			"1.1.1.1/32",
+			"1.1.1.2/31",
+			"1.1.1.4/30",
+			"1.1.1.8/29",
+			"1.1.1.16/28",
+			"1.1.1.32/32",
+			"::1:ffff:ffff/128",
+			"::2:0:0/122",
+			"::2:0:40/124",
+			"::2:0:50/125",
+			"::2:0:58/127",
 		},
+		Options: []NetworksOption{SkipAliasedNetworks},
 	},
 	{
 		Network:  "1.1.1.16/28",
@@ -176,7 +239,7 @@ func TestNetworksWithin(t *testing.T) {
 
 			_, network, err := net.ParseCIDR(v.Network)
 			assert.Nil(t, err)
-			n := reader.NetworksWithin(network)
+			n := reader.NetworksWithin(network, v.Options...)
 			var innerIPs []string
 
 			for n.Next() {