Merge pull request #1271 from sdboyer/pkgtree-hidden-trimming

sdboyer · web-flow · commit 1d73a4a22b84 · 2017-10-15T00:13:37.000-04:00
Trim hidden packages from (root) PackageTrees
diff --git a/cmd/dep/ensure.go b/cmd/dep/ensure.go
@@ -178,12 +178,12 @@ func (cmd *ensureCommand) Run(ctx *dep.Ctx, args []string) error {
 		return cmd.runVendorOnly(ctx, args, p, sm, params)
 	}
 
-	params.RootPackageTree, err = pkgtree.ListPackages(p.ResolvedAbsRoot, string(p.ImportRoot))
+	params.RootPackageTree, err = p.ParseRootPackageTree()
 	if err != nil {
-		return errors.Wrap(err, "ensure ListPackage for project")
+		return err
 	}
 
-	if fatal, err := checkErrors(params.RootPackageTree.Packages); err != nil {
+	if fatal, err := checkErrors(params.RootPackageTree.Packages, p.Manifest.IgnoredPackages()); err != nil {
 		if fatal {
 			return err
 		} else if ctx.Verbose {
@@ -761,13 +761,17 @@ func getProjectConstraint(arg string, sm gps.SourceManager) (gps.ProjectConstrai
 	return gps.ProjectConstraint{Ident: pi, Constraint: c}, arg, nil
 }
 
-func checkErrors(m map[string]pkgtree.PackageOrErr) (fatal bool, err error) {
+func checkErrors(m map[string]pkgtree.PackageOrErr, ignore *pkgtree.IgnoredRuleset) (fatal bool, err error) {
 	var (
 		noGoErrors    int
 		pkgtreeErrors = make(pkgtreeErrs, 0, len(m))
 	)
 
-	for _, poe := range m {
+	for ip, poe := range m {
+		if ignore.IsIgnored(ip) {
+			continue
+		}
+
 		if poe.Err != nil {
 			switch poe.Err.(type) {
 			case *build.NoGoError:
diff --git a/cmd/dep/ensure_test.go b/cmd/dep/ensure_test.go
@@ -135,7 +135,7 @@ func TestCheckErrors(t *testing.T) {
 
 	for _, tc := range tt {
 		t.Run(tc.name, func(t *testing.T) {
-			fatal, err := checkErrors(tc.pkgOrErrMap)
+			fatal, err := checkErrors(tc.pkgOrErrMap, nil)
 			if tc.fatal != fatal {
 				t.Fatalf("expected fatal flag to be %T, got %T", tc.fatal, fatal)
 			}
diff --git a/cmd/dep/init.go b/cmd/dep/init.go
@@ -227,9 +227,9 @@ func (cmd *initCommand) Run(ctx *dep.Ctx, args []string) error {
 }
 
 func getDirectDependencies(sm gps.SourceManager, p *dep.Project) (pkgtree.PackageTree, map[string]bool, error) {
-	pkgT, err := pkgtree.ListPackages(p.ResolvedAbsRoot, string(p.ImportRoot))
+	pkgT, err := p.ParseRootPackageTree()
 	if err != nil {
-		return pkgtree.PackageTree{}, nil, errors.Wrap(err, "gps.ListPackages")
+		return pkgtree.PackageTree{}, nil, err
 	}
 
 	directDeps := map[string]bool{}
diff --git a/cmd/dep/status.go b/cmd/dep/status.go
@@ -171,7 +171,8 @@ type dotOutput struct {
 func (out *dotOutput) BasicHeader() {
 	out.g = new(graphviz).New()
 
-	ptree, _ := pkgtree.ListPackages(out.p.ResolvedAbsRoot, string(out.p.ImportRoot))
+	ptree, _ := out.p.ParseRootPackageTree()
+	// TODO(sdboyer) should be true, true, false, out.p.Manifest.IgnoredPackages()
 	prm, _ := ptree.ToReachMap(true, false, false, nil)
 
 	out.g.createNode(string(out.p.ImportRoot), "", prm.FlattenFn(paths.IsStandardImportPath))
@@ -358,9 +359,9 @@ type MissingStatus struct {
 func runStatusAll(ctx *dep.Ctx, out outputter, p *dep.Project, sm gps.SourceManager) (hasMissingPkgs bool, errCount int, err error) {
 	// While the network churns on ListVersions() requests, statically analyze
 	// code from the current project.
-	ptree, err := pkgtree.ListPackages(p.ResolvedAbsRoot, string(p.ImportRoot))
+	ptree, err := p.ParseRootPackageTree()
 	if err != nil {
-		return false, 0, errors.Wrapf(err, "analysis of local packages failed")
+		return false, 0, err
 	}
 
 	// Set up a solver in order to check the InputHash.
@@ -437,7 +438,7 @@ func runStatusAll(ctx *dep.Ctx, out outputter, p *dep.Project, sm gps.SourceMana
 						errListPkgCh <- err
 					}
 
-					prm, _ := ptr.ToReachMap(true, false, false, nil)
+					prm, _ := ptr.ToReachMap(true, true, false, p.Manifest.IgnoredPackages())
 					bs.Children = prm.FlattenFn(paths.IsStandardImportPath)
 				}
 
diff --git a/internal/gps/_testdata/src/varied_hidden/.onlyfromtests/a.go b/internal/gps/_testdata/src/varied_hidden/.onlyfromtests/a.go
@@ -0,0 +1,18 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package onlyfromtests
+
+import (
+	"sort"
+
+	_ "varied/_secondorder"
+
+	"github.com/golang/dep/internal/gps"
+)
+
+var (
+	M = sort.Strings
+	_ = gps.Solve
+)
diff --git a/internal/gps/_testdata/src/varied_hidden/.onlyfromtests/b.go b/internal/gps/_testdata/src/varied_hidden/.onlyfromtests/b.go
@@ -0,0 +1,15 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package onlyfromtests
+
+import (
+	"os"
+	"sort"
+)
+
+var (
+	_ = sort.Strings
+	_ = os.PathSeparator
+)
diff --git a/internal/gps/_testdata/src/varied_hidden/_frommain/a.go b/internal/gps/_testdata/src/varied_hidden/_frommain/a.go
@@ -0,0 +1,16 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package frommain
+
+import (
+	"sort"
+
+	"github.com/golang/dep/internal/gps"
+)
+
+var (
+	M = sort.Strings
+	_ = gps.Solve
+)
diff --git a/internal/gps/_testdata/src/varied_hidden/_never/a.go b/internal/gps/_testdata/src/varied_hidden/_never/a.go
@@ -0,0 +1,16 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package never
+
+import (
+	"sort"
+
+	"github.com/golang/dep/internal/gps"
+)
+
+var (
+	M = sort.Strings
+	_ = gps.Solve
+)
diff --git a/internal/gps/_testdata/src/varied_hidden/_secondorder/secondorder.go b/internal/gps/_testdata/src/varied_hidden/_secondorder/secondorder.go
@@ -0,0 +1,11 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package secondorder
+
+import "hash"
+
+var (
+	H = hash.Hash
+)
diff --git a/internal/gps/_testdata/src/varied_hidden/always/always_test.go b/internal/gps/_testdata/src/varied_hidden/always/always_test.go
@@ -0,0 +1,7 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package always
+
+import _ "varied/.onlyfromtests"
diff --git a/internal/gps/_testdata/src/varied_hidden/dotdotslash/a.go b/internal/gps/_testdata/src/varied_hidden/dotdotslash/a.go
@@ -0,0 +1,13 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package dotslash
+
+import (
+	"../github.com/golang/dep/internal/gps"
+)
+
+var (
+	A = gps.Solver
+)
diff --git a/internal/gps/_testdata/src/varied_hidden/locals.go b/internal/gps/_testdata/src/varied_hidden/locals.go
@@ -0,0 +1,14 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	_ "varied/_frommain"
+	"varied/simple"
+)
+
+var (
+	_ = simple.S
+)
diff --git a/internal/gps/_testdata/src/varied_hidden/main.go b/internal/gps/_testdata/src/varied_hidden/main.go
@@ -0,0 +1,13 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"net/http"
+)
+
+var (
+	_ = http.Client
+)
diff --git a/internal/gps/_testdata/src/varied_hidden/simple/locals.go b/internal/gps/_testdata/src/varied_hidden/simple/locals.go
@@ -0,0 +1,11 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package simple
+
+import "varied/simple/testdata"
+
+var (
+	_ = testdata.H
+)
diff --git a/internal/gps/_testdata/src/varied_hidden/simple/simple.go b/internal/gps/_testdata/src/varied_hidden/simple/simple.go
@@ -0,0 +1,16 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package simple
+
+import (
+	"go/parser"
+
+	"github.com/golang/dep/internal/gps"
+)
+
+var (
+	_ = parser.ParseFile
+	S = gps.Prepare
+)
diff --git a/internal/gps/_testdata/src/varied_hidden/simple/testdata/another.go b/internal/gps/_testdata/src/varied_hidden/simple/testdata/another.go
@@ -0,0 +1,7 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package testdata
+
+import _ "varied/dotdotslash"
diff --git a/internal/gps/pkgtree/ignored_ruleset.go b/internal/gps/pkgtree/ignored_ruleset.go
@@ -0,0 +1,111 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package pkgtree
+
+import (
+	"sort"
+	"strings"
+
+	"github.com/armon/go-radix"
+)
+
+// IgnoredRuleset comprises a set of rules for ignoring import paths. It can
+// manage both literal and prefix-wildcard matches.
+type IgnoredRuleset struct {
+	t *radix.Tree
+}
+
+// NewIgnoredRuleset processes a set of strings into an IgnoredRuleset. Strings
+// that end in "*" are treated as wildcards, where any import path with a
+// matching prefix will be ignored. IgnoredRulesets are immutable once created.
+//
+// Duplicate and redundant (i.e. a literal path that has a prefix of a wildcard
+// path) declarations are discarded. Consequently, it is possible that the
+// returned IgnoredRuleset may have a smaller Len() than the input slice.
+func NewIgnoredRuleset(ig []string) *IgnoredRuleset {
+	if len(ig) == 0 {
+		return &IgnoredRuleset{}
+	}
+
+	ir := &IgnoredRuleset{
+		t: radix.New(),
+	}
+
+	// Sort the list of all the ignores in order to ensure that wildcard
+	// precedence is recorded correctly in the trie.
+	sort.Strings(ig)
+	for _, i := range ig {
+		// Skip global ignore and empty string.
+		if i == "*" || i == "" {
+			continue
+		}
+
+		_, wildi, has := ir.t.LongestPrefix(i)
+		// We may not always have a value here, but if we do, then it's a bool.
+		wild, _ := wildi.(bool)
+		// Check if it's a wildcard ignore.
+		if strings.HasSuffix(i, "*") {
+			// Check if it is ineffectual.
+			if has && wild {
+				// Skip ineffectual wildcard ignore.
+				continue
+			}
+			// Create the ignore prefix and insert in the radix tree.
+			ir.t.Insert(i[:len(i)-1], true)
+		} else if !has || !wild {
+			ir.t.Insert(i, false)
+		}
+	}
+
+	if ir.t.Len() == 0 {
+		ir.t = nil
+	}
+
+	return ir
+}
+
+// IsIgnored indicates whether the provided path should be ignored, according to
+// the ruleset.
+func (ir *IgnoredRuleset) IsIgnored(path string) bool {
+	if path == "" || ir == nil || ir.t == nil {
+		return false
+	}
+
+	prefix, wildi, has := ir.t.LongestPrefix(path)
+	return has && (wildi.(bool) || path == prefix)
+}
+
+// Len indicates the number of rules in the ruleset.
+func (ir *IgnoredRuleset) Len() int {
+	if ir == nil || ir.t == nil {
+		return 0
+	}
+
+	return ir.t.Len()
+}
+
+// ToSlice converts the contents of the IgnoredRuleset to a string slice.
+//
+// This operation is symmetrically dual to NewIgnoredRuleset.
+func (ir *IgnoredRuleset) ToSlice() []string {
+	irlen := ir.Len()
+	if irlen == 0 {
+		return nil
+	}
+
+	items := make([]string, 0, irlen)
+	ir.t.Walk(func(s string, v interface{}) bool {
+		if s != "" {
+			if v.(bool) {
+				items = append(items, s+"*")
+			} else {
+				items = append(items, s)
+			}
+		}
+		return false
+	})
+
+	return items
+}
diff --git a/internal/gps/pkgtree/ignored_ruleset_test.go b/internal/gps/pkgtree/ignored_ruleset_test.go
diff --git a/internal/gps/pkgtree/pkgtree.go b/internal/gps/pkgtree/pkgtree.go
diff --git a/internal/gps/pkgtree/pkgtree_test.go b/internal/gps/pkgtree/pkgtree_test.go
diff --git a/project.go b/project.go

Original file line number	Diff line number	Diff line change
`@@ -135,7 +135,7 @@ func TestCheckErrors(t *testing.T) {`
`135`	`135`
`136`	`136`	`for _, tc := range tt {`
`137`	`137`	`t.Run(tc.name, func(t *testing.T) {`
`138`		`- fatal, err := checkErrors(tc.pkgOrErrMap)`
	`138`	`+ fatal, err := checkErrors(tc.pkgOrErrMap, nil)`
`139`	`139`	`if tc.fatal != fatal {`
`140`	`140`	`t.Fatalf("expected fatal flag to be %T, got %T", tc.fatal, fatal)`
`141`	`141`	`}`
Original file line number	Diff line number	Diff line change
`@@ -227,9 +227,9 @@ func (cmd initCommand) Run(ctx dep.Ctx, args []string) error {`
`227`	`227`	`}`
`228`	`228`
`229`	`229`	`func getDirectDependencies(sm gps.SourceManager, p *dep.Project) (pkgtree.PackageTree, map[string]bool, error) {`
`230`		`- pkgT, err := pkgtree.ListPackages(p.ResolvedAbsRoot, string(p.ImportRoot))`
	`230`	`+ pkgT, err := p.ParseRootPackageTree()`
`231`	`231`	`if err != nil {`
`232`		`- return pkgtree.PackageTree{}, nil, errors.Wrap(err, "gps.ListPackages")`
	`232`	`+ return pkgtree.PackageTree{}, nil, err`
`233`	`233`	`}`
`234`	`234`
`235`	`235`	`directDeps := map[string]bool{}`