gps: support loading credentials from a netrc file

This enables support for private gitlab files and other places where
the URL requires basic authentication.

Initial proposal and implementation came from Johnny
(github.com/wesgur).

Fixes #2061.
Fixes #1898.

Author: kevinburke

Makefile

@@ -7,7 +7,7 @@ GOBIN := $(GOPATH)/bin
 default: build validate test
 
 get-deps:
-	go get -u golang.org/x/lint/golint honnef.co/go/tools/cmd/megacheck
+	go get -u golang.org/x/lint/golint honnef.co/go/tools/cmd/staticcheck
 
 build:
 	go fmt ./...

gps/deduce.go

@@ -8,15 +8,19 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"net/http"
 	"net/url"
+	"os"
 	"path"
+	"path/filepath"
 	"regexp"
+	"runtime"
 	"strconv"
 	"strings"
 	"sync"
 
-	"github.com/armon/go-radix"
+	radix "github.com/armon/go-radix"
 	"github.com/pkg/errors"
 )
 
@@ -26,6 +30,8 @@ var (
 	hgSchemes      = []string{"https", "ssh", "http"}
 	svnSchemes     = []string{"https", "http", "svn", "svn+ssh"}
 	gopkginSchemes = []string{"https", "http"}
+	netrc          []netrcLine
+	readNetrcOnce  sync.Once
 )
 
 const gopkgUnstableSuffix = "-unstable"
@@ -848,6 +854,8 @@ func doFetchMetadata(ctx context.Context, scheme, path string) (io.ReadCloser, e
 			return nil, errors.Wrapf(err, "unable to build HTTP request for URL %q", url)
 		}
 
+		req = addAuthFromNetrc(url, req)
+
 		resp, err := http.DefaultClient.Do(req.WithContext(ctx))
 		if err != nil {
 			return nil, errors.Wrapf(err, "failed HTTP request to URL %q", url)
@@ -859,6 +867,113 @@ func doFetchMetadata(ctx context.Context, scheme, path string) (io.ReadCloser, e
 	}
 }
 
+// See https://github.com/golang/go/blob/master/src/cmd/go/internal/web2/web.go
+// for implementation
+// Temporary netrc reader until https://github.com/golang/go/issues/31334 is solved
+type netrcLine struct {
+	machine  string
+	login    string
+	password string
+}
+
+func parseNetrc(data string) []netrcLine {
+	// See https://www.gnu.org/software/inetutils/manual/html_node/The-_002enetrc-file.html
+	// for documentation on the .netrc format.
+	var nrc []netrcLine
+	var l netrcLine
+	inMacro := false
+	for _, line := range strings.Split(data, "\n") {
+		if inMacro {
+			if line == "" {
+				inMacro = false
+			}
+			continue
+		}
+
+		f := strings.Fields(line)
+		i := 0
+		for ; i < len(f)-1; i += 2 {
+			// Reset at each "machine" token.
+			// “The auto-login process searches the .netrc file for a machine token
+			// that matches […]. Once a match is made, the subsequent .netrc tokens
+			// are processed, stopping when the end of file is reached or another
+			// machine or a default token is encountered.”
+			switch f[i] {
+			case "machine":
+				l = netrcLine{machine: f[i+1]}
+			case "login":
+				l.login = f[i+1]
+			case "password":
+				l.password = f[i+1]
+			case "macdef":
+				// “A macro is defined with the specified name; its contents begin with
+				// the next .netrc line and continue until a null line (consecutive
+				// new-line characters) is encountered.”
+				inMacro = true
+			}
+			if l.machine != "" && l.login != "" && l.password != "" {
+				nrc = append(nrc, l)
+				l = netrcLine{}
+			}
+		}
+
+		if i < len(f) && f[i] == "default" {
+			// “There can be only one default token, and it must be after all machine tokens.”
+			break
+		}
+	}
+
+	return nrc
+}
+
+func netrcPath() (string, error) {
+	if env := os.Getenv("NETRC"); env != "" {
+		return env, nil
+	}
+
+	dir := os.Getenv("HOME")
+
+	base := ".netrc"
+	if runtime.GOOS == "windows" {
+		base = "_netrc"
+	}
+	return filepath.Join(dir, base), nil
+}
+
+// readNetrc parses a user's netrc file, ignoring any errors that occur.
+func readNetrc() {
+	path, err := netrcPath()
+	if err != nil {
+		return
+	}
+
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+		return
+	}
+
+	netrc = parseNetrc(string(data))
+}
+
+// addAuthFromNetrc uses basic authentication on go-get requests
+// for private repositories.
+func addAuthFromNetrc(rawurl string, req *http.Request) *http.Request {
+	readNetrcOnce.Do(readNetrc)
+	for _, m := range netrc {
+		u, err := url.Parse(rawurl)
+		if err != nil {
+			continue
+		}
+
+		if u.Host == m.machine {
+			req.SetBasicAuth(m.login, m.password)
+			break
+		}
+	}
+
+	return req
+}
+
 // getMetadata fetches and decodes remote metadata for path.
 //
 // scheme is optional. If it's http, only http will be attempted for fetching.

hack/lint.bash

@@ -9,4 +9,4 @@ set -e
 PKGS=$(go list ./... | grep -vF /vendor/)
 go vet $PKGS
 golint $PKGS
-megacheck -unused.exported -ignore "github.com/golang/dep/internal/test/test.go:U1000 github.com/golang/dep/gps/prune.go:U1000 github.com/golang/dep/manifest.go:U1000" $PKGS
+staticcheck -ignore "github.com/golang/dep/internal/test/test.go:U1000 github.com/golang/dep/gps/prune.go:U1000 github.com/golang/dep/manifest.go:U1000" $PKGS