diff --git a/README.md b/README.md index b093ae0d..8cae8839 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,12 @@ Gitrob is a tool to help find potentially sensitive files pushed to public repos Number of repository commits to process (default 500) -debug Print debugging information +-enterprise-upload-url string + Upload URL for Github Enterprise (defaults to the URL set in -enterprise-url if any) +-enterprise-url string + URL for Github Enterprise +-enterprise-user string + Username for Github Enterprise (defaults to first target) -github-access-token string GitHub access token to use for API requests -load string @@ -54,6 +60,14 @@ A session stored in a file can be loaded with the `-load` option: Gitrob will start its web interface and serve the results for analysis. +### Use with Github Enterprise + +To configure Gitrob for Github Enterprise, the following switches can be used: + +- `enterprise-url`: Must be specified; this is the URL where the path `/api/v3/` exists. This is usually the URL where the Github web interface can be found. Example: `-enterprise-url=https://github.yourcompany.com` +- `enterprise-upload-url:` Optional, defaults to `enterprise-url`; full path to the upload URL if different from the main Github Enterprise URL. Example: `-enterprise-upload-url=https://github.yourcompany.com/api/v3/upload` +- `enterprise-user`: Optional, defaults to the first target. Example: `-enterprise-user=your.username` + ## Installation A [precompiled version is available](https://github.com/michenriksen/gitrob/releases) for each release, alternatively you can use the latest version of the source code from this repository in order to build your own binary. diff --git a/core/git.go b/core/git.go index f5abbc5c..635de023 100644 --- a/core/git.go +++ b/core/git.go @@ -7,6 +7,7 @@ import ( "gopkg.in/src-d/go-git.v4" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/object" + "gopkg.in/src-d/go-git.v4/plumbing/transport/http" "gopkg.in/src-d/go-git.v4/utils/merkletrie" ) @@ -14,20 +15,27 @@ const ( EmptyTreeCommitId = "4b825dc642cb6eb9a060e54bf8d69288fbee4904" ) -func CloneRepository(url *string, branch *string, depth int) (*git.Repository, string, error) { +func CloneRepository(url *string, branch *string, sess *Session) (*git.Repository, string, error) { urlVal := *url branchVal := *branch dir, err := ioutil.TempDir("", "gitrob") if err != nil { return nil, "", err } - repository, err := git.PlainClone(dir, false, &git.CloneOptions{ + + options := &git.CloneOptions{ URL: urlVal, - Depth: depth, + Depth: *sess.Options.CommitDepth, ReferenceName: plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", branchVal)), SingleBranch: true, Tags: git.NoTags, - }) + } + + if sess.GithubAccessToken != "" && *sess.Options.EnterpriseUser != "" { + options.Auth = &http.BasicAuth{Username: *sess.Options.EnterpriseUser, Password: sess.GithubAccessToken} + } + + repository, err := git.PlainClone(dir, false, options) if err != nil { return nil, dir, err } diff --git a/core/options.go b/core/options.go index cf610816..d4ef4cb7 100644 --- a/core/options.go +++ b/core/options.go @@ -7,6 +7,10 @@ import ( type Options struct { CommitDepth *int GithubAccessToken *string `json:"-"` + EnterpriseURL *string + EnterpriseAPI *string + EnterpriseUpload *string + EnterpriseUser *string NoExpandOrgs *bool Threads *int Save *string `json:"-"` @@ -22,6 +26,9 @@ func ParseOptions() (Options, error) { options := Options{ CommitDepth: flag.Int("commit-depth", 500, "Number of repository commits to process"), GithubAccessToken: flag.String("github-access-token", "", "GitHub access token to use for API requests"), + EnterpriseURL: flag.String("enterprise-url", "", "URL of the GitHub Enterprise instance, e.g. https://github.yourcompany.com"), + EnterpriseUpload: flag.String("enterprise-upload-url", "", "Upload URL for GitHub Enterprise, e.g. https://github.yourcompany.com/api/v3/upload"), + EnterpriseUser: flag.String("enterprise-user", "", "Username for your GitHub Enterprise account"), NoExpandOrgs: flag.Bool("no-expand-orgs", false, "Don't add members to targets when processing organizations"), Threads: flag.Int("threads", 0, "Number of concurrent threads (default number of logical CPUs)"), Save: flag.String("save", "", "Save session to file"), diff --git a/core/router.go b/core/router.go index 227e29bb..56d6fcde 100644 --- a/core/router.go +++ b/core/router.go @@ -1,8 +1,8 @@ package core import ( + "context" "fmt" - "io/ioutil" "net/http" "strings" @@ -10,10 +10,12 @@ import ( "github.com/gin-contrib/secure" "github.com/gin-contrib/static" "github.com/gin-gonic/gin" + "github.com/google/go-github/github" ) const ( - GithubBaseUri = "https://raw.githubusercontent.com" + contextKeyGithubClient = "kGithubClient" + MaximumFileSize = 102400 CspPolicy = "default-src 'none'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; font-src 'self'" ReferrerPolicy = "no-referrer" @@ -74,36 +76,26 @@ func NewRouter(s *Session) *gin.Engine { router.GET("/repositories", func(c *gin.Context) { c.JSON(200, s.Repositories) }) - router.GET("/files/:owner/:repo/:commit/*path", fetchFile) + + router.GET("/files/:owner/:repo/:commit/*path", func (c *gin.Context) { + c.Set(contextKeyGithubClient, s.GithubClient) + fetchFile(c) + }) return router } func fetchFile(c *gin.Context) { - fileUrl := fmt.Sprintf("%s/%s/%s/%s%s", GithubBaseUri, c.Param("owner"), c.Param("repo"), c.Param("commit"), c.Param("path")) - resp, err := http.Head(fileUrl) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{ - "message": err, - }) - return + client, _ := c.Get(contextKeyGithubClient) + githubClient := client.(*github.Client) + + ctx := context.Background() + options := &github.RepositoryContentGetOptions{ + Ref: c.Param("commit"), } - if resp.StatusCode == http.StatusNotFound { - c.JSON(http.StatusNotFound, gin.H{ - "message": "No content", - }) - return - } + fileResponse, _, _, err := githubClient.Repositories.GetContents(ctx, c.Param("owner"), c.Param("repo"), c.Param("path"), options) - if resp.ContentLength > MaximumFileSize { - c.JSON(http.StatusUnprocessableEntity, gin.H{ - "message": fmt.Sprintf("File size exceeds maximum of %d bytes", MaximumFileSize), - }) - return - } - - resp, err = http.Get(fileUrl) if err != nil { c.JSON(http.StatusInternalServerError, gin.H{ "message": err, @@ -111,14 +103,14 @@ func fetchFile(c *gin.Context) { return } - defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{ - "message": err, + if fileResponse.GetSize() > MaximumFileSize { + c.JSON(http.StatusUnprocessableEntity, gin.H{ + "message": fmt.Sprintf("File size exceeds maximum of %d bytes", MaximumFileSize), }) return } + + content, _ := fileResponse.GetContent() - c.String(http.StatusOK, string(body[:])) + c.String(http.StatusOK, content) } diff --git a/core/session.go b/core/session.go index bf58134e..2076bf01 100644 --- a/core/session.go +++ b/core/session.go @@ -8,6 +8,7 @@ import ( "io/ioutil" "os" "runtime" + "strings" "sync" "time" @@ -23,6 +24,9 @@ const ( StatusGathering = "gathering" StatusAnalyzing = "analyzing" StatusFinished = "finished" + + githubDotComURL = "https://github.com" + githubAPIPath = "/api/v3/" ) type Stats struct { @@ -59,6 +63,7 @@ func (s *Session) Start() { s.InitLogger() s.InitThreads() s.InitGithubAccessToken() + s.initEnterpriseConfig() s.InitGithubClient() s.InitRouter() } @@ -130,13 +135,61 @@ func (s *Session) InitGithubAccessToken() { } } +func (s *Session) initEnterpriseConfig() { + apiURL := *s.Options.EnterpriseURL + + if apiURL == "" { + return + } + + apiURL = strings.TrimSuffix(apiURL, "/") + + *s.Options.EnterpriseURL = apiURL + apiPath := apiURL + githubAPIPath + s.Options.EnterpriseAPI = &apiPath + + uploadURL := *s.Options.EnterpriseUpload + + if uploadURL == "" { + uploadURL = *s.Options.EnterpriseAPI + } else { + if !strings.HasSuffix(uploadURL, "/") { + uploadURL += "/" + *s.Options.EnterpriseUpload = uploadURL + } + } + + if *s.Options.EnterpriseUser == "" && len(s.Options.Logins) > 0 { + *s.Options.EnterpriseUser = s.Options.Logins[0] + } +} + +func (s *Session) GithubURL() string { + if s.Options.EnterpriseURL != nil && *s.Options.EnterpriseURL != "" { + return *s.Options.EnterpriseURL + } + + return githubDotComURL +} + func (s *Session) InitGithubClient() { ctx := context.Background() ts := oauth2.StaticTokenSource( &oauth2.Token{AccessToken: s.GithubAccessToken}, ) tc := oauth2.NewClient(ctx, ts) - s.GithubClient = github.NewClient(tc) + + if s.Options.EnterpriseAPI != nil && *s.Options.EnterpriseAPI != "" { + enterpriseClient, err := github.NewEnterpriseClient(*s.Options.EnterpriseAPI, *s.Options.EnterpriseUpload, tc) + if err != nil { + s.Out.Fatal("Error creating GitHub Enterprise client: %s\n", err) + } + + s.GithubClient = enterpriseClient + } else { + s.GithubClient = github.NewClient(tc) + } + s.GithubClient.UserAgent = fmt.Sprintf("%s v%s", Name, Version) } diff --git a/core/signatures.go b/core/signatures.go index fe36cc03..65b91e7d 100644 --- a/core/signatures.go +++ b/core/signatures.go @@ -59,8 +59,8 @@ type Finding struct { RepositoryUrl string } -func (f *Finding) setupUrls() { - f.RepositoryUrl = fmt.Sprintf("https://github.com/%s/%s", f.RepositoryOwner, f.RepositoryName) +func (f *Finding) setupUrls(githubURL string) { + f.RepositoryUrl = strings.Join([]string {githubURL, f.RepositoryOwner, f.RepositoryName}, "/") f.FileUrl = fmt.Sprintf("%s/blob/%s/%s", f.RepositoryUrl, f.CommitHash, f.FilePath) f.CommitUrl = fmt.Sprintf("%s/commit/%s", f.RepositoryUrl, f.CommitHash) } @@ -77,8 +77,8 @@ func (f *Finding) generateID() { f.Id = fmt.Sprintf("%x", h.Sum(nil)) } -func (f *Finding) Initialize() { - f.setupUrls() +func (f *Finding) Initialize(githubURL string) { + f.setupUrls(githubURL) f.generateID() } diff --git a/main.go b/main.go index a693ad89..b7ea8915 100644 --- a/main.go +++ b/main.go @@ -103,6 +103,8 @@ func AnalyzeRepositories(sess *core.Session) { sess.Out.Important("Analyzing %d %s...\n", len(sess.Repositories), core.Pluralize(len(sess.Repositories), "repository", "repositories")) + githubURL := sess.GithubURL() + for i := 0; i < threadNum; i++ { go func(tid int) { for { @@ -115,7 +117,7 @@ func AnalyzeRepositories(sess *core.Session) { } sess.Out.Debug("[THREAD #%d][%s] Cloning repository...\n", tid, *repo.FullName) - clone, path, err := core.CloneRepository(repo.CloneURL, repo.DefaultBranch, *sess.Options.CommitDepth) + clone, path, err := core.CloneRepository(repo.CloneURL, repo.DefaultBranch, sess) if err != nil { if err.Error() != "remote repository is empty" { sess.Out.Error("Error cloning repository %s: %s\n", *repo.FullName, err) @@ -163,7 +165,7 @@ func AnalyzeRepositories(sess *core.Session) { CommitMessage: strings.TrimSpace(commit.Message), CommitAuthor: commit.Author.String(), } - finding.Initialize() + finding.Initialize(githubURL) sess.AddFinding(finding) sess.Out.Warn(" %s: %s\n", strings.ToUpper(changeAction), finding.Description)