From 23ce0e9920d915934ea5bb60e61de707434e3b17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Marschollek?= Date: Thu, 6 Sep 2018 17:02:52 +0100 Subject: [PATCH 1/4] =?UTF-8?q?=E2=9C=A8=20Add=20support=20for=20Github=20?= =?UTF-8?q?Enterprise?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This adds support for GitHub Enterprise which can be configured using the three new command line arguments with the enterprise prefix (see documentation). Closes #145 Closes #148 --- README.md | 14 ++++++++++++++ core/git.go | 18 +++++++++++++----- core/options.go | 8 +++++++- core/session.go | 48 +++++++++++++++++++++++++++++++++++++++++++++++- main.go | 2 +- 5 files changed, 82 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index b093ae0d..6ef1d8c7 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,12 @@ Gitrob is a tool to help find potentially sensitive files pushed to public repos Number of repository commits to process (default 500) -debug Print debugging information +-enterprise-upload-url string + Upload URL for Github Enterprise (defaults to the URL set in -enterprise-url if any) +-enterprise-url string + URL for Github Enterprise (/api/v3/ will be appended if not included) +-enterprise-user string + Username for Github Enterprise (defaults to first target) -github-access-token string GitHub access token to use for API requests -load string @@ -54,6 +60,14 @@ A session stored in a file can be loaded with the `-load` option: Gitrob will start its web interface and serve the results for analysis. +### Use with Github Enterprise + +To configure Gitrob for Github Enterprise, the following switches can be used: + +- `enterprise-url`: Must be specified; this is the URL where the path `/api/v3/` exists. This is usually the URL where the Github Webinterface can be found. +- `enterprise-upload-url:` Optional, defaults to `enterprise-url`; full path to the upload URL if different from the main Github Enterprise URL. +- `enterprise-user`: Optional, defaults to the first target. + ## Installation A [precompiled version is available](https://github.com/michenriksen/gitrob/releases) for each release, alternatively you can use the latest version of the source code from this repository in order to build your own binary. diff --git a/core/git.go b/core/git.go index f5abbc5c..96c5cb9a 100644 --- a/core/git.go +++ b/core/git.go @@ -7,6 +7,7 @@ import ( "gopkg.in/src-d/go-git.v4" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/object" + "gopkg.in/src-d/go-git.v4/plumbing/transport/http" "gopkg.in/src-d/go-git.v4/utils/merkletrie" ) @@ -14,20 +15,27 @@ const ( EmptyTreeCommitId = "4b825dc642cb6eb9a060e54bf8d69288fbee4904" ) -func CloneRepository(url *string, branch *string, depth int) (*git.Repository, string, error) { +func CloneRepository(url *string, branch *string, sess *Session) (*git.Repository, string, error) { urlVal := *url branchVal := *branch dir, err := ioutil.TempDir("", "gitrob") if err != nil { return nil, "", err } - repository, err := git.PlainClone(dir, false, &git.CloneOptions{ - URL: urlVal, - Depth: depth, + + options := &git.CloneOptions{ + URL: urlVal, + Depth: *sess.Options.CommitDepth, ReferenceName: plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", branchVal)), SingleBranch: true, Tags: git.NoTags, - }) + } + + if sess.GithubAccessToken != "" && *sess.Options.EnterpriseUser != "" { + options.Auth = &http.BasicAuth{Username: *sess.Options.EnterpriseUser, Password: sess.GithubAccessToken} + } + + repository, err := git.PlainClone(dir, false, options) if err != nil { return nil, dir, err } diff --git a/core/options.go b/core/options.go index cf610816..f48c60ee 100644 --- a/core/options.go +++ b/core/options.go @@ -7,6 +7,9 @@ import ( type Options struct { CommitDepth *int GithubAccessToken *string `json:"-"` + EnterpriseAPI *string + EnterpriseUpload *string + EnterpriseUser *string NoExpandOrgs *bool Threads *int Save *string `json:"-"` @@ -21,7 +24,10 @@ type Options struct { func ParseOptions() (Options, error) { options := Options{ CommitDepth: flag.Int("commit-depth", 500, "Number of repository commits to process"), - GithubAccessToken: flag.String("github-access-token", "", "GitHub access token to use for API requests"), + GithubAccessToken: flag.String("github-access-token", "", "GitHub access token to use for API requests"), + EnterpriseAPI: flag.String("enterprise-url", "", "Base URL of the GitHub Enterprise API"), + EnterpriseUpload: flag.String("enterprise-upload-url", "", "Upload URL for GitHub Enterprise"), + EnterpriseUser: flag.String("enterprise-user", "", "Username for your GitHub Enterprise account"), NoExpandOrgs: flag.Bool("no-expand-orgs", false, "Don't add members to targets when processing organizations"), Threads: flag.Int("threads", 0, "Number of concurrent threads (default number of logical CPUs)"), Save: flag.String("save", "", "Save session to file"), diff --git a/core/session.go b/core/session.go index bf58134e..d327c764 100644 --- a/core/session.go +++ b/core/session.go @@ -8,6 +8,7 @@ import ( "io/ioutil" "os" "runtime" + "strings" "sync" "time" @@ -59,6 +60,7 @@ func (s *Session) Start() { s.InitLogger() s.InitThreads() s.InitGithubAccessToken() + s.initEnterpriseConfig() s.InitGithubClient() s.InitRouter() } @@ -130,13 +132,57 @@ func (s *Session) InitGithubAccessToken() { } } +func (s *Session) initEnterpriseConfig() { + apiURL := *s.Options.EnterpriseAPI + + if apiURL == "" { + return + } + + if !strings.HasSuffix(apiURL, "/") { + apiURL += "/" + } + + if !strings.HasSuffix(apiURL, "/api/v3/") { + apiURL += "/api/v3/" + } + + *s.Options.EnterpriseAPI = apiURL + + uploadURL := *s.Options.EnterpriseUpload + + if uploadURL == "" { + uploadURL = *s.Options.EnterpriseAPI + } else { + if !strings.HasSuffix(uploadURL, "/") { + uploadURL += "/" + *s.Options.EnterpriseUpload = uploadURL + } + } + + if *s.Options.EnterpriseUser == "" && len(s.Options.Logins) > 0 { + *s.Options.EnterpriseUser = s.Options.Logins[0] + } + } + func (s *Session) InitGithubClient() { ctx := context.Background() ts := oauth2.StaticTokenSource( &oauth2.Token{AccessToken: s.GithubAccessToken}, ) tc := oauth2.NewClient(ctx, ts) - s.GithubClient = github.NewClient(tc) + + if *s.Options.EnterpriseAPI != "" { + enterpriseClient, err := github.NewEnterpriseClient(*s.Options.EnterpriseAPI, *s.Options.EnterpriseUpload, tc) + if err != nil { + s.Out.Fatal("Error creating GitHub Enterprise client: %s\n", err) + } + + s.GithubClient = enterpriseClient + } else { + s.GithubClient = github.NewClient(tc) + } + s.GithubClient.UserAgent = fmt.Sprintf("%s v%s", Name, Version) } diff --git a/main.go b/main.go index a693ad89..06f682cf 100644 --- a/main.go +++ b/main.go @@ -115,7 +115,7 @@ func AnalyzeRepositories(sess *core.Session) { } sess.Out.Debug("[THREAD #%d][%s] Cloning repository...\n", tid, *repo.FullName) - clone, path, err := core.CloneRepository(repo.CloneURL, repo.DefaultBranch, *sess.Options.CommitDepth) + clone, path, err := core.CloneRepository(repo.CloneURL, repo.DefaultBranch, sess) if err != nil { if err.Error() != "remote repository is empty" { sess.Out.Error("Error cloning repository %s: %s\n", *repo.FullName, err) From 85ad655c8662bce74a01d9991b4b83b3f685fb24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Marschollek?= Date: Sat, 20 Oct 2018 12:13:56 +0100 Subject: [PATCH 2/4] =?UTF-8?q?=F0=9F=90=9B=20Fix=20preview=20of=20files?= =?UTF-8?q?=20on=20Github=20Enterprise?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes download of files when using Github Enterprise. It now uses the existing Github client to download the files instead of making a manual HTTP request. --- README.md | 8 +++---- core/options.go | 7 ++++--- core/router.go | 52 ++++++++++++++++++++-------------------------- core/session.go | 29 ++++++++++++++++---------- core/signatures.go | 8 +++---- main.go | 4 +++- 6 files changed, 55 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index 6ef1d8c7..3438314b 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Gitrob is a tool to help find potentially sensitive files pushed to public repos -enterprise-upload-url string Upload URL for Github Enterprise (defaults to the URL set in -enterprise-url if any) -enterprise-url string - URL for Github Enterprise (/api/v3/ will be appended if not included) + URL for Github Enterprise -enterprise-user string Username for Github Enterprise (defaults to first target) -github-access-token string @@ -64,9 +64,9 @@ Gitrob will start its web interface and serve the results for analysis. To configure Gitrob for Github Enterprise, the following switches can be used: -- `enterprise-url`: Must be specified; this is the URL where the path `/api/v3/` exists. This is usually the URL where the Github Webinterface can be found. -- `enterprise-upload-url:` Optional, defaults to `enterprise-url`; full path to the upload URL if different from the main Github Enterprise URL. -- `enterprise-user`: Optional, defaults to the first target. +- `enterprise-url`: Must be specified; this is the URL where the path `/api/v3/` exists. This is usually the URL where the Github Webinterface can be found. Example: `-enterprise-url=https://github.yourcompany.com` +- `enterprise-upload-url:` Optional, defaults to `enterprise-url`; full path to the upload URL if different from the main Github Enterprise URL. Example: `-enterprise-upload-url=https://github.yourcompany.com/api/v3/upload` +- `enterprise-user`: Optional, defaults to the first target. Example: `-enterprise-user=your.username` ## Installation diff --git a/core/options.go b/core/options.go index f48c60ee..ff820eae 100644 --- a/core/options.go +++ b/core/options.go @@ -7,6 +7,7 @@ import ( type Options struct { CommitDepth *int GithubAccessToken *string `json:"-"` + EnterpriseURL *string EnterpriseAPI *string EnterpriseUpload *string EnterpriseUser *string @@ -24,9 +25,9 @@ type Options struct { func ParseOptions() (Options, error) { options := Options{ CommitDepth: flag.Int("commit-depth", 500, "Number of repository commits to process"), - GithubAccessToken: flag.String("github-access-token", "", "GitHub access token to use for API requests"), - EnterpriseAPI: flag.String("enterprise-url", "", "Base URL of the GitHub Enterprise API"), - EnterpriseUpload: flag.String("enterprise-upload-url", "", "Upload URL for GitHub Enterprise"), + GithubAccessToken: flag.String("github-access-token", "", "GitHub access token to use for API requests"), + EnterpriseURL: flag.String("enterprise-url", "", "URL of the GitHub Enterprise instance, e.g. https://github.yourcompany.com"), + EnterpriseUpload: flag.String("enterprise-upload-url", "", "Upload URL for GitHub Enterprise, e.g. https://github.yourcompany.com/api/v3/upload"), EnterpriseUser: flag.String("enterprise-user", "", "Username for your GitHub Enterprise account"), NoExpandOrgs: flag.Bool("no-expand-orgs", false, "Don't add members to targets when processing organizations"), Threads: flag.Int("threads", 0, "Number of concurrent threads (default number of logical CPUs)"), diff --git a/core/router.go b/core/router.go index 227e29bb..56d6fcde 100644 --- a/core/router.go +++ b/core/router.go @@ -1,8 +1,8 @@ package core import ( + "context" "fmt" - "io/ioutil" "net/http" "strings" @@ -10,10 +10,12 @@ import ( "github.com/gin-contrib/secure" "github.com/gin-contrib/static" "github.com/gin-gonic/gin" + "github.com/google/go-github/github" ) const ( - GithubBaseUri = "https://raw.githubusercontent.com" + contextKeyGithubClient = "kGithubClient" + MaximumFileSize = 102400 CspPolicy = "default-src 'none'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; font-src 'self'" ReferrerPolicy = "no-referrer" @@ -74,36 +76,26 @@ func NewRouter(s *Session) *gin.Engine { router.GET("/repositories", func(c *gin.Context) { c.JSON(200, s.Repositories) }) - router.GET("/files/:owner/:repo/:commit/*path", fetchFile) + + router.GET("/files/:owner/:repo/:commit/*path", func (c *gin.Context) { + c.Set(contextKeyGithubClient, s.GithubClient) + fetchFile(c) + }) return router } func fetchFile(c *gin.Context) { - fileUrl := fmt.Sprintf("%s/%s/%s/%s%s", GithubBaseUri, c.Param("owner"), c.Param("repo"), c.Param("commit"), c.Param("path")) - resp, err := http.Head(fileUrl) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{ - "message": err, - }) - return + client, _ := c.Get(contextKeyGithubClient) + githubClient := client.(*github.Client) + + ctx := context.Background() + options := &github.RepositoryContentGetOptions{ + Ref: c.Param("commit"), } - if resp.StatusCode == http.StatusNotFound { - c.JSON(http.StatusNotFound, gin.H{ - "message": "No content", - }) - return - } + fileResponse, _, _, err := githubClient.Repositories.GetContents(ctx, c.Param("owner"), c.Param("repo"), c.Param("path"), options) - if resp.ContentLength > MaximumFileSize { - c.JSON(http.StatusUnprocessableEntity, gin.H{ - "message": fmt.Sprintf("File size exceeds maximum of %d bytes", MaximumFileSize), - }) - return - } - - resp, err = http.Get(fileUrl) if err != nil { c.JSON(http.StatusInternalServerError, gin.H{ "message": err, @@ -111,14 +103,14 @@ func fetchFile(c *gin.Context) { return } - defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{ - "message": err, + if fileResponse.GetSize() > MaximumFileSize { + c.JSON(http.StatusUnprocessableEntity, gin.H{ + "message": fmt.Sprintf("File size exceeds maximum of %d bytes", MaximumFileSize), }) return } + + content, _ := fileResponse.GetContent() - c.String(http.StatusOK, string(body[:])) + c.String(http.StatusOK, content) } diff --git a/core/session.go b/core/session.go index d327c764..2076bf01 100644 --- a/core/session.go +++ b/core/session.go @@ -24,6 +24,9 @@ const ( StatusGathering = "gathering" StatusAnalyzing = "analyzing" StatusFinished = "finished" + + githubDotComURL = "https://github.com" + githubAPIPath = "/api/v3/" ) type Stats struct { @@ -133,21 +136,17 @@ func (s *Session) InitGithubAccessToken() { } func (s *Session) initEnterpriseConfig() { - apiURL := *s.Options.EnterpriseAPI + apiURL := *s.Options.EnterpriseURL if apiURL == "" { return } - if !strings.HasSuffix(apiURL, "/") { - apiURL += "/" - } - - if !strings.HasSuffix(apiURL, "/api/v3/") { - apiURL += "/api/v3/" - } + apiURL = strings.TrimSuffix(apiURL, "/") - *s.Options.EnterpriseAPI = apiURL + *s.Options.EnterpriseURL = apiURL + apiPath := apiURL + githubAPIPath + s.Options.EnterpriseAPI = &apiPath uploadURL := *s.Options.EnterpriseUpload @@ -163,8 +162,16 @@ func (s *Session) initEnterpriseConfig() { if *s.Options.EnterpriseUser == "" && len(s.Options.Logins) > 0 { *s.Options.EnterpriseUser = s.Options.Logins[0] } +} + +func (s *Session) GithubURL() string { + if s.Options.EnterpriseURL != nil && *s.Options.EnterpriseURL != "" { + return *s.Options.EnterpriseURL } + return githubDotComURL +} + func (s *Session) InitGithubClient() { ctx := context.Background() ts := oauth2.StaticTokenSource( @@ -172,7 +179,7 @@ func (s *Session) InitGithubClient() { ) tc := oauth2.NewClient(ctx, ts) - if *s.Options.EnterpriseAPI != "" { + if s.Options.EnterpriseAPI != nil && *s.Options.EnterpriseAPI != "" { enterpriseClient, err := github.NewEnterpriseClient(*s.Options.EnterpriseAPI, *s.Options.EnterpriseUpload, tc) if err != nil { s.Out.Fatal("Error creating GitHub Enterprise client: %s\n", err) @@ -180,7 +187,7 @@ func (s *Session) InitGithubClient() { s.GithubClient = enterpriseClient } else { - s.GithubClient = github.NewClient(tc) + s.GithubClient = github.NewClient(tc) } s.GithubClient.UserAgent = fmt.Sprintf("%s v%s", Name, Version) diff --git a/core/signatures.go b/core/signatures.go index fe36cc03..65b91e7d 100644 --- a/core/signatures.go +++ b/core/signatures.go @@ -59,8 +59,8 @@ type Finding struct { RepositoryUrl string } -func (f *Finding) setupUrls() { - f.RepositoryUrl = fmt.Sprintf("https://github.com/%s/%s", f.RepositoryOwner, f.RepositoryName) +func (f *Finding) setupUrls(githubURL string) { + f.RepositoryUrl = strings.Join([]string {githubURL, f.RepositoryOwner, f.RepositoryName}, "/") f.FileUrl = fmt.Sprintf("%s/blob/%s/%s", f.RepositoryUrl, f.CommitHash, f.FilePath) f.CommitUrl = fmt.Sprintf("%s/commit/%s", f.RepositoryUrl, f.CommitHash) } @@ -77,8 +77,8 @@ func (f *Finding) generateID() { f.Id = fmt.Sprintf("%x", h.Sum(nil)) } -func (f *Finding) Initialize() { - f.setupUrls() +func (f *Finding) Initialize(githubURL string) { + f.setupUrls(githubURL) f.generateID() } diff --git a/main.go b/main.go index 06f682cf..b7ea8915 100644 --- a/main.go +++ b/main.go @@ -103,6 +103,8 @@ func AnalyzeRepositories(sess *core.Session) { sess.Out.Important("Analyzing %d %s...\n", len(sess.Repositories), core.Pluralize(len(sess.Repositories), "repository", "repositories")) + githubURL := sess.GithubURL() + for i := 0; i < threadNum; i++ { go func(tid int) { for { @@ -163,7 +165,7 @@ func AnalyzeRepositories(sess *core.Session) { CommitMessage: strings.TrimSpace(commit.Message), CommitAuthor: commit.Author.String(), } - finding.Initialize() + finding.Initialize(githubURL) sess.AddFinding(finding) sess.Out.Warn(" %s: %s\n", strings.ToUpper(changeAction), finding.Description) From 24ad583da61708b578b6e4b857d501bf16c91adb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Marschollek?= Date: Sat, 20 Oct 2018 14:49:35 +0100 Subject: [PATCH 3/4] =?UTF-8?q?=F0=9F=93=9D=20Fix=20a=20typo=20in=20the=20?= =?UTF-8?q?README?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3438314b..8cae8839 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ Gitrob will start its web interface and serve the results for analysis. To configure Gitrob for Github Enterprise, the following switches can be used: -- `enterprise-url`: Must be specified; this is the URL where the path `/api/v3/` exists. This is usually the URL where the Github Webinterface can be found. Example: `-enterprise-url=https://github.yourcompany.com` +- `enterprise-url`: Must be specified; this is the URL where the path `/api/v3/` exists. This is usually the URL where the Github web interface can be found. Example: `-enterprise-url=https://github.yourcompany.com` - `enterprise-upload-url:` Optional, defaults to `enterprise-url`; full path to the upload URL if different from the main Github Enterprise URL. Example: `-enterprise-upload-url=https://github.yourcompany.com/api/v3/upload` - `enterprise-user`: Optional, defaults to the first target. Example: `-enterprise-user=your.username` From 7385f86a8212b07c11ffa4d91f5c07bbaa3e670a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Marschollek?= Date: Sat, 20 Oct 2018 14:57:39 +0100 Subject: [PATCH 4/4] =?UTF-8?q?=F0=9F=8E=A8=20Fix=20(some=20of)=20the=20in?= =?UTF-8?q?dentation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/git.go | 2 +- core/options.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/git.go b/core/git.go index 96c5cb9a..635de023 100644 --- a/core/git.go +++ b/core/git.go @@ -24,7 +24,7 @@ func CloneRepository(url *string, branch *string, sess *Session) (*git.Repositor } options := &git.CloneOptions{ - URL: urlVal, + URL: urlVal, Depth: *sess.Options.CommitDepth, ReferenceName: plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", branchVal)), SingleBranch: true, diff --git a/core/options.go b/core/options.go index ff820eae..d4ef4cb7 100644 --- a/core/options.go +++ b/core/options.go @@ -25,7 +25,7 @@ type Options struct { func ParseOptions() (Options, error) { options := Options{ CommitDepth: flag.Int("commit-depth", 500, "Number of repository commits to process"), - GithubAccessToken: flag.String("github-access-token", "", "GitHub access token to use for API requests"), + GithubAccessToken: flag.String("github-access-token", "", "GitHub access token to use for API requests"), EnterpriseURL: flag.String("enterprise-url", "", "URL of the GitHub Enterprise instance, e.g. https://github.yourcompany.com"), EnterpriseUpload: flag.String("enterprise-upload-url", "", "Upload URL for GitHub Enterprise, e.g. https://github.yourcompany.com/api/v3/upload"), EnterpriseUser: flag.String("enterprise-user", "", "Username for your GitHub Enterprise account"),