diff --git a/README.md b/README.md index b093ae0d..e8c70624 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,8 @@ Gitrob is a tool to help find potentially sensitive files pushed to public repos Print debugging information -github-access-token string GitHub access token to use for API requests +-include-branches + Include repository branches in scan -load string Load session file -no-expand-orgs diff --git a/core/git.go b/core/git.go index f5abbc5c..c19aa951 100644 --- a/core/git.go +++ b/core/git.go @@ -54,7 +54,9 @@ func GetRepositoryHistory(repository *git.Repository) ([]*object.Commit, error) func GetChanges(commit *object.Commit, repo *git.Repository) (object.Changes, error) { parentCommit, err := GetParentCommit(commit, repo) if err != nil { - return nil, err + //this may be the parent commit + parentCommit = commit + //return nil, err } commitTree, err := commit.Tree() @@ -67,7 +69,14 @@ func GetChanges(commit *object.Commit, repo *git.Repository) (object.Changes, er return nil, err } - changes, err := object.DiffTree(parentCommitTree, commitTree) + //changes, err := object.DiffTree(parentCommitTree, commitTree) + var changes object.Changes + if parentCommit == commit { + changes, err = object.DiffTree(nil, parentCommitTree) + } else { + changes, err = object.DiffTree(parentCommitTree, commitTree) + } + if err != nil { return nil, err } diff --git a/core/github.go b/core/github.go index a1941701..e10a5988 100644 --- a/core/github.go +++ b/core/github.go @@ -28,6 +28,7 @@ type GithubRepository struct { CloneURL *string URL *string DefaultBranch *string + BranchName *string Description *string Homepage *string } @@ -53,14 +54,14 @@ func GetUserOrOrganization(login string, client *github.Client) (*GithubOwner, e }, nil } -func GetRepositoriesFromOwner(login *string, client *github.Client) ([]*GithubRepository, error) { +func GetRepositoriesFromOwner(login *string, client *github.Client, includeBranches *bool) ([]*GithubRepository, error) { var allRepos []*GithubRepository loginVal := *login ctx := context.Background() opt := &github.RepositoryListOptions{ Type: "sources", } - + opts := &github.ListOptions{} for { repos, resp, err := client.Repositories.List(ctx, loginVal, opt) if err != nil { @@ -68,7 +69,7 @@ func GetRepositoriesFromOwner(login *string, client *github.Client) ([]*GithubRe } for _, repo := range repos { if !*repo.Fork { - r := GithubRepository{ + t := GithubRepository{ Owner: repo.Owner.Login, ID: repo.ID, Name: repo.Name, @@ -76,10 +77,34 @@ func GetRepositoriesFromOwner(login *string, client *github.Client) ([]*GithubRe CloneURL: repo.CloneURL, URL: repo.HTMLURL, DefaultBranch: repo.DefaultBranch, + BranchName: repo.DefaultBranch, Description: repo.Description, Homepage: repo.Homepage, } - allRepos = append(allRepos, &r) + branches, resp, err := client.Repositories.ListBranches(ctx, *t.Owner, *t.Name, opts) + if err != nil { + return allRepos, err + } + for _, branch := range branches { + r := GithubRepository{ + Owner: t.Owner, + ID: t.ID, + Name: t.Name, + FullName: t.FullName, + CloneURL: t.CloneURL, + URL: t.URL, + DefaultBranch: t.DefaultBranch, + BranchName: branch.Name, + Description: t.Description, + Homepage: t.Homepage, + } + if *includeBranches { + allRepos = append(allRepos, &r) + } else { + allRepos = append(allRepos, &t) + } + } + opts.Page = resp.NextPage } } if resp.NextPage == 0 { diff --git a/core/options.go b/core/options.go index cf610816..06f7eeaf 100644 --- a/core/options.go +++ b/core/options.go @@ -15,6 +15,7 @@ type Options struct { Port *int Silent *bool Debug *bool + IncludeBranches *bool Logins []string } @@ -30,6 +31,7 @@ func ParseOptions() (Options, error) { Port: flag.Int("port", 9393, "Port to run web server on"), Silent: flag.Bool("silent", false, "Suppress all output except for errors"), Debug: flag.Bool("debug", false, "Print debugging information"), + IncludeBranches: flag.Bool("include-branches", false, "Include repository branches in scan"), } flag.Parse() diff --git a/core/session.go b/core/session.go index bf58134e..2c003948 100644 --- a/core/session.go +++ b/core/session.go @@ -90,6 +90,17 @@ func (s *Session) AddRepository(repository *GithubRepository) { s.Repositories = append(s.Repositories, repository) } +func (s *Session) AddBranches(repository *GithubRepository) { + s.Lock() + defer s.Unlock() + for _, r := range s.Repositories { + if *repository.BranchName == *r.BranchName { + return + } + } + s.Repositories = append(s.Repositories, repository) +} + func (s *Session) AddFinding(finding *Finding) { s.Lock() defer s.Unlock() diff --git a/core/signatures.go b/core/signatures.go index fe36cc03..245a5124 100644 --- a/core/signatures.go +++ b/core/signatures.go @@ -51,6 +51,7 @@ type Finding struct { Comment string RepositoryOwner string RepositoryName string + BranchName string CommitHash string CommitMessage string CommitAuthor string @@ -71,6 +72,7 @@ func (f *Finding) generateID() { io.WriteString(h, f.Action) io.WriteString(h, f.RepositoryOwner) io.WriteString(h, f.RepositoryName) + io.WriteString(h, f.BranchName) io.WriteString(h, f.CommitHash) io.WriteString(h, f.CommitMessage) io.WriteString(h, f.CommitAuthor) diff --git a/main.go b/main.go index a693ad89..f1e5489c 100644 --- a/main.go +++ b/main.go @@ -62,7 +62,7 @@ func GatherRepositories(sess *core.Session) { wg.Done() return } - repos, err := core.GetRepositoriesFromOwner(target.Login, sess.GithubClient) + repos, err := core.GetRepositoriesFromOwner(target.Login, sess.GithubClient, sess.Options.IncludeBranches) if err != nil { sess.Out.Error(" Failed to retrieve repositories from %s: %s\n", *target.Login, err) } @@ -72,6 +72,7 @@ func GatherRepositories(sess *core.Session) { for _, repo := range repos { sess.Out.Debug(" Retrieved repository: %s\n", *repo.FullName) sess.AddRepository(repo) + sess.AddBranches(repo) } sess.Stats.IncrementTargets() sess.Out.Info(" Retrieved %d %s from %s\n", len(repos), core.Pluralize(len(repos), "repository", "repositories"), *target.Login) @@ -101,7 +102,7 @@ func AnalyzeRepositories(sess *core.Session) { wg.Add(threadNum) sess.Out.Debug("Threads for repository analysis: %d\n", threadNum) - sess.Out.Important("Analyzing %d %s...\n", len(sess.Repositories), core.Pluralize(len(sess.Repositories), "repository", "repositories")) + sess.Out.Important("Analyzing %d %s...\n", len(sess.Repositories), core.Pluralize(len(sess.Repositories), "branch", "branches")) for i := 0; i < threadNum; i++ { go func(tid int) { @@ -115,7 +116,7 @@ func AnalyzeRepositories(sess *core.Session) { } sess.Out.Debug("[THREAD #%d][%s] Cloning repository...\n", tid, *repo.FullName) - clone, path, err := core.CloneRepository(repo.CloneURL, repo.DefaultBranch, *sess.Options.CommitDepth) + clone, path, err := core.CloneRepository(repo.CloneURL, repo.BranchName, *sess.Options.CommitDepth) if err != nil { if err.Error() != "remote repository is empty" { sess.Out.Error("Error cloning repository %s: %s\n", *repo.FullName, err) @@ -159,6 +160,7 @@ func AnalyzeRepositories(sess *core.Session) { Comment: signature.Comment(), RepositoryOwner: *repo.Owner, RepositoryName: *repo.Name, + BranchName: *repo.BranchName, CommitHash: commit.Hash.String(), CommitMessage: strings.TrimSpace(commit.Message), CommitAuthor: commit.Author.String(), @@ -169,6 +171,7 @@ func AnalyzeRepositories(sess *core.Session) { sess.Out.Warn(" %s: %s\n", strings.ToUpper(changeAction), finding.Description) sess.Out.Info(" Path.......: %s\n", finding.FilePath) sess.Out.Info(" Repo.......: %s\n", *repo.FullName) + sess.Out.Info(" Branch.....: %s\n", *repo.BranchName) sess.Out.Info(" Message....: %s\n", core.TruncateString(finding.CommitMessage, 100)) sess.Out.Info(" Author.....: %s\n", finding.CommitAuthor) if finding.Comment != "" {