diff --git a/main.go b/main.go index 04d372f33..cbb263bd2 100644 --- a/main.go +++ b/main.go @@ -463,6 +463,7 @@ func run(state overseer.State) { // OSS Default simplified gitlab enumeration feature.UseSimplifiedGitlabEnumeration.Store(true) + feature.GitlabProjectsPerPage.Store(100) conf := &config.Config{} if *configFilename != "" { diff --git a/pkg/engine/gitlab.go b/pkg/engine/gitlab.go index dab05c293..810057058 100644 --- a/pkg/engine/gitlab.go +++ b/pkg/engine/gitlab.go @@ -63,6 +63,7 @@ func (e *Engine) ScanGitLab(ctx context.Context, c sources.GitlabConfig) (source } connection.NoCleanup = c.NoCleanup + connection.PrintLegacyJson = c.PrintLegacyJSON var conn anypb.Any diff --git a/pkg/feature/feature.go b/pkg/feature/feature.go index 53a67230c..3c030a631 100644 --- a/pkg/feature/feature.go +++ b/pkg/feature/feature.go @@ -9,7 +9,8 @@ var ( EnableAPKHandler atomic.Bool UserAgentSuffix AtomicString UseSimplifiedGitlabEnumeration atomic.Bool - UseGitMirror atomic.Bool + UseGitMirror atomic.Bool + GitlabProjectsPerPage atomic.Int64 ) type AtomicString struct { diff --git a/pkg/sources/gitlab/gitlab.go b/pkg/sources/gitlab/gitlab.go index f3f515a13..b8c9b5fe6 100644 --- a/pkg/sources/gitlab/gitlab.go +++ b/pkg/sources/gitlab/gitlab.go @@ -66,9 +66,12 @@ type Source struct { useAuthInUrl bool - clonePath string - noCleanup bool + clonePath string + noCleanup bool + printLegacyJSON bool + + projectsPerPage int } // WithCustomContentWriter sets the useCustomContentWriter flag on the source. @@ -171,6 +174,11 @@ func (s *Source) Init(ctx context.Context, name string, jobId sources.JobID, sou s.clonePath = conn.GetClonePath() s.noCleanup = conn.GetNoCleanup() s.printLegacyJSON = conn.GetPrintLegacyJson() + s.projectsPerPage = int(feature.GitlabProjectsPerPage.Load()) + + if s.projectsPerPage > 100 { + return fmt.Errorf("invalid config: maximum allowed projects per page for gitlab is 100") + } // configuration uses the inverse logic of the `useAuthInUrl` flag. s.useAuthInUrl = !conn.RemoveAuthInUrl @@ -546,10 +554,10 @@ func (s *Source) getAllProjectRepos( } const ( - orderBy = "id" // TODO: Use keyset pagination (https://docs.gitlab.com/ee/api/rest/index.html#keyset-based-pagination) - paginationLimit = 100 // Default is 20, max is 100. + orderBy = "id" ) - listOpts := gitlab.ListOptions{PerPage: paginationLimit} + // Trufflehog default per page 100 unless set to other value through feature flag. If 0 provided in feature flag gitlab default it to 20 + listOpts := gitlab.ListOptions{PerPage: s.projectsPerPage} projectQueryOptions := &gitlab.ListProjectsOptions{OrderBy: gitlab.Ptr(orderBy), ListOptions: listOpts} for { @@ -654,14 +662,13 @@ func (s *Source) getAllProjectReposV2( ) error { gitlabReposEnumerated.WithLabelValues(s.name).Set(0) - const paginationLimit = 100 // default is 20, max is 100. - // example: https://gitlab.com/gitlab-org/api/client-go/-/blob/main/examples/pagination.go#L55 listOpts := gitlab.ListOptions{ OrderBy: "id", Pagination: "keyset", // https://docs.gitlab.com/api/rest/#keyset-based-pagination - PerPage: paginationLimit, - Sort: "asc", + // Trufflehog default per page 100 unless set to other value through feature flag. If 0 provided in feature flag gitlab default it to 20 + PerPage: s.projectsPerPage, + Sort: "asc", } projectQueryOptions := &gitlab.ListProjectsOptions{ @@ -756,11 +763,10 @@ func (s *Source) getAllProjectReposInGroups( var projectsWithNamespace []string const ( - orderBy = "id" - paginationLimit = 100 + orderBy = "id" ) - listOpts := gitlab.ListOptions{PerPage: paginationLimit} + listOpts := gitlab.ListOptions{PerPage: s.projectsPerPage} projectOpts := &gitlab.ListGroupProjectsOptions{ ListOptions: listOpts, OrderBy: gitlab.Ptr(orderBy),