diff --git a/pkg/sources/jenkins/jenkins.go b/pkg/sources/jenkins/jenkins.go index 22c5f4e45..73f60eb67 100644 --- a/pkg/sources/jenkins/jenkins.go +++ b/pkg/sources/jenkins/jenkins.go @@ -1,6 +1,7 @@ package jenkins import ( + "encoding/base64" "encoding/json" "fmt" "io" @@ -46,8 +47,14 @@ type header struct { value string } -// Ensure the Source satisfies the interface at compile time -var _ sources.Source = (*Source)(nil) +// Ensure the Source satisfies the interfaces at compile time. +// SourceUnitChunker is deliberately not implemented yet, which keeps +// SourceManager on the non-unit Chunks path for scans. +var ( + _ sources.Source = (*Source)(nil) + _ sources.SourceUnitEnumerator = (*Source)(nil) + _ sources.SourceUnitUnmarshaller = (*Source)(nil) +) // Type returns the type of source. // It is used for matching source types in configuration and job input. @@ -193,42 +200,93 @@ func (s *Source) GetJenkinsJobs(ctx context.Context) (JenkinsJobResponse, error) } func (s *Source) RecursivelyGetJenkinsObjectsForPath(ctx context.Context, absolutePath string) (JenkinsJobResponse, error) { + collector := new(jobCollector) + err := s.walkJobs(ctx, absolutePath, collector) + return JenkinsJobResponse{Jobs: collector.jobs}, err +} + +// jobCollector is the UnitReporter the non-unit path uses to gather the walk's +// jobs into a slice. UnitErr returns the error it is given, which aborts the +// walk, preserving the fail-fast behavior callers of GetJenkinsJobs have always +// had. +type jobCollector struct { + jobs []JenkinsJob +} + +var _ sources.UnitReporter = (*jobCollector)(nil) + +func (c *jobCollector) UnitOk(_ context.Context, unit sources.SourceUnit) error { + job, ok := unit.(JenkinsJob) + if !ok { + return fmt.Errorf("expected JenkinsJob, got %T", unit) + } + c.jobs = append(c.jobs, job) + return nil +} + +func (c *jobCollector) UnitErr(_ context.Context, err error) error { return err } + +// walkJobs traverses the Jenkins object tree beneath absolutePath, reporting +// every scannable job it finds to the UnitReporter. Recursion continues unless +// the reporter returns an error, which aborts the walk and is returned to the +// caller. +func (s *Source) walkJobs(ctx context.Context, absolutePath string, reporter sources.UnitReporter) error { ctx.Logger().V(3).Info("getting objects", "path", absolutePath) - jobs := JenkinsJobResponse{} objects, err := s.GetJenkinsObjectsForPath(ctx, absolutePath) if err != nil { - return jobs, err + return reporter.UnitErr(ctx, errors.WrapPrefix(err, fmt.Sprintf("failed to get Jenkins objects for path %q", absolutePath), 0)) } ctx.Logger().V(3).Info("got objects", "path", absolutePath, "count", len(objects.Jobs)) for _, job := range objects.Jobs { - ctx.Logger().V(3).Info("processing object", + if common.IsDone(ctx) { + return ctx.Err() + } + + ctx := context.WithValues(ctx, "object_name", job.Name, "object_class", job.Class, "object_url", job.Url) + ctx.Logger().V(3).Info("processing object") if job.Class == "com.cloudbees.hudson.plugins.folder.Folder" { u, err := url.Parse(job.Url) if err != nil { - return jobs, err + if err := reporter.UnitErr(ctx, fmt.Errorf("failed to parse folder URL %q: %w", job.Url, err)); err != nil { + return err + } + continue } - objects, err := s.RecursivelyGetJenkinsObjectsForPath(ctx, u.Path) - if err != nil { - return jobs, err - } - jobs.Jobs = append(jobs.Jobs, objects.Jobs...) - } else { - if job.Class == "hudson.model.FreeStyleProject" || - job.Class == "org.jenkinsci.plugins.workflow.job.WorkflowJob" { - jobs.Jobs = append(jobs.Jobs, job) + if err := s.walkJobs(ctx, u.Path, reporter); err != nil { + return err } + continue + } + + if job.Class != "hudson.model.FreeStyleProject" && + job.Class != "org.jenkinsci.plugins.workflow.job.WorkflowJob" { + continue + } + + parsedUrl, err := url.Parse(job.Url) + if err != nil { + // Skipped rather than reported, so that Enumerate and the walk + // Chunks uses agree on the set of jobs. Chunks skips a job it + // cannot parse a URL for too. + ctx.Logger().Error(err, "failed to parse job URL; skipping job") + continue + } + job.Path = strings.TrimSuffix(parsedUrl.Path, "/") + + if err := reporter.UnitOk(ctx, job); err != nil { + return err } } - return jobs, nil + return nil } func (s *Source) GetJenkinsObjectsForPath(ctx context.Context, absolutePath string) (JenkinsJobResponse, error) { @@ -341,13 +399,9 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ . s.SetProgressComplete(i, len(jobs.Jobs), fmt.Sprintf("Project: %s", project.Name), "") - parsedUrl, err := url.Parse(project.Url) - if err != nil { - ctx.Logger().Error(err, "failed to parse job URL; skipping job") - continue - } + // Path is derived from the job URL by the walk above. projectURL := *s.url - projectURL.Path = parsedUrl.Path + projectURL.Path = project.Path builds, err := s.GetJenkinsBuilds(ctx, projectURL.Path) if err != nil { @@ -433,12 +487,46 @@ func (s *Source) chunkBuild( return handlers.HandleFile(ctx, resp.Body, chunkSkel, sources.ChanReporter{Ch: chunksChan}) } +// Enumerate implements the SourceUnitEnumerator interface. It reports one unit +// per Jenkins job, walking folders recursively. A folder that cannot be listed +// is reported to the UnitReporter and the rest of the walk continues. +func (s *Source) Enumerate(ctx context.Context, reporter sources.UnitReporter) error { + baseUrl := *s.url + return s.walkJobs(ctx, baseUrl.Path, reporter) +} + +// UnmarshalSourceUnit implements the SourceUnitUnmarshaller interface. It +// accepts three shapes: the persisted envelope carrying unit_data (round-trips +// the unit exactly), the envelope without unit_data (rebuilt from id), and a +// bare JenkinsJob. +func (s *Source) UnmarshalSourceUnit(data []byte) (sources.SourceUnit, error) { + var envelope unitEnvelope + if err := json.Unmarshal(data, &envelope); err == nil && envelope.Kind == string(SourceUnitKindJob) { + if envelope.UnitData != "" { + if decoded, err := base64.StdEncoding.DecodeString(envelope.UnitData); err == nil { + var unit JenkinsJob + if json.Unmarshal(decoded, &unit) == nil && unit.Path != "" { + return unit, nil + } + } + } + if envelope.ID != "" { + return JenkinsJob{Path: envelope.ID}, nil + } + } + + var unit JenkinsJob + if err := json.Unmarshal(data, &unit); err != nil { + return nil, err + } + if unit.Path == "" { + return nil, errors.New("not a Jenkins job unit") + } + return unit, nil +} + type JenkinsJobResponse struct { - Jobs []struct { - Class string `json:"_class"` - Name string `json:"name"` - Url string `json:"url"` - } `json:"jobs"` + Jobs []JenkinsJob `json:"jobs"` } type JenkinsBuildResponse struct { diff --git a/pkg/sources/jenkins/unit.go b/pkg/sources/jenkins/unit.go new file mode 100644 index 000000000..62fd15b3a --- /dev/null +++ b/pkg/sources/jenkins/unit.go @@ -0,0 +1,69 @@ +package jenkins + +import ( + "strings" + + "github.com/trufflesecurity/trufflehog/v3/pkg/sources" +) + +const SourceUnitKindJob sources.SourceUnitKind = "job" + +// jobPathSegment is the path segment Jenkins puts before every job or folder +// name in a job URL, e.g. /job/folder1/job/build. +const jobPathSegment = "job" + +// JenkinsJob is a job as listed by the Jenkins API, and is the source unit the +// Jenkins source enumerates. +type JenkinsJob struct { + Class string `json:"_class"` + Name string `json:"name"` + Url string `json:"url"` + // Path is the job's path relative to the Jenkins instance. Jenkins reports + // absolute job URLs whose host comes from the instance's own root URL + // setting, which can differ from the endpoint being scanned, so only the + // path is retained as the identity. Chunks does the same when it scans a + // job. Path is not part of the Jenkins API response; walkJobs derives it + // from Url, so it is empty on a value decoded straight from a listing. + Path string `json:"path"` +} + +var _ sources.SourceUnit = JenkinsJob{} + +func (j JenkinsJob) SourceUnitID() (string, sources.SourceUnitKind) { + return j.Path, SourceUnitKindJob +} + +// Display returns the job's folder path without the "job" segments Jenkins +// interleaves, e.g. /jenkins/job/folder1/job/build becomes folder1/build. It is +// derived from Path rather than from Name so that a unit rebuilt from its ID +// alone still renders a name. +func (j JenkinsJob) Display() string { + segments := strings.Split(strings.Trim(j.Path, "/"), "/") + + // Walk in pairs so that a job named "job" and any instance base path + // (/jenkins/) are both handled: a name is only ever the segment directly + // after a "job" segment. + names := make([]string, 0, len(segments)/2) + for i := 0; i < len(segments)-1; i++ { + if segments[i] == jobPathSegment { + names = append(names, segments[i+1]) + i++ + } + } + + if len(names) == 0 { + return j.Path + } + return strings.Join(names, "/") +} + +// unitEnvelope is the JSON shape a host application persists for an +// enumerated unit between an enumerate pass and a later scan pass: the +// SourceUnit proto marshalled with encoding/json, where unit_data carries the +// original unit as base64-encoded bytes. +type unitEnvelope struct { + ID string `json:"id"` + Kind string `json:"kind,omitempty"` + Display string `json:"display,omitempty"` + UnitData string `json:"unit_data,omitempty"` +} diff --git a/pkg/sources/jenkins/unit_test.go b/pkg/sources/jenkins/unit_test.go new file mode 100644 index 000000000..f768c6664 --- /dev/null +++ b/pkg/sources/jenkins/unit_test.go @@ -0,0 +1,401 @@ +package jenkins + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "runtime" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/anypb" + + "github.com/trufflesecurity/trufflehog/v3/pkg/context" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" + "github.com/trufflesecurity/trufflehog/v3/pkg/sources" +) + +// testUnitReporter records everything Enumerate reports. +type testUnitReporter struct { + mu sync.Mutex + units []sources.SourceUnit + errs []error +} + +func (r *testUnitReporter) UnitOk(_ context.Context, unit sources.SourceUnit) error { + r.mu.Lock() + defer r.mu.Unlock() + r.units = append(r.units, unit) + return nil +} + +func (r *testUnitReporter) UnitErr(_ context.Context, err error) error { + r.mu.Lock() + defer r.mu.Unlock() + r.errs = append(r.errs, err) + return nil +} + +func (r *testUnitReporter) unitIDs(t *testing.T) []string { + t.Helper() + ids := make([]string, 0, len(r.units)) + for _, unit := range r.units { + id, kind := unit.SourceUnitID() + assert.Equal(t, SourceUnitKindJob, kind, "every Jenkins unit must use the job kind") + ids = append(ids, id) + } + return ids +} + +// createMockJenkinsTreeServer serves a nested job tree. The existing mock in +// jenkins_test.go has no folders, so folder recursion is only covered here. +// +// /job/top-job WorkflowJob +// /job/ignored-class MatrixProject, unsupported, dropped +// /job/folder1/ Folder +// /job/folder1/job/nested-job FreeStyleProject +// /job/folder1/job/folder2/ Folder +// /job/folder1/job/folder2/job/deep-job WorkflowJob +// /job/broken/ Folder whose listing is forbidden +func createMockJenkinsTreeServer(withBrokenFolder bool) *httptest.Server { + mux := http.NewServeMux() + server := httptest.NewServer(mux) + + object := func(class, name, path string) string { + return fmt.Sprintf(`{"_class":%q,"name":%q,"url":"%s%s"}`, class, name, server.URL, path) + } + jobsResponse := func(w http.ResponseWriter, r *http.Request, objects ...string) { + w.Header().Set("Content-Type", "application/json") + if !strings.Contains(r.URL.RawQuery, "tree=jobs") { + _, _ = fmt.Fprint(w, `{"jobs":[]}`) + return + } + _, _ = fmt.Fprintf(w, `{"jobs":[%s]}`, strings.Join(objects, ",")) + } + + root := []string{ + object("org.jenkinsci.plugins.workflow.job.WorkflowJob", "top-job", "/job/top-job/"), + object("hudson.matrix.MatrixProject", "ignored-class", "/job/ignored-class/"), + object("com.cloudbees.hudson.plugins.folder.Folder", "folder1", "/job/folder1/"), + } + if withBrokenFolder { + root = append(root, object("com.cloudbees.hudson.plugins.folder.Folder", "broken", "/job/broken/")) + } + + mux.HandleFunc("/api/json", func(w http.ResponseWriter, r *http.Request) { + jobsResponse(w, r, root...) + }) + mux.HandleFunc("/job/folder1/api/json", func(w http.ResponseWriter, r *http.Request) { + jobsResponse(w, r, + object("hudson.model.FreeStyleProject", "nested-job", "/job/folder1/job/nested-job/"), + object("com.cloudbees.hudson.plugins.folder.Folder", "folder2", "/job/folder1/job/folder2/")) + }) + mux.HandleFunc("/job/folder1/job/folder2/api/json", func(w http.ResponseWriter, r *http.Request) { + jobsResponse(w, r, + object("org.jenkinsci.plugins.workflow.job.WorkflowJob", "deep-job", "/job/folder1/job/folder2/job/deep-job/")) + }) + // 403 rather than 500 so the client's 5XX retry policy doesn't stall the + // test for the full 90s retry budget. + mux.HandleFunc("/job/broken/api/json", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusForbidden) + }) + + return server +} + +func newTestSource(t *testing.T, ctx context.Context, endpoint, name string) *Source { + t.Helper() + + conn, err := anypb.New(&sourcespb.Jenkins{ + Endpoint: endpoint, + Credential: &sourcespb.Jenkins_BasicAuth{ + BasicAuth: &credentialspb.BasicAuth{ + Username: "testuser", + Password: "testpass", + }, + }, + }) + require.NoError(t, err) + + s := new(Source) + require.NoError(t, s.Init(ctx, name, 0, 1, false, conn, runtime.NumCPU())) + return s +} + +// wantedUnitIDs are the jobs the mock tree exposes, in the order the walk finds +// them. The unsupported MatrixProject and the folders themselves are excluded. +var wantedUnitIDs = []string{ + "/job/top-job", + "/job/folder1/job/nested-job", + "/job/folder1/job/folder2/job/deep-job", +} + +func TestEnumerate(t *testing.T) { + t.Parallel() + + server := createMockJenkinsTreeServer(false) + defer server.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + s := newTestSource(t, ctx, server.URL, "test-jenkins-enumerate") + + reporter := new(testUnitReporter) + require.NoError(t, s.Enumerate(ctx, reporter)) + + assert.Equal(t, wantedUnitIDs, reporter.unitIDs(t)) + assert.Empty(t, reporter.errs) +} + +func TestEnumerateStableAcrossRuns(t *testing.T) { + t.Parallel() + + server := createMockJenkinsTreeServer(false) + defer server.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + s := newTestSource(t, ctx, server.URL, "test-jenkins-stable") + + first := new(testUnitReporter) + require.NoError(t, s.Enumerate(ctx, first)) + second := new(testUnitReporter) + require.NoError(t, s.Enumerate(ctx, second)) + + assert.Equal(t, first.unitIDs(t), second.unitIDs(t)) +} + +func TestEnumerateContinuesPastFolderError(t *testing.T) { + t.Parallel() + + server := createMockJenkinsTreeServer(true) + defer server.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + s := newTestSource(t, ctx, server.URL, "test-jenkins-folder-error") + + reporter := new(testUnitReporter) + require.NoError(t, s.Enumerate(ctx, reporter)) + + // The unreadable folder is reported but does not stop the other jobs from + // being enumerated. + assert.Equal(t, wantedUnitIDs, reporter.unitIDs(t)) + require.Len(t, reporter.errs, 1) + assert.Contains(t, reporter.errs[0].Error(), "/job/broken/") +} + +// TestEnumerateMatchesChunkTraversal guards against Enumerate and the walk +// Chunks uses reporting different sets of jobs. +func TestEnumerateMatchesChunkTraversal(t *testing.T) { + t.Parallel() + + server := createMockJenkinsTreeServer(false) + defer server.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + s := newTestSource(t, ctx, server.URL, "test-jenkins-parity") + + jobs, err := s.GetJenkinsJobs(ctx) + require.NoError(t, err) + + reporter := new(testUnitReporter) + require.NoError(t, s.Enumerate(ctx, reporter)) + + require.Len(t, reporter.units, len(jobs.Jobs)) + for i, job := range jobs.Jobs { + id, _ := reporter.units[i].SourceUnitID() + assert.Equal(t, job.Path, id, "unit does not match the job Chunks would scan (%q)", job.Url) + assert.True(t, strings.HasSuffix(job.Url, id+"/"), + "job path %q was not derived from the job URL %q", job.Path, job.Url) + } +} + +// TestUnparseableJobURLSkippedByBothPaths pins that a job whose URL cannot be +// parsed drops out of enumeration and out of the walk Chunks uses, rather than +// failing either one. +func TestUnparseableJobURLSkippedByBothPaths(t *testing.T) { + t.Parallel() + + mux := http.NewServeMux() + server := httptest.NewServer(mux) + defer server.Close() + + mux.HandleFunc("/api/json", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if !strings.Contains(r.URL.RawQuery, "tree=jobs") { + _, _ = fmt.Fprint(w, `{"jobs":[]}`) + return + } + // The second URL has an invalid percent escape, which url.Parse rejects. + _, _ = fmt.Fprintf(w, `{"jobs":[`+ + `{"_class":"hudson.model.FreeStyleProject","name":"good","url":"%s/job/good/"},`+ + `{"_class":"hudson.model.FreeStyleProject","name":"bad","url":"%s/job/%%zz/"}`+ + `]}`, server.URL, server.URL) + }) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + s := newTestSource(t, ctx, server.URL, "test-jenkins-bad-job-url") + + reporter := new(testUnitReporter) + require.NoError(t, s.Enumerate(ctx, reporter)) + assert.Equal(t, []string{"/job/good"}, reporter.unitIDs(t)) + assert.Empty(t, reporter.errs) + + jobs, err := s.GetJenkinsJobs(ctx) + require.NoError(t, err) + require.Len(t, jobs.Jobs, 1) + assert.Equal(t, "/job/good", jobs.Jobs[0].Path) +} + +// TestGetJenkinsJobsStillFailsFast pins the pre-existing behavior of the walk +// Chunks relies on: an unreadable folder aborts the traversal. +func TestGetJenkinsJobsStillFailsFast(t *testing.T) { + t.Parallel() + + server := createMockJenkinsTreeServer(true) + defer server.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + s := newTestSource(t, ctx, server.URL, "test-jenkins-fail-fast") + + _, err := s.GetJenkinsJobs(ctx) + require.Error(t, err) +} + +func TestJenkinsJobDisplay(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + path string + want string + }{ + {name: "top level", path: "/job/build", want: "build"}, + {name: "nested", path: "/job/folder1/job/sub/job/build", want: "folder1/sub/build"}, + // A folder holding many jobs is the normal case, so the leaf name has + // to survive into the display string or siblings would all render the + // same. See TestJenkinsJobDisplaySiblingsDiffer. + {name: "sibling in same folder", path: "/job/folder1/job/first", want: "folder1/first"}, + {name: "other sibling in same folder", path: "/job/folder1/job/second", want: "folder1/second"}, + {name: "instance base path", path: "/jenkins/job/folder1/job/build", want: "folder1/build"}, + {name: "job named job", path: "/job/job/job/build", want: "job/build"}, + {name: "trailing slash", path: "/job/build/", want: "build"}, + {name: "no job segment falls back", path: "/some/other/path", want: "/some/other/path"}, + {name: "empty falls back", path: "", want: ""}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + assert.Equal(t, tt.want, JenkinsJob{Path: tt.path}.Display()) + }) + } +} + +// TestJenkinsJobDisplaySiblingsDiffer pins the property that distinct jobs never +// collapse to the same display string, including jobs sharing a folder. +func TestJenkinsJobDisplaySiblingsDiffer(t *testing.T) { + t.Parallel() + + paths := []string{ + "/job/folder1/job/a", + "/job/folder1/job/b", + "/job/folder1/job/sub/job/a", + "/job/folder2/job/a", + "/job/a", + } + + seen := make(map[string]string, len(paths)) + for _, path := range paths { + display := JenkinsJob{Path: path}.Display() + if previous, ok := seen[display]; ok { + t.Errorf("paths %q and %q both display as %q", previous, path, display) + } + seen[display] = path + } +} + +func TestJenkinsJobSourceUnitID(t *testing.T) { + t.Parallel() + + id, kind := JenkinsJob{Path: "/job/folder1/job/build"}.SourceUnitID() + assert.Equal(t, "/job/folder1/job/build", id) + assert.Equal(t, SourceUnitKindJob, kind) +} + +func TestUnmarshalSourceUnit(t *testing.T) { + t.Parallel() + + const jobPath = "/job/folder1/job/build" + + unitData, err := json.Marshal(JenkinsJob{Path: jobPath}) + require.NoError(t, err) + + envelopeWithData, err := json.Marshal(unitEnvelope{ + ID: jobPath, + Kind: string(SourceUnitKindJob), + Display: "folder1/build", + UnitData: base64.StdEncoding.EncodeToString(unitData), + }) + require.NoError(t, err) + + envelopeIDOnly, err := json.Marshal(unitEnvelope{ + ID: jobPath, + Kind: string(SourceUnitKindJob), + Display: "folder1/build", + }) + require.NoError(t, err) + + tests := []struct { + name string + data []byte + want string + wantErr bool + }{ + {name: "envelope with unit data", data: envelopeWithData, want: jobPath}, + {name: "envelope without unit data", data: envelopeIDOnly, want: jobPath}, + {name: "bare unit", data: unitData, want: jobPath}, + { + name: "envelope with wrong kind", + data: fmt.Appendf(nil, `{"id":%q,"kind":"bucket"}`, jobPath), + wantErr: true, + }, + {name: "empty path", data: []byte(`{"path":""}`), wantErr: true}, + {name: "not json", data: []byte(`not json`), wantErr: true}, + } + + s := new(Source) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + unit, err := s.UnmarshalSourceUnit(tt.data) + if tt.wantErr { + assert.Error(t, err) + return + } + require.NoError(t, err) + + id, kind := unit.SourceUnitID() + assert.Equal(t, tt.want, id) + assert.Equal(t, SourceUnitKindJob, kind) + }) + } +}