diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go index ffc69e91b..1802e89ae 100644 --- a/pkg/engine/engine.go +++ b/pkg/engine/engine.go @@ -1170,6 +1170,12 @@ func (e *Engine) filterResults( return results } +// processResult generates a detectors.ResultWithMetadata from the provided chunk and result and puts it on the results +// channel, unless the result exists on a line with an ignore tag, in which case no result is generated. +// +// CMR: The provided chunk is wrapped in a detectableChunk, but I'm pretty sure that's purely out of convenience +// (because that's what this function's callers are using when they call this function). We're past detection at this +// point in the engine, so we should probably refactor that parameter into a less confusing data type. func (e *Engine) processResult( ctx context.Context, data detectableChunk, diff --git a/pkg/engine/engine_test.go b/pkg/engine/engine_test.go index 4e2888e99..18800715f 100644 --- a/pkg/engine/engine_test.go +++ b/pkg/engine/engine_test.go @@ -12,6 +12,7 @@ import ( "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/trufflesecurity/trufflehog/v3/pkg/config" "github.com/trufflesecurity/trufflehog/v3/pkg/context" @@ -26,6 +27,7 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" "github.com/trufflesecurity/trufflehog/v3/pkg/sources" + "github.com/trufflesecurity/trufflehog/v3/pkg/verificationcache" ) const fakeDetectorKeyword = "fakedetector" @@ -1318,3 +1320,223 @@ def test_something(): }) } } + +type passthroughDetector struct { + detectorType detectorspb.DetectorType + keywords []string + secret string +} + +func (p passthroughDetector) FromData(_ aCtx.Context, verify bool, data []byte) ([]detectors.Result, error) { + raw := data + if p.secret != "" { + raw = []byte(p.secret) + } + return []detectors.Result{ + { + Raw: raw, + Verified: verify, + }, + }, nil +} + +func (p passthroughDetector) Keywords() []string { return p.keywords } +func (p passthroughDetector) Type() detectorspb.DetectorType { return p.detectorType } +func (p passthroughDetector) Description() string { return "fake detector for testing" } + +type passthroughDecoder struct{} + +func (p passthroughDecoder) FromChunk(chunk *sources.Chunk) *decoders.DecodableChunk { + return &decoders.DecodableChunk{ + Chunk: chunk, + DecoderType: detectorspb.DecoderType(-1), + } +} + +func (p passthroughDecoder) Type() detectorspb.DecoderType { return detectorspb.DecoderType(-1) } + +func TestEngine_DetectChunk_UsesVerifyFlag(t *testing.T) { + ctx := context.Background() + + // Arrange: Create a minimal engine. + e := &Engine{ + results: make(chan detectors.ResultWithMetadata, 1), + verificationCache: verificationcache.New(nil, &verificationcache.InMemoryMetrics{}), + } + + // Arrange: Create a detector match. We can't create one directly, so we have to use a minimal A-H core. + ahcore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{passthroughDetector{keywords: []string{"keyword"}}}) + detectorMatches := ahcore.FindDetectorMatches([]byte("keyword")) + require.Len(t, detectorMatches, 1) + + // Arrange: Create a chunk to detect. + chunk := detectableChunk{ + chunk: sources.Chunk{ + Verify: true, + }, + detector: detectorMatches[0], + wgDoneFn: func() {}, + } + + // Act + e.detectChunk(ctx, chunk) + close(e.results) + + // Assert: Confirm that a result was generated and that it has the expected verify flag. + select { + case result := <-e.results: + assert.True(t, result.Result.Verified) + default: + t.Errorf("expected a result but did not get one") + } +} + +func TestEngine_ScannerWorker_DetectableChunkHasCorrectVerifyFlag(t *testing.T) { + ctx := context.Background() + + // Arrange: Create a minimal engine. + detector := &passthroughDetector{keywords: []string{"keyword"}} + e := &Engine{ + AhoCorasickCore: ahocorasick.NewAhoCorasickCore([]detectors.Detector{detector}), + decoders: []decoders.Decoder{passthroughDecoder{}}, + detectableChunksChan: make(chan detectableChunk, 1), + sourceManager: sources.NewManager(), + verify: true, + } + + // Arrange: Create a chunk to scan. + chunk := sources.Chunk{ + Data: []byte("keyword"), + Verify: true, + } + + // Arrange: Enqueue a chunk to be scanned. + e.sourceManager.ScanChunk(&chunk) + + // Act + go e.scannerWorker(ctx) + + // Assert: Confirm that a chunk was generated and that it has the expected verify flag. + select { + case chunk := <-e.detectableChunksChan: + assert.True(t, chunk.chunk.Verify) + case <-time.After(1 * time.Second): + t.Errorf("expected a detectableChunk but did not get one") + } +} + +func TestEngine_VerificationOverlapWorker_DetectableChunkHasCorrectVerifyFlag(t *testing.T) { + ctx := context.Background() + + t.Run("overlap", func(t *testing.T) { + // Arrange: Create a minimal engine + e := &Engine{ + detectableChunksChan: make(chan detectableChunk, 2), + results: make(chan detectors.ResultWithMetadata, 2), + retainFalsePositives: true, + verificationOverlapChunksChan: make(chan verificationOverlapChunk, 2), + verify: true, + } + + // Arrange: Set up a fake detectableChunk processor so that any chunks (incorrectly) sent to + // e.detectableChunksChan don't block the test. + processedDetectableChunks := make(chan detectableChunk, 2) + go func() { + for chunk := range e.detectableChunksChan { + chunk.wgDoneFn() + processedDetectableChunks <- chunk + } + }() + + // Arrange: Create a chunk to "scan." + chunk := sources.Chunk{ + Data: []byte("keyword ;oahpow8heg;blaisd"), + Verify: true, + } + + // Arrange: Create overlapping detector matches. We can't create them directly, so we have to use a minimal A-H + // core. + ahcore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{ + passthroughDetector{detectorType: detectorspb.DetectorType(-1), keywords: []string{"keyw"}}, + passthroughDetector{detectorType: detectorspb.DetectorType(-2), keywords: []string{"keyword"}}, + }) + detectorMatches := ahcore.FindDetectorMatches(chunk.Data) + require.Len(t, detectorMatches, 2) + + // Arrange: Enqueue a verification overlap chunk + e.verificationOverlapChunksChan <- verificationOverlapChunk{ + chunk: chunk, + detectors: detectorMatches, + verificationOverlapWgDoneFn: func() { close(e.verificationOverlapChunksChan) }, + } + + // Act + e.verificationOverlapWorker(ctx) + close(e.results) + close(e.detectableChunksChan) + close(processedDetectableChunks) + + // Assert: Confirm that every generated result is unverified (because overlap detection precluded it). + for result := range e.results { + assert.False(t, result.Result.Verified) + } + + // Assert: Confirm that every generated detectable chunk carries the original Verify flag. + // CMR: There should be not be any of these chunks. However, due to what I believe is an unrelated bug, there + // are. This test ensures that even in that erroneous case, their Verify flag is correct. + for detectableChunk := range processedDetectableChunks { + assert.True(t, detectableChunk.chunk.Verify) + } + }) + t.Run("no overlap", func(t *testing.T) { + // Arrange: Create a minimal engine + e := &Engine{ + detectableChunksChan: make(chan detectableChunk, 2), + retainFalsePositives: true, + verificationOverlapChunksChan: make(chan verificationOverlapChunk, 2), + verify: true, + } + + // Arrange: Set up a fake detectableChunk processor so that any chunks sent to e.detectableChunksChan don't + // block the test. + processedDetectableChunks := make(chan detectableChunk, 2) + go func() { + for chunk := range e.detectableChunksChan { + chunk.wgDoneFn() + processedDetectableChunks <- chunk + } + }() + + // Arrange: Create a chunk to "scan." + chunk := sources.Chunk{ + Data: []byte("keyword ;oahpow8heg;blaisd"), + Verify: true, + } + + // Arrange: Create non-overlapping detector matches. We can't create them directly, so we have to use a minimal + // A-H core. + ahcore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{ + passthroughDetector{detectorType: detectorspb.DetectorType(-1), keywords: []string{"keyw"}, secret: "oahpow"}, + passthroughDetector{detectorType: detectorspb.DetectorType(-2), keywords: []string{"keyword"}, secret: "blaisd"}, + }) + detectorMatches := ahcore.FindDetectorMatches(chunk.Data) + require.Len(t, detectorMatches, 2) + + // Arrange: Enqueue a verification overlap chunk + e.verificationOverlapChunksChan <- verificationOverlapChunk{ + chunk: chunk, + detectors: detectorMatches, + verificationOverlapWgDoneFn: func() { close(e.verificationOverlapChunksChan) }, + } + + // Act + e.verificationOverlapWorker(ctx) + close(e.detectableChunksChan) + close(processedDetectableChunks) + + // Assert: Confirm that every generated detectable chunk carries the original Verify flag. + for detectableChunk := range processedDetectableChunks { + assert.True(t, detectableChunk.chunk.Verify) + } + }) +}