diff --git a/pkg/decoders/escaped_unicode.go b/pkg/decoders/escaped_unicode.go index b8574bc10..074373f96 100644 --- a/pkg/decoders/escaped_unicode.go +++ b/pkg/decoders/escaped_unicode.go @@ -105,6 +105,7 @@ func (d *EscapedUnicode) FromChunk(chunk *sources.Chunk) *DecodableChunk { DecoderType: d.Type(), Chunk: &sources.Chunk{ Data: chunkData, + OriginalData: chunk.OriginalData, SourceName: chunk.SourceName, SourceID: chunk.SourceID, JobID: chunk.JobID, diff --git a/pkg/detectors/copy_metadata_test.go b/pkg/detectors/copy_metadata_test.go new file mode 100644 index 000000000..2eb9f6626 --- /dev/null +++ b/pkg/detectors/copy_metadata_test.go @@ -0,0 +1,40 @@ +package detectors + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/trufflesecurity/trufflehog/v3/pkg/sources" +) + +func TestCopyMetadata_ChunkDataFromOriginalData(t *testing.T) { + chunk := &sources.Chunk{ + Data: []byte("decoded-data"), + OriginalData: []byte("original-source-data"), + SourceName: "test-source", + } + result := Result{ + DetectorType: 1, + Raw: []byte("secret"), + } + + rwm := CopyMetadata(chunk, result) + + assert.Equal(t, "original-source-data", string(rwm.ChunkData)) +} + +func TestCopyMetadata_ChunkDataFallsBackToData(t *testing.T) { + chunk := &sources.Chunk{ + Data: []byte("only-data"), + SourceName: "test-source", + } + result := Result{ + DetectorType: 1, + Raw: []byte("secret"), + } + + rwm := CopyMetadata(chunk, result) + + assert.Equal(t, "only-data", string(rwm.ChunkData)) +} diff --git a/pkg/detectors/detectors.go b/pkg/detectors/detectors.go index cf1ff4649..38f1ad446 100644 --- a/pkg/detectors/detectors.go +++ b/pkg/detectors/detectors.go @@ -217,10 +217,19 @@ type ResultWithMetadata struct { DetectorDescription string // DecoderType is the type of decoder that was used to generate this result's data. DecoderType detectorspb.DecoderType + // ChunkData holds the original pre-decode source chunk data, preserved + // for secret storage encryption in the dispatcher. + ChunkData []byte } // CopyMetadata returns a detector result with included metadata from the source chunk. func CopyMetadata(chunk *sources.Chunk, result Result) ResultWithMetadata { + // OriginalData may be nil when CopyMetadata is called outside the engine + // pipeline (e.g., in tests or external consumers that construct chunks directly). + chunkData := chunk.OriginalData + if chunkData == nil { + chunkData = chunk.Data + } return ResultWithMetadata{ SourceMetadata: chunk.SourceMetadata, SourceID: chunk.SourceID, @@ -229,6 +238,7 @@ func CopyMetadata(chunk *sources.Chunk, result Result) ResultWithMetadata { SourceType: chunk.SourceType, SourceName: chunk.SourceName, Result: result, + ChunkData: chunkData, } } diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go index 5e64ba63e..25f8270aa 100644 --- a/pkg/engine/engine.go +++ b/pkg/engine/engine.go @@ -854,6 +854,7 @@ func (e *Engine) scannerWorker(ctx context.Context) { startTime := time.Now() sourceVerify := chunk.SourceVerify + chunk.OriginalData = chunk.Data decoded := iterativeDecode(chunk, e.decoders, e.maxDecodeDepth) for _, d := range decoded { diff --git a/pkg/sources/sources.go b/pkg/sources/sources.go index 761b7b0ac..6a8c2b1d5 100644 --- a/pkg/sources/sources.go +++ b/pkg/sources/sources.go @@ -21,14 +21,13 @@ type ( ) // Chunk contains data to be decoded and scanned along with context on where it came from. -// -// **Important:** The order of the fields in this struct is specifically designed to optimize -// struct alignment and minimize memory usage. Do not change the field order without carefully considering -// the potential impact on memory consumption. -// Ex: https://go.dev/play/p/Azf4a7O-DhC type Chunk struct { // Data is the data to decode and scan. Data []byte + // OriginalData holds the pre-decode source data, preserved for secret + // storage. Set before iterative decoding so it retains the original + // content even after Data is replaced with decoded forms. + OriginalData []byte // SourceName is the name of the Source that produced the chunk. SourceName string diff --git a/pkg/sources/sources_test.go b/pkg/sources/sources_test.go index 49fd853fb..6db3fccbc 100644 --- a/pkg/sources/sources_test.go +++ b/pkg/sources/sources_test.go @@ -7,8 +7,10 @@ import ( "github.com/stretchr/testify/assert" ) -// TestChunkSize ensures that the Chunk struct does not exceed 80 bytes. +// TestChunkSize ensures that the Chunk struct does not exceed 104 bytes. +// Size increased from 80 to 104 with the addition of OriginalData []byte +// (24-byte slice header) for secret storage chunk threading. func TestChunkSize(t *testing.T) { t.Parallel() - assert.Equal(t, unsafe.Sizeof(Chunk{}), uintptr(80), "Chunk struct size exceeds 80 bytes") + assert.Equal(t, unsafe.Sizeof(Chunk{}), uintptr(104), "Chunk struct size exceeds 104 bytes") }