Thread original chunk data through engine pipeline (#4780)

* [secret-storage] Thread original chunk data through engine pipeline

Adds OriginalData/ChunkData fields to preserve pre-decode source data
through the scan pipeline:

1. Chunk.OriginalData: captures chunk.Data before iterativeDecode
2. engine.go: sets chunk.OriginalData = chunk.Data before decode
3. ResultWithMetadata.ChunkData: populated by CopyMetadata from
   OriginalData (falls back to Data when nil)

This enables downstream consumers (e.g. the dispatcher in thog) to
access the original source data for secret storage encryption.

* Update TestChunkSize for OriginalData field addition

Chunk struct grew from 80 to 104 bytes with the OriginalData []byte
slice header (24 bytes). Field placement is already optimal (adjacent
to Data []byte).

* fix: preserve OriginalData field in EscapedUnicode decoder

The EscapedUnicode decoder constructed a new sources.Chunk manually
copying fields but omitted OriginalData. This caused CopyMetadata to
fall back to the decoded Data instead of the original pre-decode content,
defeating the purpose of preserving original chunk data for secret
storage encryption.

* Address PR review feedback: use testify/assert, add nil-guard comment, remove stale alignment comment

---------

Co-authored-by: Cursor Agent <[email protected]>

pkg/decoders/escaped_unicode.go

@@ -105,6 +105,7 @@ func (d *EscapedUnicode) FromChunk(chunk *sources.Chunk) *DecodableChunk {
 			DecoderType: d.Type(),
 			Chunk: &sources.Chunk{
 				Data:           chunkData,
+				OriginalData:   chunk.OriginalData,
 				SourceName:     chunk.SourceName,
 				SourceID:       chunk.SourceID,
 				JobID:          chunk.JobID,

pkg/detectors/copy_metadata_test.go

@@ -0,0 +1,40 @@
+package detectors
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+)
+
+func TestCopyMetadata_ChunkDataFromOriginalData(t *testing.T) {
+	chunk := &sources.Chunk{
+		Data:         []byte("decoded-data"),
+		OriginalData: []byte("original-source-data"),
+		SourceName:   "test-source",
+	}
+	result := Result{
+		DetectorType: 1,
+		Raw:          []byte("secret"),
+	}
+
+	rwm := CopyMetadata(chunk, result)
+
+	assert.Equal(t, "original-source-data", string(rwm.ChunkData))
+}
+
+func TestCopyMetadata_ChunkDataFallsBackToData(t *testing.T) {
+	chunk := &sources.Chunk{
+		Data:       []byte("only-data"),
+		SourceName: "test-source",
+	}
+	result := Result{
+		DetectorType: 1,
+		Raw:          []byte("secret"),
+	}
+
+	rwm := CopyMetadata(chunk, result)
+
+	assert.Equal(t, "only-data", string(rwm.ChunkData))
+}

pkg/detectors/detectors.go

@@ -217,10 +217,19 @@ type ResultWithMetadata struct {
 	DetectorDescription string
 	// DecoderType is the type of decoder that was used to generate this result's data.
 	DecoderType detectorspb.DecoderType
+	// ChunkData holds the original pre-decode source chunk data, preserved
+	// for secret storage encryption in the dispatcher.
+	ChunkData []byte
 }
 
 // CopyMetadata returns a detector result with included metadata from the source chunk.
 func CopyMetadata(chunk *sources.Chunk, result Result) ResultWithMetadata {
+	// OriginalData may be nil when CopyMetadata is called outside the engine
+	// pipeline (e.g., in tests or external consumers that construct chunks directly).
+	chunkData := chunk.OriginalData
+	if chunkData == nil {
+		chunkData = chunk.Data
+	}
 	return ResultWithMetadata{
 		SourceMetadata: chunk.SourceMetadata,
 		SourceID:       chunk.SourceID,
@@ -229,6 +238,7 @@ func CopyMetadata(chunk *sources.Chunk, result Result) ResultWithMetadata {
 		SourceType:     chunk.SourceType,
 		SourceName:     chunk.SourceName,
 		Result:         result,
+		ChunkData:      chunkData,
 	}
 }
 

pkg/engine/engine.go

@@ -854,6 +854,7 @@ func (e *Engine) scannerWorker(ctx context.Context) {
 		startTime := time.Now()
 		sourceVerify := chunk.SourceVerify
 
+		chunk.OriginalData = chunk.Data
 		decoded := iterativeDecode(chunk, e.decoders, e.maxDecodeDepth)
 
 		for _, d := range decoded {

pkg/sources/sources.go

@@ -21,14 +21,13 @@ type (
 )
 
 // Chunk contains data to be decoded and scanned along with context on where it came from.
-//
-// **Important:** The order of the fields in this struct is specifically designed to optimize
-// struct alignment and minimize memory usage. Do not change the field order without carefully considering
-// the potential impact on memory consumption.
-// Ex: https://go.dev/play/p/Azf4a7O-DhC
 type Chunk struct {
 	// Data is the data to decode and scan.
 	Data []byte
+	// OriginalData holds the pre-decode source data, preserved for secret
+	// storage. Set before iterative decoding so it retains the original
+	// content even after Data is replaced with decoded forms.
+	OriginalData []byte
 
 	// SourceName is the name of the Source that produced the chunk.
 	SourceName string

pkg/sources/sources_test.go

@@ -7,8 +7,10 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-// TestChunkSize ensures that the Chunk struct does not exceed 80 bytes.
+// TestChunkSize ensures that the Chunk struct does not exceed 104 bytes.
+// Size increased from 80 to 104 with the addition of OriginalData []byte
+// (24-byte slice header) for secret storage chunk threading.
 func TestChunkSize(t *testing.T) {
 	t.Parallel()
-	assert.Equal(t, unsafe.Sizeof(Chunk{}), uintptr(80), "Chunk struct size exceeds 80 bytes")
+	assert.Equal(t, unsafe.Sizeof(Chunk{}), uintptr(104), "Chunk struct size exceeds 104 bytes")
 }