[INS-465] Skip unverified JWT Detector results when feature flag is enabled (#5072)

* Skip unverified jwt results when  feature flag is enabled

* update man page

docs/man/trufflehog.1

@@ -116,6 +116,9 @@ Skip additional references.
 \fB--user-agent-suffix=USER-AGENT-SUFFIX\fR
 Suffix to add to User-Agent.
 .TP
+\fB--drop-unverified-jwt-results\fR
+Drop unverified results without any verification errors from the JWT detector.
+.TP
 \fB--version\fR
 Show application version.
 .SH COMMANDS

main.go

@@ -87,11 +87,12 @@ var (
 	noVerificationCache = cli.Flag("no-verification-cache", "Disable verification caching").Bool()
 
 	// Add feature flags
-	forceSkipBinaries  = cli.Flag("force-skip-binaries", "Force skipping binaries.").Bool()
-	forceSkipArchives  = cli.Flag("force-skip-archives", "Force skipping archives.").Bool()
-	gitCloneTimeout    = cli.Flag("git-clone-timeout", "Maximum time to spend cloning a repository, as a duration.").Hidden().Duration()
-	skipAdditionalRefs = cli.Flag("skip-additional-refs", "Skip additional references.").Bool()
-	userAgentSuffix    = cli.Flag("user-agent-suffix", "Suffix to add to User-Agent.").String()
+	forceSkipBinaries        = cli.Flag("force-skip-binaries", "Force skipping binaries.").Bool()
+	forceSkipArchives        = cli.Flag("force-skip-archives", "Force skipping archives.").Bool()
+	gitCloneTimeout          = cli.Flag("git-clone-timeout", "Maximum time to spend cloning a repository, as a duration.").Hidden().Duration()
+	skipAdditionalRefs       = cli.Flag("skip-additional-refs", "Skip additional references.").Bool()
+	userAgentSuffix          = cli.Flag("user-agent-suffix", "Suffix to add to User-Agent.").String()
+	dropUnverifiedJWTResults = cli.Flag("drop-unverified-jwt-results", "Drop unverified results without any verification errors from the JWT detector.").Bool()
 
 	gitScan                = cli.Command("git", "Find credentials in git repositories.")
 	gitScanURI             = gitScan.Arg("uri", "Git repository URL. https://, file://, or ssh:// schema expected.").Required().String()
@@ -516,6 +517,10 @@ func run(state overseer.State, logSync func() error) {
 		feature.UserAgentSuffix.Store(*userAgentSuffix)
 	}
 
+	if *dropUnverifiedJWTResults {
+		feature.DropUnverifiedJWTResults.Store(true)
+	}
+
 	// OSS Default APK handling on
 	feature.EnableAPKHandler.Store(true)
 

pkg/detectors/jwt/jwt.go

@@ -16,6 +16,7 @@ import (
 	regexp "github.com/wasilibs/go-re2"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/feature"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb"
 )
 
@@ -78,6 +79,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 	client := detectors.DetectorHttpClientWithNoLocalAddresses
 
 	seenMatches := make(map[string]struct{})
+	skipUnverified := feature.DropUnverifiedJWTResults.Load()
 
 	for _, matchGroups := range keyPat.FindAllStringSubmatch(string(data), -1) {
 		match := matchGroups[1]
@@ -144,6 +146,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1.SetVerificationError(verificationErr, match)
 		}
 
+		// Remove unverified results from jwt detector output when the "drop-unverified-jwt-results" feature flag is enabled.
+		if skipUnverified && verify && !s1.Verified && s1.VerificationError() == nil {
+			continue
+		}
 		results = append(results, s1)
 	}
 

pkg/detectors/jwt/jwt_test.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/feature"
 )
 
 // This tests the JWT detector for a number of different cases (mostly HMAC-based) without verification enabled.
@@ -135,3 +136,69 @@ func TestJwt_Pattern(t *testing.T) {
 		})
 	}
 }
+
+func TestJwt_SkipUnverified(t *testing.T) {
+	d := Scanner{}
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
+	tests := []struct {
+		name           string
+		input          string
+		want           []string
+		skipUnverified bool
+	}{
+		{
+			name:           "expired token - results should be skipped when skipUnverified is true",
+			input:          "eyJhbGciOiJFZERTQSIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRydWUsImlhdCI6MTcwNDA2NzIwMCwiZXhwIjoxNzA0MDcwODAwfQ.9z15Ku3lWBC8rFPlvxkX3wx6wRalhoN7PlBRtE7fUXyoHqVLiZG2l5eMo6KwBlAu1L2hFtkvQwiJ79G5o3a4Bw",
+			want:           []string{},
+			skipUnverified: true,
+		},
+		{
+			name:           "expired token - results should not be skipped when skipUnverified is false",
+			input:          "eyJhbGciOiJFZERTQSIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRydWUsImlhdCI6MTcwNDA2NzIwMCwiZXhwIjoxNzA0MDcwODAwfQ.9z15Ku3lWBC8rFPlvxkX3wx6wRalhoN7PlBRtE7fUXyoHqVLiZG2l5eMo6KwBlAu1L2hFtkvQwiJ79G5o3a4Bw",
+			want:           []string{"eyJhbGciOiJFZERTQSIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRydWUsImlhdCI6MTcwNDA2NzIwMCwiZXhwIjoxNzA0MDcwODAwfQ.9z15Ku3lWBC8rFPlvxkX3wx6wRalhoN7PlBRtE7fUXyoHqVLiZG2l5eMo6KwBlAu1L2hFtkvQwiJ79G5o3a4Bw"},
+			skipUnverified: false,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+			if len(matchedDetectors) == 0 {
+				t.Errorf("test %q failed: expected keywords %v to be found in the input", test.name, d.Keywords())
+				return
+			}
+
+			if test.skipUnverified {
+				feature.DropUnverifiedJWTResults.Store(true)
+			} else {
+				feature.DropUnverifiedJWTResults.Store(false)
+			}
+
+			results, err := d.FromData(context.Background(), true, []byte(test.input))
+			require.NoError(t, err)
+
+			if len(results) != len(test.want) {
+				t.Errorf("mismatch in result count: expected %d, got %d", len(test.want), len(results))
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+			}
+		})
+	}
+}

pkg/feature/feature.go

@@ -28,6 +28,7 @@ var (
 	BraintrustDetectorEnabled       atomic.Bool
 	PgAnalyzeReadKeyDetectorEnabled atomic.Bool
 	RedHatPyxisDetectorEnabled      atomic.Bool
+  DropUnverifiedJWTResults        atomic.Bool
 )
 
 type AtomicString struct {