From 6dd65a515659cdb1a657151f3bba7bc2bad609bf Mon Sep 17 00:00:00 2001 From: Kashif Khan <70996046+kashifkhan0771@users.noreply.github.com> Date: Wed, 27 Aug 2025 11:25:48 +0500 Subject: [PATCH] Ignore known common prefix matches for Github V1 detector (#4379) Co-authored-by: Amaan Ullah --- pkg/detectors/github/v1/github_old.go | 27 +++++--- pkg/detectors/github/v1/github_old_test.go | 77 +++++++++++++++------- 2 files changed, 74 insertions(+), 30 deletions(-) diff --git a/pkg/detectors/github/v1/github_old.go b/pkg/detectors/github/v1/github_old.go index 337b21ffa..4165c19f7 100644 --- a/pkg/detectors/github/v1/github_old.go +++ b/pkg/detectors/github/v1/github_old.go @@ -28,8 +28,8 @@ func (Scanner) CloudEndpoint() string { return "https://api.github.com" } var ( // Oauth token // https://developer.github.com/v3/#oauth2-token-sent-in-a-header - // the middle regex `\b[a-zA-Z0-9.\/?=&]{0,40}` is to match the prefix of token match to avoid processing common known patterns - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"github", "gh", "pat", "token"}) + `\b[a-zA-Z0-9.\/?=&]{0,40}` + `\b([a-f0-9]{40})\b`) + // the middle regex `(?:[a-zA-Z0-9.\/?=&:-]{0,40})` is to match the prefix of token match to avoid processing common known patterns + keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"github", "gh", "pat", "token"}) + `\b(?:[a-zA-Z0-9.\/?=&:-]{0,40})([a-f0-9]{40})\b`) // TODO: Oauth2 client_id and client_secret // https://developer.github.com/v3/#oauth2-keysecret @@ -73,7 +73,15 @@ var ghFalsePositives = map[detectors.FalsePositive]struct{}{ } var ghKnownNonSensitivePrefixes = []string{ - "avatars.githubusercontent.com", // github avatar urls prefix + "avatars.githubusercontent.com", // GitHub avatar URLs + "actions/", // GitHub Actions paths + "raw.githubusercontent.com/", // Raw file URLs from GitHub + "api.github.com/repos/", // GitHub API repository endpoints + "gist.github.com/", // GitHub Gist URLs + "sha256:", // SHA256 hash prefix + "github.com/", // General GitHub repo URLs + "pipelines.actions.githubusercontent.com/", // GitHub Actions infrastructure + "ghcr.io/", // GitHub Container Registry } // FromData will find and optionally verify GitHub secrets in a given set of bytes. @@ -93,11 +101,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result continue } - // to avoid false positives - if isKnownNonSensitiveCommonPrefix(matchPrefix) { - continue - } - if detectors.StringShannonEntropy(token) < 3.5 { continue } @@ -116,6 +119,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result client := common.SaneHttpClient() isVerified, userResponse, headers, err := s.VerifyGithub(ctx, client, token) + + if !isVerified { + // to avoid false positives for unverified findings + if isKnownNonSensitiveCommonPrefix(matchPrefix) { + continue + } + } + s1.Verified = isVerified s1.SetVerificationError(err, token) diff --git a/pkg/detectors/github/v1/github_old_test.go b/pkg/detectors/github/v1/github_old_test.go index f4a4eea48..d3a319ab7 100644 --- a/pkg/detectors/github/v1/github_old_test.go +++ b/pkg/detectors/github/v1/github_old_test.go @@ -5,29 +5,10 @@ import ( "testing" "github.com/google/go-cmp/cmp" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -var ( - validPattern = `[{ - "_id": "1a8d0cca-e1a9-4318-bc2f-f5658ab2dcb5", - "name": "Github", - "type": "Detector", - "api": true, - "authentication_type": "", - "verification_url": "https://api.example.com/example", - "test_secrets": { - "github_secret": "abc123def4567890abcdef1234567890abcdef12" - }, - "expected_response": "200", - "method": "GET", - "deprecated": false - }]` - secret = "abc123def4567890abcdef1234567890abcdef12" -) - func TestGithub_Pattern(t *testing.T) { d := Scanner{} ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) @@ -38,9 +19,22 @@ func TestGithub_Pattern(t *testing.T) { want []string }{ { - name: "valid pattern", - input: validPattern, - want: []string{secret}, + name: "valid pattern", + input: `[{ + "_id": "1a8d0cca-e1a9-4318-bc2f-f5658ab2dcb5", + "name": "Github", + "type": "Detector", + "api": true, + "authentication_type": "", + "verification_url": "https://api.example.com/example", + "test_secrets": { + "github_secret": "abc123def4567890abcdef1234567890abcdef12" + }, + "expected_response": "200", + "method": "GET", + "deprecated": false + }]`, + want: []string{"abc123def4567890abcdef1234567890abcdef12"}, }, } @@ -86,3 +80,42 @@ func TestGithub_Pattern(t *testing.T) { }) } } + +func Test_isKnownNonSensitiveCommonPrefix(t *testing.T) { + type args struct { + matchPrefix string + } + tests := []struct { + name string + args args + isKnownPrefix bool + }{ + { + name: "repo url", + args: args{matchPrefix: "github.com/repos/symfony/monolog-bridge/zipball/9d14621e59f22c2b6d030d92d37ffe5ae1e60452"}, + isKnownPrefix: true, + }, + { + name: "sha256 hash", + args: args{matchPrefix: "Digest: sha256:f9a92af4d46ca171bffa5c00509414a19d9887c9ed4fe98d1f43757b52600e39"}, + isKnownPrefix: true, + }, + { + name: "real looking token", + args: args{matchPrefix: "github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e"}, + isKnownPrefix: false, + }, + { + name: "github url", + args: args{matchPrefix: "github.com/wrandelshofer/FastDoubleParser/blob/39e123b15b71f29a38a087d16a0bc620fc879aa6/bigint-LICENSE"}, + isKnownPrefix: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isKnownNonSensitiveCommonPrefix(tt.args.matchPrefix); got != tt.isKnownPrefix { + t.Errorf("isKnownNonSensitiveCommonPrefix() = %v, want %v", got, tt.isKnownPrefix) + } + }) + } +}