Ignore known common prefix matches for Github V1 detector (#4379)

Co-authored-by: Amaan Ullah <[email protected]>

pkg/detectors/github/v1/github_old.go

@@ -28,8 +28,8 @@ func (Scanner) CloudEndpoint() string { return "https://api.github.com" }
 var (
 	// Oauth token
 	// https://developer.github.com/v3/#oauth2-token-sent-in-a-header
-	// the middle regex `\b[a-zA-Z0-9.\/?=&]{0,40}` is to match the prefix of token match to avoid processing common known patterns
-	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"github", "gh", "pat", "token"}) + `\b[a-zA-Z0-9.\/?=&]{0,40}` + `\b([a-f0-9]{40})\b`)
+	// the middle regex `(?:[a-zA-Z0-9.\/?=&:-]{0,40})` is to match the prefix of token match to avoid processing common known patterns
+	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"github", "gh", "pat", "token"}) + `\b(?:[a-zA-Z0-9.\/?=&:-]{0,40})([a-f0-9]{40})\b`)
 
 	// TODO: Oauth2 client_id and client_secret
 	// https://developer.github.com/v3/#oauth2-keysecret
@@ -73,7 +73,15 @@ var ghFalsePositives = map[detectors.FalsePositive]struct{}{
 }
 
 var ghKnownNonSensitivePrefixes = []string{
-	"avatars.githubusercontent.com", // github avatar urls prefix
+	"avatars.githubusercontent.com",            // GitHub avatar URLs
+	"actions/",                                 // GitHub Actions paths
+	"raw.githubusercontent.com/",               // Raw file URLs from GitHub
+	"api.github.com/repos/",                    // GitHub API repository endpoints
+	"gist.github.com/",                         // GitHub Gist URLs
+	"sha256:",                                  // SHA256 hash prefix
+	"github.com/",                              // General GitHub repo URLs
+	"pipelines.actions.githubusercontent.com/", // GitHub Actions infrastructure
+	"ghcr.io/",                                 // GitHub Container Registry
 }
 
 // FromData will find and optionally verify GitHub secrets in a given set of bytes.
@@ -93,11 +101,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			continue
 		}
 
-		// to avoid false positives
-		if isKnownNonSensitiveCommonPrefix(matchPrefix) {
-			continue
-		}
-
 		if detectors.StringShannonEntropy(token) < 3.5 {
 			continue
 		}
@@ -116,6 +119,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			client := common.SaneHttpClient()
 
 			isVerified, userResponse, headers, err := s.VerifyGithub(ctx, client, token)
+
+			if !isVerified {
+				// to avoid false positives for unverified findings
+				if isKnownNonSensitiveCommonPrefix(matchPrefix) {
+					continue
+				}
+			}
+
 			s1.Verified = isVerified
 			s1.SetVerificationError(err, token)
 

pkg/detectors/github/v1/github_old_test.go

@@ -5,29 +5,10 @@ import (
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
-
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
 )
 
-var (
-	validPattern = `[{
-		"_id": "1a8d0cca-e1a9-4318-bc2f-f5658ab2dcb5",
-		"name": "Github",
-		"type": "Detector",
-		"api": true,
-		"authentication_type": "",
-		"verification_url": "https://api.example.com/example",
-		"test_secrets": {
-			"github_secret": "abc123def4567890abcdef1234567890abcdef12"
-		},
-		"expected_response": "200",
-		"method": "GET",
-		"deprecated": false
-	}]`
-	secret = "abc123def4567890abcdef1234567890abcdef12"
-)
-
 func TestGithub_Pattern(t *testing.T) {
 	d := Scanner{}
 	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
@@ -38,9 +19,22 @@ func TestGithub_Pattern(t *testing.T) {
 		want  []string
 	}{
 		{
-			name:  "valid pattern",
-			input: validPattern,
-			want:  []string{secret},
+			name: "valid pattern",
+			input: `[{
+				"_id": "1a8d0cca-e1a9-4318-bc2f-f5658ab2dcb5",
+				"name": "Github",
+				"type": "Detector",
+				"api": true,
+				"authentication_type": "",
+				"verification_url": "https://api.example.com/example",
+				"test_secrets": {
+					"github_secret": "abc123def4567890abcdef1234567890abcdef12"
+				},
+				"expected_response": "200",
+				"method": "GET",
+				"deprecated": false
+			}]`,
+			want: []string{"abc123def4567890abcdef1234567890abcdef12"},
 		},
 	}
 
@@ -86,3 +80,42 @@ func TestGithub_Pattern(t *testing.T) {
 		})
 	}
 }
+
+func Test_isKnownNonSensitiveCommonPrefix(t *testing.T) {
+	type args struct {
+		matchPrefix string
+	}
+	tests := []struct {
+		name          string
+		args          args
+		isKnownPrefix bool
+	}{
+		{
+			name:          "repo url",
+			args:          args{matchPrefix: "github.com/repos/symfony/monolog-bridge/zipball/9d14621e59f22c2b6d030d92d37ffe5ae1e60452"},
+			isKnownPrefix: true,
+		},
+		{
+			name:          "sha256 hash",
+			args:          args{matchPrefix: "Digest: sha256:f9a92af4d46ca171bffa5c00509414a19d9887c9ed4fe98d1f43757b52600e39"},
+			isKnownPrefix: true,
+		},
+		{
+			name:          "real looking token",
+			args:          args{matchPrefix: "github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e"},
+			isKnownPrefix: false,
+		},
+		{
+			name:          "github url",
+			args:          args{matchPrefix: "github.com/wrandelshofer/FastDoubleParser/blob/39e123b15b71f29a38a087d16a0bc620fc879aa6/bigint-LICENSE"},
+			isKnownPrefix: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := isKnownNonSensitiveCommonPrefix(tt.args.matchPrefix); got != tt.isKnownPrefix {
+				t.Errorf("isKnownNonSensitiveCommonPrefix() = %v, want %v", got, tt.isKnownPrefix)
+			}
+		})
+	}
+}