[INS-472] [INS-515] Add user detector to defaults.go, gate it behind feat flag, update verification logic and add custom ep configuration (#5007)

* Rebase on top of origin/main * updated detector regex * resolved comments * implmented endpoint customizer interface
2026-06-15 18:13:35 +05:00
parent 84a2b33c9f
commit 7bcf376421
7 changed files with 203 additions and 52 deletions
@@ -537,6 +537,7 @@ func run(state overseer.State, logSync func() error) {
 	feature.TlyDetectorEnabled.Store(true)
 	feature.WitDetectorEnabled.Store(true)
 	feature.RevDetectorEnabled.Store(true)
+	feature.UserDetectorEnabled.Store(true)

 	conf := &config.Config{}
 	if *configFilename != "" {
@@ -2,71 +2,140 @@ package user

 import (
 	"context"
+	"errors"
 	"fmt"
 	"net/http"
+	"net/url"
 	"strings"

 	regexp "github.com/wasilibs/go-re2"

-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb"
 )

-type Scanner struct{}
-
-// Ensure the Scanner satisfies the interface at compile time.
-var _ detectors.Detector = (*Scanner)(nil)
+type Scanner struct {
+	client *http.Client
+	detectors.DefaultMultiPartCredentialProvider
+	detectors.EndpointSetter
+}

 var (
-	client = common.SaneHttpClient()
+	// Ensure the Scanner satisfies the interface at compile time.
+	_ detectors.Detector           = (*Scanner)(nil)
+	_ detectors.EndpointCustomizer = (*Scanner)(nil)

-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"user"}) + `\b([a-zA-Z0-9-._+=]{64})\b`)
+	defaultClient = detectors.NewClientWithDedup(detectors.DetectorHttpClientWithNoLocalAddresses)
+
+	keyPat     = regexp.MustCompile(detectors.PrefixRegex([]string{"user"}) + `\b([A-Za-z0-9]{64})\b`)
+	userURLPat = regexp.MustCompile(`\b([a-z0-9-]+\.user\.com)\b`)
+
+	invalidHosts = simple.NewCache[struct{}]()
+	errNoHost    = errors.New("no such host")
 )

+func (Scanner) CloudEndpoint() string { return "" }
+
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
 func (s Scanner) Keywords() []string {
-	return []string{"user"}
+	return []string{"user.com"}
+}
+
+func (s Scanner) getClient() *http.Client {
+	if s.client != nil {
+		return s.client
+	}
+	return defaultClient
 }

 // FromData will find and optionally verify User secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
 	dataStr := string(data)

-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	var uniqueTokens, uniqueURLs = make(map[string]struct{}), make(map[string]struct{})

-	for _, match := range matches {
-		resMatch := strings.TrimSpace(match[1])
+	for _, match := range keyPat.FindAllStringSubmatch(dataStr, -1) {
+		uniqueTokens[strings.TrimSpace(match[1])] = struct{}{}
+	}

-		s1 := detectors.Result{
-			DetectorType: detector_typepb.DetectorType_User,
-			Raw:          []byte(resMatch),
-			SecretParts:  map[string]string{"key": resMatch},
-		}
+	var foundURLs = make([]string, 0)
+	for _, match := range userURLPat.FindAllStringSubmatch(dataStr, -1) {
+		foundURLs = append(foundURLs, match[1])
+	}

-		if verify {
-			req, err := http.NewRequestWithContext(ctx, "GET", "https://secretscanner.user.com/api/public/users/", nil)
-			if err != nil {
+	// Merge found endpoints with any user-configured endpoints.
+	// Callers must call UseFoundEndpoints(true) to include endpoints extracted from data.
+	for _, endpoint := range s.Endpoints(foundURLs...) {
+		endpoint = strings.TrimPrefix(endpoint, "https://")
+		uniqueURLs[endpoint] = struct{}{}
+	}
+
+	for token := range uniqueTokens {
+		for hostname := range uniqueURLs {
+			if invalidHosts.Exists(hostname) {
+				delete(uniqueURLs, hostname)
 				continue
 			}
-			req.Header.Add("Authorization", fmt.Sprintf("Token %s", resMatch))
-			res, err := client.Do(req)
-			if err == nil {
-				defer func() { _ = res.Body.Close() }()
-				if res.StatusCode >= 200 && res.StatusCode < 300 {
-					s1.Verified = true
+
+			endpointURL := url.URL{Scheme: "https", Host: hostname}
+			s1 := detectors.Result{
+				DetectorType: detector_typepb.DetectorType_User,
+				Raw:          []byte(token),
+				RawV2:        []byte(token + ":" + endpointURL.String()),
+				SecretParts: map[string]string{
+					"key":      token,
+					"endpoint": endpointURL.String(),
+				},
+			}
+
+			if verify {
+				isVerified, verificationErr := verifyUserToken(ctx, s.getClient(), hostname, token)
+				s1.Verified = isVerified
+				if verificationErr != nil {
+					if errors.Is(verificationErr, errNoHost) {
+						invalidHosts.Set(hostname, struct{}{})
+						continue
+					}
+					s1.SetVerificationError(verificationErr, token)
 				}
 			}
-		}

-		results = append(results, s1)
+			results = append(results, s1)
+		}
 	}

 	return results, nil
 }

+func verifyUserToken(ctx context.Context, client *http.Client, hostname, token string) (bool, error) {
+	u := url.URL{Scheme: "https", Host: hostname, Path: "/api/public/users/"}
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), http.NoBody)
+	if err != nil {
+		return false, err
+	}
+	req.Header.Set("Authorization", fmt.Sprintf("Token %s", token))
+
+	res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_User, token+":"+hostname, req)
+	if err != nil {
+		if strings.Contains(err.Error(), "no such host") {
+			return false, errNoHost
+		}
+		return false, err
+	}
+	defer func() { _ = res.Body.Close() }()
+
+	switch res.StatusCode {
+	case http.StatusOK:
+		return true, nil
+	case http.StatusUnauthorized, http.StatusForbidden, http.StatusGone:
+		return false, nil
+	default:
+		return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
+	}
+}
+
 func (s Scanner) Type() detector_typepb.DetectorType {
 	return detector_typepb.DetectorType_User
 }
@@ -9,7 +9,9 @@ import (
 	"testing"
 	"time"

-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/stretchr/testify/require"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"

 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -25,6 +27,7 @@ func TestUser_FromChunk(t *testing.T) {
 	}
 	secret := testSecrets.MustGetField("USER")
 	inactiveSecret := testSecrets.MustGetField("USER_INACTIVE")
+	endpoint := testSecrets.MustGetField("USER_ENDPOINT")

 	type args struct {
 		ctx    context.Context
@@ -43,13 +46,19 @@ func TestUser_FromChunk(t *testing.T) {
 			s:    Scanner{},
 			args: args{
 				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a user secret %s within", secret)),
+				data:   []byte(fmt.Sprintf("user token = %s\nurl = %s", secret, endpoint)),
 				verify: true,
 			},
 			want: []detectors.Result{
 				{
 					DetectorType: detector_typepb.DetectorType_User,
 					Verified:     true,
+					Raw:          []byte(secret),
+					RawV2:        []byte(secret + ":" + endpoint),
+					SecretParts: map[string]string{
+						"key":      secret,
+						"endpoint": endpoint,
+					},
 				},
 			},
 			wantErr: false,
@@ -59,13 +68,19 @@ func TestUser_FromChunk(t *testing.T) {
 			s:    Scanner{},
 			args: args{
 				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a user secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				data:   []byte(fmt.Sprintf("user token = %s\nurl = %s", inactiveSecret, endpoint)),
 				verify: true,
 			},
 			want: []detectors.Result{
 				{
 					DetectorType: detector_typepb.DetectorType_User,
 					Verified:     false,
+					Raw:          []byte(inactiveSecret),
+					RawV2:        []byte(inactiveSecret + ":" + endpoint),
+					SecretParts: map[string]string{
+						"key":      inactiveSecret,
+						"endpoint": endpoint,
+					},
 				},
 			},
 			wantErr: false,
@@ -84,8 +99,9 @@ func TestUser_FromChunk(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			s := Scanner{}
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			tt.s.UseFoundEndpoints(true)
+
+			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("User.FromData() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -94,15 +110,62 @@ func TestUser_FromChunk(t *testing.T) {
 				if len(got[i].Raw) == 0 {
 					t.Fatalf("no raw secret present: \n %+v", got[i])
 				}
-				got[i].Raw = nil
+				gotErr := ""
+				if got[i].VerificationError() != nil {
+					gotErr = got[i].VerificationError().Error()
+				}
+				wantErr := ""
+				if tt.want[i].VerificationError() != nil {
+					wantErr = tt.want[i].VerificationError().Error()
+				}
+				if gotErr != wantErr {
+					t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())
+				}
 			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("User.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			ignoreOpts := cmpopts.IgnoreFields(
+				detectors.Result{},
+				"ExtraData",
+				"verificationError",
+				"primarySecret",
+				"chunkOffset",
+				"chunkOffsetSet",
+			)
+			if diff := cmp.Diff(tt.want, got, ignoreOpts); diff != "" {
+				t.Errorf("User.FromData() %s diff: (-want +got)\n%s", tt.name, diff)
 			}
 		})
 	}
 }

+func TestUser_FromChunk_WithCustomEndpoint(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("USER")
+	endpoint := testSecrets.MustGetField("USER_ENDPOINT")
+
+	s := Scanner{}
+	s.UseFoundEndpoints(true)
+	if err := s.SetConfiguredEndpoints(endpoint); err != nil {
+		t.Fatal("Error in setting configured endpoint")
+	}
+
+	data := []byte(fmt.Sprintf("user token = %s", secret))
+
+	got, err := s.FromData(ctx, true, data)
+
+	require.NoError(t, err, "unexpected error from FromData")
+	require.Greater(t, len(got), 0, "expected at least 1 result")
+
+	expectedRawV2 := []byte(secret + ":" + endpoint)
+	if string(got[0].RawV2) != string(expectedRawV2) {
+		t.Errorf("User.FromData() rawV2 mismatch: got %s, want %s", string(got[0].RawV2), string(expectedRawV2))
+	}
+}
+
 func BenchmarkFromData(benchmark *testing.B) {
 	ctx := context.Background()
 	s := Scanner{}
@@ -12,9 +12,10 @@ import (
 )

 var (
-	validPattern   = "OmFxWjhZvCpOeMsgTJdZMas+dlUpr=fa7+.QOKKYvi7RKWyBeHtaLa7_rzMhLrRd"
-	invalidPattern = "OmFxWjhZvCpOeMsgTJdZMas+dlUpr=fa?+.QOKKYvi7RKWyBeHtaLa7_rzMhLrRd"
-	keyword        = "user"
+	validKey   = "9UgOsRfud4RTyBQpJQFOQiwNQcfeLGHH1DDoxhgzCvBmccmVQ7MYB0ai3LXGZNMf"
+	validURL   = "https://testdetector.user.com"
+	invalidKey = "OmFxWjhZvCpOeMsgTJdZMas+dlUpr=fa?+.QOKKYvi7RKWyBeHtaLa7_rzMhLrRd"
+	keyword    = "user.com"
 )

 func TestUser_Pattern(t *testing.T) {
@@ -26,23 +27,33 @@ func TestUser_Pattern(t *testing.T) {
 		want  []string
 	}{
 		{
-			name:  "valid pattern - with keyword user",
-			input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
-			want:  []string{validPattern},
+			name:  "valid pattern - key and url present",
+			input: fmt.Sprintf("user token = '%s'\nurl = %s", validKey, validURL),
+			want:  []string{validKey + ":" + validURL},
 		},
 		{
-			name:  "valid pattern - ignore duplicate",
-			input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
-			want:  []string{validPattern},
+			name:  "valid pattern - ignore duplicate keys",
+			input: fmt.Sprintf("user token = '%s' | '%s'\nurl = %s", validKey, validKey, validURL),
+			want:  []string{validKey + ":" + validURL},
 		},
 		{
-			name:  "valid pattern - key out of prefix range",
-			input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+			name:  "no result - key without url",
+			input: fmt.Sprintf("user token = '%s'", validKey),
 			want:  []string{},
 		},
 		{
-			name:  "invalid pattern",
-			input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+			name:  "no result - url without key",
+			input: fmt.Sprintf("%s url = %s", keyword, validURL),
+			want:  []string{},
+		},
+		{
+			name:  "no result - key out of prefix range",
+			input: fmt.Sprintf("user keyword is not close to the real key in the data\n = '%s'\nurl = %s", validKey, validURL),
+			want:  []string{},
+		},
+		{
+			name:  "no result - invalid key format",
+			input: fmt.Sprintf("user = '%s'\nurl = %s", invalidKey, validURL),
 			want:  []string{},
 		},
 	}
@@ -50,12 +61,14 @@ func TestUser_Pattern(t *testing.T) {
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-			if len(matchedDetectors) == 0 {
+			if len(matchedDetectors) == 0 && len(test.want) > 0 {
 				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}

-			results, err := d.FromData(context.Background(), false, []byte(test.input))
+			s := Scanner{}
+			s.UseFoundEndpoints(true)
+			results, err := s.FromData(context.Background(), false, []byte(test.input))
 			if err != nil {
 				t.Errorf("error = %v", err)
 				return
@@ -811,6 +811,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/upwave"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/uri"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/urlscan"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/user"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/userflow"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/userstack"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/vagrantcloudpersonaltoken"
@@ -1713,6 +1714,7 @@ func buildDetectorList() []detectors.Detector {
 		&upwave.Scanner{},
 		&uri.Scanner{},
 		&urlscan.Scanner{},
+		&user.Scanner{},
 		&userflow.Scanner{},
 		&userstack.Scanner{},
 		&vagrantcloudpersonaltoken.Scanner{},
@@ -1796,6 +1798,8 @@ func buildDetectorList() []detectors.Detector {
 			return !feature.WitDetectorEnabled.Load()
 		case *rev.Scanner:
 			return !feature.RevDetectorEnabled.Load()
+		case *user.Scanner:
+			return !feature.UserDetectorEnabled.Load()
 		default:
 			return false
 		}
@@ -122,7 +122,6 @@ var excludedFromDefaultList = map[detector_typepb.DetectorType]struct{}{
 	detector_typepb.DetectorType_IPInfo: {},
 	detector_typepb.DetectorType_Lob:    {},
 	detector_typepb.DetectorType_Tru:    {},
-	detector_typepb.DetectorType_User:   {},

 	// Feature flag gated detectors
 	// These should be removed from this list when we remove the feature flag
@@ -134,6 +133,7 @@ var excludedFromDefaultList = map[detector_typepb.DetectorType]struct{}{
 	detector_typepb.DetectorType_TLy:           {},
 	detector_typepb.DetectorType_Wit:           {},
 	detector_typepb.DetectorType_Rev:           {},
+	detector_typepb.DetectorType_User:          {},

 	// Reserved / special types.
 	detector_typepb.DetectorType_CustomRegex: {}, // added dynamically via engine config, not via buildDetectorList()
@@ -24,6 +24,7 @@ var (
 	TlyDetectorEnabled             atomic.Bool
 	WitDetectorEnabled             atomic.Bool
 	RevDetectorEnabled             atomic.Bool
+	UserDetectorEnabled            atomic.Bool
 )

 type AtomicString struct {