diff --git a/main.go b/main.go index 7395c43f1..549b942e7 100644 --- a/main.go +++ b/main.go @@ -537,6 +537,7 @@ func run(state overseer.State, logSync func() error) { feature.TlyDetectorEnabled.Store(true) feature.WitDetectorEnabled.Store(true) feature.RevDetectorEnabled.Store(true) + feature.UserDetectorEnabled.Store(true) conf := &config.Config{} if *configFilename != "" { diff --git a/pkg/detectors/user/user.go b/pkg/detectors/user/user.go index 953610140..d0ec714ec 100644 --- a/pkg/detectors/user/user.go +++ b/pkg/detectors/user/user.go @@ -2,71 +2,140 @@ package user import ( "context" + "errors" "fmt" "net/http" + "net/url" "strings" regexp "github.com/wasilibs/go-re2" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" ) -type Scanner struct{} - -// Ensure the Scanner satisfies the interface at compile time. -var _ detectors.Detector = (*Scanner)(nil) +type Scanner struct { + client *http.Client + detectors.DefaultMultiPartCredentialProvider + detectors.EndpointSetter +} var ( - client = common.SaneHttpClient() + // Ensure the Scanner satisfies the interface at compile time. + _ detectors.Detector = (*Scanner)(nil) + _ detectors.EndpointCustomizer = (*Scanner)(nil) - // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"user"}) + `\b([a-zA-Z0-9-._+=]{64})\b`) + defaultClient = detectors.NewClientWithDedup(detectors.DetectorHttpClientWithNoLocalAddresses) + + keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"user"}) + `\b([A-Za-z0-9]{64})\b`) + userURLPat = regexp.MustCompile(`\b([a-z0-9-]+\.user\.com)\b`) + + invalidHosts = simple.NewCache[struct{}]() + errNoHost = errors.New("no such host") ) +func (Scanner) CloudEndpoint() string { return "" } + // Keywords are used for efficiently pre-filtering chunks. // Use identifiers in the secret preferably, or the provider name. func (s Scanner) Keywords() []string { - return []string{"user"} + return []string{"user.com"} +} + +func (s Scanner) getClient() *http.Client { + if s.client != nil { + return s.client + } + return defaultClient } // FromData will find and optionally verify User secrets in a given set of bytes. func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { dataStr := string(data) - matches := keyPat.FindAllStringSubmatch(dataStr, -1) + var uniqueTokens, uniqueURLs = make(map[string]struct{}), make(map[string]struct{}) - for _, match := range matches { - resMatch := strings.TrimSpace(match[1]) + for _, match := range keyPat.FindAllStringSubmatch(dataStr, -1) { + uniqueTokens[strings.TrimSpace(match[1])] = struct{}{} + } - s1 := detectors.Result{ - DetectorType: detector_typepb.DetectorType_User, - Raw: []byte(resMatch), - SecretParts: map[string]string{"key": resMatch}, - } + var foundURLs = make([]string, 0) + for _, match := range userURLPat.FindAllStringSubmatch(dataStr, -1) { + foundURLs = append(foundURLs, match[1]) + } - if verify { - req, err := http.NewRequestWithContext(ctx, "GET", "https://secretscanner.user.com/api/public/users/", nil) - if err != nil { + // Merge found endpoints with any user-configured endpoints. + // Callers must call UseFoundEndpoints(true) to include endpoints extracted from data. + for _, endpoint := range s.Endpoints(foundURLs...) { + endpoint = strings.TrimPrefix(endpoint, "https://") + uniqueURLs[endpoint] = struct{}{} + } + + for token := range uniqueTokens { + for hostname := range uniqueURLs { + if invalidHosts.Exists(hostname) { + delete(uniqueURLs, hostname) continue } - req.Header.Add("Authorization", fmt.Sprintf("Token %s", resMatch)) - res, err := client.Do(req) - if err == nil { - defer func() { _ = res.Body.Close() }() - if res.StatusCode >= 200 && res.StatusCode < 300 { - s1.Verified = true + + endpointURL := url.URL{Scheme: "https", Host: hostname} + s1 := detectors.Result{ + DetectorType: detector_typepb.DetectorType_User, + Raw: []byte(token), + RawV2: []byte(token + ":" + endpointURL.String()), + SecretParts: map[string]string{ + "key": token, + "endpoint": endpointURL.String(), + }, + } + + if verify { + isVerified, verificationErr := verifyUserToken(ctx, s.getClient(), hostname, token) + s1.Verified = isVerified + if verificationErr != nil { + if errors.Is(verificationErr, errNoHost) { + invalidHosts.Set(hostname, struct{}{}) + continue + } + s1.SetVerificationError(verificationErr, token) } } - } - results = append(results, s1) + results = append(results, s1) + } } return results, nil } +func verifyUserToken(ctx context.Context, client *http.Client, hostname, token string) (bool, error) { + u := url.URL{Scheme: "https", Host: hostname, Path: "/api/public/users/"} + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), http.NoBody) + if err != nil { + return false, err + } + req.Header.Set("Authorization", fmt.Sprintf("Token %s", token)) + + res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_User, token+":"+hostname, req) + if err != nil { + if strings.Contains(err.Error(), "no such host") { + return false, errNoHost + } + return false, err + } + defer func() { _ = res.Body.Close() }() + + switch res.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusUnauthorized, http.StatusForbidden, http.StatusGone: + return false, nil + default: + return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) + } +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_User } diff --git a/pkg/detectors/user/user_integration_test.go b/pkg/detectors/user/user_integration_test.go index b48a6ba4a..420edbb63 100644 --- a/pkg/detectors/user/user_integration_test.go +++ b/pkg/detectors/user/user_integration_test.go @@ -9,7 +9,9 @@ import ( "testing" "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/stretchr/testify/require" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/common" @@ -25,6 +27,7 @@ func TestUser_FromChunk(t *testing.T) { } secret := testSecrets.MustGetField("USER") inactiveSecret := testSecrets.MustGetField("USER_INACTIVE") + endpoint := testSecrets.MustGetField("USER_ENDPOINT") type args struct { ctx context.Context @@ -43,13 +46,19 @@ func TestUser_FromChunk(t *testing.T) { s: Scanner{}, args: args{ ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a user secret %s within", secret)), + data: []byte(fmt.Sprintf("user token = %s\nurl = %s", secret, endpoint)), verify: true, }, want: []detectors.Result{ { DetectorType: detector_typepb.DetectorType_User, Verified: true, + Raw: []byte(secret), + RawV2: []byte(secret + ":" + endpoint), + SecretParts: map[string]string{ + "key": secret, + "endpoint": endpoint, + }, }, }, wantErr: false, @@ -59,13 +68,19 @@ func TestUser_FromChunk(t *testing.T) { s: Scanner{}, args: args{ ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a user secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + data: []byte(fmt.Sprintf("user token = %s\nurl = %s", inactiveSecret, endpoint)), verify: true, }, want: []detectors.Result{ { DetectorType: detector_typepb.DetectorType_User, Verified: false, + Raw: []byte(inactiveSecret), + RawV2: []byte(inactiveSecret + ":" + endpoint), + SecretParts: map[string]string{ + "key": inactiveSecret, + "endpoint": endpoint, + }, }, }, wantErr: false, @@ -84,8 +99,9 @@ func TestUser_FromChunk(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + tt.s.UseFoundEndpoints(true) + + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) if (err != nil) != tt.wantErr { t.Errorf("User.FromData() error = %v, wantErr %v", err, tt.wantErr) return @@ -94,15 +110,62 @@ func TestUser_FromChunk(t *testing.T) { if len(got[i].Raw) == 0 { t.Fatalf("no raw secret present: \n %+v", got[i]) } - got[i].Raw = nil + gotErr := "" + if got[i].VerificationError() != nil { + gotErr = got[i].VerificationError().Error() + } + wantErr := "" + if tt.want[i].VerificationError() != nil { + wantErr = tt.want[i].VerificationError().Error() + } + if gotErr != wantErr { + t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError()) + } } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("User.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + ignoreOpts := cmpopts.IgnoreFields( + detectors.Result{}, + "ExtraData", + "verificationError", + "primarySecret", + "chunkOffset", + "chunkOffsetSet", + ) + if diff := cmp.Diff(tt.want, got, ignoreOpts); diff != "" { + t.Errorf("User.FromData() %s diff: (-want +got)\n%s", tt.name, diff) } }) } } +func TestUser_FromChunk_WithCustomEndpoint(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("USER") + endpoint := testSecrets.MustGetField("USER_ENDPOINT") + + s := Scanner{} + s.UseFoundEndpoints(true) + if err := s.SetConfiguredEndpoints(endpoint); err != nil { + t.Fatal("Error in setting configured endpoint") + } + + data := []byte(fmt.Sprintf("user token = %s", secret)) + + got, err := s.FromData(ctx, true, data) + + require.NoError(t, err, "unexpected error from FromData") + require.Greater(t, len(got), 0, "expected at least 1 result") + + expectedRawV2 := []byte(secret + ":" + endpoint) + if string(got[0].RawV2) != string(expectedRawV2) { + t.Errorf("User.FromData() rawV2 mismatch: got %s, want %s", string(got[0].RawV2), string(expectedRawV2)) + } +} + func BenchmarkFromData(benchmark *testing.B) { ctx := context.Background() s := Scanner{} diff --git a/pkg/detectors/user/user_test.go b/pkg/detectors/user/user_test.go index 1847ee565..1fbbcf37f 100644 --- a/pkg/detectors/user/user_test.go +++ b/pkg/detectors/user/user_test.go @@ -12,9 +12,10 @@ import ( ) var ( - validPattern = "OmFxWjhZvCpOeMsgTJdZMas+dlUpr=fa7+.QOKKYvi7RKWyBeHtaLa7_rzMhLrRd" - invalidPattern = "OmFxWjhZvCpOeMsgTJdZMas+dlUpr=fa?+.QOKKYvi7RKWyBeHtaLa7_rzMhLrRd" - keyword = "user" + validKey = "9UgOsRfud4RTyBQpJQFOQiwNQcfeLGHH1DDoxhgzCvBmccmVQ7MYB0ai3LXGZNMf" + validURL = "https://testdetector.user.com" + invalidKey = "OmFxWjhZvCpOeMsgTJdZMas+dlUpr=fa?+.QOKKYvi7RKWyBeHtaLa7_rzMhLrRd" + keyword = "user.com" ) func TestUser_Pattern(t *testing.T) { @@ -26,23 +27,33 @@ func TestUser_Pattern(t *testing.T) { want []string }{ { - name: "valid pattern - with keyword user", - input: fmt.Sprintf("%s token = '%s'", keyword, validPattern), - want: []string{validPattern}, + name: "valid pattern - key and url present", + input: fmt.Sprintf("user token = '%s'\nurl = %s", validKey, validURL), + want: []string{validKey + ":" + validURL}, }, { - name: "valid pattern - ignore duplicate", - input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern), - want: []string{validPattern}, + name: "valid pattern - ignore duplicate keys", + input: fmt.Sprintf("user token = '%s' | '%s'\nurl = %s", validKey, validKey, validURL), + want: []string{validKey + ":" + validURL}, }, { - name: "valid pattern - key out of prefix range", - input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern), + name: "no result - key without url", + input: fmt.Sprintf("user token = '%s'", validKey), want: []string{}, }, { - name: "invalid pattern", - input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern), + name: "no result - url without key", + input: fmt.Sprintf("%s url = %s", keyword, validURL), + want: []string{}, + }, + { + name: "no result - key out of prefix range", + input: fmt.Sprintf("user keyword is not close to the real key in the data\n = '%s'\nurl = %s", validKey, validURL), + want: []string{}, + }, + { + name: "no result - invalid key format", + input: fmt.Sprintf("user = '%s'\nurl = %s", invalidKey, validURL), want: []string{}, }, } @@ -50,12 +61,14 @@ func TestUser_Pattern(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) - if len(matchedDetectors) == 0 { + if len(matchedDetectors) == 0 && len(test.want) > 0 { t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - results, err := d.FromData(context.Background(), false, []byte(test.input)) + s := Scanner{} + s.UseFoundEndpoints(true) + results, err := s.FromData(context.Background(), false, []byte(test.input)) if err != nil { t.Errorf("error = %v", err) return diff --git a/pkg/engine/defaults/defaults.go b/pkg/engine/defaults/defaults.go index a22401c80..f3604c038 100644 --- a/pkg/engine/defaults/defaults.go +++ b/pkg/engine/defaults/defaults.go @@ -811,6 +811,7 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/upwave" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/uri" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/urlscan" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/user" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/userflow" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/userstack" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/vagrantcloudpersonaltoken" @@ -1713,6 +1714,7 @@ func buildDetectorList() []detectors.Detector { &upwave.Scanner{}, &uri.Scanner{}, &urlscan.Scanner{}, + &user.Scanner{}, &userflow.Scanner{}, &userstack.Scanner{}, &vagrantcloudpersonaltoken.Scanner{}, @@ -1796,6 +1798,8 @@ func buildDetectorList() []detectors.Detector { return !feature.WitDetectorEnabled.Load() case *rev.Scanner: return !feature.RevDetectorEnabled.Load() + case *user.Scanner: + return !feature.UserDetectorEnabled.Load() default: return false } diff --git a/pkg/engine/defaults/defaults_test.go b/pkg/engine/defaults/defaults_test.go index 0e02533f2..1983e7b4f 100644 --- a/pkg/engine/defaults/defaults_test.go +++ b/pkg/engine/defaults/defaults_test.go @@ -122,7 +122,6 @@ var excludedFromDefaultList = map[detector_typepb.DetectorType]struct{}{ detector_typepb.DetectorType_IPInfo: {}, detector_typepb.DetectorType_Lob: {}, detector_typepb.DetectorType_Tru: {}, - detector_typepb.DetectorType_User: {}, // Feature flag gated detectors // These should be removed from this list when we remove the feature flag @@ -134,6 +133,7 @@ var excludedFromDefaultList = map[detector_typepb.DetectorType]struct{}{ detector_typepb.DetectorType_TLy: {}, detector_typepb.DetectorType_Wit: {}, detector_typepb.DetectorType_Rev: {}, + detector_typepb.DetectorType_User: {}, // Reserved / special types. detector_typepb.DetectorType_CustomRegex: {}, // added dynamically via engine config, not via buildDetectorList() diff --git a/pkg/feature/feature.go b/pkg/feature/feature.go index f7be41070..a44e0526d 100644 --- a/pkg/feature/feature.go +++ b/pkg/feature/feature.go @@ -24,6 +24,7 @@ var ( TlyDetectorEnabled atomic.Bool WitDetectorEnabled atomic.Bool RevDetectorEnabled atomic.Bool + UserDetectorEnabled atomic.Bool ) type AtomicString struct {