diff --git a/pkg/detectors/postgres/postgres.go b/pkg/detectors/postgres/postgres.go index f22291a6e..3a7449e05 100644 --- a/pkg/detectors/postgres/postgres.go +++ b/pkg/detectors/postgres/postgres.go @@ -55,6 +55,33 @@ var ( type Scanner struct { detectors.DefaultMultiPartCredentialProvider detectLoopback bool // Automated tests run against localhost, but we want to ignore those results in the wild + ignorePatterns []*regexp.Regexp +} + +func New(opts ...func(*Scanner)) *Scanner { + scanner := &Scanner{ + ignorePatterns: []*regexp.Regexp{}, + } + for _, opt := range opts { + opt(scanner) + } + + return scanner +} + +func WithIgnorePattern(ignoreStrings []string) func(*Scanner) { + return func(s *Scanner) { + var ignorePatterns []*regexp.Regexp + for _, ignoreString := range ignoreStrings { + ignorePattern, err := regexp.Compile(ignoreString) + if err != nil { + panic(fmt.Sprintf("%s is not a valid regex, error received: %v", ignoreString, err)) + } + ignorePatterns = append(ignorePatterns, ignorePattern) + } + + s.ignorePatterns = ignorePatterns + } } var _ detectors.Detector = (*Scanner)(nil) @@ -66,7 +93,7 @@ func (s Scanner) Keywords() []string { func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) ([]detectors.Result, error) { var results []detectors.Result - candidateParamSets := findUriMatches(data) + candidateParamSets := findUriMatches(data, s.ignorePatterns) for _, params := range candidateParamSets { if common.IsDone(ctx) { @@ -161,9 +188,12 @@ func (s Scanner) IsFalsePositive(_ detectors.Result) (bool, string) { return false, "" } -func findUriMatches(data []byte) []map[string]string { +func findUriMatches(data []byte, ignorePatterns []*regexp.Regexp) []map[string]string { var matches []map[string]string for _, uri := range uriPattern.FindAll(data, -1) { + if shouldIgnore(uri, ignorePatterns) { + continue + } // Capture the database type (e.g., "postgres" or "postgresql") dbTypeMatch := uriPattern.FindSubmatch(uri) if len(dbTypeMatch) < 2 { @@ -188,6 +218,15 @@ func findUriMatches(data []byte) []map[string]string { return matches } +func shouldIgnore(uri []byte, ignorePatterns []*regexp.Regexp) bool { + for _, ignore := range ignorePatterns { + if ignore.Match(uri) { + return true + } + } + return false +} + // getDeadlineInSeconds gets the deadline from the context in seconds. If there // is no deadline, false is returned. If the deadline is already exceeded, a // negative or 0 value will be returned. diff --git a/pkg/detectors/postgres/postgres_test.go b/pkg/detectors/postgres/postgres_test.go index 0570cca9a..85f5b82eb 100644 --- a/pkg/detectors/postgres/postgres_test.go +++ b/pkg/detectors/postgres/postgres_test.go @@ -81,3 +81,18 @@ func TestPostgres_Pattern(t *testing.T) { }) } } + +func TestPostgres_FromDataWithIgnorePattern(t *testing.T) { + s := New( + WithIgnorePattern([]string{ + `1\.2\.3\.4`, + })) + got, err := s.FromData(context.Background(), false, []byte(validUriPattern)) + if err != nil { + t.Errorf("FromData() error = %v", err) + return + } + if len(got) != 0 { + t.Errorf("expected no results, but got %d", len(got)) + } +} diff --git a/pkg/detectors/sqlserver/sqlserver.go b/pkg/detectors/sqlserver/sqlserver.go index 283710752..78d6bcd6a 100644 --- a/pkg/detectors/sqlserver/sqlserver.go +++ b/pkg/detectors/sqlserver/sqlserver.go @@ -18,7 +18,35 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" ) -type Scanner struct{} +type Scanner struct { + ignorePatterns []*regexp.Regexp +} + +func New(opts ...func(*Scanner)) *Scanner { + scanner := &Scanner{ + ignorePatterns: []*regexp.Regexp{}, + } + for _, opt := range opts { + opt(scanner) + } + + return scanner +} + +func WithIgnorePattern(ignoreStrings []string) func(*Scanner) { + return func(s *Scanner) { + var ignorePatterns []*regexp.Regexp + for _, ignoreString := range ignoreStrings { + ignorePattern, err := regexp.Compile(ignoreString) + if err != nil { + panic(fmt.Sprintf("%s is not a valid regex, error received: %v", ignoreString, err)) + } + ignorePatterns = append(ignorePatterns, ignorePattern) + } + + s.ignorePatterns = ignorePatterns + } +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) @@ -38,6 +66,9 @@ func (s Scanner) Keywords() []string { func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { matches := pattern.FindAllStringSubmatch(string(data), -1) for _, match := range matches { + if shouldIgnore(match[1], s.ignorePatterns) { + continue + } paramsUnsafe, err := msdsn.Parse(match[1]) if err != nil { continue @@ -80,6 +111,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func shouldIgnore(uri string, ignorePatterns []*regexp.Regexp) bool { + for _, ignore := range ignorePatterns { + if ignore.MatchString(uri) { + return true + } + } + return false +} + var ping = func(ctx context.Context, config msdsn.Config) (bool, error) { // TCP connectivity check to prevent indefinite hangs address := net.JoinHostPort(config.Host, strconv.Itoa(int(config.Port))) diff --git a/pkg/detectors/sqlserver/sqlserver_test.go b/pkg/detectors/sqlserver/sqlserver_test.go index fc5748b71..8c376eb06 100644 --- a/pkg/detectors/sqlserver/sqlserver_test.go +++ b/pkg/detectors/sqlserver/sqlserver_test.go @@ -1,7 +1,13 @@ package sqlserver import ( + "context" + "fmt" "testing" + + "github.com/google/go-cmp/cmp" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) func TestSQLServer_Pattern(t *testing.T) { @@ -18,3 +24,87 @@ func TestSQLServer_Pattern(t *testing.T) { t.Errorf("SQLServer.pattern: did not find connection string in xml format") } } + +const ( + validConnStr = `builder.Services.AddDbContext(optionsBuilder => optionsBuilder.UseSqlServer("Server=localhost;Initial Catalog=master;User ID=sa;Password=P@ssw0rd!;Persist Security Info=true;MultipleActiveResultSets=true;"));` + validParsedStr = `sqlserver://sa:P%40ssw0rd%21@localhost?database=master&dial+timeout=15&disableretry=false` + invalidConnStr = `some random text without connection string` +) + +func TestSqlServer_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { + name string + input string + want []string + }{ + { + name: "valid pattern - with keyword sql", + input: fmt.Sprintf(`sql - %s`, validConnStr), + want: []string{validParsedStr}, + }, + { + name: "invalid pattern", + input: fmt.Sprintf("sql=%s", invalidConnStr), + want: []string{}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) + return + } + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return + } + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} + } + } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } + }) + } +} + +func TestSqlServer_FromDataWithIgnorePattern(t *testing.T) { + s := New( + WithIgnorePattern([]string{ + `^Server=localhost`, + })) + got, err := s.FromData(context.Background(), false, []byte("Server=localhost;Initial Catalog=master;User ID=sa;Password=P@ssw0rd!;Persist Security Info=true;MultipleActiveResultSets=true")) + if err != nil { + t.Errorf("FromData() error = %v", err) + return + } + if len(got) != 0 { + t.Errorf("expected no results, but got %d", len(got)) + } +}