From d86254ee022cc6533192c5601c143a42706c1004 Mon Sep 17 00:00:00 2001 From: Mario Corchero Date: Mon, 18 May 2026 13:24:40 +0200 Subject: [PATCH] feat: add host, db and username to ExtraData for database detectors (#4849) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Populate ExtraData with parsed fields for all database connection string detectors (MongoDB, PostgreSQL, Redis, JDBC). This surfaces useful metadata about detected credentials. The parsing logic already existed in each detector — this change exposes the extracted values in the result's ExtraData map alongside any pre-existing fields (rotation_guide, sslmode, etc.). --- pkg/detectors/jdbc/jdbc.go | 44 ++++++++---- pkg/detectors/jdbc/jdbc_integration_test.go | 2 + pkg/detectors/jdbc/jdbc_test.go | 70 +++++++++++++++++++ pkg/detectors/mongodb/mongodb.go | 29 ++++++-- .../mongodb/mongodb_integration_test.go | 4 +- pkg/detectors/mongodb/mongodb_test.go | 61 ++++++++++++++++ pkg/detectors/postgres/postgres.go | 13 ++++ .../postgres/postgres_integration_test.go | 2 +- pkg/detectors/postgres/postgres_test.go | 57 +++++++++++++-- pkg/detectors/redis/redis.go | 17 ++++- pkg/detectors/redis/redis_integration_test.go | 4 ++ pkg/detectors/redis/redis_test.go | 42 +++++++++++ 12 files changed, 311 insertions(+), 34 deletions(-) diff --git a/pkg/detectors/jdbc/jdbc.go b/pkg/detectors/jdbc/jdbc.go index e3ef7a235..e07ac81ac 100644 --- a/pkg/detectors/jdbc/jdbc.go +++ b/pkg/detectors/jdbc/jdbc.go @@ -86,24 +86,38 @@ matchLoop: SecretParts: map[string]string{"connection_string": jdbcConn}, } - if verify { - j, err := NewJDBC(logCtx, jdbcConn) - if err != nil { - continue + // Try to parse connection info for ExtraData regardless of verification. + if j, parseErr := NewJDBC(logCtx, jdbcConn); parseErr == nil { + if info := j.GetConnectionInfo(); info != nil { + extraData := make(map[string]string) + if info.Host != "" { + extraData["host"] = info.Host + } + if info.User != "" { + extraData["username"] = info.User + } + if info.Database != "" { + extraData["database"] = info.Database + } + if len(extraData) > 0 { + result.ExtraData = extraData + } } - ctx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - pingRes := j.ping(ctx) - result.Verified = pingRes.err == nil - // If there's a ping error that is marked as "determinate" we throw it away. We do this because this was the - // behavior before tri-state verification was introduced and preserving it allows us to gradually migrate - // detectors to use tri-state verification. - if pingRes.err != nil && !pingRes.determinate { - err = pingRes.err - result.SetVerificationError(err, jdbcConn) + if verify { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + pingRes := j.ping(ctx) + result.Verified = pingRes.err == nil + // If there's a ping error that is marked as "determinate" we throw it away. We do this because this was the + // behavior before tri-state verification was introduced and preserving it allows us to gradually migrate + // detectors to use tri-state verification. + if pingRes.err != nil && !pingRes.determinate { + result.SetVerificationError(pingRes.err, jdbcConn) + } } - // TODO: specialized redaction + } else if verify { + continue } results = append(results, result) diff --git a/pkg/detectors/jdbc/jdbc_integration_test.go b/pkg/detectors/jdbc/jdbc_integration_test.go index f20b2a1fd..ea2e71d04 100644 --- a/pkg/detectors/jdbc/jdbc_integration_test.go +++ b/pkg/detectors/jdbc/jdbc_integration_test.go @@ -195,6 +195,7 @@ func TestJdbcVerified(t *testing.T) { t.Fatal("no raw secret present") } got[i].Raw = nil + got[i].ExtraData = nil } if diff := pretty.Compare(got, tt.want); diff != "" { t.Errorf("Jdbc.FromData() %s diff: (-got +want)\n%s", tt.name, diff) @@ -323,6 +324,7 @@ func TestJdbc_FromChunk(t *testing.T) { } got[i].Raw = nil got[i].SecretParts = nil + got[i].ExtraData = nil } if diff := pretty.Compare(got, tt.want); diff != "" { t.Errorf("Jdbc.FromData() %s diff: (-got +want)\n%s", tt.name, diff) diff --git a/pkg/detectors/jdbc/jdbc_test.go b/pkg/detectors/jdbc/jdbc_test.go index 2ba0c937a..267346caf 100644 --- a/pkg/detectors/jdbc/jdbc_test.go +++ b/pkg/detectors/jdbc/jdbc_test.go @@ -151,6 +151,76 @@ func TestJdbc_Pattern(t *testing.T) { } } +func TestJdbc_ExtraData(t *testing.T) { + tests := []struct { + name string + data string + wantHost string + wantUsername string + wantDatabase string + }{ + { + name: "mysql with basic auth", + data: `jdbc:mysql://root:password@localhost:3306/testdb`, + wantHost: "tcp(localhost:3306)", + wantUsername: "root", + wantDatabase: "testdb", + }, + { + name: "postgresql with basic auth", + data: `jdbc:postgresql://postgres:secret@dbhost:5432/mydb`, + wantHost: "dbhost:5432", + wantUsername: "postgres", + wantDatabase: "mydb", + }, + { + name: "sqlserver with semicolon params", + data: `jdbc:sqlserver://server.example.com:1433;database=testdb;user=sa;password=Pass123`, + wantHost: "server.example.com:1433", + wantUsername: "sa", + wantDatabase: "testdb", + }, + { + name: "mysql with query params for credentials", + data: `jdbc:mysql://dbhost:3307/testdb?user=admin&password=secret`, + wantHost: "tcp(dbhost:3307)", + wantUsername: "admin", + wantDatabase: "testdb", + }, + { + name: "postgresql with query params for credentials", + data: `jdbc:postgresql://localhost:1521/testdb?sslmode=disable&password=testpassword&user=testuser`, + wantHost: "localhost:1521", + wantUsername: "testuser", + wantDatabase: "testdb", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + results, err := s.FromData(context.Background(), false, []byte(tt.data)) + require.NoError(t, err) + require.NotEmpty(t, results, "expected at least one result") + + r := results[0] + assert.Equal(t, tt.wantHost, r.ExtraData["host"]) + assert.Equal(t, tt.wantUsername, r.ExtraData["username"]) + assert.Equal(t, tt.wantDatabase, r.ExtraData["database"]) + }) + } +} + +func TestJdbc_ExtraData_UnsupportedSubprotocol(t *testing.T) { + // For unsupported subprotocols (e.g., sqlite), ExtraData should be nil + // because we can't parse connection info, but the result should still be returned. + s := Scanner{} + results, err := s.FromData(context.Background(), false, []byte(`jdbc:sqlite:/data/test.db`)) + require.NoError(t, err) + require.NotEmpty(t, results, "expected at least one result") + assert.Nil(t, results[0].ExtraData, "expected nil ExtraData for unsupported subprotocol") +} + func TestJdbc_FromDataWithIgnorePattern(t *testing.T) { type args struct { ctx context.Context diff --git a/pkg/detectors/mongodb/mongodb.go b/pkg/detectors/mongodb/mongodb.go index 810b46fb2..0aee27c5c 100644 --- a/pkg/detectors/mongodb/mongodb.go +++ b/pkg/detectors/mongodb/mongodb.go @@ -45,7 +45,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result logger := logContext.AddLogger(ctx).Logger().WithName("mongodb") dataStr := string(data) - uniqueMatches := make(map[string]string) + type mongoMatch struct { + password string + parsedURL *url.URL + } + uniqueMatches := make(map[string]mongoMatch) for _, match := range connStrPat.FindAllStringSubmatch(dataStr, -1) { // Filter out common placeholder passwords. password := match[3] @@ -78,16 +82,27 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result connUrl.RawQuery = params.Encode() connStr = connUrl.String() - uniqueMatches[connStr] = password + uniqueMatches[connStr] = mongoMatch{password: password, parsedURL: connUrl} } - for connStr, password := range uniqueMatches { + for connStr, m := range uniqueMatches { + extraData := map[string]string{ + "rotation_guide": "https://howtorotate.com/docs/tutorials/mongo/", + } + if m.parsedURL.Host != "" { + extraData["host"] = m.parsedURL.Host + } + if m.parsedURL.User != nil && m.parsedURL.User.Username() != "" { + extraData["username"] = m.parsedURL.User.Username() + } + if db := strings.TrimPrefix(m.parsedURL.Path, "/"); db != "" { + extraData["database"] = db + } + r := detectors.Result{ DetectorType: detector_typepb.DetectorType_MongoDB, Raw: []byte(connStr), - ExtraData: map[string]string{ - "rotation_guide": "https://howtorotate.com/docs/tutorials/mongo/", - }, + ExtraData: extraData, SecretParts: map[string]string{"key": connStr}, } @@ -102,7 +117,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if isErrDeterminate(vErr) { continue } - r.SetVerificationError(vErr, password) + r.SetVerificationError(vErr, m.password) } results = append(results, r) } diff --git a/pkg/detectors/mongodb/mongodb_integration_test.go b/pkg/detectors/mongodb/mongodb_integration_test.go index 60a6b5e65..fa1557f5c 100644 --- a/pkg/detectors/mongodb/mongodb_integration_test.go +++ b/pkg/detectors/mongodb/mongodb_integration_test.go @@ -173,7 +173,7 @@ func TestIntegrationMongoDB_FromChunk(t *testing.T) { t.Fatalf("wantVerificationErr = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError()) } } - ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "verificationError") + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "verificationError", "SecretParts", "ExtraData") if diff := cmp.Diff(tt.want, got, ignoreOpts); diff != "" { t.Errorf("MongoDB.FromData() %s diff: (-got +want)\n%s", tt.name, diff) } @@ -310,7 +310,7 @@ func TestMongoDB_FromChunk(t *testing.T) { t.Fatalf("wantVerificationErr = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError()) } } - ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "verificationError") + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "verificationError", "SecretParts", "ExtraData") if diff := cmp.Diff(tt.want, got, ignoreOpts); diff != "" { t.Errorf("MongoDB.FromData() %s diff: (-got +want)\n%s", tt.name, diff) } diff --git a/pkg/detectors/mongodb/mongodb_test.go b/pkg/detectors/mongodb/mongodb_test.go index 9482fcca7..2836bcff0 100644 --- a/pkg/detectors/mongodb/mongodb_test.go +++ b/pkg/detectors/mongodb/mongodb_test.go @@ -3,8 +3,69 @@ package mongodb import ( "context" "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +func TestMongoDB_ExtraData(t *testing.T) { + tests := []struct { + name string + data string + wantHost string + wantUsername string + wantDatabase string + }{ + { + name: "single host with port", + data: `mongodb://myDBReader:D1fficultP%40ssw0rd@mongodb0.example.com:27017`, + wantHost: "mongodb0.example.com:27017", + wantUsername: "myDBReader", + }, + { + name: "single host without port", + data: `mongodb://myDBReader:D1fficultP%40ssw0rd@mongodb0.example.com`, + wantHost: "mongodb0.example.com", + wantUsername: "myDBReader", + }, + { + name: "with options and no database", + data: `mongodb://username:password@host.docker.internal:27018/?authMechanism=PLAIN&tls=true`, + wantHost: "host.docker.internal:27018", + wantUsername: "username", + }, + { + name: "cosmos db style with database", + data: `mongodb://agenda-live:m21w7PFfRXQwfHZU1Fgx0rTX29ZBQaWMODLeAjsmyslVcMmcmy6CnLyu3byVDtdLYcCokze8lIE4KyAgSCGZxQ==@agenda-live.mongo.cosmos.azure.com:10255/csb-db?retryWrites=false&ssl=true&replicaSet=globaldb&maxIdleTimeMS=120000&appName=@agenda-live@`, + wantHost: "agenda-live.mongo.cosmos.azure.com:10255", + wantUsername: "agenda-live", + wantDatabase: "csb-db", + }, + { + name: "with database in path", + data: `mongodb://db-user:db-password@mongodb-instance:27017/db-name`, + wantHost: "mongodb-instance:27017", + wantUsername: "db-user", + wantDatabase: "db-name", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + results, err := s.FromData(context.Background(), false, []byte(tt.data)) + require.NoError(t, err) + require.NotEmpty(t, results, "expected at least one result") + + r := results[0] + assert.Equal(t, tt.wantHost, r.ExtraData["host"]) + assert.Equal(t, tt.wantUsername, r.ExtraData["username"]) + assert.Equal(t, tt.wantDatabase, r.ExtraData["database"]) + assert.NotEmpty(t, r.ExtraData["rotation_guide"], "ExtraData[rotation_guide] should still be present") + }) + } +} + func TestMongoDB_Pattern(t *testing.T) { tests := []struct { name string diff --git a/pkg/detectors/postgres/postgres.go b/pkg/detectors/postgres/postgres.go index ab7990f87..c69655fc9 100644 --- a/pkg/detectors/postgres/postgres.go +++ b/pkg/detectors/postgres/postgres.go @@ -175,6 +175,19 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) ([]dete result.ExtraData = map[string]string{ pgSslmode: sslmode, } + if host != "" { + if port != "" { + result.ExtraData["host"] = host + ":" + port + } else { + result.ExtraData["host"] = host + } + } + if user != "" { + result.ExtraData["username"] = user + } + if dbname := params[pgDbname]; dbname != "" { + result.ExtraData["database"] = dbname + } results = append(results, result) } diff --git a/pkg/detectors/postgres/postgres_integration_test.go b/pkg/detectors/postgres/postgres_integration_test.go index d78b5cc7d..26cbda1d3 100644 --- a/pkg/detectors/postgres/postgres_integration_test.go +++ b/pkg/detectors/postgres/postgres_integration_test.go @@ -335,7 +335,7 @@ func TestPostgres_FromChunk(t *testing.T) { t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError()) } } - ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "verificationError", "SecretParts") + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "verificationError", "SecretParts", "ExtraData") if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { t.Errorf("Postgres.FromData() %s diff: (-got +want)\n%s", tt.name, diff) } diff --git a/pkg/detectors/postgres/postgres_test.go b/pkg/detectors/postgres/postgres_test.go index 85f5b82eb..62e1bd881 100644 --- a/pkg/detectors/postgres/postgres_test.go +++ b/pkg/detectors/postgres/postgres_test.go @@ -6,6 +6,8 @@ import ( "testing" "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" @@ -82,17 +84,58 @@ func TestPostgres_Pattern(t *testing.T) { } } +func TestPostgres_ExtraData(t *testing.T) { + tests := []struct { + name string + data string + wantHost string + wantUsername string + wantDatabase string + }{ + { + name: "standard URI with database", + data: "postgres://myuser:mypass@dbhost.example.com:5432/mydb", + wantHost: "dbhost.example.com:5432", + wantUsername: "myuser", + wantDatabase: "mydb", + }, + { + name: "postgresql scheme", + data: "postgresql://admin:secret@10.0.0.1:5433/production", + wantHost: "10.0.0.1:5433", + wantUsername: "admin", + wantDatabase: "production", + }, + { + name: "without database", + data: "postgres://sN19x:d7N8bs@1.2.3.4:5432?sslmode=require", + wantHost: "1.2.3.4:5432", + wantUsername: "sN19x", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{detectLoopback: true} + results, err := s.FromData(context.Background(), false, []byte(tt.data)) + require.NoError(t, err) + require.NotEmpty(t, results, "expected at least one result") + + r := results[0] + assert.Equal(t, tt.wantHost, r.ExtraData["host"]) + assert.Equal(t, tt.wantUsername, r.ExtraData["username"]) + assert.Equal(t, tt.wantDatabase, r.ExtraData["database"]) + assert.Contains(t, r.ExtraData, "sslmode", "ExtraData[sslmode] should still be present") + }) + } +} + func TestPostgres_FromDataWithIgnorePattern(t *testing.T) { s := New( WithIgnorePattern([]string{ `1\.2\.3\.4`, })) got, err := s.FromData(context.Background(), false, []byte(validUriPattern)) - if err != nil { - t.Errorf("FromData() error = %v", err) - return - } - if len(got) != 0 { - t.Errorf("expected no results, but got %d", len(got)) - } + require.NoError(t, err) + assert.Empty(t, got) } diff --git a/pkg/detectors/redis/redis.go b/pkg/detectors/redis/redis.go index 35f19659a..2c6fb70ee 100644 --- a/pkg/detectors/redis/redis.go +++ b/pkg/detectors/redis/redis.go @@ -65,7 +65,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result "host": parsedURL.Host, "password": password, }, - Redacted: redact, + Redacted: redact, + ExtraData: extraDataFromURL(parsedURL), } if verify { @@ -108,7 +109,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result "host": parsedURL.Host, "password": password, }, - Redacted: redact, + Redacted: redact, + ExtraData: extraDataFromURL(parsedURL), } if verify { @@ -128,6 +130,17 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func extraDataFromURL(u *url.URL) map[string]string { + extraData := make(map[string]string) + if u.Host != "" { + extraData["host"] = u.Host + } + if u.User != nil && u.User.Username() != "" { + extraData["username"] = u.User.Username() + } + return extraData +} + func verifyRedis(ctx context.Context, u *url.URL) bool { opt, err := redis.ParseURL(u.String()) if err != nil { diff --git a/pkg/detectors/redis/redis_integration_test.go b/pkg/detectors/redis/redis_integration_test.go index 44380084e..8f21d3c72 100644 --- a/pkg/detectors/redis/redis_integration_test.go +++ b/pkg/detectors/redis/redis_integration_test.go @@ -147,6 +147,8 @@ func TestRedisIntegration_FromChunk(t *testing.T) { } for i := range got { got[i].Raw = nil + got[i].SecretParts = nil + got[i].ExtraData = nil } if diff := pretty.Compare(got, tt.want); diff != "" { t.Errorf("URI.FromData() %s diff: (-got +want)\n%s", tt.name, diff) @@ -206,6 +208,8 @@ func TestURI_FromChunk(t *testing.T) { } for i := range got { got[i].Raw = nil + got[i].SecretParts = nil + got[i].ExtraData = nil } if diff := pretty.Compare(got, tt.want); diff != "" { t.Errorf("URI.FromData() %s diff: (-got +want)\n%s", tt.name, diff) diff --git a/pkg/detectors/redis/redis_test.go b/pkg/detectors/redis/redis_test.go index fccd171ab..c9050a921 100644 --- a/pkg/detectors/redis/redis_test.go +++ b/pkg/detectors/redis/redis_test.go @@ -6,6 +6,8 @@ import ( "testing" "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" @@ -22,6 +24,46 @@ var ( keyword = "redis" ) +func TestRedis_ExtraData(t *testing.T) { + tests := []struct { + name string + data string + wantHost string + wantUsername string + }{ + { + name: "standard redis URI", + data: `redis://myuser:mysecretpass@redis.example.com:6379/0`, + wantHost: "redis.example.com:6379", + wantUsername: "myuser", + }, + { + name: "redis URI with default username", + data: `redis://default:mysecretpass@redis.example.com:6379`, + wantHost: "redis.example.com:6379", + wantUsername: "default", + }, + { + name: "azure redis pattern without username", + data: `mycache.redis.cache.windows.net:6380,password=Xcc3S9d7And6aMdfOcUc0acHJh3CiDh3l9DsapNwGwyS,ssl=True,abortConnect=False`, + wantHost: "mycache.redis.cache.windows.net:6380", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + results, err := s.FromData(context.Background(), false, []byte(tt.data)) + require.NoError(t, err) + require.NotEmpty(t, results, "expected at least one result") + + r := results[0] + assert.Equal(t, tt.wantHost, r.ExtraData["host"]) + assert.Equal(t, tt.wantUsername, r.ExtraData["username"]) + }) + } +} + func TestRedisIntegration_Pattern(t *testing.T) { d := Scanner{} ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})