diff --git a/pkg/detectors/jdbc/jdbc.go b/pkg/detectors/jdbc/jdbc.go index e3ef7a235..e07ac81ac 100644 --- a/pkg/detectors/jdbc/jdbc.go +++ b/pkg/detectors/jdbc/jdbc.go @@ -86,24 +86,38 @@ matchLoop: SecretParts: map[string]string{"connection_string": jdbcConn}, } - if verify { - j, err := NewJDBC(logCtx, jdbcConn) - if err != nil { - continue + // Try to parse connection info for ExtraData regardless of verification. + if j, parseErr := NewJDBC(logCtx, jdbcConn); parseErr == nil { + if info := j.GetConnectionInfo(); info != nil { + extraData := make(map[string]string) + if info.Host != "" { + extraData["host"] = info.Host + } + if info.User != "" { + extraData["username"] = info.User + } + if info.Database != "" { + extraData["database"] = info.Database + } + if len(extraData) > 0 { + result.ExtraData = extraData + } } - ctx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - pingRes := j.ping(ctx) - result.Verified = pingRes.err == nil - // If there's a ping error that is marked as "determinate" we throw it away. We do this because this was the - // behavior before tri-state verification was introduced and preserving it allows us to gradually migrate - // detectors to use tri-state verification. - if pingRes.err != nil && !pingRes.determinate { - err = pingRes.err - result.SetVerificationError(err, jdbcConn) + if verify { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + pingRes := j.ping(ctx) + result.Verified = pingRes.err == nil + // If there's a ping error that is marked as "determinate" we throw it away. We do this because this was the + // behavior before tri-state verification was introduced and preserving it allows us to gradually migrate + // detectors to use tri-state verification. + if pingRes.err != nil && !pingRes.determinate { + result.SetVerificationError(pingRes.err, jdbcConn) + } } - // TODO: specialized redaction + } else if verify { + continue } results = append(results, result) diff --git a/pkg/detectors/jdbc/jdbc_integration_test.go b/pkg/detectors/jdbc/jdbc_integration_test.go index f20b2a1fd..ea2e71d04 100644 --- a/pkg/detectors/jdbc/jdbc_integration_test.go +++ b/pkg/detectors/jdbc/jdbc_integration_test.go @@ -195,6 +195,7 @@ func TestJdbcVerified(t *testing.T) { t.Fatal("no raw secret present") } got[i].Raw = nil + got[i].ExtraData = nil } if diff := pretty.Compare(got, tt.want); diff != "" { t.Errorf("Jdbc.FromData() %s diff: (-got +want)\n%s", tt.name, diff) @@ -323,6 +324,7 @@ func TestJdbc_FromChunk(t *testing.T) { } got[i].Raw = nil got[i].SecretParts = nil + got[i].ExtraData = nil } if diff := pretty.Compare(got, tt.want); diff != "" { t.Errorf("Jdbc.FromData() %s diff: (-got +want)\n%s", tt.name, diff) diff --git a/pkg/detectors/jdbc/jdbc_test.go b/pkg/detectors/jdbc/jdbc_test.go index 2ba0c937a..267346caf 100644 --- a/pkg/detectors/jdbc/jdbc_test.go +++ b/pkg/detectors/jdbc/jdbc_test.go @@ -151,6 +151,76 @@ func TestJdbc_Pattern(t *testing.T) { } } +func TestJdbc_ExtraData(t *testing.T) { + tests := []struct { + name string + data string + wantHost string + wantUsername string + wantDatabase string + }{ + { + name: "mysql with basic auth", + data: `jdbc:mysql://root:password@localhost:3306/testdb`, + wantHost: "tcp(localhost:3306)", + wantUsername: "root", + wantDatabase: "testdb", + }, + { + name: "postgresql with basic auth", + data: `jdbc:postgresql://postgres:secret@dbhost:5432/mydb`, + wantHost: "dbhost:5432", + wantUsername: "postgres", + wantDatabase: "mydb", + }, + { + name: "sqlserver with semicolon params", + data: `jdbc:sqlserver://server.example.com:1433;database=testdb;user=sa;password=Pass123`, + wantHost: "server.example.com:1433", + wantUsername: "sa", + wantDatabase: "testdb", + }, + { + name: "mysql with query params for credentials", + data: `jdbc:mysql://dbhost:3307/testdb?user=admin&password=secret`, + wantHost: "tcp(dbhost:3307)", + wantUsername: "admin", + wantDatabase: "testdb", + }, + { + name: "postgresql with query params for credentials", + data: `jdbc:postgresql://localhost:1521/testdb?sslmode=disable&password=testpassword&user=testuser`, + wantHost: "localhost:1521", + wantUsername: "testuser", + wantDatabase: "testdb", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + results, err := s.FromData(context.Background(), false, []byte(tt.data)) + require.NoError(t, err) + require.NotEmpty(t, results, "expected at least one result") + + r := results[0] + assert.Equal(t, tt.wantHost, r.ExtraData["host"]) + assert.Equal(t, tt.wantUsername, r.ExtraData["username"]) + assert.Equal(t, tt.wantDatabase, r.ExtraData["database"]) + }) + } +} + +func TestJdbc_ExtraData_UnsupportedSubprotocol(t *testing.T) { + // For unsupported subprotocols (e.g., sqlite), ExtraData should be nil + // because we can't parse connection info, but the result should still be returned. + s := Scanner{} + results, err := s.FromData(context.Background(), false, []byte(`jdbc:sqlite:/data/test.db`)) + require.NoError(t, err) + require.NotEmpty(t, results, "expected at least one result") + assert.Nil(t, results[0].ExtraData, "expected nil ExtraData for unsupported subprotocol") +} + func TestJdbc_FromDataWithIgnorePattern(t *testing.T) { type args struct { ctx context.Context diff --git a/pkg/detectors/mongodb/mongodb.go b/pkg/detectors/mongodb/mongodb.go index 810b46fb2..0aee27c5c 100644 --- a/pkg/detectors/mongodb/mongodb.go +++ b/pkg/detectors/mongodb/mongodb.go @@ -45,7 +45,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result logger := logContext.AddLogger(ctx).Logger().WithName("mongodb") dataStr := string(data) - uniqueMatches := make(map[string]string) + type mongoMatch struct { + password string + parsedURL *url.URL + } + uniqueMatches := make(map[string]mongoMatch) for _, match := range connStrPat.FindAllStringSubmatch(dataStr, -1) { // Filter out common placeholder passwords. password := match[3] @@ -78,16 +82,27 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result connUrl.RawQuery = params.Encode() connStr = connUrl.String() - uniqueMatches[connStr] = password + uniqueMatches[connStr] = mongoMatch{password: password, parsedURL: connUrl} } - for connStr, password := range uniqueMatches { + for connStr, m := range uniqueMatches { + extraData := map[string]string{ + "rotation_guide": "https://howtorotate.com/docs/tutorials/mongo/", + } + if m.parsedURL.Host != "" { + extraData["host"] = m.parsedURL.Host + } + if m.parsedURL.User != nil && m.parsedURL.User.Username() != "" { + extraData["username"] = m.parsedURL.User.Username() + } + if db := strings.TrimPrefix(m.parsedURL.Path, "/"); db != "" { + extraData["database"] = db + } + r := detectors.Result{ DetectorType: detector_typepb.DetectorType_MongoDB, Raw: []byte(connStr), - ExtraData: map[string]string{ - "rotation_guide": "https://howtorotate.com/docs/tutorials/mongo/", - }, + ExtraData: extraData, SecretParts: map[string]string{"key": connStr}, } @@ -102,7 +117,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if isErrDeterminate(vErr) { continue } - r.SetVerificationError(vErr, password) + r.SetVerificationError(vErr, m.password) } results = append(results, r) } diff --git a/pkg/detectors/mongodb/mongodb_integration_test.go b/pkg/detectors/mongodb/mongodb_integration_test.go index 60a6b5e65..fa1557f5c 100644 --- a/pkg/detectors/mongodb/mongodb_integration_test.go +++ b/pkg/detectors/mongodb/mongodb_integration_test.go @@ -173,7 +173,7 @@ func TestIntegrationMongoDB_FromChunk(t *testing.T) { t.Fatalf("wantVerificationErr = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError()) } } - ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "verificationError") + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "verificationError", "SecretParts", "ExtraData") if diff := cmp.Diff(tt.want, got, ignoreOpts); diff != "" { t.Errorf("MongoDB.FromData() %s diff: (-got +want)\n%s", tt.name, diff) } @@ -310,7 +310,7 @@ func TestMongoDB_FromChunk(t *testing.T) { t.Fatalf("wantVerificationErr = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError()) } } - ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "verificationError") + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "verificationError", "SecretParts", "ExtraData") if diff := cmp.Diff(tt.want, got, ignoreOpts); diff != "" { t.Errorf("MongoDB.FromData() %s diff: (-got +want)\n%s", tt.name, diff) } diff --git a/pkg/detectors/mongodb/mongodb_test.go b/pkg/detectors/mongodb/mongodb_test.go index 9482fcca7..2836bcff0 100644 --- a/pkg/detectors/mongodb/mongodb_test.go +++ b/pkg/detectors/mongodb/mongodb_test.go @@ -3,8 +3,69 @@ package mongodb import ( "context" "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +func TestMongoDB_ExtraData(t *testing.T) { + tests := []struct { + name string + data string + wantHost string + wantUsername string + wantDatabase string + }{ + { + name: "single host with port", + data: `mongodb://myDBReader:D1fficultP%40ssw0rd@mongodb0.example.com:27017`, + wantHost: "mongodb0.example.com:27017", + wantUsername: "myDBReader", + }, + { + name: "single host without port", + data: `mongodb://myDBReader:D1fficultP%40ssw0rd@mongodb0.example.com`, + wantHost: "mongodb0.example.com", + wantUsername: "myDBReader", + }, + { + name: "with options and no database", + data: `mongodb://username:password@host.docker.internal:27018/?authMechanism=PLAIN&tls=true`, + wantHost: "host.docker.internal:27018", + wantUsername: "username", + }, + { + name: "cosmos db style with database", + data: `mongodb://agenda-live:m21w7PFfRXQwfHZU1Fgx0rTX29ZBQaWMODLeAjsmyslVcMmcmy6CnLyu3byVDtdLYcCokze8lIE4KyAgSCGZxQ==@agenda-live.mongo.cosmos.azure.com:10255/csb-db?retryWrites=false&ssl=true&replicaSet=globaldb&maxIdleTimeMS=120000&appName=@agenda-live@`, + wantHost: "agenda-live.mongo.cosmos.azure.com:10255", + wantUsername: "agenda-live", + wantDatabase: "csb-db", + }, + { + name: "with database in path", + data: `mongodb://db-user:db-password@mongodb-instance:27017/db-name`, + wantHost: "mongodb-instance:27017", + wantUsername: "db-user", + wantDatabase: "db-name", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + results, err := s.FromData(context.Background(), false, []byte(tt.data)) + require.NoError(t, err) + require.NotEmpty(t, results, "expected at least one result") + + r := results[0] + assert.Equal(t, tt.wantHost, r.ExtraData["host"]) + assert.Equal(t, tt.wantUsername, r.ExtraData["username"]) + assert.Equal(t, tt.wantDatabase, r.ExtraData["database"]) + assert.NotEmpty(t, r.ExtraData["rotation_guide"], "ExtraData[rotation_guide] should still be present") + }) + } +} + func TestMongoDB_Pattern(t *testing.T) { tests := []struct { name string diff --git a/pkg/detectors/postgres/postgres.go b/pkg/detectors/postgres/postgres.go index ab7990f87..c69655fc9 100644 --- a/pkg/detectors/postgres/postgres.go +++ b/pkg/detectors/postgres/postgres.go @@ -175,6 +175,19 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) ([]dete result.ExtraData = map[string]string{ pgSslmode: sslmode, } + if host != "" { + if port != "" { + result.ExtraData["host"] = host + ":" + port + } else { + result.ExtraData["host"] = host + } + } + if user != "" { + result.ExtraData["username"] = user + } + if dbname := params[pgDbname]; dbname != "" { + result.ExtraData["database"] = dbname + } results = append(results, result) } diff --git a/pkg/detectors/postgres/postgres_integration_test.go b/pkg/detectors/postgres/postgres_integration_test.go index d78b5cc7d..26cbda1d3 100644 --- a/pkg/detectors/postgres/postgres_integration_test.go +++ b/pkg/detectors/postgres/postgres_integration_test.go @@ -335,7 +335,7 @@ func TestPostgres_FromChunk(t *testing.T) { t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError()) } } - ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "verificationError", "SecretParts") + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "verificationError", "SecretParts", "ExtraData") if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { t.Errorf("Postgres.FromData() %s diff: (-got +want)\n%s", tt.name, diff) } diff --git a/pkg/detectors/postgres/postgres_test.go b/pkg/detectors/postgres/postgres_test.go index 85f5b82eb..62e1bd881 100644 --- a/pkg/detectors/postgres/postgres_test.go +++ b/pkg/detectors/postgres/postgres_test.go @@ -6,6 +6,8 @@ import ( "testing" "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" @@ -82,17 +84,58 @@ func TestPostgres_Pattern(t *testing.T) { } } +func TestPostgres_ExtraData(t *testing.T) { + tests := []struct { + name string + data string + wantHost string + wantUsername string + wantDatabase string + }{ + { + name: "standard URI with database", + data: "postgres://myuser:mypass@dbhost.example.com:5432/mydb", + wantHost: "dbhost.example.com:5432", + wantUsername: "myuser", + wantDatabase: "mydb", + }, + { + name: "postgresql scheme", + data: "postgresql://admin:secret@10.0.0.1:5433/production", + wantHost: "10.0.0.1:5433", + wantUsername: "admin", + wantDatabase: "production", + }, + { + name: "without database", + data: "postgres://sN19x:d7N8bs@1.2.3.4:5432?sslmode=require", + wantHost: "1.2.3.4:5432", + wantUsername: "sN19x", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{detectLoopback: true} + results, err := s.FromData(context.Background(), false, []byte(tt.data)) + require.NoError(t, err) + require.NotEmpty(t, results, "expected at least one result") + + r := results[0] + assert.Equal(t, tt.wantHost, r.ExtraData["host"]) + assert.Equal(t, tt.wantUsername, r.ExtraData["username"]) + assert.Equal(t, tt.wantDatabase, r.ExtraData["database"]) + assert.Contains(t, r.ExtraData, "sslmode", "ExtraData[sslmode] should still be present") + }) + } +} + func TestPostgres_FromDataWithIgnorePattern(t *testing.T) { s := New( WithIgnorePattern([]string{ `1\.2\.3\.4`, })) got, err := s.FromData(context.Background(), false, []byte(validUriPattern)) - if err != nil { - t.Errorf("FromData() error = %v", err) - return - } - if len(got) != 0 { - t.Errorf("expected no results, but got %d", len(got)) - } + require.NoError(t, err) + assert.Empty(t, got) } diff --git a/pkg/detectors/redis/redis.go b/pkg/detectors/redis/redis.go index 35f19659a..2c6fb70ee 100644 --- a/pkg/detectors/redis/redis.go +++ b/pkg/detectors/redis/redis.go @@ -65,7 +65,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result "host": parsedURL.Host, "password": password, }, - Redacted: redact, + Redacted: redact, + ExtraData: extraDataFromURL(parsedURL), } if verify { @@ -108,7 +109,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result "host": parsedURL.Host, "password": password, }, - Redacted: redact, + Redacted: redact, + ExtraData: extraDataFromURL(parsedURL), } if verify { @@ -128,6 +130,17 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func extraDataFromURL(u *url.URL) map[string]string { + extraData := make(map[string]string) + if u.Host != "" { + extraData["host"] = u.Host + } + if u.User != nil && u.User.Username() != "" { + extraData["username"] = u.User.Username() + } + return extraData +} + func verifyRedis(ctx context.Context, u *url.URL) bool { opt, err := redis.ParseURL(u.String()) if err != nil { diff --git a/pkg/detectors/redis/redis_integration_test.go b/pkg/detectors/redis/redis_integration_test.go index 44380084e..8f21d3c72 100644 --- a/pkg/detectors/redis/redis_integration_test.go +++ b/pkg/detectors/redis/redis_integration_test.go @@ -147,6 +147,8 @@ func TestRedisIntegration_FromChunk(t *testing.T) { } for i := range got { got[i].Raw = nil + got[i].SecretParts = nil + got[i].ExtraData = nil } if diff := pretty.Compare(got, tt.want); diff != "" { t.Errorf("URI.FromData() %s diff: (-got +want)\n%s", tt.name, diff) @@ -206,6 +208,8 @@ func TestURI_FromChunk(t *testing.T) { } for i := range got { got[i].Raw = nil + got[i].SecretParts = nil + got[i].ExtraData = nil } if diff := pretty.Compare(got, tt.want); diff != "" { t.Errorf("URI.FromData() %s diff: (-got +want)\n%s", tt.name, diff) diff --git a/pkg/detectors/redis/redis_test.go b/pkg/detectors/redis/redis_test.go index fccd171ab..c9050a921 100644 --- a/pkg/detectors/redis/redis_test.go +++ b/pkg/detectors/redis/redis_test.go @@ -6,6 +6,8 @@ import ( "testing" "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" @@ -22,6 +24,46 @@ var ( keyword = "redis" ) +func TestRedis_ExtraData(t *testing.T) { + tests := []struct { + name string + data string + wantHost string + wantUsername string + }{ + { + name: "standard redis URI", + data: `redis://myuser:mysecretpass@redis.example.com:6379/0`, + wantHost: "redis.example.com:6379", + wantUsername: "myuser", + }, + { + name: "redis URI with default username", + data: `redis://default:mysecretpass@redis.example.com:6379`, + wantHost: "redis.example.com:6379", + wantUsername: "default", + }, + { + name: "azure redis pattern without username", + data: `mycache.redis.cache.windows.net:6380,password=Xcc3S9d7And6aMdfOcUc0acHJh3CiDh3l9DsapNwGwyS,ssl=True,abortConnect=False`, + wantHost: "mycache.redis.cache.windows.net:6380", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + results, err := s.FromData(context.Background(), false, []byte(tt.data)) + require.NoError(t, err) + require.NotEmpty(t, results, "expected at least one result") + + r := results[0] + assert.Equal(t, tt.wantHost, r.ExtraData["host"]) + assert.Equal(t, tt.wantUsername, r.ExtraData["username"]) + }) + } +} + func TestRedisIntegration_Pattern(t *testing.T) { d := Scanner{} ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})