trufflesecurity/trufflehog
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: add host, db and username to ExtraData for database detectors (#4849)

Browse Source 
Populate ExtraData with parsed fields for all
database connection string detectors (MongoDB, PostgreSQL, Redis, JDBC).
This surfaces useful metadata about detected credentials.

The parsing logic already existed in each detector — this change
exposes the extracted values in the result's ExtraData map alongside
any pre-existing fields (rotation_guide, sslmode, etc.).
This commit is contained in:
Mario Corchero
2026-05-18 16:24:40 +05:00
committed by GitHub
parent 0fa069c12f
commit d86254ee02
12 changed files with 311 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/detectors/jdbc/jdbc.go
+29 -15
View File
@@ -86,24 +86,38 @@ matchLoop:
SecretParts: map[string]string{"connection_string": jdbcConn},
}
if verify {
j, err := NewJDBC(logCtx, jdbcConn)
if err != nil {
continue
// Try to parse connection info for ExtraData regardless of verification.
if j, parseErr := NewJDBC(logCtx, jdbcConn); parseErr == nil {
if info := j.GetConnectionInfo(); info != nil {
extraData := make(map[string]string)
if info.Host != "" {
extraData["host"] = info.Host
}
if info.User != "" {
extraData["username"] = info.User
}
if info.Database != "" {
extraData["database"] = info.Database
}
if len(extraData) > 0 {
result.ExtraData = extraData
}
}
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
pingRes := j.ping(ctx)
result.Verified = pingRes.err == nil
// If there's a ping error that is marked as "determinate" we throw it away. We do this because this was the
// behavior before tri-state verification was introduced and preserving it allows us to gradually migrate
// detectors to use tri-state verification.
if pingRes.err != nil && !pingRes.determinate {
err = pingRes.err
result.SetVerificationError(err, jdbcConn)
if verify {
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
pingRes := j.ping(ctx)
result.Verified = pingRes.err == nil
// If there's a ping error that is marked as "determinate" we throw it away. We do this because this was the
// behavior before tri-state verification was introduced and preserving it allows us to gradually migrate
// detectors to use tri-state verification.
if pingRes.err != nil && !pingRes.determinate {
result.SetVerificationError(pingRes.err, jdbcConn)
}
}
// TODO: specialized redaction
} else if verify {
continue
}
results = append(results, result)
pkg/detectors/jdbc/jdbc_integration_test.go
+2
View File
@@ -195,6 +195,7 @@ func TestJdbcVerified(t *testing.T) {
t.Fatal("no raw secret present")
}
got[i].Raw = nil
got[i].ExtraData = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("Jdbc.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
@@ -323,6 +324,7 @@ func TestJdbc_FromChunk(t *testing.T) {
}
got[i].Raw = nil
got[i].SecretParts = nil
got[i].ExtraData = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("Jdbc.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
pkg/detectors/jdbc/jdbc_test.go
+70
View File
@@ -151,6 +151,76 @@ func TestJdbc_Pattern(t *testing.T) {
}
}
func TestJdbc_ExtraData(t *testing.T) {
tests := []struct {
name string
data string
wantHost string
wantUsername string
wantDatabase string
}{
{
name: "mysql with basic auth",
data: `jdbc:mysql://root:password@localhost:3306/testdb`,
wantHost: "tcp(localhost:3306)",
wantUsername: "root",
wantDatabase: "testdb",
},
{
name: "postgresql with basic auth",
data: `jdbc:postgresql://postgres:secret@dbhost:5432/mydb`,
wantHost: "dbhost:5432",
wantUsername: "postgres",
wantDatabase: "mydb",
},
{
name: "sqlserver with semicolon params",
data: `jdbc:sqlserver://server.example.com:1433;database=testdb;user=sa;password=Pass123`,
wantHost: "server.example.com:1433",
wantUsername: "sa",
wantDatabase: "testdb",
},
{
name: "mysql with query params for credentials",
data: `jdbc:mysql://dbhost:3307/testdb?user=admin&password=secret`,
wantHost: "tcp(dbhost:3307)",
wantUsername: "admin",
wantDatabase: "testdb",
},
{
name: "postgresql with query params for credentials",
data: `jdbc:postgresql://localhost:1521/testdb?sslmode=disable&password=testpassword&user=testuser`,
wantHost: "localhost:1521",
wantUsername: "testuser",
wantDatabase: "testdb",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
results, err := s.FromData(context.Background(), false, []byte(tt.data))
require.NoError(t, err)
require.NotEmpty(t, results, "expected at least one result")
r := results[0]
assert.Equal(t, tt.wantHost, r.ExtraData["host"])
assert.Equal(t, tt.wantUsername, r.ExtraData["username"])
assert.Equal(t, tt.wantDatabase, r.ExtraData["database"])
})
}
}
func TestJdbc_ExtraData_UnsupportedSubprotocol(t *testing.T) {
// For unsupported subprotocols (e.g., sqlite), ExtraData should be nil
// because we can't parse connection info, but the result should still be returned.
s := Scanner{}
results, err := s.FromData(context.Background(), false, []byte(`jdbc:sqlite:/data/test.db`))
require.NoError(t, err)
require.NotEmpty(t, results, "expected at least one result")
assert.Nil(t, results[0].ExtraData, "expected nil ExtraData for unsupported subprotocol")
}
func TestJdbc_FromDataWithIgnorePattern(t *testing.T) {
type args struct {
ctx context.Context
pkg/detectors/mongodb/mongodb.go
+22 -7
View File
@@ -45,7 +45,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
logger := logContext.AddLogger(ctx).Logger().WithName("mongodb")
dataStr := string(data)
uniqueMatches := make(map[string]string)
type mongoMatch struct {
password string
parsedURL *url.URL
}
uniqueMatches := make(map[string]mongoMatch)
for _, match := range connStrPat.FindAllStringSubmatch(dataStr, -1) {
// Filter out common placeholder passwords.
password := match[3]
@@ -78,16 +82,27 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
connUrl.RawQuery = params.Encode()
connStr = connUrl.String()
uniqueMatches[connStr] = password
uniqueMatches[connStr] = mongoMatch{password: password, parsedURL: connUrl}
}
for connStr, password := range uniqueMatches {
for connStr, m := range uniqueMatches {
extraData := map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/mongo/",
}
if m.parsedURL.Host != "" {
extraData["host"] = m.parsedURL.Host
}
if m.parsedURL.User != nil && m.parsedURL.User.Username() != "" {
extraData["username"] = m.parsedURL.User.Username()
}
if db := strings.TrimPrefix(m.parsedURL.Path, "/"); db != "" {
extraData["database"] = db
}
r := detectors.Result{
DetectorType: detector_typepb.DetectorType_MongoDB,
Raw: []byte(connStr),
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/mongo/",
},
ExtraData: extraData,
SecretParts: map[string]string{"key": connStr},
}
@@ -102,7 +117,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if isErrDeterminate(vErr) {
continue
}
r.SetVerificationError(vErr, password)
r.SetVerificationError(vErr, m.password)
}
results = append(results, r)
}
pkg/detectors/mongodb/mongodb_integration_test.go
+2 -2
View File
@@ -173,7 +173,7 @@ func TestIntegrationMongoDB_FromChunk(t *testing.T) {
t.Fatalf("wantVerificationErr = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
}
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "verificationError")
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "verificationError", "SecretParts", "ExtraData")
if diff := cmp.Diff(tt.want, got, ignoreOpts); diff != "" {
t.Errorf("MongoDB.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
@@ -310,7 +310,7 @@ func TestMongoDB_FromChunk(t *testing.T) {
t.Fatalf("wantVerificationErr = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
}
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "verificationError")
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "verificationError", "SecretParts", "ExtraData")
if diff := cmp.Diff(tt.want, got, ignoreOpts); diff != "" {
t.Errorf("MongoDB.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
pkg/detectors/mongodb/mongodb_test.go
+61
View File
@@ -3,8 +3,69 @@ package mongodb
import (
"context"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestMongoDB_ExtraData(t *testing.T) {
tests := []struct {
name string
data string
wantHost string
wantUsername string
wantDatabase string
}{
{
name: "single host with port",
data: `mongodb://myDBReader:D1fficultP%[email protected]:27017`,
wantHost: "mongodb0.example.com:27017",
wantUsername: "myDBReader",
},
{
name: "single host without port",
data: `mongodb://myDBReader:D1fficultP%[email protected]`,
wantHost: "mongodb0.example.com",
wantUsername: "myDBReader",
},
{
name: "with options and no database",
data: `mongodb://username:[email protected]:27018/?authMechanism=PLAIN&tls=true`,
wantHost: "host.docker.internal:27018",
wantUsername: "username",
},
{
name: "cosmos db style with database",
data: `mongodb://agenda-live:m21w7PFfRXQwfHZU1Fgx0rTX29ZBQaWMODLeAjsmyslVcMmcmy6CnLyu3byVDtdLYcCokze8lIE4KyAgSCGZxQ==@agenda-live.mongo.cosmos.azure.com:10255/csb-db?retryWrites=false&ssl=true&replicaSet=globaldb&maxIdleTimeMS=120000&appName=@agenda-live@`,
wantHost: "agenda-live.mongo.cosmos.azure.com:10255",
wantUsername: "agenda-live",
wantDatabase: "csb-db",
},
{
name: "with database in path",
data: `mongodb://db-user:db-password@mongodb-instance:27017/db-name`,
wantHost: "mongodb-instance:27017",
wantUsername: "db-user",
wantDatabase: "db-name",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
results, err := s.FromData(context.Background(), false, []byte(tt.data))
require.NoError(t, err)
require.NotEmpty(t, results, "expected at least one result")
r := results[0]
assert.Equal(t, tt.wantHost, r.ExtraData["host"])
assert.Equal(t, tt.wantUsername, r.ExtraData["username"])
assert.Equal(t, tt.wantDatabase, r.ExtraData["database"])
assert.NotEmpty(t, r.ExtraData["rotation_guide"], "ExtraData[rotation_guide] should still be present")
})
}
}
func TestMongoDB_Pattern(t *testing.T) {
tests := []struct {
name string
pkg/detectors/postgres/postgres.go
+13
View File
@@ -175,6 +175,19 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) ([]dete
result.ExtraData = map[string]string{
pgSslmode: sslmode,
}
if host != "" {
if port != "" {
result.ExtraData["host"] = host + ":" + port
} else {
result.ExtraData["host"] = host
}
}
if user != "" {
result.ExtraData["username"] = user
}
if dbname := params[pgDbname]; dbname != "" {
result.ExtraData["database"] = dbname
}
results = append(results, result)
}
pkg/detectors/postgres/postgres_integration_test.go
+1 -1
View File
@@ -335,7 +335,7 @@ func TestPostgres_FromChunk(t *testing.T) {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())
}
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "verificationError", "SecretParts")
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "verificationError", "SecretParts", "ExtraData")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Postgres.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
pkg/detectors/postgres/postgres_test.go
+50 -7
View File
@@ -6,6 +6,8 @@ import (
"testing"
"github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
@@ -82,17 +84,58 @@ func TestPostgres_Pattern(t *testing.T) {
}
}
func TestPostgres_ExtraData(t *testing.T) {
tests := []struct {
name string
data string
wantHost string
wantUsername string
wantDatabase string
}{
{
name: "standard URI with database",
data: "postgres://myuser:[email protected]:5432/mydb",
wantHost: "dbhost.example.com:5432",
wantUsername: "myuser",
wantDatabase: "mydb",
},
{
name: "postgresql scheme",
data: "postgresql://admin:[email protected]:5433/production",
wantHost: "10.0.0.1:5433",
wantUsername: "admin",
wantDatabase: "production",
},
{
name: "without database",
data: "postgres://sN19x:[email protected]:5432?sslmode=require",
wantHost: "1.2.3.4:5432",
wantUsername: "sN19x",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{detectLoopback: true}
results, err := s.FromData(context.Background(), false, []byte(tt.data))
require.NoError(t, err)
require.NotEmpty(t, results, "expected at least one result")
r := results[0]
assert.Equal(t, tt.wantHost, r.ExtraData["host"])
assert.Equal(t, tt.wantUsername, r.ExtraData["username"])
assert.Equal(t, tt.wantDatabase, r.ExtraData["database"])
assert.Contains(t, r.ExtraData, "sslmode", "ExtraData[sslmode] should still be present")
})
}
}
func TestPostgres_FromDataWithIgnorePattern(t *testing.T) {
s := New(
WithIgnorePattern([]string{
`1\.2\.3\.4`,
}))
got, err := s.FromData(context.Background(), false, []byte(validUriPattern))
if err != nil {
t.Errorf("FromData() error = %v", err)
return
}
if len(got) != 0 {
t.Errorf("expected no results, but got %d", len(got))
}
require.NoError(t, err)
assert.Empty(t, got)
}
pkg/detectors/redis/redis.go
+15 -2
View File
@@ -65,7 +65,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
"host": parsedURL.Host,
"password": password,
},
Redacted: redact,
Redacted: redact,
ExtraData: extraDataFromURL(parsedURL),
}
if verify {
@@ -108,7 +109,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
"host": parsedURL.Host,
"password": password,
},
Redacted: redact,
Redacted: redact,
ExtraData: extraDataFromURL(parsedURL),
}
if verify {
@@ -128,6 +130,17 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
return results, nil
}
func extraDataFromURL(u *url.URL) map[string]string {
extraData := make(map[string]string)
if u.Host != "" {
extraData["host"] = u.Host
}
if u.User != nil && u.User.Username() != "" {
extraData["username"] = u.User.Username()
}
return extraData
}
func verifyRedis(ctx context.Context, u *url.URL) bool {
opt, err := redis.ParseURL(u.String())
if err != nil {
pkg/detectors/redis/redis_integration_test.go
+4
View File
@@ -147,6 +147,8 @@ func TestRedisIntegration_FromChunk(t *testing.T) {
}
for i := range got {
got[i].Raw = nil
got[i].SecretParts = nil
got[i].ExtraData = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("URI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
@@ -206,6 +208,8 @@ func TestURI_FromChunk(t *testing.T) {
}
for i := range got {
got[i].Raw = nil
got[i].SecretParts = nil
got[i].ExtraData = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("URI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
pkg/detectors/redis/redis_test.go
+42
View File
@@ -6,6 +6,8 @@ import (
"testing"
"github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
@@ -22,6 +24,46 @@ var (
keyword = "redis"
)
func TestRedis_ExtraData(t *testing.T) {
tests := []struct {
name string
data string
wantHost string
wantUsername string
}{
{
name: "standard redis URI",
data: `redis://myuser:[email protected]:6379/0`,
wantHost: "redis.example.com:6379",
wantUsername: "myuser",
},
{
name: "redis URI with default username",
data: `redis://default:[email protected]:6379`,
wantHost: "redis.example.com:6379",
wantUsername: "default",
},
{
name: "azure redis pattern without username",
data: `mycache.redis.cache.windows.net:6380,password=Xcc3S9d7And6aMdfOcUc0acHJh3CiDh3l9DsapNwGwyS,ssl=True,abortConnect=False`,
wantHost: "mycache.redis.cache.windows.net:6380",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
results, err := s.FromData(context.Background(), false, []byte(tt.data))
require.NoError(t, err)
require.NotEmpty(t, results, "expected at least one result")
r := results[0]
assert.Equal(t, tt.wantHost, r.ExtraData["host"])
assert.Equal(t, tt.wantUsername, r.ExtraData["username"])
})
}
}
func TestRedisIntegration_Pattern(t *testing.T) {
d := Scanner{}
ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})