custom_detectors: add OAuth2 token acquisition and inline verification

Build ROPC token sources from the new VerifierAuth proto config at
detector initialization time. Each verifier is paired with its optional
token source in a customDetectorVerifier struct, eliminating parallel
slices. BuildTokenSource is exported so external callers can
reuse the same grant-type dispatch without duplicating it.

The inline verification loop in createResults now builds an
OAuth2-authenticated HTTP client (via oauth2.NewClient) when a token
source is present. Token acquisition, verify requests, and outcomes
are logged at Info level with the oauth2_trace correlation ID so a
single trace value connects the full lifecycle.

Token endpoint URLs are validated against the same unsafe flag as the
verification endpoint — sending credentials over plaintext requires
the same explicit opt-in.

Co-authored-by: Cursor <[email protected]>
This commit is contained in:
Dave Hubbard
2026-09-17 16:33:27 -04:00
co-authored by Cursor
parent 68beec721e
commit 4ff164ee9d
3 changed files with 514 additions and 106 deletions
+243 -99
View File
@@ -1,20 +1,20 @@
package custom_detectors
import (
"bytes"
"context"
"encoding/json"
"errors"
"io"
"fmt"
"maps"
"net/http"
"regexp" //nolint:depguard // used instead of github.com/wasilibs/go-re2 due to differences in utf-8 handling
"slices"
"strings"
"golang.org/x/oauth2"
"golang.org/x/sync/errgroup"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/custom_detectorspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb"
@@ -25,11 +25,186 @@ import (
// for poorly defined regexps.
const maxTotalMatches = 100
// ─── OAuth2 token acquisition ────────────────────────────────────────
// ropcTokenSource implements oauth2.TokenSource for the Resource Owner
// Password Credentials grant (RFC 6749 Section 4.3). Caching and
// expiry are handled by oauth2.ReuseTokenSource; this type only
// performs the token exchange.
type ropcTokenSource struct {
conf *oauth2.Config
username string
password string
// httpCtx carries both the HTTP client (via the oauth2.HTTPClient
// context key) and the structured logger with oauth2_trace set.
// Token acquisition logs inherit the trace automatically.
//
// This context is rooted in Background rather than the per-scan
// detection context because oauth2.TokenSource.Token() accepts no
// context parameter — there is no way to thread a caller-provided
// context through the interface. The SaneHttpClient timeout bounds
// the worst case for a stuck token endpoint.
httpCtx context.Context
}
func (s *ropcTokenSource) Token() (*oauth2.Token, error) {
logger := logContext.AddLogger(s.httpCtx).Logger()
logger.Info("requesting ROPC token from endpoint",
"token_endpoint", s.conf.Endpoint.TokenURL,
"client_id", s.conf.ClientID,
)
tok, err := s.conf.PasswordCredentialsToken(s.httpCtx, s.username, s.password)
if err != nil {
logger.Error(err, "ROPC token acquisition failed",
"token_endpoint", s.conf.Endpoint.TokenURL,
"client_id", s.conf.ClientID,
)
return nil, err
}
logger.Info("ROPC token acquired successfully",
"token_endpoint", s.conf.Endpoint.TokenURL,
"token_type", tok.TokenType,
"expires_in", tok.Expiry,
)
return tok, nil
}
// newROPCTokenSource builds an oauth2.TokenSource for the ROPC grant,
// wrapped in ReuseTokenSource for automatic caching with a 10-second
// expiry buffer. The provided ctx already carries oauth2_trace on its
// logger; the HTTP client is added here for token endpoint requests.
func newROPCTokenSource(ctx logContext.Context, oc *custom_detectorspb.OAuth2Config, ropc *custom_detectorspb.ROPCConfig) oauth2.TokenSource {
conf := &oauth2.Config{
ClientID: ropc.GetClientId(),
ClientSecret: ropc.GetClientSecret(),
Endpoint: oauth2.Endpoint{
TokenURL: oc.GetTokenEndpoint(),
AuthStyle: oauth2.AuthStyleInParams,
},
}
if scope := ropc.GetScope(); scope != "" {
conf.Scopes = strings.Split(scope, " ")
}
// Add the SaneHttpClient for TLS/timeout settings. The HTTPClient
// key is non-string, so it passes through WithValue without
// polluting the structured logger.
httpCtx := logContext.WithValue(ctx, oauth2.HTTPClient, httpClient)
base := &ropcTokenSource{
conf: conf,
username: ropc.GetUsername(),
password: ropc.GetPassword(),
httpCtx: httpCtx,
}
return oauth2.ReuseTokenSource(nil, base)
}
// customDetectorVerifier binds a VerifierConfig to an optional
// OAuth2TokenSource. This avoids parallel slices and ensures the auth
// config can never get out of sync with its verifier.
type customDetectorVerifier struct {
config *custom_detectorspb.VerifierConfig
tokenSource detectors.OAuth2TokenSource // nil when no auth is configured
}
// BuildTokenSource creates the appropriate OAuth2TokenSource for a
// VerifierConfig's auth block. Returns (nil, nil) when no auth is
// configured. Returns a non-nil error when auth is configured but
// the grant type is unrecognized or unusable. Exported so the
// external callers can build token sources from proto config
// without duplicating grant-type logic.
func BuildTokenSource(auth *custom_detectorspb.VerifierAuth) (detectors.OAuth2TokenSource, error) {
if auth == nil {
return nil, nil
}
switch ac := auth.GetAuthConfig().(type) {
case *custom_detectorspb.VerifierAuth_Oauth2:
return buildOAuth2TokenSource(ac.Oauth2)
default:
return nil, fmt.Errorf("unrecognized auth config type: %T", auth.GetAuthConfig())
}
}
// buildOAuth2TokenSource dispatches to the correct grant-type builder
// based on which oneof field is set on the OAuth2Config.
func buildOAuth2TokenSource(oc *custom_detectorspb.OAuth2Config) (detectors.OAuth2TokenSource, error) {
if oc == nil {
return nil, fmt.Errorf("oauth2 config is nil")
}
// Generate the trace ID once. It rides on the TracedTokenSource
// wrapper so verification sites can extract it, and on the token
// source's internal httpCtx so token acquisition logs inherit it.
ts := &detectors.TracedTokenSource{Trace: common.RandomID(8)}
ctx := ts.EnrichContext(logContext.Background())
logger := ctx.Logger()
switch cfg := oc.GetGrantConfig().(type) {
case *custom_detectorspb.OAuth2Config_Ropc:
logger.Info("building ROPC token source",
"token_endpoint", oc.GetTokenEndpoint(),
"client_id", cfg.Ropc.GetClientId(),
"has_scope", cfg.Ropc.GetScope() != "",
)
ts.TokenSource = newROPCTokenSource(ctx, oc, cfg.Ropc)
return ts, nil
default:
return nil, fmt.Errorf("unrecognized OAuth2 grant type: %T", oc.GetGrantConfig())
}
}
// ─── Request body template ──────────────────────────────────────────
// knownTokens lists the $token references supported in a
// VerifyRequestBody template. Values are substituted at verification
// time from the detected result and (optionally) the OAuth2 token.
var knownTokens = map[string]bool{
"$secret": true,
"$detector_type": true,
"$detector_name": true,
"$token": true,
}
// validateRequestBody checks that every $-prefixed value in the body
// template references a known token. Called at init time so typos
// surface as startup errors rather than silent verification failures.
func validateRequestBody(body map[string]string) error {
for key, val := range body {
if strings.HasPrefix(val, "$") && !knownTokens[val] {
return fmt.Errorf("unknown token %q in request body field %q; supported tokens: $secret, $detector_type, $detector_name, $token", val, key)
}
}
return nil
}
// ResolveRequestBody substitutes $token references in the body
// template with their runtime values and returns the JSON-encoded
// result. Non-token values pass through as-is. Exported so the
// engine OAuth verification path can use the same logic.
func ResolveRequestBody(template map[string]string, vars map[string]string) ([]byte, error) {
resolved := make(map[string]string, len(template))
for key, val := range template {
if strings.HasPrefix(val, "$") {
if replacement, ok := vars[val]; ok {
resolved[key] = replacement
} else {
// Validated at init, so this should not happen.
resolved[key] = ""
}
} else {
resolved[key] = val
}
}
return json.Marshal(resolved)
}
// ─── Custom detector ─────────────────────────────────────────────────
// CustomRegexWebhook is a CustomRegex with webhook validation that is
// guaranteed to be valid (assuming the data is not changed after
// initialization).
type CustomRegexWebhook struct {
*custom_detectorspb.CustomRegex
// verifiers pairs each VerifierConfig with its optional token source.
// Built once in NewWebhookCustomRegex; used in createResults.
verifiers []customDetectorVerifier
}
// Ensure the Scanner satisfies the interface at compile time.
@@ -62,6 +237,15 @@ func NewWebhookCustomRegex(pb *custom_detectorspb.CustomRegex) (*CustomRegexWebh
if err := ValidateVerifyEndpoint(verify.Endpoint, verify.Unsafe); err != nil {
return nil, err
}
// The verifier's unsafe flag covers the entire communication
// chain, including the auth token endpoint. Sending credentials
// over plaintext is at least as dangerous as verification over
// plaintext, so both are gated by the same flag.
if oc := verify.GetAuth().GetOauth2(); oc != nil {
if err := ValidateVerifyEndpoint(oc.GetTokenEndpoint(), verify.Unsafe); err != nil {
return nil, fmt.Errorf("auth token endpoint: %w", err)
}
}
if err := ValidateVerifyHeaders(verify.Headers); err != nil {
return nil, err
}
@@ -76,8 +260,32 @@ func NewWebhookCustomRegex(pb *custom_detectorspb.CustomRegex) (*CustomRegexWebh
// Ensure primary regex name is set.
ensurePrimaryRegexNameSet(pb)
// Build the verifier slice, pairing each VerifierConfig with its
// token source (nil when auth isn't configured).
verifiers := make([]customDetectorVerifier, 0, len(pb.GetVerify()))
for _, vc := range pb.GetVerify() {
ts, err := BuildTokenSource(vc.GetAuth())
if err != nil {
return nil, fmt.Errorf("verifier %q: %w", vc.GetEndpoint(), err)
}
// Validate request body template tokens at init time so
// typos are caught at startup rather than at verification.
if req := vc.GetRequest(); req != nil && len(req.GetBody()) > 0 {
if err := validateRequestBody(req.GetBody()); err != nil {
return nil, fmt.Errorf("verifier %q: %w", vc.GetEndpoint(), err)
}
}
verifiers = append(verifiers, customDetectorVerifier{
config: vc,
tokenSource: ts,
})
}
// TODO: Copy only necessary data out of pb.
return &CustomRegexWebhook{pb}, nil
return &CustomRegexWebhook{
CustomRegex: pb,
verifiers: verifiers,
}, nil
}
var httpClient = common.SaneHttpClient()
@@ -274,7 +482,10 @@ func (c *CustomRegexWebhook) createResults(ctx context.Context, match map[string
}
}
// Verify via webhook.
jsonBody, err := json.Marshal(map[string]map[string][]string{
// Default body uses the existing format: detector name → regex match
// groups. Verifiers with a custom request.body template override this
// per-verifier inside the shared verification loop.
defaultBody, err := json.Marshal(map[string]map[string][]string{
c.GetName(): match,
})
if err != nil {
@@ -283,90 +494,37 @@ func (c *CustomRegexWebhook) createResults(ctx context.Context, match map[string
return nil
}
var (
definitive bool
rangesInEffect bool
)
// Try each config until we get a definitive answer.
for _, verifyConfig := range c.GetVerify() {
if common.IsDone(ctx) {
return ctx.Err()
// Build the shared VerifyEndpoint slice from the detector's
// proto verifier configs. Each verifier can have its own endpoint,
// headers, status code ranges, body template, and token source.
endpoints := make([]VerifyEndpoint, len(c.verifiers))
for i, v := range c.verifiers {
ep := VerifyEndpoint{
URL: v.config.GetEndpoint(),
Headers: v.config.GetHeaders(),
SuccessRanges: v.config.GetSuccessRanges(),
RotatedRanges: v.config.GetRotatedRanges(),
TokenSource: v.tokenSource,
}
req, err := http.NewRequestWithContext(ctx, "POST", verifyConfig.GetEndpoint(), bytes.NewReader(jsonBody))
if err != nil {
continue
if req := v.config.GetRequest(); req != nil {
ep.Body = req.GetBody()
}
for _, header := range verifyConfig.GetHeaders() {
key, value, found := strings.Cut(header, ":")
if !found {
continue
}
req.Header.Add(key, strings.TrimLeft(value, "\t\n\v\f\r "))
}
if req.Header.Get("Content-Type") == "" {
req.Header.Set("Content-Type", "application/json")
}
resp, err := httpClient.Do(req)
if err != nil {
continue
}
defer func() {
_, _ = io.Copy(io.Discard, resp.Body)
_ = resp.Body.Close()
}()
successRanges := verifyConfig.GetSuccessRanges()
rotatedRanges := verifyConfig.GetRotatedRanges()
if len(successRanges) == 0 && len(rotatedRanges) == 0 {
// Backward compat: no ranges configured, use legacy behavior.
if resp.StatusCode == http.StatusOK {
result.Verified = true
definitive = true
storeResponseBody(resp, result.ExtraData)
break
}
// Legacy non-200 is a meaningful response (verifier said "no");
// mark definitive so a prior ranged verifier with rangesInEffect
// does not cause a spurious verification error.
definitive = true
continue
}
rangesInEffect = true
bothConfigured := len(successRanges) > 0 && len(rotatedRanges) > 0
if StatusCodeMatchesRanges(resp.StatusCode, successRanges) {
result.Verified = true
definitive = true
storeResponseBody(resp, result.ExtraData)
break
}
if StatusCodeMatchesRanges(resp.StatusCode, rotatedRanges) {
definitive = true
break
}
// Status matched neither configured range.
if !bothConfigured {
// Only one side was configured: the non-matching response is
// treated as the opposite state.
// successRanges only -> non-match means rotated
// rotatedRanges only -> non-match means live
definitive = true
if len(rotatedRanges) > 0 {
result.Verified = true
storeResponseBody(resp, result.ExtraData)
}
break
}
// Both configured but neither matched -- try the next verifier.
endpoints[i] = ep
}
if rangesInEffect && !definitive {
vars := map[string]string{
"$secret": string(result.Raw),
"$detector_type": result.DetectorType.String(),
"$detector_name": result.DetectorName,
}
outcome := VerifyCredential(ctx, httpClient, endpoints, defaultBody, vars)
result.Verified = outcome.Verified
if outcome.RespBody != "" {
result.ExtraData["response"] = outcome.RespBody
}
if !outcome.Definitive && outcome.RangesInEffect {
// At least one endpoint used ranges but none matched.
result.SetVerificationError(errors.New("verification response status code did not match any configured successRanges or rotatedRanges"))
}
@@ -378,20 +536,6 @@ func (c *CustomRegexWebhook) createResults(ctx context.Context, match map[string
}
}
const maxResponseLen = 200
func storeResponseBody(resp *http.Response, extraData map[string]string) {
body, err := io.ReadAll(resp.Body)
if err != nil {
return
}
responseStr := string(body)
if len(responseStr) > maxResponseLen {
responseStr = responseStr[:maxResponseLen]
}
extraData["response"] = responseStr
}
func (c *CustomRegexWebhook) Keywords() []string {
return c.GetKeywords()
}
@@ -137,7 +137,7 @@ func TestCustomDetectorsParsing(t *testing.T) {
func TestFromData_InvalidRegEx(t *testing.T) {
c := &CustomRegexWebhook{
&custom_detectorspb.CustomRegex{
CustomRegex: &custom_detectorspb.CustomRegex{
Name: "Internal bi tool",
Keywords: []string{"secret_v1_", "pat_v2_"},
Regex: map[string]string{
@@ -832,12 +832,12 @@ func TestVerificationWithConfigurableRanges(t *testing.T) {
t.Parallel()
tests := []struct {
name string
serverStatus int
successRanges []string
rotatedRanges []string
wantVerified bool
wantVerifyErr bool
name string
serverStatus int
successRanges []string
rotatedRanges []string
wantVerified bool
wantVerifyErr bool
}{
{
name: "backward compat: no ranges, 200 -> verified",
+264
View File
@@ -0,0 +1,264 @@
// verify.go provides the shared credential verification loop used by
// both the inline custom detector path (createResults) and the engine
// OAuth verification path (OAuthVerify). A single implementation
// ensures consistent behavior: body template resolution, header
// application, OAuth client wrapping, status code interpretation,
// endpoint fallback, response capture, and outcome logging.
package custom_detectors
import (
"bytes"
"context"
"io"
"net/http"
"strings"
"golang.org/x/oauth2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
)
// VerifyEndpoint holds everything needed to attempt verification against
// a single endpoint. Both the inline custom detector path and the engine
// OAuth path populate this from their respective config sources.
type VerifyEndpoint struct {
URL string
Headers []string
SuccessRanges []string
RotatedRanges []string
// Body is the customer-defined request body template with $token
// references. Nil means use the caller-provided defaultBody.
Body map[string]string
// TokenSource supplies OAuth2 Bearer tokens for this endpoint.
// Nil means no OAuth wrapping — the request is sent as-is.
TokenSource oauth2.TokenSource
}
// VerifyOutcome captures the result of attempting verification across
// one or more endpoints.
type VerifyOutcome struct {
// Verified is true when the credential was confirmed active.
Verified bool
// Definitive is true when at least one endpoint gave a conclusive
// answer (verified or not). False means no endpoint matched any
// configured range — the caller should treat this as an error.
Definitive bool
// RangesInEffect is true when at least one endpoint had
// successRanges or rotatedRanges configured. When true and
// Definitive is false, it means a range-based endpoint failed
// to match — a configuration or server error. When false and
// Definitive is false, no endpoint returned 200 in default mode
// — "not verified" but not an error.
RangesInEffect bool
// StatusCode is the HTTP status from the endpoint that produced
// the definitive answer, or the last status code seen.
StatusCode int
// RespBody holds the response body from the definitive endpoint
// (truncated to maxResponseLen). Empty when no definitive answer.
RespBody string
}
// maxRespBodyLen caps the response body captured in VerifyOutcome so
// large responses don't bloat result metadata.
const maxRespBodyLen = 200
// VerifyCredential tries each endpoint in order until a definitive
// answer is reached. It handles OAuth2 client wrapping, body template
// resolution, header application, status code interpretation (default
// and range-based with single-range inference), endpoint fallback,
// response body capture, and outcome logging under the caller's trace.
//
// The vars map supplies runtime values for $token references in body
// templates ($secret, $detector_type, $detector_name). The $token var
// is resolved per-endpoint from the endpoint's TokenSource.
//
// When no endpoint produces a definitive answer, Verified is false and
// Definitive is false — the caller should set a verification error.
func VerifyCredential(
ctx context.Context,
baseClient *http.Client,
endpoints []VerifyEndpoint,
defaultBody []byte,
vars map[string]string,
) VerifyOutcome {
if baseClient == nil {
baseClient = common.SaneHttpClient()
}
var (
outcome VerifyOutcome
rangesInEffect bool
traced bool
lastCtx logContext.Context
)
for _, ep := range endpoints {
if common.IsDone(ctx) {
return outcome
}
// Per-endpoint context enrichment: if this endpoint has an
// OAuth2 token source carrying a trace, set it on the context
// so all downstream log messages inherit the correlation ID.
epCtx := logContext.AddLogger(ctx)
if ts, ok := ep.TokenSource.(*detectors.TracedTokenSource); ok {
epCtx = ts.EnrichContext(epCtx)
traced = true
}
logger := epCtx.Logger()
// Build the request body. When the endpoint has a custom
// template, resolve $token references. The $token var is
// fetched per-endpoint since each can have its own token source.
body := defaultBody
if len(ep.Body) > 0 {
epVars := make(map[string]string, len(vars)+1)
for k, v := range vars {
epVars[k] = v
}
if ep.TokenSource != nil {
if tok, err := ep.TokenSource.Token(); err == nil {
epVars["$token"] = tok.AccessToken
}
}
resolved, err := ResolveRequestBody(ep.Body, epVars)
if err != nil {
logger.Error(err, "failed to resolve request body template",
"endpoint", ep.URL,
)
continue
}
body = resolved
}
req, err := http.NewRequestWithContext(epCtx, "POST", ep.URL, bytes.NewReader(body))
if err != nil {
logger.Error(err, "failed to create verify request",
"endpoint", ep.URL,
)
continue
}
// Apply customer-defined headers, then default Content-Type.
for _, h := range ep.Headers {
key, value, found := strings.Cut(h, ":")
if !found {
continue
}
req.Header.Add(key, strings.TrimLeft(value, "\t\n\v\f\r "))
}
if req.Header.Get("Content-Type") == "" {
req.Header.Set("Content-Type", "application/json")
}
// If this endpoint has OAuth2 auth, wrap the base client with
// oauth2.Transport so the Bearer token is added transparently.
client := baseClient
if ep.TokenSource != nil {
logger.Info("sending OAuth2-authenticated verify request",
"endpoint", ep.URL,
)
client = oauth2.NewClient(
context.WithValue(epCtx, oauth2.HTTPClient, baseClient),
ep.TokenSource,
)
}
resp, err := client.Do(req)
if err != nil {
logger.Error(err, "verify request failed",
"endpoint", ep.URL,
)
continue
}
// Read and close the response body. Capture it for the outcome
// in case this endpoint produces the definitive answer.
respBody, _ := io.ReadAll(resp.Body)
_ = resp.Body.Close()
lastCtx = epCtx
outcome.StatusCode = resp.StatusCode
// ── Status code interpretation ───────────────────────────
successRanges := ep.SuccessRanges
rotatedRanges := ep.RotatedRanges
if len(successRanges) == 0 && len(rotatedRanges) == 0 {
// Default: no ranges configured, 200 means verified.
// Non-200 is a meaningful "not verified" — mark definitive
// so a prior ranged verifier's rangesInEffect flag doesn't
// cause a spurious error, then try the next endpoint.
if resp.StatusCode == http.StatusOK {
outcome.Verified = true
outcome.Definitive = true
outcome.RespBody = truncateBody(respBody)
break
}
outcome.Definitive = true
continue
}
rangesInEffect = true
outcome.RangesInEffect = true
bothConfigured := len(successRanges) > 0 && len(rotatedRanges) > 0
if StatusCodeMatchesRanges(resp.StatusCode, successRanges) {
outcome.Verified = true
outcome.Definitive = true
outcome.RespBody = truncateBody(respBody)
break
}
if StatusCodeMatchesRanges(resp.StatusCode, rotatedRanges) {
outcome.Definitive = true
break
}
// Single-range inference: when only one side is configured and
// the status didn't match, infer the opposite state.
if !bothConfigured {
outcome.Definitive = true
if len(rotatedRanges) > 0 {
// Only rotatedRanges configured, didn't match → verified.
outcome.Verified = true
outcome.RespBody = truncateBody(respBody)
}
break
}
// Both configured but neither matched — try the next endpoint.
}
// Outcome logging under the caller's trace context so the full
// lifecycle (token acquisition → verify → result) is visible.
if traced && lastCtx != nil {
logger := lastCtx.Logger()
if rangesInEffect && !outcome.Definitive {
logger.Error(nil, "OAuth2 verification inconclusive; no status range matched",
"status_code", outcome.StatusCode,
)
} else if outcome.Verified {
logger.Info("OAuth2 verification succeeded",
"status_code", outcome.StatusCode,
)
} else {
logger.Info("OAuth2 verification completed, credential not verified",
"status_code", outcome.StatusCode,
)
}
}
return outcome
}
func truncateBody(b []byte) string {
s := string(b)
if len(s) > maxRespBodyLen {
return s[:maxRespBodyLen]
}
return s
}