trufflesecurity/trufflehog
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Support exclude regexes, excludewords, and entropy filters for custom detectors (#3860)
Lint / golangci-lint (push) Waiting to run
Lint / semgrep (push) Waiting to run
Release / Release (push) Waiting to run
Scan for secrets / test (push) Waiting to run
Test / test (push) Waiting to run
Test / test-community (push) Waiting to run

Browse Source 
* Simple implementation of exclude regexes, stopwords, and entropy checks for customdetectors

* better name

* readme blurb and example

* link
This commit is contained in:
Zachary Rice
2025-01-30 12:34:07 -06:00
committed by GitHub
parent 9ecaf075c5
commit 853e1e8d24
6 changed files with 1639 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+6
View File
@@ -656,6 +656,10 @@ TruffleHog will send a JSON POST request containing the regex matches to a
configured webhook endpoint. If the endpoint responds with a `200 OK` response
status code, the secret is considered verified.
Custom Detectors support a few different filtering mechanisms: entropy, regex targeting the entire match, regex targeting the captured secret,
and excluded word lists checked against the secret (captured group if present, entire match if capture group is not present). Note that if
your custom detector has multiple `regex` set (in this example `hogID`, and `hogToken`), then the filters get applied to each regex. [Here](examples/generic_with_filters.yml) is an example of a custom detector using these filters.
**NB:** This feature is alpha and subject to change.
## Regex Detector Example
@@ -749,6 +753,8 @@ with HTTPServer(('', 8000), Verifier) as server:
pass
```
## :mag: Analyze
TruffleHog supports running a deeper analysis of a credential to view its permissions and the resources it has access to.
examples/generic_with_filters.yml
+1498
View File
File diff suppressed because it is too large Load Diff
pkg/custom_detectors/custom_detectors.go
+60
View File
@@ -65,6 +65,28 @@ func (c *CustomRegexWebhook) FromData(ctx context.Context, verify bool, data []b
dataStr := string(data)
regexMatches := make(map[string][][]string, len(c.GetRegex()))
// Compile exclude regexes targeting the capture group
excludeRegexesCapture := make([]*regexp.Regexp, 0, len(c.GetExcludeRegexesCapture()))
for _, exclude := range c.GetExcludeRegexesCapture() {
regex, err := regexp.Compile(exclude)
if err != nil {
// This will only happen if the regex is invalid.
return nil, err
}
excludeRegexesCapture = append(excludeRegexesCapture, regex)
}
// Compile exclude regexes targeting the entire match
excludeRegexes := make([]*regexp.Regexp, 0, len(c.GetExcludeRegexesMatch()))
for _, exclude := range c.GetExcludeRegexesMatch() {
regex, err := regexp.Compile(exclude)
if err != nil {
// This will only happen if the regex is invalid.
return nil, err
}
excludeRegexes = append(excludeRegexes, regex)
}
// Find all submatches for each regex.
for name, regex := range c.GetRegex() {
regex, err := regexp.Compile(regex)
@@ -91,7 +113,45 @@ func (c *CustomRegexWebhook) FromData(ctx context.Context, verify bool, data []b
// Create result object and test for verification.
resultsCh := make(chan detectors.Result, maxTotalMatches)
MatchLoop:
for _, match := range matches {
for _, values := range match {
// attempt to use capture group
secret := values[0]
if len(values) > 1 {
secret = values[1]
}
// check entropy
entropy := c.GetEntropy()
if entropy > 0.0 && detectors.StringShannonEntropy(secret) < float64(entropy) {
continue MatchLoop
}
// check for exclude words
for _, excludeWord := range c.GetExcludeWords() {
if strings.Contains(strings.ToLower(secret), excludeWord) {
continue MatchLoop
}
}
// exclude checks
for _, excludeMatch := range excludeRegexes {
if excludeMatch.MatchString(values[0]) {
continue MatchLoop
}
}
// exclude secret (capture group), or if no capture group is set,
// check against entire match.
for _, excludeSecret := range excludeRegexesCapture {
if excludeSecret.MatchString(secret) {
continue MatchLoop
}
}
}
g.Go(func() error {
return c.createResults(ctx, match, verify, resultsCh)
})
pkg/pb/custom_detectorspb/custom_detectors.pb.go
+68 -25
View File
@@ -73,11 +73,15 @@ type CustomRegex struct {
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
Keywords []string `protobuf:"bytes,2,rep,name=keywords,proto3" json:"keywords,omitempty"`
Regex map[string]string `protobuf:"bytes,3,rep,name=regex,proto3" json:"regex,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
Verify []*VerifierConfig `protobuf:"bytes,4,rep,name=verify,proto3" json:"verify,omitempty"`
Description string `protobuf:"bytes,5,opt,name=description,proto3" json:"description,omitempty"`
Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
Keywords []string `protobuf:"bytes,2,rep,name=keywords,proto3" json:"keywords,omitempty"`
Regex map[string]string `protobuf:"bytes,3,rep,name=regex,proto3" json:"regex,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
Verify []*VerifierConfig `protobuf:"bytes,4,rep,name=verify,proto3" json:"verify,omitempty"`
Description string `protobuf:"bytes,5,opt,name=description,proto3" json:"description,omitempty"`
ExcludeRegexesCapture []string `protobuf:"bytes,6,rep,name=exclude_regexes_capture,json=excludeRegexesCapture,proto3" json:"exclude_regexes_capture,omitempty"`
ExcludeWords []string `protobuf:"bytes,7,rep,name=exclude_words,json=excludeWords,proto3" json:"exclude_words,omitempty"`
Entropy float32 `protobuf:"fixed32,8,opt,name=entropy,proto3" json:"entropy,omitempty"`
ExcludeRegexesMatch []string `protobuf:"bytes,9,rep,name=exclude_regexes_match,json=excludeRegexesMatch,proto3" json:"exclude_regexes_match,omitempty"`
}
func (x *CustomRegex) Reset() {
@@ -147,6 +151,34 @@ func (x *CustomRegex) GetDescription() string {
return ""
}
func (x *CustomRegex) GetExcludeRegexesCapture() []string {
if x != nil {
return x.ExcludeRegexesCapture
}
return nil
}
func (x *CustomRegex) GetExcludeWords() []string {
if x != nil {
return x.ExcludeWords
}
return nil
}
func (x *CustomRegex) GetEntropy() float32 {
if x != nil {
return x.Entropy
}
return 0
}
func (x *CustomRegex) GetExcludeRegexesMatch() []string {
if x != nil {
return x.ExcludeRegexesMatch
}
return nil
}
type VerifierConfig struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
@@ -230,7 +262,7 @@ var file_custom_detectors_proto_rawDesc = []byte{
0x6f, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x63, 0x75, 0x73, 0x74,
0x6f, 0x6d, 0x5f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x2e, 0x43, 0x75, 0x73,
0x74, 0x6f, 0x6d, 0x52, 0x65, 0x67, 0x65, 0x78, 0x52, 0x09, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74,
0x6f, 0x72, 0x73, 0x22, 0x93, 0x02, 0x0a, 0x0b, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x52, 0x65,
0x6f, 0x72, 0x73, 0x22, 0xbe, 0x03, 0x0a, 0x0b, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x52, 0x65,
0x67, 0x65, 0x78, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28,
0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x77, 0x6f,
0x72, 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x77, 0x6f,
@@ -243,25 +275,36 @@ var file_custom_detectors_proto_rawDesc = []byte{
0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x2e, 0x56, 0x65, 0x72, 0x69, 0x66, 0x69, 0x65, 0x72, 0x43,
0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x06, 0x76, 0x65, 0x72, 0x69, 0x66, 0x79, 0x12, 0x20, 0x0a,
0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01,
0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x1a,
0x38, 0x0a, 0x0a, 0x52, 0x65, 0x67, 0x65, 0x78, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a,
0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12,
0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05,
0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x8e, 0x01, 0x0a, 0x0e, 0x56, 0x65,
0x72, 0x69, 0x66, 0x69, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x24, 0x0a, 0x08,
0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08,
0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69,
0x6e, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x75, 0x6e, 0x73, 0x61, 0x66, 0x65, 0x18, 0x02, 0x20, 0x01,
0x28, 0x08, 0x52, 0x06, 0x75, 0x6e, 0x73, 0x61, 0x66, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x68, 0x65,
0x61, 0x64, 0x65, 0x72, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x68, 0x65, 0x61,
0x64, 0x65, 0x72, 0x73, 0x12, 0x24, 0x0a, 0x0d, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52,
0x61, 0x6e, 0x67, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x73, 0x75, 0x63,
0x63, 0x65, 0x73, 0x73, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x42, 0x44, 0x5a, 0x42, 0x67, 0x69,
0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65,
0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65,
0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x63, 0x75,
0x73, 0x74, 0x6f, 0x6d, 0x5f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x70, 0x62,
0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12,
0x36, 0x0a, 0x17, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x72, 0x65, 0x67, 0x65, 0x78,
0x65, 0x73, 0x5f, 0x63, 0x61, 0x70, 0x74, 0x75, 0x72, 0x65, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09,
0x52, 0x15, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x52, 0x65, 0x67, 0x65, 0x78, 0x65, 0x73,
0x43, 0x61, 0x70, 0x74, 0x75, 0x72, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x65, 0x78, 0x63, 0x6c, 0x75,
0x64, 0x65, 0x5f, 0x77, 0x6f, 0x72, 0x64, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c,
0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x57, 0x6f, 0x72, 0x64, 0x73, 0x12, 0x18, 0x0a, 0x07,
0x65, 0x6e, 0x74, 0x72, 0x6f, 0x70, 0x79, 0x18, 0x08, 0x20, 0x01, 0x28, 0x02, 0x52, 0x07, 0x65,
0x6e, 0x74, 0x72, 0x6f, 0x70, 0x79, 0x12, 0x32, 0x0a, 0x15, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64,
0x65, 0x5f, 0x72, 0x65, 0x67, 0x65, 0x78, 0x65, 0x73, 0x5f, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x18,
0x09, 0x20, 0x03, 0x28, 0x09, 0x52, 0x13, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x52, 0x65,
0x67, 0x65, 0x78, 0x65, 0x73, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x1a, 0x38, 0x0a, 0x0a, 0x52, 0x65,
0x67, 0x65, 0x78, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18,
0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61,
0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65,
0x3a, 0x02, 0x38, 0x01, 0x22, 0x8e, 0x01, 0x0a, 0x0e, 0x56, 0x65, 0x72, 0x69, 0x66, 0x69, 0x65,
0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f,
0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03,
0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x16, 0x0a,
0x06, 0x75, 0x6e, 0x73, 0x61, 0x66, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x75,
0x6e, 0x73, 0x61, 0x66, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73,
0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x12,
0x24, 0x0a, 0x0d, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x73,
0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52,
0x61, 0x6e, 0x67, 0x65, 0x73, 0x42, 0x44, 0x5a, 0x42, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e,
0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72,
0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76,
0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5f,
0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f,
0x74, 0x6f, 0x33,
}
var (
pkg/pb/custom_detectorspb/custom_detectors.pb.validate.go
+2
View File
@@ -231,6 +231,8 @@ func (m *CustomRegex) validate(all bool) error {
// no validation rules for Description
// no validation rules for Entropy
if len(errors) > 0 {
return CustomRegexMultiError(errors)
}
proto/custom_detectors.proto
+5
View File
@@ -16,8 +16,13 @@ message CustomRegex {
map<string, string> regex = 3;
repeated VerifierConfig verify = 4;
string description = 5;
repeated string exclude_regexes_capture = 6;
repeated string exclude_words = 7;
float entropy = 8;
repeated string exclude_regexes_match = 9;
}
message VerifierConfig {
string endpoint = 1 [(validate.rules).string.uri_ref = true];
bool unsafe = 2;