diff --git a/pkg/detectors/netlify/netlify.go b/pkg/detectors/netlify/v1/netlify_v1.go similarity index 93% rename from pkg/detectors/netlify/netlify.go rename to pkg/detectors/netlify/v1/netlify_v1.go index 6995b81ca..9d1fa62d3 100644 --- a/pkg/detectors/netlify/netlify.go +++ b/pkg/detectors/netlify/v1/netlify_v1.go @@ -3,10 +3,12 @@ package netlify import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" + "strconv" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" @@ -16,6 +18,7 @@ type Scanner struct{} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) +var _ detectors.Versioner = (*Scanner)(nil) var ( client = common.SaneHttpClient() @@ -23,6 +26,8 @@ var ( keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"netlify"}) + `\b([A-Za-z0-9_-]{43,45})\b`) ) +func (Scanner) Version() int { return 1 } + // Keywords are used for efficiently pre-filtering chunks. // Use identifiers in the secret preferably, or the provider name. func (s Scanner) Keywords() []string { @@ -44,6 +49,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } s1.ExtraData = map[string]string{ "rotation_guide": "https://howtorotate.com/docs/tutorials/netlify/", + "version": strconv.Itoa(s.Version()), } if verify { diff --git a/pkg/detectors/netlify/netlify_integration_test.go b/pkg/detectors/netlify/v1/netlify_v1_integration_test.go similarity index 98% rename from pkg/detectors/netlify/netlify_integration_test.go rename to pkg/detectors/netlify/v1/netlify_v1_integration_test.go index 5129a95c2..4cab93a48 100644 --- a/pkg/detectors/netlify/netlify_integration_test.go +++ b/pkg/detectors/netlify/v1/netlify_v1_integration_test.go @@ -52,6 +52,7 @@ func TestNetlify_FromChunk(t *testing.T) { Verified: true, ExtraData: map[string]string{ "rotation_guide": "https://howtorotate.com/docs/tutorials/netlify/", + "version": "1", }, }, }, @@ -71,6 +72,7 @@ func TestNetlify_FromChunk(t *testing.T) { Verified: false, ExtraData: map[string]string{ "rotation_guide": "https://howtorotate.com/docs/tutorials/netlify/", + "version": "1", }, }, }, diff --git a/pkg/detectors/netlify/netlify_test.go b/pkg/detectors/netlify/v1/netlify_v1_test.go similarity index 100% rename from pkg/detectors/netlify/netlify_test.go rename to pkg/detectors/netlify/v1/netlify_v1_test.go diff --git a/pkg/detectors/netlify/v2/netlify_v2.go b/pkg/detectors/netlify/v2/netlify_v2.go new file mode 100644 index 000000000..5985979f1 --- /dev/null +++ b/pkg/detectors/netlify/v2/netlify_v2.go @@ -0,0 +1,82 @@ +package netlify + +import ( + "context" + "fmt" + "net/http" + "strconv" + "strings" + + regexp "github.com/wasilibs/go-re2" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +type Scanner struct{} + +// Ensure the Scanner satisfies the interface at compile time. +var _ detectors.Detector = (*Scanner)(nil) +var _ detectors.Versioner = (*Scanner)(nil) + +var ( + client = common.SaneHttpClient() + + keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"netlify"}) + `\b(nfp_[a-zA-Z0-9_]{36})\b`) +) + +func (Scanner) Version() int { return 2 } + +// Keywords are used for efficiently pre-filtering chunks. +// Use identifiers in the secret preferably, or the provider name. +func (s Scanner) Keywords() []string { + return []string{"netlify"} +} + +// FromData will find and optionally verify Netlify secrets in a given set of bytes. +func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { + dataStr := string(data) + + matches := keyPat.FindAllStringSubmatch(dataStr, -1) + + for _, match := range matches { + resMatch := strings.TrimSpace(match[1]) + + s1 := detectors.Result{ + DetectorType: detectorspb.DetectorType_Netlify, + Raw: []byte(resMatch), + } + s1.ExtraData = map[string]string{ + "rotation_guide": "https://howtorotate.com/docs/tutorials/netlify/", + "version": strconv.Itoa(s.Version()), + } + + if verify { + req, err := http.NewRequestWithContext(ctx, "GET", "https://api.netlify.com/api/v1/sites", nil) + if err != nil { + continue + } + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) + res, err := client.Do(req) + if err == nil { + defer res.Body.Close() + if res.StatusCode >= 200 && res.StatusCode < 300 { + s1.Verified = true + } + } + } + + results = append(results, s1) + } + + return results, nil +} + +func (s Scanner) Type() detectorspb.DetectorType { + return detectorspb.DetectorType_Netlify +} + +func (s Scanner) Description() string { + return "Netlify is a cloud platform for web developers that provides hosting and serverless backend services for web applications and static websites. Netlify API keys can be used to manage sites, deploy applications, and access various services offered by the platform." +} diff --git a/pkg/detectors/netlify/v2/netlify_v2_integration_test.go b/pkg/detectors/netlify/v2/netlify_v2_integration_test.go new file mode 100644 index 000000000..2fe5fde88 --- /dev/null +++ b/pkg/detectors/netlify/v2/netlify_v2_integration_test.go @@ -0,0 +1,128 @@ +//go:build detectors +// +build detectors + +package netlify + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestNetlify_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("NETLIFY_TOKEN") + inactiveSecret := testSecrets.MustGetField("NETLIFY_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a netlify secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Netlify, + Verified: true, + ExtraData: map[string]string{ + "rotation_guide": "https://howtorotate.com/docs/tutorials/netlify/", + "version": "2", + }, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a netlify secret %s within but unverified", inactiveSecret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Netlify, + Verified: false, + ExtraData: map[string]string{ + "rotation_guide": "https://howtorotate.com/docs/tutorials/netlify/", + "version": "2", + }, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Netlify.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Netlify.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/netlify/v2/netlify_v2_test.go b/pkg/detectors/netlify/v2/netlify_v2_test.go new file mode 100644 index 000000000..44d5590da --- /dev/null +++ b/pkg/detectors/netlify/v2/netlify_v2_test.go @@ -0,0 +1,91 @@ +package netlify + +import ( + "context" + "fmt" + "testing" + + "github.com/google/go-cmp/cmp" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) + +var ( + validPattern = "nfp_gsTPZr8vTqmit69DY8u8es68N3XvxqW20330" + invalidPattern = "nfp_?sTPZr8vTqmit69DY8u8es68N3XvxqW20330" + keyword = "netlify" +) + +func TestNetlify_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { + name string + input string + want []string + }{ + { + name: "valid pattern - with keyword netlify", + input: fmt.Sprintf("%s token = '%s'", keyword, validPattern), + want: []string{validPattern}, + }, + { + name: "valid pattern - ignore duplicate", + input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern), + want: []string{validPattern}, + }, + { + name: "valid pattern - key out of prefix range", + input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern), + want: []string{}, + }, + { + name: "invalid pattern", + input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern), + want: []string{}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) + return + } + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return + } + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} + } + } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } + }) + } +} diff --git a/pkg/engine/defaults/defaults.go b/pkg/engine/defaults/defaults.go index 4be9926d0..2e6c4d54e 100644 --- a/pkg/engine/defaults/defaults.go +++ b/pkg/engine/defaults/defaults.go @@ -468,7 +468,8 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/myfreshworks" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/myintervals" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/nethunt" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/netlify" + netlifyv1 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/netlify/v1" + netlifyv2 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/netlify/v2" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/netsuite" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/neutrinoapi" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/newrelicpersonalapikey" @@ -1320,7 +1321,8 @@ func buildDetectorList() []detectors.Detector { &myintervals.Scanner{}, // &nasdaqdatalink.Scanner{}, &nethunt.Scanner{}, - &netlify.Scanner{}, + &netlifyv1.Scanner{}, + &netlifyv2.Scanner{}, &netsuite.Scanner{}, &neutrinoapi.Scanner{}, &newrelicpersonalapikey.Scanner{},