[Feat] Detector implementation for Azure API Management Repository key (#3955)

* detector for azure repository initial commit

* included pattern test and integration test for azure api management repository key detector

* remove unused http.Client field.

* generate proto

pkg/detectors/azureapimanagement/repositorykey/repositorykey.go

@@ -0,0 +1,161 @@
+package repositorykey
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/url"
+	"os/exec"
+	"strconv"
+	"strings"
+
+	regexp "github.com/wasilibs/go-re2"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple"
+	logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+type Scanner struct {
+	detectors.DefaultMultiPartCredentialProvider
+}
+
+// Ensure the Scanner satisfies the interface at compile time.
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
+	urlPat      = regexp.MustCompile(detectors.PrefixRegex([]string{"azure", "url"}) + `([a-z0-9][a-z0-9-]{0,48}[a-z0-9]\.scm\.azure-api\.net)`)
+	passwordPat = regexp.MustCompile(detectors.PrefixRegex([]string{"azure", "password"}) + `\b(git&[0-9]{12}&[a-zA-Z0-9\/+]{85}[a-zA-Z0-9]==)`)
+
+	invalidHosts  = simple.NewCache[struct{}]()
+	noSuchHostErr = errors.New("Could not resolve host")
+)
+
+const (
+	azureGitUsername = "apim"
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"azure", ".scm.azure-api.net"}
+}
+
+// FromData will find and optionally verify AzureDevopsPersonalAccessToken secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+	logger := logContext.AddLogger(ctx).Logger().WithName("azurecr")
+	dataStr := string(data)
+
+	// Deduplicate matches.
+	uniqueUrlsMatches := make(map[string]struct{})
+	uniquePasswordMatches := make(map[string]struct{})
+
+	for _, matches := range urlPat.FindAllStringSubmatch(dataStr, -1) {
+		uniqueUrlsMatches[strings.TrimSpace(matches[1])] = struct{}{}
+	}
+
+	for _, matches := range passwordPat.FindAllStringSubmatch(dataStr, -1) {
+		uniquePasswordMatches[strings.TrimSpace(matches[1])] = struct{}{}
+	}
+
+EndpointLoop:
+	for urlMatch := range uniqueUrlsMatches {
+		for passwordMatch := range uniquePasswordMatches {
+			s1 := detectors.Result{
+				DetectorType: detectorspb.DetectorType_AzureApiManagementRepositoryKey,
+				Raw:          []byte(passwordMatch),
+				RawV2:        []byte(urlMatch + passwordMatch),
+			}
+
+			if verify {
+				if invalidHosts.Exists(urlMatch) {
+					logger.V(3).Info("Skipping invalid registry", "url", urlMatch)
+					continue EndpointLoop
+				}
+
+				isVerified, err := verifyUrlPassword(ctx, urlMatch, azureGitUsername, passwordMatch)
+				s1.Verified = isVerified
+				if err != nil {
+					if errors.Is(err, noSuchHostErr) {
+						invalidHosts.Set(urlMatch, struct{}{})
+						continue EndpointLoop
+					}
+					s1.SetVerificationError(err, urlMatch)
+				}
+			}
+			results = append(results, s1)
+		}
+	}
+
+	return results, nil
+}
+
+func (s Scanner) Type() detectorspb.DetectorType {
+	return detectorspb.DetectorType_AzureApiManagementRepositoryKey
+}
+
+func (s Scanner) Description() string {
+	return "Azure API Management Repository Keys provide access to the API Management (APIM) configuration repository, allowing users to directly interact with and modify API definitions, policies, and settings. These keys enable programmatic access to APIM's Git-based repository, where configurations can be cloned, edited, and pushed back to apply changes. They are primarily used for managing API configurations as code, automating deployments, and synchronizing APIM settings across environments."
+}
+
+func gitCmdCheck() error {
+	if errors.Is(exec.Command("git").Run(), exec.ErrNotFound) {
+		return fmt.Errorf("'git' command not found in $PATH. Make sure git is installed and included in $PATH")
+	}
+
+	// Check the version is greater than or equal to 2.20.0
+	out, err := exec.Command("git", "--version").Output()
+	if err != nil {
+		return fmt.Errorf("failed to check git version: %w", err)
+	}
+
+	// Extract the version string using a regex to find the version numbers
+	var regex = regexp.MustCompile(`\d+\.\d+\.\d+`)
+
+	versionStr := regex.FindString(string(out))
+	versionParts := strings.Split(versionStr, ".")
+
+	// Parse version numbers
+	major, _ := strconv.Atoi(versionParts[0])
+	minor, _ := strconv.Atoi(versionParts[1])
+
+	// Compare with version 2.20.0<=x<3.0.0
+	if major == 2 && minor >= 20 {
+		return nil
+	}
+	return fmt.Errorf("git version is %s, but must be greater than or equal to 2.20.0, and less than 3.0.0", versionStr)
+}
+
+func verifyUrlPassword(_ context.Context, repoUrl, user, password string) (bool, error) {
+	if err := gitCmdCheck(); err != nil {
+		return false, err
+	}
+
+	parsedURL, err := url.Parse(repoUrl)
+	if err != nil {
+		return false, err
+	}
+
+	if parsedURL.User == nil {
+		parsedURL.User = url.UserPassword(user, password)
+	}
+	parsedURL.Scheme = "https" // Force HTTPS
+
+	fakeRef := "TRUFFLEHOG_CHECK_GIT_REMOTE_URL_REACHABILITY"
+	gitArgs := []string{"ls-remote", parsedURL.String(), "--quiet", fakeRef}
+	cmd := exec.Command("git", gitArgs...)
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		outputString := string(output)
+		if strings.Contains(outputString, "Authentication failed") {
+			return false, nil
+		} else if strings.Contains(outputString, "Could not resolve host") {
+			return false, noSuchHostErr
+		}
+		return false, err
+	}
+
+	return true, nil
+}

pkg/detectors/azureapimanagement/repositorykey/repositorykey_integration_test.go

@@ -0,0 +1,135 @@
+//go:build detectors
+// +build detectors
+
+package repositorykey
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAxonaut_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	url := testSecrets.MustGetField("AZURE_API_MGMT_REPOSITORY_KEY_URL")
+	inactiveUrl := testSecrets.MustGetField("AZURE_API_MGMT_REPOSITORY_KEY_URL_INACTIVE")
+	password := testSecrets.MustGetField("AZURE_API_MGMT_REPOSITORY_KEY_PASSWORD")
+	inactivePassword := testSecrets.MustGetField("AZURE_API_MGMT_REPOSITORY_KEY_PASSWORD_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a azure repository url %s password %s", url, password)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_AzureApiManagementRepositoryKey,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a azure repository url %s password %s but unverified", url, inactivePassword)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_AzureApiManagementRepositoryKey,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+		{
+			name: "found, host not resolved",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a azure repository url %s password %s but unverified", inactiveUrl, inactivePassword)),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("AzureApiManagementRepositoryKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+			}
+
+			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError", "ExtraData", "AnalysisInfo")
+			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+				t.Errorf("AzureApiManagementRepositoryKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/azureapimanagement/repositorykey/repositorykey_test.go

@@ -0,0 +1,105 @@
+package repositorykey
+
+import (
+	"context"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
+
+var (
+	validPattern = `
+		AZURE_URL=https://test.scm.azure-api.net
+		PASSWORD=git&202503251200&R2xlVEmqi+OW130dxWIDhfw1K6XKw/gxc5P9te3cwWBtnK2XkZq5k+VUAdnuX1Y0T/I5CRK9fJyBJr31SmFEYw==
+	`
+	invalidHostPattern = `
+		AZURE_URL=https://test.scm.azure.net
+		PASSWORD=git&202503251200&R2xlVEmqi+OW130dxWIDhfw1K6XKw/gxc5P9te3cwWBtnK2XkZq5k+VUAdnuX1Y0T/I5CRK9fJyBJr31SmFEYw==
+	`
+	invalidPasswordPattern1 = `
+		AZURE_URL=https://test.scm.azure-api.net
+		PASSWORD=git&202503251200&R2xlVEmqi+OW130dxWIDhfw1K6XKw/gxc5P9te3cwWBtnK2XkZq5k+VUAdnuX1Y0T/I5CRK9fJyBJr31SmFEYw=
+	`
+	invalidPasswordPattern2 = `
+		AZURE_URL=https://test.scm.azure-api.net
+		PASSWORD=git&20250325&R2xlVEmqi+OW130dxWIDhfw1K6XKw/gxc5P9te3cwWBtnK2XkZq5k+VUAdnuX1Y0T/I5CRK9fJyBJr31SmFEYw==
+	`
+)
+
+func TestAzureAPIManagementRepositoryKey_Pattern(t *testing.T) {
+	d := Scanner{}
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
+	tests := []struct {
+		name  string
+		input string
+		want  []string
+	}{
+		{
+			name:  `valid pattern`,
+			input: validPattern,
+			want:  []string{"test.scm.azure-api.net" + "git&202503251200&R2xlVEmqi+OW130dxWIDhfw1K6XKw/gxc5P9te3cwWBtnK2XkZq5k+VUAdnuX1Y0T/I5CRK9fJyBJr31SmFEYw=="},
+		},
+		{
+			name:  `invalid host pattern`,
+			input: invalidHostPattern,
+			want:  []string{},
+		},
+		{
+			name:  `invalid password pattern without ==`,
+			input: invalidPasswordPattern1,
+			want:  []string{},
+		},
+		{
+			name:  `invalid password pattern with wrong expiry date`,
+			input: invalidPasswordPattern2,
+			want:  []string{},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+			if len(matchedDetectors) == 0 {
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
+				return
+			}
+
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
+			if err != nil {
+				t.Errorf("error = %v", err)
+				return
+			}
+
+			if len(results) != len(test.want) {
+				if len(results) == 0 {
+					t.Errorf("did not receive result")
+				} else {
+					if diff := cmp.Diff(test.want, results); diff != "" {
+						t.Errorf("%s expected %d results, received %d: (-want +got)\n%s", test.name, len(test.want), len(results), diff)
+					}
+				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+			}
+		})
+	}
+}

pkg/engine/defaults/defaults.go

@@ -72,6 +72,7 @@ import (
 	azure_entra_serviceprincipal_v2 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azure_entra/serviceprincipal/v2"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azure_openai"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azure_storage"
+	azurerepositorykey "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azureapimanagement/repositorykey"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azurecontainerregistry"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azuredevopspersonalaccesstoken"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azuresastoken"
@@ -911,6 +912,7 @@ func buildDetectorList() []detectors.Detector {
 		&azuresearchadminkey.Scanner{},
 		&azuresearchquerykey.Scanner{},
 		&azure_storage.Scanner{},
+		&azurerepositorykey.Scanner{},
 		&bannerbear.Scanner{},
 		&baremetrics.Scanner{},
 		&beamer.Scanner{},

pkg/pb/detectorspb/detectors.pb.go

@@ -1122,6 +1122,7 @@ const (
 	DetectorType_AirtablePersonalAccessToken             DetectorType = 1015
 	DetectorType_StoryblokPersonalAccessToken            DetectorType = 1016
 	DetectorType_SentryOrgToken                          DetectorType = 1017
+	DetectorType_AzureApiManagementRepositoryKey         DetectorType = 1018
 )
 
 // Enum value maps for DetectorType.
@@ -2141,6 +2142,7 @@ var (
 		1015: "AirtablePersonalAccessToken",
 		1016: "StoryblokPersonalAccessToken",
 		1017: "SentryOrgToken",
+		1018: "AzureApiManagementRepositoryKey",
 	}
 	DetectorType_value = map[string]int32{
 		"Alibaba":                               0,
@@ -3157,6 +3159,7 @@ var (
 		"AirtablePersonalAccessToken":      1015,
 		"StoryblokPersonalAccessToken":     1016,
 		"SentryOrgToken":                   1017,
+		"AzureApiManagementRepositoryKey":  1018,
 	}
 )
 
@@ -3610,7 +3613,7 @@ var file_detectors_proto_rawDesc = []byte{
 	0x4c, 0x41, 0x49, 0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34,
 	0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x12, 0x13, 0x0a,
 	0x0f, 0x45, 0x53, 0x43, 0x41, 0x50, 0x45, 0x44, 0x5f, 0x55, 0x4e, 0x49, 0x43, 0x4f, 0x44, 0x45,
-	0x10, 0x04, 0x2a, 0xcd, 0x82, 0x01, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72,
+	0x10, 0x04, 0x2a, 0xf3, 0x82, 0x01, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72,
 	0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41, 0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10,
 	0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41,
 	0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x10, 0x03, 0x12,
@@ -4655,11 +4658,13 @@ var file_detectors_proto_rawDesc = []byte{
 	0x74, 0x6f, 0x72, 0x79, 0x62, 0x6c, 0x6f, 0x6b, 0x50, 0x65, 0x72, 0x73, 0x6f, 0x6e, 0x61, 0x6c,
 	0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x10, 0xf8, 0x07, 0x12, 0x13,
 	0x0a, 0x0e, 0x53, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x4f, 0x72, 0x67, 0x54, 0x6f, 0x6b, 0x65, 0x6e,
-	0x10, 0xf9, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f,
-	0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74,
-	0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f,
-	0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73,
-	0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x10, 0xf9, 0x07, 0x12, 0x24, 0x0a, 0x1f, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x41, 0x70, 0x69, 0x4d,
+	0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74,
+	0x6f, 0x72, 0x79, 0x4b, 0x65, 0x79, 0x10, 0xfa, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74,
+	0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73,
+	0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68,
+	0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74,
+	0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (

proto/detectors.proto

@@ -1027,6 +1027,7 @@ enum DetectorType {
   AirtablePersonalAccessToken = 1015;
   StoryblokPersonalAccessToken = 1016;
   SentryOrgToken = 1017;
+  AzureApiManagementRepositoryKey = 1018;
 }
 
 message Result {