[INS-470] Add Tly detector to defaults.go, gate it behind feat flag and update its verification logic (#5006)

* refactored tly detector

main.go

@@ -534,6 +534,7 @@ func run(state overseer.State, logSync func() error) {
 	feature.GitLabOAuthDetectorEnabled.Store(true)
 	feature.EnigmaDetectorEnabled.Store(true)
 	feature.DatadogApiKeyDetectorEnabled.Store(true)
+	feature.TlyDetectorEnabled.Store(true)
 
 	conf := &config.Config{}
 	if *configFilename != "" {

pkg/detectors/tly/tly.go

@@ -2,6 +2,8 @@ package tly
 
 import (
 	"context"
+	"fmt"
+	"io"
 	"net/http"
 	"strings"
 
@@ -31,38 +33,76 @@ func (s Scanner) Keywords() []string {
 }
 
 // FromData will find and optionally verify TLy secrets in a given set of bytes.
-func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+func (s Scanner) FromData(
+	ctx context.Context,
+	verify bool,
+	data []byte,
+) (results []detectors.Result, err error) {
+
 	dataStr := string(data)
 
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	uniqueKeys := make(map[string]struct{})
+	for _, match := range keyPat.FindAllStringSubmatch(dataStr, -1) {
+		uniqueKeys[strings.TrimSpace(match[1])] = struct{}{}
+	}
 
-	for _, match := range matches {
-		resMatch := strings.TrimSpace(match[1])
-
-		s1 := detectors.Result{
+	for key := range uniqueKeys {
+		result := detectors.Result{
 			DetectorType: detector_typepb.DetectorType_TLy,
-			Raw:          []byte(resMatch),
-			SecretParts:  map[string]string{"key": resMatch},
+			Raw:          []byte(key),
+			SecretParts: map[string]string{
+				"key": key,
+			},
 		}
 
 		if verify {
-			req, err := http.NewRequestWithContext(ctx, "GET", "https://t.ly/api/v1/link/stats?api_token="+resMatch+"&short_url=https://t.ly/h9YS", nil)
-			if err != nil {
-				continue
-			}
-			res, err := client.Do(req)
-			if err == nil {
-				defer func() { _ = res.Body.Close() }()
-				if res.StatusCode >= 200 && res.StatusCode < 300 {
-					s1.Verified = true
-				}
-			}
+			verified, verificationErr := verifyTLyKey(ctx, client, key)
+			result.SetVerificationError(verificationErr, key)
+			result.Verified = verified
 		}
 
-		results = append(results, s1)
+		results = append(results, result)
 	}
 
-	return results, nil
+	return
+}
+
+func verifyTLyKey(
+	ctx context.Context,
+	client *http.Client,
+	key string,
+) (bool, error) {
+
+	req, err := http.NewRequestWithContext(
+		ctx,
+		http.MethodGet,
+		"https://api.t.ly/api/v1/link/list",
+		http.NoBody,
+	)
+	if err != nil {
+		return false, err
+	}
+
+	req.Header.Set("Authorization", "Bearer "+key)
+	req.Header.Set("Accept", "application/json")
+
+	res, err := client.Do(req)
+	if err != nil {
+		return false, err
+	}
+	defer func() {
+		_, _ = io.Copy(io.Discard, res.Body)
+		_ = res.Body.Close()
+	}()
+
+	switch res.StatusCode {
+	case http.StatusOK:
+		return true, nil
+	case http.StatusUnauthorized, http.StatusForbidden:
+		return false, nil
+	default:
+		return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
+	}
 }
 
 func (s Scanner) Type() detector_typepb.DetectorType {

pkg/detectors/tly/tly_integration_test.go

@@ -9,8 +9,8 @@ import (
 	"testing"
 	"time"
 
-	"github.com/kylelemons/godebug/pretty"
-
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb"
@@ -96,8 +96,19 @@ func TestTLy_FromChunk(t *testing.T) {
 				}
 				got[i].Raw = nil
 			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("TLy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+			ignoreOpts := cmpopts.IgnoreFields(
+				detectors.Result{},
+				"ExtraData",
+				"verificationError",
+				"primarySecret",
+				"SecretParts",
+				"chunkOffset",
+				"chunkOffsetSet",
+			)
+
+			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+				t.Errorf("Tly.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}
 		})
 	}

pkg/engine/defaults/defaults.go

@@ -773,6 +773,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/timecamp"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/timezoneapi"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/tineswebhook"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/tly"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/tmetric"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/todoist"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/tokeet"
@@ -1671,6 +1672,7 @@ func buildDetectorList() []detectors.Detector {
 		&timecamp.Scanner{},
 		&timezoneapi.Scanner{},
 		&tineswebhook.Scanner{},
+		&tly.Scanner{},
 		&tmetric.Scanner{},
 		&todoist.Scanner{},
 		// &toggltrack.Scanner{},
@@ -1784,6 +1786,8 @@ func buildDetectorList() []detectors.Detector {
 			return !feature.EnigmaDetectorEnabled.Load()
 		case *datadogapikey.Scanner:
 			return !feature.DatadogApiKeyDetectorEnabled.Load()
+		case *tly.Scanner:
+			return !feature.TlyDetectorEnabled.Load()
 		default:
 			return false
 		}

pkg/engine/defaults/defaults_test.go

@@ -122,7 +122,6 @@ var excludedFromDefaultList = map[detector_typepb.DetectorType]struct{}{
 	detector_typepb.DetectorType_IPInfo: {},
 	detector_typepb.DetectorType_Lob:    {},
 	detector_typepb.DetectorType_Rev:    {},
-	detector_typepb.DetectorType_TLy:    {},
 	detector_typepb.DetectorType_Tru:    {},
 	detector_typepb.DetectorType_User:   {},
 	detector_typepb.DetectorType_Wit:    {},
@@ -134,6 +133,7 @@ var excludedFromDefaultList = map[detector_typepb.DetectorType]struct{}{
 	detector_typepb.DetectorType_Enigma:        {},
 	detector_typepb.DetectorType_GitLabOauth2:  {},
 	detector_typepb.DetectorType_Pinecone:      {},
+	detector_typepb.DetectorType_TLy:           {},
 
 	// Reserved / special types.
 	detector_typepb.DetectorType_CustomRegex: {}, // added dynamically via engine config, not via buildDetectorList()

pkg/feature/feature.go

@@ -21,6 +21,7 @@ var (
 	GitLabOAuthDetectorEnabled     atomic.Bool
 	EnigmaDetectorEnabled          atomic.Bool
 	DatadogApiKeyDetectorEnabled   atomic.Bool
+	TlyDetectorEnabled             atomic.Bool
 )
 
 type AtomicString struct {