From ce1ce29b905758e16cd3b63950b1a03deb45e714 Mon Sep 17 00:00:00 2001 From: ahrav Date: Wed, 5 Jun 2024 13:28:19 -0700 Subject: [PATCH] [feat] - Optimize detector performance by reducing data passed to regex (#2812) * optimize maching detetors * update method name * updates * update naming * updates * update comment * updates * remove testcase * update default match len to 512 * update * update test * add support for multpart cred provider * add ability to scan entire chunk * encapsulate matches logic within FindDetectorMatches * use []byte directly * nil chunk data * use []byte * set hidden flag to true * remove * [refactor] - multi part detectors (#2906) * Detectors beginning w/ a * Detectors beginning w/ b * Detectors beginning w/ c * Detectors beginning w/ d * Detectors beginning w/ e * Detectors beginning w/ f * Detectors beginning w/ f&g * fix * Detectors beginning w/ i-l * Detectors beginning w/ m-p * Detectors beginning w/ r-s * Detectors beginning w/ t * Detectors beginning w/ u-z * revert alconst * remaining fixes * lint * [feat] - Add Support for `compareDetectionStrategies` Mode (#2918) * Detector comparison mode * remove else * return error if results dont match * update default hidden flag to not scan entire chunks * fix tests * enhance encapsulation by including methods on DetectorMatch to handle merging and extracting * remove space * fix * update detector * updates * remove else * run comparison concurrently --- main.go | 266 +++++++++++++----- pkg/detectors/adobeio/adobeio.go | 4 +- pkg/detectors/adzuna/adzuna.go | 1 + pkg/detectors/aeroworkflow/aeroworkflow.go | 1 + pkg/detectors/agora/agora.go | 1 + pkg/detectors/aha/aha.go | 1 + .../airbrakeprojectkey/airbrakeprojectkey.go | 4 +- .../airtableapikey/airtableapikey.go | 4 +- pkg/detectors/alchemy/alchemy_test.go | 5 +- pkg/detectors/alegra/alegra.go | 4 +- .../algoliaadminkey/algoliaadminkey.go | 4 +- pkg/detectors/alibaba/alibaba.go | 1 + pkg/detectors/amadeus/amadeus.go | 4 +- .../amplitudeapikey/amplitudeapikey.go | 4 +- pkg/detectors/anypoint/anypoint.go | 4 +- pkg/detectors/apideck/apideck.go | 4 +- pkg/detectors/apiflash/apiflash.go | 4 +- pkg/detectors/apifonica/apifonica.go | 3 +- pkg/detectors/apimatic/apimatic.go | 4 +- pkg/detectors/appcues/appcues.go | 4 +- pkg/detectors/appfollow/appfollow.go | 3 +- pkg/detectors/apptivo/apptivo.go | 4 +- pkg/detectors/artifactory/artifactory.go | 1 + pkg/detectors/artsy/artsy.go | 4 +- .../auth0managementapitoken.go | 4 +- pkg/detectors/auth0oauth/auth0oauth.go | 4 +- pkg/detectors/autodesk/autodesk.go | 4 +- pkg/detectors/aws/aws.go | 2 + pkg/detectors/awssessionkeys/awssessionkey.go | 1 + pkg/detectors/aylien/aylien.go | 4 +- pkg/detectors/azure/azure.go | 4 +- pkg/detectors/azurebatch/azurebatch.go | 1 + .../azurecontainerregistry.go | 1 + .../azuredevopspersonalaccesstoken.go | 1 + .../azurefunctionkey/azurefunctionkey.go | 1 + .../azuresearchadminkey.go | 1 + .../azuresearchquerykey.go | 1 + pkg/detectors/billomat/billomat.go | 4 +- pkg/detectors/bitfinex/bitfinex.go | 4 +- pkg/detectors/bitmex/bitmex.go | 4 +- pkg/detectors/blazemeter/blazemeter.go | 3 +- .../braintreepayments/braintreepayments.go | 1 + pkg/detectors/browserstack/browserstack.go | 1 + pkg/detectors/buildkitev2/buildkite.go | 3 +- pkg/detectors/bulksms/bulksms.go | 4 +- pkg/detectors/captaindata/captaindata.go | 4 +- pkg/detectors/cashboard/cashboard.go | 4 +- pkg/detectors/caspio/caspio.go | 4 +- pkg/detectors/censys/censys.go | 4 +- pkg/detectors/cexio/cexio.go | 4 +- pkg/detectors/chatbot/chatbot.go | 3 +- pkg/detectors/checkout/checkout.go | 4 +- pkg/detectors/checkvist/checkvist.go | 4 +- pkg/detectors/clickhelp/clickhelp.go | 4 +- pkg/detectors/clicksendsms/clicksendsms.go | 4 +- pkg/detectors/clockworksms/clockworksms.go | 4 +- pkg/detectors/cloudconvert/cloudconvert.go | 9 +- pkg/detectors/cloudelements/cloudelements.go | 4 +- .../cloudflareglobalapikey.go | 4 +- pkg/detectors/cloudsmith/cloudsmith.go | 5 +- pkg/detectors/cloze/cloze.go | 4 +- pkg/detectors/coinbase_waas/coinbase_waas.go | 8 +- pkg/detectors/collect2/collect2.go | 3 +- pkg/detectors/commodities/commodities.go | 3 +- pkg/detectors/companyhub/companyhub.go | 4 +- pkg/detectors/confluent/confluent.go | 4 +- pkg/detectors/copper/copper.go | 4 +- pkg/detectors/couchbase/couchbase.go | 4 +- pkg/detectors/currencycloud/currencycloud.go | 4 +- pkg/detectors/customerguru/customerguru.go | 4 +- pkg/detectors/customerio/customerio.go | 4 +- .../databrickstoken/databrickstoken.go | 1 + pkg/detectors/datadogtoken/datadogtoken.go | 1 + pkg/detectors/demio/demio.go | 4 +- pkg/detectors/deputy/deputy.go | 4 +- pkg/detectors/detectify/detectify.go | 3 +- pkg/detectors/detectors.go | 15 + .../discordbottoken/discordbottoken.go | 4 +- pkg/detectors/dnscheck/dnscheck.go | 4 +- pkg/detectors/dockerhub/v1/dockerhub.go | 1 + pkg/detectors/dockerhub/v2/dockerhub.go | 1 + pkg/detectors/docusign/docusign.go | 4 +- pkg/detectors/dotmailer/dotmailer.go | 4 +- pkg/detectors/dovico/dovico.go | 4 +- pkg/detectors/duply/duply.go | 4 +- pkg/detectors/dwolla/dwolla.go | 4 +- .../eagleeyenetworks/eagleeyenetworks.go | 4 +- pkg/detectors/easyinsight/easyinsight.go | 4 +- pkg/detectors/ecostruxureit/ecostruxureit.go | 3 +- pkg/detectors/edamam/edamam.go | 4 +- pkg/detectors/eightxeight/eightxeight.go | 4 +- pkg/detectors/enablex/enablex.go | 4 +- pkg/detectors/everhour/everhour.go | 3 +- pkg/detectors/exportsdk/exportsdk.go | 4 +- pkg/detectors/facebookoauth/facebookoauth.go | 4 +- pkg/detectors/faceplusplus/faceplusplus.go | 4 +- pkg/detectors/fibery/fibery.go | 4 +- pkg/detectors/flightstats/flightstats.go | 4 +- pkg/detectors/flowflu/flowflu.go | 4 +- pkg/detectors/fmfw/fmfw.go | 4 +- pkg/detectors/formbucket/formbucket.go | 3 +- pkg/detectors/formsite/formsite.go | 4 +- pkg/detectors/foursquare/foursquare.go | 4 +- pkg/detectors/freshbooks/freshbooks.go | 4 +- pkg/detectors/freshdesk/freshdesk.go | 4 +- pkg/detectors/front/front.go | 3 +- pkg/detectors/gcp/gcp.go | 6 + .../gcpapplicationdefaultcredentials.go | 11 +- .../gcpapplicationdefaultcredentials_test.go | 2 +- pkg/detectors/gemini/gemini.go | 4 +- pkg/detectors/gengo/gengo.go | 4 +- pkg/detectors/geocodio/geocodio.go | 4 +- pkg/detectors/getemails/getemails.go | 4 +- pkg/detectors/getsandbox/getsandbox.go | 4 +- pkg/detectors/github_oauth2/github_oauth2.go | 13 +- pkg/detectors/githubapp/githubapp.go | 8 +- pkg/detectors/gocanvas/gocanvas.go | 4 +- .../grafanaserviceaccount.go | 1 + pkg/detectors/graphcms/graphcms.go | 4 +- pkg/detectors/guru/guru.go | 4 +- pkg/detectors/harvest/harvest.go | 4 +- pkg/detectors/heroku/heroku.go | 3 +- pkg/detectors/hive/hive.go | 4 +- pkg/detectors/hypertrack/hypertrack.go | 4 +- pkg/detectors/instamojo/instamojo.go | 4 +- pkg/detectors/intra42/intra42.go | 1 + pkg/detectors/invoiceocean/invoiceocean.go | 4 +- pkg/detectors/ip2location/ip2location.go | 3 +- pkg/detectors/ipinfodb/ipinfodb.go | 3 +- pkg/detectors/jdbc/jdbc.go | 1 + pkg/detectors/jiratoken/v1/jiratoken.go | 1 + pkg/detectors/jiratoken/v2/jiratoken_v2.go | 1 + pkg/detectors/kanban/kanban.go | 4 +- pkg/detectors/kanbantool/kanbantool.go | 4 +- pkg/detectors/keenio/keenio.go | 4 +- pkg/detectors/kraken/kraken.go | 4 +- pkg/detectors/kucoin/kucoin.go | 4 +- pkg/detectors/ldap/ldap.go | 4 +- pkg/detectors/lemonsqueezy/lemonsqueezy.go | 3 +- pkg/detectors/lendflow/lendflow.go | 7 +- .../lessannoyingcrm/lessannoyingcrm.go | 3 +- pkg/detectors/liveagent/liveagent.go | 4 +- pkg/detectors/loggly/loggly.go | 1 + pkg/detectors/loginradius/loginradius.go | 3 +- pkg/detectors/luno/luno.go | 4 +- pkg/detectors/magicbell/magicbell.go | 4 +- pkg/detectors/mailgun/mailgun.go | 4 +- pkg/detectors/mailmodo/mailmodo.go | 3 +- pkg/detectors/mapbox/mapbox.go | 7 +- .../mattermostpersonaltoken.go | 4 +- .../maxmindlicense/v1/maxmindlicense.go | 4 +- pkg/detectors/metaapi/metaapi.go | 4 +- pkg/detectors/metabase/metabase.go | 4 +- .../microsoftteamswebhook.go | 4 +- pkg/detectors/mite/mite.go | 4 +- pkg/detectors/mixpanel/mixpanel.go | 4 +- pkg/detectors/moderation/moderation.go | 3 +- pkg/detectors/monday/monday.go | 3 +- pkg/detectors/mrticktock/mrticktock.go | 4 +- .../multi_part_credential_provider.go | 28 ++ .../multi_part_credential_provider_test.go | 35 +++ pkg/detectors/mux/mux.go | 7 +- pkg/detectors/myfreshworks/myfreshworks.go | 1 + pkg/detectors/nethunt/nethunt.go | 4 +- pkg/detectors/neutrinoapi/neutrinoapi.go | 4 +- pkg/detectors/nexmoapikey/nexmoapikey.go | 4 +- pkg/detectors/ngc/ngc.go | 4 +- pkg/detectors/nozbeteams/nozbeteams.go | 3 +- pkg/detectors/nutritionix/nutritionix.go | 4 +- pkg/detectors/okta/okta.go | 4 +- pkg/detectors/onedesk/onedesk.go | 7 +- pkg/detectors/onelogin/onelogin.go | 4 +- pkg/detectors/onepagecrm/onepagecrm.go | 4 +- pkg/detectors/onfleet/onfleet_test.go | 5 +- pkg/detectors/openvpn/openvpn.go | 6 +- pkg/detectors/pagarme/pagarme_test.go | 5 +- pkg/detectors/paymongo/paymongo.go | 3 +- pkg/detectors/paypaloauth/paypaloauth.go | 4 +- pkg/detectors/pinata/pinata.go | 4 +- pkg/detectors/plaidkey/plaidkey.go | 4 +- pkg/detectors/planetscale/planetscale.go | 1 + pkg/detectors/planetscaledb/planetscaledb.go | 4 +- .../planviewleankit/planviewleankit.go | 4 +- pkg/detectors/plivo/plivo.go | 4 +- pkg/detectors/pollsapi/pollsapi.go | 3 +- pkg/detectors/poloniex/poloniex.go | 4 +- pkg/detectors/portainer/portainer.go | 1 + .../portainertoken/portainertoken.go | 1 + pkg/detectors/postgres/postgres.go | 1 + pkg/detectors/privatekey/privatekey.go | 9 +- pkg/detectors/prospectcrm/prospectcrm.go | 3 +- .../pubnubpublishkey/pubnubpublishkey.go | 1 + .../pusherchannelkey/pusherchannelkey.go | 4 +- pkg/detectors/ramp/ramp.go | 1 + pkg/detectors/razorpay/razorpay.go | 4 +- .../rechargepayments/rechargepayments.go | 4 +- pkg/detectors/redis/redis.go | 4 +- pkg/detectors/repairshopr/repairshopr.go | 4 +- pkg/detectors/rev/rev.go | 4 +- pkg/detectors/revampcrm/revampcrm.go | 4 +- pkg/detectors/ringcentral/ringcentral.go | 4 +- pkg/detectors/roaring/roaring.go | 4 +- pkg/detectors/roninapp/roninapp.go | 4 +- pkg/detectors/rownd/rownd.go | 4 +- pkg/detectors/runrunit/runrunit.go | 4 +- pkg/detectors/salesforce/salesforce.go | 1 + pkg/detectors/salesmate/salesmate.go | 4 +- .../satismeterprojectkey.go | 4 +- pkg/detectors/saucelabs/saucelabs.go | 4 +- pkg/detectors/scalr/scalr.go | 4 +- pkg/detectors/sendbird/sendbird.go | 1 + pkg/detectors/sheety/sheety.go | 7 +- pkg/detectors/shopify/shopify.go | 4 +- pkg/detectors/shutterstock/shutterstock.go | 4 +- pkg/detectors/signable/signable.go | 1 + pkg/detectors/signalwire/signalwire.go | 4 +- pkg/detectors/sigopt/sigopt.go | 3 +- pkg/detectors/sinchmessage/sinchmessage.go | 4 +- pkg/detectors/sirv/sirv.go | 4 +- pkg/detectors/siteleaf/siteleaf.go | 4 +- pkg/detectors/skybiometry/skybiometry.go | 4 +- pkg/detectors/slack/slack.go | 1 + pkg/detectors/slackwebhook/slackwebhook.go | 1 + pkg/detectors/smartystreets/smartystreets.go | 4 +- pkg/detectors/smooch/smooch.go | 4 +- pkg/detectors/snowflake/snowflake.go | 1 + pkg/detectors/snykkey/snykkey_test.go | 10 +- pkg/detectors/spotifykey/spotifykey.go | 4 +- pkg/detectors/sqlserver/sqlserver.go | 1 + pkg/detectors/squareapp/squareapp.go | 4 +- pkg/detectors/strava/strava.go | 4 +- pkg/detectors/stytch/stytch.go | 4 +- pkg/detectors/sugester/sugester.go | 4 +- pkg/detectors/sumologickey/sumologickey.go | 4 +- pkg/detectors/supernotesapi/supernotesapi.go | 3 +- .../surveyanyplace/surveyanyplace.go | 4 +- pkg/detectors/swell/swell.go | 4 +- pkg/detectors/tallyfy/tallyfy.go | 7 +- pkg/detectors/teamgate/teamgate.go | 4 +- pkg/detectors/testingbot/testingbot.go | 4 +- pkg/detectors/textmagic/textmagic.go | 4 +- pkg/detectors/thinkific/thinkific.go | 4 +- pkg/detectors/thousandeyes/thousandeyes.go | 4 +- pkg/detectors/tineswebhook/tineswebhook.go | 3 +- pkg/detectors/tokeet/tokeet.go | 4 +- pkg/detectors/trelloapikey/trelloapikey.go | 4 +- pkg/detectors/tru/tru.go | 4 +- .../trufflehogenterprise.go | 4 +- pkg/detectors/twilio/twilio.go | 1 + pkg/detectors/twitch/twitch.go | 1 + pkg/detectors/twitter/twitter.go | 4 +- pkg/detectors/typetalk/typetalk.go | 4 +- pkg/detectors/uploadcare/uploadcare.go | 4 +- pkg/detectors/uptimerobot/uptimerobot.go | 3 +- pkg/detectors/user/user.go | 3 +- pkg/detectors/verifier/verifier.go | 4 +- pkg/detectors/verimail/verimail.go | 3 +- pkg/detectors/veriphone/veriphone.go | 3 +- pkg/detectors/viewneo/viewneo.go | 8 +- pkg/detectors/vouchery/vouchery.go | 4 +- pkg/detectors/webex/webex.go | 4 +- pkg/detectors/websitepulse/websitepulse.go | 4 +- pkg/detectors/wepay/wepay.go | 4 +- pkg/detectors/wiz/wiz.go | 1 + pkg/detectors/wiz/wiz_test.go | 5 +- .../worldcoinindex/worldcoinindex.go | 3 +- pkg/detectors/youtubeapikey/youtubeapikey.go | 4 +- pkg/detectors/zendeskapi/zendeskapi.go | 4 +- pkg/detectors/zipapi/zipapi.go | 4 +- pkg/detectors/zipbooks/zipbooks.go | 4 +- pkg/detectors/zulipchat/zulipchat.go | 1 + pkg/engine/ahocorasick/ahocorasickcore.go | 202 +++++++++++-- .../ahocorasick/ahocorasickcore_test.go | 156 ++++++++-- pkg/engine/engine.go | 193 +++++++------ pkg/engine/engine_test.go | 73 ++++- 275 files changed, 1469 insertions(+), 472 deletions(-) create mode 100644 pkg/detectors/multi_part_credential_provider.go create mode 100644 pkg/detectors/multi_part_credential_provider_test.go diff --git a/main.go b/main.go index 8b522a9c3..8d63a236d 100644 --- a/main.go +++ b/main.go @@ -10,6 +10,7 @@ import ( "runtime" "strconv" "strings" + "sync" "syscall" "time" @@ -50,10 +51,12 @@ var ( onlyVerified = cli.Flag("only-verified", "Only output verified results.").Bool() results = cli.Flag("results", "Specifies which type(s) of results to output: verified, unknown, unverified. Defaults to all types.").Hidden().String() - allowVerificationOverlap = cli.Flag("allow-verification-overlap", "Allow verification of similar credentials across detectors").Bool() - filterUnverified = cli.Flag("filter-unverified", "Only output first unverified result per chunk per detector if there are more than one results.").Bool() - filterEntropy = cli.Flag("filter-entropy", "Filter unverified results with Shannon entropy. Start with 3.0.").Float64() - configFilename = cli.Flag("config", "Path to configuration file.").ExistingFile() + allowVerificationOverlap = cli.Flag("allow-verification-overlap", "Allow verification of similar credentials across detectors").Bool() + filterUnverified = cli.Flag("filter-unverified", "Only output first unverified result per chunk per detector if there are more than one results.").Bool() + filterEntropy = cli.Flag("filter-entropy", "Filter unverified results with Shannon entropy. Start with 3.0.").Float64() + scanEntireChunk = cli.Flag("scan-entire-chunk", "Scan the entire chunk for secrets.").Hidden().Default("false").Bool() + compareDetectionStrategies = cli.Flag("compare-detection-strategies", "Compare different detection strategies for matching spans").Hidden().Default("false").Bool() + configFilename = cli.Flag("config", "Path to configuration file.").ExistingFile() // rules = cli.Flag("rules", "Path to file with custom rules.").String() printAvgDetectorTime = cli.Flag("print-avg-detector-time", "Print the average time spent on each detector.").Bool() noUpdate = cli.Flag("no-update", "Don't check for updates.").Bool() @@ -463,30 +466,158 @@ func run(state overseer.State) { logFatal(err, "failed to configure results flag") } - e, err := engine.Start(ctx, - engine.WithConcurrency(*concurrency), - engine.WithDecoders(decoders.DefaultDecoders()...), - engine.WithDetectors(engine.DefaultDetectors()...), - engine.WithDetectors(conf.Detectors...), - engine.WithVerify(!*noVerification), - engine.WithFilterDetectors(includeFilter), - engine.WithFilterDetectors(excludeFilter), - engine.WithFilterDetectors(endpointCustomizer), - engine.WithFilterUnverified(*filterUnverified), - engine.WithResults(parsedResults), - engine.WithPrintAvgDetectorTime(*printAvgDetectorTime), - engine.WithPrinter(printer), - engine.WithFilterEntropy(*filterEntropy), - engine.WithVerificationOverlap(*allowVerificationOverlap), - engine.WithJobReportWriter(jobReportWriter), - ) - if err != nil { - logFatal(err, "error initializing engine") + scanConfig := scanConfig{ + Command: cmd, + Concurrency: *concurrency, + Decoders: decoders.DefaultDecoders(), + Conf: conf, + IncludeFilter: includeFilter, + ExcludeFilter: excludeFilter, + EndpointCustomizer: endpointCustomizer, + NoVerification: *noVerification, + PrintAvgDetectorTime: *printAvgDetectorTime, + FilterUnverified: *filterUnverified, + FilterEntropy: *filterEntropy, + ScanEntireChunk: *scanEntireChunk, + JobReportWriter: jobReportWriter, + AllowVerificationOverlap: *allowVerificationOverlap, + ParsedResults: parsedResults, + Printer: printer, } - switch cmd { + if *compareDetectionStrategies { + err := compareScans(ctx, scanConfig) + if err != nil { + logFatal(err, "error comparing detection strategies") + } + return + } + + metrics, err := runSingleScan(ctx, scanConfig, *scanEntireChunk) + if err != nil { + logFatal(err, "error running scan") + } + + // Print results. + logger.Info("finished scanning", + "chunks", metrics.ChunksScanned, + "bytes", metrics.BytesScanned, + "verified_secrets", metrics.VerifiedSecretsFound, + "unverified_secrets", metrics.UnverifiedSecretsFound, + "scan_duration", metrics.ScanDuration.String(), + "trufflehog_version", version.BuildVersion, + ) + + if metrics.hasFoundResults && *fail { + logger.V(2).Info("exiting with code 183 because results were found") + os.Exit(183) + } +} + +type scanConfig struct { + Command string + Concurrency int + Decoders []decoders.Decoder + Conf *config.Config + IncludeFilter func(detectors.Detector) bool + ExcludeFilter func(detectors.Detector) bool + EndpointCustomizer func(detectors.Detector) bool + NoVerification bool + PrintAvgDetectorTime bool + FilterUnverified bool + FilterEntropy float64 + ScanEntireChunk bool + JobReportWriter io.WriteCloser + AllowVerificationOverlap bool + ParsedResults map[string]struct{} + Printer engine.Printer +} + +func compareScans(ctx context.Context, cfg scanConfig) error { + var ( + entireMetrics metrics + maxLengthMetrics metrics + err error + ) + + var wg sync.WaitGroup + wg.Add(1) + + go func() { + defer wg.Done() + // Run scan with entire chunk span calculator. + entireMetrics, err = runSingleScan(ctx, cfg, true) + if err != nil { + ctx.Logger().Error(err, "error running scan with entire chunk span calculator") + } + }() + + // Run scan with max-length span calculator. + maxLengthMetrics, err = runSingleScan(ctx, cfg, false) + if err != nil { + return fmt.Errorf("error running scan with custom span calculator: %v", err) + } + + wg.Wait() + + return compareMetrics(maxLengthMetrics.Metrics, entireMetrics.Metrics) +} + +func compareMetrics(customMetrics, entireMetrics engine.Metrics) error { + fmt.Printf("Comparison of scan results: \n") + fmt.Printf("Custom span - Chunks: %d, Bytes: %d, Verified Secrets: %d, Unverified Secrets: %d, Duration: %s\n", + customMetrics.ChunksScanned, customMetrics.BytesScanned, customMetrics.VerifiedSecretsFound, customMetrics.UnverifiedSecretsFound, customMetrics.ScanDuration.String()) + fmt.Printf("Entire chunk - Chunks: %d, Bytes: %d, Verified Secrets: %d, Unverified Secrets: %d, Duration: %s\n", + entireMetrics.ChunksScanned, entireMetrics.BytesScanned, entireMetrics.VerifiedSecretsFound, entireMetrics.UnverifiedSecretsFound, entireMetrics.ScanDuration.String()) + + // Check for differences in scan metrics. + if customMetrics.ChunksScanned != entireMetrics.ChunksScanned || + customMetrics.BytesScanned != entireMetrics.BytesScanned || + customMetrics.VerifiedSecretsFound != entireMetrics.VerifiedSecretsFound { + return fmt.Errorf("scan metrics do not match") + } + + return nil +} + +type metrics struct { + engine.Metrics + hasFoundResults bool +} + +func runSingleScan(ctx context.Context, cfg scanConfig, scanEntireChunk bool) (metrics, error) { + eng, err := engine.Start(ctx, + engine.WithConcurrency(cfg.Concurrency), + engine.WithDecoders(cfg.Decoders...), + engine.WithDetectors(engine.DefaultDetectors()...), + engine.WithDetectors(cfg.Conf.Detectors...), + engine.WithVerify(!cfg.NoVerification), + engine.WithFilterDetectors(cfg.IncludeFilter), + engine.WithFilterDetectors(cfg.ExcludeFilter), + engine.WithFilterDetectors(cfg.EndpointCustomizer), + engine.WithFilterUnverified(cfg.FilterUnverified), + engine.WithResults(cfg.ParsedResults), + engine.WithPrintAvgDetectorTime(cfg.PrintAvgDetectorTime), + engine.WithPrinter(cfg.Printer), + engine.WithFilterEntropy(cfg.FilterEntropy), + engine.WithVerificationOverlap(cfg.AllowVerificationOverlap), + engine.WithEntireChunkScan(scanEntireChunk), + ) + if err != nil { + return metrics{}, fmt.Errorf("error initializing engine: %v", err) + } + + defer func() { + // Clean up temporary artifacts. + if err := cleantemp.CleanTempArtifacts(ctx); err != nil { + ctx.Logger().Error(err, "error cleaning temp artifacts") + } + }() + + var scanMetrics metrics + switch cfg.Command { case gitScan.FullCommand(): - cfg := sources.GitConfig{ + gitCfg := sources.GitConfig{ URI: *gitScanURI, IncludePathsFile: *gitScanIncludePaths, ExcludePathsFile: *gitScanExcludePaths, @@ -496,16 +627,16 @@ func run(state overseer.State) { Bare: *gitScanBare, ExcludeGlobs: *gitScanExcludeGlobs, } - if err = e.ScanGit(ctx, cfg); err != nil { - logFatal(err, "Failed to scan Git.") + if err = eng.ScanGit(ctx, gitCfg); err != nil { + return scanMetrics, fmt.Errorf("failed to scan Git: %v", err) } case githubScan.FullCommand(): filter, err := common.FilterFromFiles(*githubScanIncludePaths, *githubScanExcludePaths) if err != nil { - logFatal(err, "could not create filter") + return scanMetrics, fmt.Errorf("could not create filter: %v", err) } if len(*githubScanOrgs) == 0 && len(*githubScanRepos) == 0 { - logFatal(fmt.Errorf("invalid config"), "You must specify at least one organization or repository.") + return scanMetrics, fmt.Errorf("invalid config: you must specify at least one organization or repository") } cfg := sources.GithubConfig{ @@ -524,13 +655,13 @@ func run(state overseer.State) { IncludeGistComments: *githubScanGistComments, Filter: filter, } - if err := e.ScanGitHub(ctx, cfg); err != nil { - logFatal(err, "Failed to scan Github.") + if err := eng.ScanGitHub(ctx, cfg); err != nil { + return scanMetrics, fmt.Errorf("failed to scan Github: %v", err) } case gitlabScan.FullCommand(): filter, err := common.FilterFromFiles(*gitlabScanIncludePaths, *gitlabScanExcludePaths) if err != nil { - logFatal(err, "could not create filter") + return scanMetrics, fmt.Errorf("could not create filter: %v", err) } cfg := sources.GitlabConfig{ @@ -539,8 +670,8 @@ func run(state overseer.State) { Repos: *gitlabScanRepos, Filter: filter, } - if err := e.ScanGitLab(ctx, cfg); err != nil { - logFatal(err, "Failed to scan GitLab.") + if err := eng.ScanGitLab(ctx, cfg); err != nil { + return scanMetrics, fmt.Errorf("failed to scan GitLab: %v", err) } case filesystemScan.FullCommand(): if len(*filesystemDirectories) > 0 { @@ -554,8 +685,8 @@ func run(state overseer.State) { IncludePathsFile: *filesystemScanIncludePaths, ExcludePathsFile: *filesystemScanExcludePaths, } - if err = e.ScanFileSystem(ctx, cfg); err != nil { - logFatal(err, "Failed to scan filesystem") + if err = eng.ScanFileSystem(ctx, cfg); err != nil { + return scanMetrics, fmt.Errorf("failed to scan filesystem: %v", err) } case s3Scan.FullCommand(): cfg := sources.S3Config{ @@ -568,8 +699,8 @@ func run(state overseer.State) { CloudCred: *s3ScanCloudEnv, MaxObjectSize: int64(*s3ScanMaxObjectSize), } - if err := e.ScanS3(ctx, cfg); err != nil { - logFatal(err, "Failed to scan S3.") + if err := eng.ScanS3(ctx, cfg); err != nil { + return scanMetrics, fmt.Errorf("failed to scan S3: %v", err) } case syslogScan.FullCommand(): cfg := sources.SyslogConfig{ @@ -580,16 +711,16 @@ func run(state overseer.State) { KeyPath: *syslogTLSKey, Concurrency: *concurrency, } - if err := e.ScanSyslog(ctx, cfg); err != nil { - logFatal(err, "Failed to scan syslog.") + if err := eng.ScanSyslog(ctx, cfg); err != nil { + return scanMetrics, fmt.Errorf("failed to scan syslog: %v", err) } case circleCiScan.FullCommand(): - if err := e.ScanCircleCI(ctx, *circleCiScanToken); err != nil { - logFatal(err, "Failed to scan CircleCI.") + if err := eng.ScanCircleCI(ctx, *circleCiScanToken); err != nil { + return scanMetrics, fmt.Errorf("failed to scan CircleCI: %v", err) } case travisCiScan.FullCommand(): - if err := e.ScanTravisCI(ctx, *travisCiScanToken); err != nil { - logFatal(err, "Failed to scan TravisCI.") + if err := eng.ScanTravisCI(ctx, *travisCiScanToken); err != nil { + return scanMetrics, fmt.Errorf("failed to scan TravisCI: %v", err) } case gcsScan.FullCommand(): cfg := sources.GCSConfig{ @@ -605,8 +736,8 @@ func run(state overseer.State) { Concurrency: *concurrency, MaxObjectSize: int64(*gcsMaxObjectSize), } - if err := e.ScanGCS(ctx, cfg); err != nil { - logFatal(err, "Failed to scan GCS.") + if err := eng.ScanGCS(ctx, cfg); err != nil { + return scanMetrics, fmt.Errorf("failed to scan GCS: %v", err) } case dockerScan.FullCommand(): cfg := sources.DockerConfig{ @@ -614,8 +745,8 @@ func run(state overseer.State) { Images: *dockerScanImages, UseDockerKeychain: *dockerScanToken == "", } - if err := e.ScanDocker(ctx, cfg); err != nil { - logFatal(err, "Failed to scan Docker.") + if err := eng.ScanDocker(ctx, cfg); err != nil { + return scanMetrics, fmt.Errorf("failed to scan Docker: %v", err) } case postmanScan.FullCommand(): // handle deprecated flag @@ -651,8 +782,8 @@ func run(state overseer.State) { WorkspacePaths: *postmanWorkspacePaths, EnvironmentPaths: *postmanEnvironmentPaths, } - if err := e.ScanPostman(ctx, cfg); err != nil { - logFatal(err, "Failed to scan Postman.") + if err := eng.ScanPostman(ctx, cfg); err != nil { + return scanMetrics, fmt.Errorf("failed to scan Postman: %v", err) } case elasticsearchScan.FullCommand(): cfg := sources.ElasticsearchConfig{ @@ -667,8 +798,8 @@ func run(state overseer.State) { SinceTimestamp: *elasticsearchSinceTimestamp, BestEffortScan: *elasticsearchBestEffortScan, } - if err := e.ScanElasticsearch(ctx, cfg); err != nil { - logFatal(err, "Failed to scan Elasticsearch.") + if err := eng.ScanElasticsearch(ctx, cfg); err != nil { + return scanMetrics, fmt.Errorf("failed to scan Elasticsearch: %v", err) } case jenkinsScan.FullCommand(): cfg := engine.JenkinsConfig{ @@ -677,40 +808,23 @@ func run(state overseer.State) { Username: *jenkinsUsername, Password: *jenkinsPassword, } - if err := e.ScanJenkins(ctx, cfg); err != nil { - logFatal(err, "Failed to scan Jenkins.") + if err := eng.ScanJenkins(ctx, cfg); err != nil { + return scanMetrics, fmt.Errorf("failed to scan Jenkins: %v", err) } default: - logFatal(fmt.Errorf("invalid command"), "Command not recognized.") + return scanMetrics, fmt.Errorf("invalid command: %s", cfg.Command) } // Wait for all workers to finish. - if err = e.Finish(ctx); err != nil { - logFatal(err, "engine failed to finish execution") + if err = eng.Finish(ctx); err != nil { + return scanMetrics, fmt.Errorf("engine failed to finish execution: %v", err) } - if err := cleantemp.CleanTempArtifacts(ctx); err != nil { - ctx.Logger().Error(err, "error cleaning temp artifacts") - } - - metrics := e.GetMetrics() - // Print results. - logger.Info("finished scanning", - "chunks", metrics.ChunksScanned, - "bytes", metrics.BytesScanned, - "verified_secrets", metrics.VerifiedSecretsFound, - "unverified_secrets", metrics.UnverifiedSecretsFound, - "scan_duration", metrics.ScanDuration.String(), - "trufflehog_version", version.BuildVersion, - ) if *printAvgDetectorTime { - printAverageDetectorTime(e) + printAverageDetectorTime(eng) } - if e.HasFoundResults() && *fail { - logger.V(2).Info("exiting with code 183 because results were found") - os.Exit(183) - } + return metrics{Metrics: eng.GetMetrics(), hasFoundResults: eng.HasFoundResults()}, nil } // parseResults ensures that users provide valid CSV input to `--results`. diff --git a/pkg/detectors/adobeio/adobeio.go b/pkg/detectors/adobeio/adobeio.go index 1efe07cbb..0e8667e81 100644 --- a/pkg/detectors/adobeio/adobeio.go +++ b/pkg/detectors/adobeio/adobeio.go @@ -11,7 +11,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/adzuna/adzuna.go b/pkg/detectors/adzuna/adzuna.go index 5fb06ba2d..71ac40a1b 100644 --- a/pkg/detectors/adzuna/adzuna.go +++ b/pkg/detectors/adzuna/adzuna.go @@ -14,6 +14,7 @@ import ( type Scanner struct { client *http.Client + detectors.DefaultMultiPartCredentialProvider } const adzunaURL = "https://api.adzuna.com" diff --git a/pkg/detectors/aeroworkflow/aeroworkflow.go b/pkg/detectors/aeroworkflow/aeroworkflow.go index 1bbff9e03..6c015ac88 100644 --- a/pkg/detectors/aeroworkflow/aeroworkflow.go +++ b/pkg/detectors/aeroworkflow/aeroworkflow.go @@ -14,6 +14,7 @@ import ( ) type Scanner struct { + detectors.DefaultMultiPartCredentialProvider client *http.Client } diff --git a/pkg/detectors/agora/agora.go b/pkg/detectors/agora/agora.go index fe7fb55c6..8a2a6d6fe 100644 --- a/pkg/detectors/agora/agora.go +++ b/pkg/detectors/agora/agora.go @@ -14,6 +14,7 @@ import ( ) type Scanner struct { + detectors.DefaultMultiPartCredentialProvider client *http.Client } diff --git a/pkg/detectors/aha/aha.go b/pkg/detectors/aha/aha.go index 9126386c9..6788794aa 100644 --- a/pkg/detectors/aha/aha.go +++ b/pkg/detectors/aha/aha.go @@ -14,6 +14,7 @@ import ( ) type Scanner struct { + detectors.DefaultMultiPartCredentialProvider client *http.Client } diff --git a/pkg/detectors/airbrakeprojectkey/airbrakeprojectkey.go b/pkg/detectors/airbrakeprojectkey/airbrakeprojectkey.go index 1c98fd610..82efb1de1 100644 --- a/pkg/detectors/airbrakeprojectkey/airbrakeprojectkey.go +++ b/pkg/detectors/airbrakeprojectkey/airbrakeprojectkey.go @@ -11,7 +11,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/airtableapikey/airtableapikey.go b/pkg/detectors/airtableapikey/airtableapikey.go index 01ca34972..38b98d640 100644 --- a/pkg/detectors/airtableapikey/airtableapikey.go +++ b/pkg/detectors/airtableapikey/airtableapikey.go @@ -13,7 +13,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/alchemy/alchemy_test.go b/pkg/detectors/alchemy/alchemy_test.go index e3465e5e9..cd655da7d 100644 --- a/pkg/detectors/alchemy/alchemy_test.go +++ b/pkg/detectors/alchemy/alchemy_test.go @@ -35,9 +35,8 @@ func TestAlchemy_Pattern(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - chunkSpecificDetectors := make(map[ahocorasick.DetectorKey]detectors.Detector, 2) - ahoCorasickCore.PopulateMatchingDetectors(test.input, chunkSpecificDetectors) - if len(chunkSpecificDetectors) == 0 { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } diff --git a/pkg/detectors/alegra/alegra.go b/pkg/detectors/alegra/alegra.go index c23b5cb12..3e6355dd1 100644 --- a/pkg/detectors/alegra/alegra.go +++ b/pkg/detectors/alegra/alegra.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/algoliaadminkey/algoliaadminkey.go b/pkg/detectors/algoliaadminkey/algoliaadminkey.go index ee0739ac9..0287589ae 100644 --- a/pkg/detectors/algoliaadminkey/algoliaadminkey.go +++ b/pkg/detectors/algoliaadminkey/algoliaadminkey.go @@ -11,7 +11,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/alibaba/alibaba.go b/pkg/detectors/alibaba/alibaba.go index 0a5f5a035..f98f9ff82 100644 --- a/pkg/detectors/alibaba/alibaba.go +++ b/pkg/detectors/alibaba/alibaba.go @@ -22,6 +22,7 @@ import ( ) type Scanner struct { + detectors.DefaultMultiPartCredentialProvider client *http.Client } diff --git a/pkg/detectors/amadeus/amadeus.go b/pkg/detectors/amadeus/amadeus.go index 37893d126..6f994dcae 100644 --- a/pkg/detectors/amadeus/amadeus.go +++ b/pkg/detectors/amadeus/amadeus.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/amplitudeapikey/amplitudeapikey.go b/pkg/detectors/amplitudeapikey/amplitudeapikey.go index c48cbc107..25e3a9e45 100644 --- a/pkg/detectors/amplitudeapikey/amplitudeapikey.go +++ b/pkg/detectors/amplitudeapikey/amplitudeapikey.go @@ -11,7 +11,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/anypoint/anypoint.go b/pkg/detectors/anypoint/anypoint.go index f001a7bfe..c30e69709 100644 --- a/pkg/detectors/anypoint/anypoint.go +++ b/pkg/detectors/anypoint/anypoint.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/apideck/apideck.go b/pkg/detectors/apideck/apideck.go index 68f037e3d..655ee86c9 100644 --- a/pkg/detectors/apideck/apideck.go +++ b/pkg/detectors/apideck/apideck.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/apiflash/apiflash.go b/pkg/detectors/apiflash/apiflash.go index 9a6e1aee8..50dc97ccb 100644 --- a/pkg/detectors/apiflash/apiflash.go +++ b/pkg/detectors/apiflash/apiflash.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/apifonica/apifonica.go b/pkg/detectors/apifonica/apifonica.go index bf770b3e3..0f50379db 100644 --- a/pkg/detectors/apifonica/apifonica.go +++ b/pkg/detectors/apifonica/apifonica.go @@ -4,10 +4,11 @@ import ( "context" b64 "encoding/base64" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/apimatic/apimatic.go b/pkg/detectors/apimatic/apimatic.go index 0e843c80f..d55508fc9 100644 --- a/pkg/detectors/apimatic/apimatic.go +++ b/pkg/detectors/apimatic/apimatic.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/appcues/appcues.go b/pkg/detectors/appcues/appcues.go index 7e66993f8..51e27841c 100644 --- a/pkg/detectors/appcues/appcues.go +++ b/pkg/detectors/appcues/appcues.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/appfollow/appfollow.go b/pkg/detectors/appfollow/appfollow.go index 37fc5a9ad..962278df8 100644 --- a/pkg/detectors/appfollow/appfollow.go +++ b/pkg/detectors/appfollow/appfollow.go @@ -2,10 +2,11 @@ package appfollow import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/apptivo/apptivo.go b/pkg/detectors/apptivo/apptivo.go index 2be5fe94d..07010e044 100644 --- a/pkg/detectors/apptivo/apptivo.go +++ b/pkg/detectors/apptivo/apptivo.go @@ -13,7 +13,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/artifactory/artifactory.go b/pkg/detectors/artifactory/artifactory.go index 8b5236446..afe9dd123 100644 --- a/pkg/detectors/artifactory/artifactory.go +++ b/pkg/detectors/artifactory/artifactory.go @@ -15,6 +15,7 @@ import ( type Scanner struct { client *http.Client + detectors.DefaultMultiPartCredentialProvider } var ( diff --git a/pkg/detectors/artsy/artsy.go b/pkg/detectors/artsy/artsy.go index 77ab08fa2..bc0ed8041 100644 --- a/pkg/detectors/artsy/artsy.go +++ b/pkg/detectors/artsy/artsy.go @@ -11,7 +11,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/auth0managementapitoken/auth0managementapitoken.go b/pkg/detectors/auth0managementapitoken/auth0managementapitoken.go index e4081d292..3f5e279a6 100644 --- a/pkg/detectors/auth0managementapitoken/auth0managementapitoken.go +++ b/pkg/detectors/auth0managementapitoken/auth0managementapitoken.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/auth0oauth/auth0oauth.go b/pkg/detectors/auth0oauth/auth0oauth.go index 25ce7d035..c35e14e4e 100644 --- a/pkg/detectors/auth0oauth/auth0oauth.go +++ b/pkg/detectors/auth0oauth/auth0oauth.go @@ -13,7 +13,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/autodesk/autodesk.go b/pkg/detectors/autodesk/autodesk.go index 1c8dbffea..75abd2aae 100644 --- a/pkg/detectors/autodesk/autodesk.go +++ b/pkg/detectors/autodesk/autodesk.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/aws/aws.go b/pkg/detectors/aws/aws.go index 110a06566..cf0cff0f7 100644 --- a/pkg/detectors/aws/aws.go +++ b/pkg/detectors/aws/aws.go @@ -26,6 +26,7 @@ import ( type scanner struct { verificationClient *http.Client skipIDs map[string]struct{} + detectors.DefaultMultiPartCredentialProvider } // resourceTypes derived from: https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html#identifiers-unique-ids @@ -102,6 +103,7 @@ func WithSkipIDs(skipIDs []string) func(*scanner) { // Ensure the scanner satisfies the interface at compile time. var _ detectors.Detector = (*scanner)(nil) +var _ detectors.MultiPartCredentialProvider = (*scanner)(nil) var ( defaultVerificationClient = common.SaneHttpClient() diff --git a/pkg/detectors/awssessionkeys/awssessionkey.go b/pkg/detectors/awssessionkeys/awssessionkey.go index 643626a25..4974f3d76 100644 --- a/pkg/detectors/awssessionkeys/awssessionkey.go +++ b/pkg/detectors/awssessionkeys/awssessionkey.go @@ -19,6 +19,7 @@ import ( ) type scanner struct { + detectors.DefaultMultiPartCredentialProvider verificationClient *http.Client skipIDs map[string]struct{} } diff --git a/pkg/detectors/aylien/aylien.go b/pkg/detectors/aylien/aylien.go index cfe20a098..28195ecaa 100644 --- a/pkg/detectors/aylien/aylien.go +++ b/pkg/detectors/aylien/aylien.go @@ -11,7 +11,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/azure/azure.go b/pkg/detectors/azure/azure.go index 99ce2ddc3..2319ce97c 100644 --- a/pkg/detectors/azure/azure.go +++ b/pkg/detectors/azure/azure.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/azurebatch/azurebatch.go b/pkg/detectors/azurebatch/azurebatch.go index 508c0833a..342b37d13 100644 --- a/pkg/detectors/azurebatch/azurebatch.go +++ b/pkg/detectors/azurebatch/azurebatch.go @@ -19,6 +19,7 @@ import ( type Scanner struct { client *http.Client + detectors.DefaultMultiPartCredentialProvider } // Ensure the Scanner satisfies the interface at compile time. diff --git a/pkg/detectors/azurecontainerregistry/azurecontainerregistry.go b/pkg/detectors/azurecontainerregistry/azurecontainerregistry.go index d5053f6e8..0126d4745 100644 --- a/pkg/detectors/azurecontainerregistry/azurecontainerregistry.go +++ b/pkg/detectors/azurecontainerregistry/azurecontainerregistry.go @@ -15,6 +15,7 @@ import ( type Scanner struct { client *http.Client + detectors.DefaultMultiPartCredentialProvider } // Ensure the Scanner satisfies the interface at compile time. diff --git a/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken.go b/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken.go index d1996bb2e..6f8fe0782 100644 --- a/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken.go +++ b/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken.go @@ -15,6 +15,7 @@ import ( type Scanner struct { client *http.Client + detectors.DefaultMultiPartCredentialProvider } // Ensure the Scanner satisfies the interface at compile time. diff --git a/pkg/detectors/azurefunctionkey/azurefunctionkey.go b/pkg/detectors/azurefunctionkey/azurefunctionkey.go index fb994cd31..7af938ea2 100644 --- a/pkg/detectors/azurefunctionkey/azurefunctionkey.go +++ b/pkg/detectors/azurefunctionkey/azurefunctionkey.go @@ -15,6 +15,7 @@ import ( type Scanner struct { client *http.Client + detectors.DefaultMultiPartCredentialProvider } // Ensure the Scanner satisfies the interface at compile time. diff --git a/pkg/detectors/azuresearchadminkey/azuresearchadminkey.go b/pkg/detectors/azuresearchadminkey/azuresearchadminkey.go index 6dd0673be..f20271b19 100644 --- a/pkg/detectors/azuresearchadminkey/azuresearchadminkey.go +++ b/pkg/detectors/azuresearchadminkey/azuresearchadminkey.go @@ -15,6 +15,7 @@ import ( type Scanner struct { client *http.Client + detectors.DefaultMultiPartCredentialProvider } // Ensure the Scanner satisfies the interface at compile time. diff --git a/pkg/detectors/azuresearchquerykey/azuresearchquerykey.go b/pkg/detectors/azuresearchquerykey/azuresearchquerykey.go index e6781286c..5e7b9497e 100644 --- a/pkg/detectors/azuresearchquerykey/azuresearchquerykey.go +++ b/pkg/detectors/azuresearchquerykey/azuresearchquerykey.go @@ -15,6 +15,7 @@ import ( type Scanner struct { client *http.Client + detectors.DefaultMultiPartCredentialProvider } // Ensure the Scanner satisfies the interface at compile time. diff --git a/pkg/detectors/billomat/billomat.go b/pkg/detectors/billomat/billomat.go index cf2eba3d3..a431c9f7e 100644 --- a/pkg/detectors/billomat/billomat.go +++ b/pkg/detectors/billomat/billomat.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/bitfinex/bitfinex.go b/pkg/detectors/bitfinex/bitfinex.go index 7ec56f865..534d886d8 100644 --- a/pkg/detectors/bitfinex/bitfinex.go +++ b/pkg/detectors/bitfinex/bitfinex.go @@ -14,7 +14,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/bitmex/bitmex.go b/pkg/detectors/bitmex/bitmex.go index 01adc8d33..b82bee561 100644 --- a/pkg/detectors/bitmex/bitmex.go +++ b/pkg/detectors/bitmex/bitmex.go @@ -17,7 +17,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/blazemeter/blazemeter.go b/pkg/detectors/blazemeter/blazemeter.go index 615b7b5f5..51acf8c9d 100644 --- a/pkg/detectors/blazemeter/blazemeter.go +++ b/pkg/detectors/blazemeter/blazemeter.go @@ -3,10 +3,11 @@ package blazemeter import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/braintreepayments/braintreepayments.go b/pkg/detectors/braintreepayments/braintreepayments.go index b156783b9..2a8daf391 100644 --- a/pkg/detectors/braintreepayments/braintreepayments.go +++ b/pkg/detectors/braintreepayments/braintreepayments.go @@ -14,6 +14,7 @@ import ( ) type Scanner struct { + detectors.DefaultMultiPartCredentialProvider client *http.Client useTestURL bool } diff --git a/pkg/detectors/browserstack/browserstack.go b/pkg/detectors/browserstack/browserstack.go index d3f5e1930..4d2947789 100644 --- a/pkg/detectors/browserstack/browserstack.go +++ b/pkg/detectors/browserstack/browserstack.go @@ -15,6 +15,7 @@ import ( ) type Scanner struct { + detectors.DefaultMultiPartCredentialProvider client *http.Client } diff --git a/pkg/detectors/buildkitev2/buildkite.go b/pkg/detectors/buildkitev2/buildkite.go index 54b98106e..e5eca0491 100644 --- a/pkg/detectors/buildkitev2/buildkite.go +++ b/pkg/detectors/buildkitev2/buildkite.go @@ -3,10 +3,11 @@ package buildkitev2 import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/bulksms/bulksms.go b/pkg/detectors/bulksms/bulksms.go index 632fd40b9..a55122230 100644 --- a/pkg/detectors/bulksms/bulksms.go +++ b/pkg/detectors/bulksms/bulksms.go @@ -13,7 +13,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/captaindata/captaindata.go b/pkg/detectors/captaindata/captaindata.go index 82b0b2abb..6e12a59d8 100644 --- a/pkg/detectors/captaindata/captaindata.go +++ b/pkg/detectors/captaindata/captaindata.go @@ -11,7 +11,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/cashboard/cashboard.go b/pkg/detectors/cashboard/cashboard.go index 5bdb9e0aa..d3f444fa7 100644 --- a/pkg/detectors/cashboard/cashboard.go +++ b/pkg/detectors/cashboard/cashboard.go @@ -13,7 +13,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/caspio/caspio.go b/pkg/detectors/caspio/caspio.go index 6dfe2e6d2..b022e1123 100644 --- a/pkg/detectors/caspio/caspio.go +++ b/pkg/detectors/caspio/caspio.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/censys/censys.go b/pkg/detectors/censys/censys.go index e07ec0cce..83f8d7fa9 100644 --- a/pkg/detectors/censys/censys.go +++ b/pkg/detectors/censys/censys.go @@ -11,7 +11,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/cexio/cexio.go b/pkg/detectors/cexio/cexio.go index 5821ae8a6..c4390260f 100644 --- a/pkg/detectors/cexio/cexio.go +++ b/pkg/detectors/cexio/cexio.go @@ -19,7 +19,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/chatbot/chatbot.go b/pkg/detectors/chatbot/chatbot.go index c71f80ca9..709c828a8 100644 --- a/pkg/detectors/chatbot/chatbot.go +++ b/pkg/detectors/chatbot/chatbot.go @@ -3,10 +3,11 @@ package chatbot import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/checkout/checkout.go b/pkg/detectors/checkout/checkout.go index 35b11b31d..9bf8ac3c3 100644 --- a/pkg/detectors/checkout/checkout.go +++ b/pkg/detectors/checkout/checkout.go @@ -11,7 +11,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/checkvist/checkvist.go b/pkg/detectors/checkvist/checkvist.go index d35fbd4be..ce470dcfe 100644 --- a/pkg/detectors/checkvist/checkvist.go +++ b/pkg/detectors/checkvist/checkvist.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/clickhelp/clickhelp.go b/pkg/detectors/clickhelp/clickhelp.go index 42267d49d..6ea90af11 100644 --- a/pkg/detectors/clickhelp/clickhelp.go +++ b/pkg/detectors/clickhelp/clickhelp.go @@ -13,7 +13,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/clicksendsms/clicksendsms.go b/pkg/detectors/clicksendsms/clicksendsms.go index d8574775e..b2f96fb69 100644 --- a/pkg/detectors/clicksendsms/clicksendsms.go +++ b/pkg/detectors/clicksendsms/clicksendsms.go @@ -13,7 +13,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/clockworksms/clockworksms.go b/pkg/detectors/clockworksms/clockworksms.go index e48e1a8be..37a12d5af 100644 --- a/pkg/detectors/clockworksms/clockworksms.go +++ b/pkg/detectors/clockworksms/clockworksms.go @@ -11,7 +11,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/cloudconvert/cloudconvert.go b/pkg/detectors/cloudconvert/cloudconvert.go index 8ffadc9fb..8f47162dc 100644 --- a/pkg/detectors/cloudconvert/cloudconvert.go +++ b/pkg/detectors/cloudconvert/cloudconvert.go @@ -3,10 +3,11 @@ package cloudconvert import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" @@ -16,6 +17,7 @@ type Scanner struct{} // Ensure the Scanner satisfies the interface at compile time var _ detectors.Detector = (*Scanner)(nil) +var _ detectors.MaxSecretSizeProvider = (*Scanner)(nil) var ( client = common.SaneHttpClient() @@ -30,6 +32,11 @@ func (s Scanner) Keywords() []string { return []string{"cloudconvert"} } +const maxJWTSize = 1300 + +// MaxSecretSize returns the maximum size of a secret that this detector can find. +func (s Scanner) MaxSecretSize() int64 { return maxJWTSize } + // FromData will find and optionally verify CloudConvert secrets in a given set of bytes. func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { dataStr := string(data) diff --git a/pkg/detectors/cloudelements/cloudelements.go b/pkg/detectors/cloudelements/cloudelements.go index 3ba8c1e1c..4c1181b5d 100644 --- a/pkg/detectors/cloudelements/cloudelements.go +++ b/pkg/detectors/cloudelements/cloudelements.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/cloudflareglobalapikey/cloudflareglobalapikey.go b/pkg/detectors/cloudflareglobalapikey/cloudflareglobalapikey.go index 7e364a43a..850aa92a2 100644 --- a/pkg/detectors/cloudflareglobalapikey/cloudflareglobalapikey.go +++ b/pkg/detectors/cloudflareglobalapikey/cloudflareglobalapikey.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/cloudsmith/cloudsmith.go b/pkg/detectors/cloudsmith/cloudsmith.go index 953f65840..bca1eeaa3 100644 --- a/pkg/detectors/cloudsmith/cloudsmith.go +++ b/pkg/detectors/cloudsmith/cloudsmith.go @@ -3,10 +3,11 @@ package cloudsmith import ( "context" "encoding/json" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" @@ -70,7 +71,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if r.Authenticated { s1.Verified = true } - } + } } } diff --git a/pkg/detectors/cloze/cloze.go b/pkg/detectors/cloze/cloze.go index 3e90294b4..cc94e13e6 100644 --- a/pkg/detectors/cloze/cloze.go +++ b/pkg/detectors/cloze/cloze.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/coinbase_waas/coinbase_waas.go b/pkg/detectors/coinbase_waas/coinbase_waas.go index ac7ab6ddf..401512007 100644 --- a/pkg/detectors/coinbase_waas/coinbase_waas.go +++ b/pkg/detectors/coinbase_waas/coinbase_waas.go @@ -6,10 +6,11 @@ import ( "crypto/x509" "encoding/pem" "errors" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/coinbase/waas-client-library-go/auth" "github.com/coinbase/waas-client-library-go/clients" v1clients "github.com/coinbase/waas-client-library-go/clients/v1" @@ -48,6 +49,11 @@ func (s Scanner) Keywords() []string { return []string{"organizations", "apiKeys", "begin ec"} } +const maxPrivateKeySize = 4096 + +// MaxSecretSize returns the maximum size of a secret that this detector can find. +func (s Scanner) MaxSecretSize() int64 { return maxPrivateKeySize } + // FromData will find and optionally verify CoinbaseWaaS secrets in a given set of bytes. func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { dataStr := string(data) diff --git a/pkg/detectors/collect2/collect2.go b/pkg/detectors/collect2/collect2.go index 8ad52883b..51f2eaac6 100644 --- a/pkg/detectors/collect2/collect2.go +++ b/pkg/detectors/collect2/collect2.go @@ -3,10 +3,11 @@ package collect2 import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/commodities/commodities.go b/pkg/detectors/commodities/commodities.go index 126215fe0..113551abb 100644 --- a/pkg/detectors/commodities/commodities.go +++ b/pkg/detectors/commodities/commodities.go @@ -2,12 +2,13 @@ package commodities import ( "context" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" "time" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/companyhub/companyhub.go b/pkg/detectors/companyhub/companyhub.go index d79e32704..08763bdde 100644 --- a/pkg/detectors/companyhub/companyhub.go +++ b/pkg/detectors/companyhub/companyhub.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/confluent/confluent.go b/pkg/detectors/confluent/confluent.go index b6a07eb82..0e1f2d223 100644 --- a/pkg/detectors/confluent/confluent.go +++ b/pkg/detectors/confluent/confluent.go @@ -13,7 +13,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/copper/copper.go b/pkg/detectors/copper/copper.go index 6d1dc61cc..d0d1f3770 100644 --- a/pkg/detectors/copper/copper.go +++ b/pkg/detectors/copper/copper.go @@ -11,7 +11,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/couchbase/couchbase.go b/pkg/detectors/couchbase/couchbase.go index 3f18bdf47..284e10497 100644 --- a/pkg/detectors/couchbase/couchbase.go +++ b/pkg/detectors/couchbase/couchbase.go @@ -15,7 +15,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/currencycloud/currencycloud.go b/pkg/detectors/currencycloud/currencycloud.go index 63617a191..7775cb645 100644 --- a/pkg/detectors/currencycloud/currencycloud.go +++ b/pkg/detectors/currencycloud/currencycloud.go @@ -13,7 +13,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/customerguru/customerguru.go b/pkg/detectors/customerguru/customerguru.go index 8d339794f..8c6c3e135 100644 --- a/pkg/detectors/customerguru/customerguru.go +++ b/pkg/detectors/customerguru/customerguru.go @@ -11,7 +11,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/customerio/customerio.go b/pkg/detectors/customerio/customerio.go index 4dc8036b4..13804cae2 100644 --- a/pkg/detectors/customerio/customerio.go +++ b/pkg/detectors/customerio/customerio.go @@ -13,7 +13,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/databrickstoken/databrickstoken.go b/pkg/detectors/databrickstoken/databrickstoken.go index 724f72cf6..866fcbb99 100644 --- a/pkg/detectors/databrickstoken/databrickstoken.go +++ b/pkg/detectors/databrickstoken/databrickstoken.go @@ -14,6 +14,7 @@ import ( type Scanner struct { client *http.Client + detectors.DefaultMultiPartCredentialProvider } // Ensure the Scanner satisfies the interface at compile time. diff --git a/pkg/detectors/datadogtoken/datadogtoken.go b/pkg/detectors/datadogtoken/datadogtoken.go index c1f737f8a..e4743701c 100644 --- a/pkg/detectors/datadogtoken/datadogtoken.go +++ b/pkg/detectors/datadogtoken/datadogtoken.go @@ -15,6 +15,7 @@ import ( type Scanner struct { detectors.EndpointSetter + detectors.DefaultMultiPartCredentialProvider } // Ensure the Scanner satisfies the interface at compile time. diff --git a/pkg/detectors/demio/demio.go b/pkg/detectors/demio/demio.go index e1b4a7046..32aadf9a3 100644 --- a/pkg/detectors/demio/demio.go +++ b/pkg/detectors/demio/demio.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/deputy/deputy.go b/pkg/detectors/deputy/deputy.go index e0d23f4d9..ed0438fa5 100644 --- a/pkg/detectors/deputy/deputy.go +++ b/pkg/detectors/deputy/deputy.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/detectify/detectify.go b/pkg/detectors/detectify/detectify.go index 07258812c..8824e16cf 100644 --- a/pkg/detectors/detectify/detectify.go +++ b/pkg/detectors/detectify/detectify.go @@ -2,10 +2,11 @@ package detectify import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/detectors.go b/pkg/detectors/detectors.go index 400dbcecf..0fb10b321 100644 --- a/pkg/detectors/detectors.go +++ b/pkg/detectors/detectors.go @@ -32,6 +32,21 @@ type Versioner interface { Version() int } +// MaxSecretSizeProvider is an optional interface that a detector can implement to +// provide a custom max size for the secret it finds. +type MaxSecretSizeProvider interface { + MaxSecretSize() int64 +} + +// MultiPartCredentialProvider is an optional interface that a detector can implement +// to indicate its compatibility with multi-part credentials and provide the maximum +// secret size for the credential it finds. +type MultiPartCredentialProvider interface { + // MaxCredentialSpan returns the maximum span or range of characters that the + // detector should consider when searching for a multi-part credential. + MaxCredentialSpan() int64 +} + // EndpointCustomizer is an optional interface that a detector can implement to // support verifying against user-supplied endpoints. type EndpointCustomizer interface { diff --git a/pkg/detectors/discordbottoken/discordbottoken.go b/pkg/detectors/discordbottoken/discordbottoken.go index bee617a07..fd6384160 100644 --- a/pkg/detectors/discordbottoken/discordbottoken.go +++ b/pkg/detectors/discordbottoken/discordbottoken.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/dnscheck/dnscheck.go b/pkg/detectors/dnscheck/dnscheck.go index bd744c76a..fcb38ce54 100644 --- a/pkg/detectors/dnscheck/dnscheck.go +++ b/pkg/detectors/dnscheck/dnscheck.go @@ -11,7 +11,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/dockerhub/v1/dockerhub.go b/pkg/detectors/dockerhub/v1/dockerhub.go index b990bd827..2811b8005 100644 --- a/pkg/detectors/dockerhub/v1/dockerhub.go +++ b/pkg/detectors/dockerhub/v1/dockerhub.go @@ -19,6 +19,7 @@ import ( type Scanner struct { client *http.Client + detectors.DefaultMultiPartCredentialProvider } func (s Scanner) Version() int { return 1 } diff --git a/pkg/detectors/dockerhub/v2/dockerhub.go b/pkg/detectors/dockerhub/v2/dockerhub.go index bb738cbd1..6f183d187 100644 --- a/pkg/detectors/dockerhub/v2/dockerhub.go +++ b/pkg/detectors/dockerhub/v2/dockerhub.go @@ -19,6 +19,7 @@ import ( type Scanner struct { client *http.Client + detectors.DefaultMultiPartCredentialProvider } func (s Scanner) Version() int { return 2 } diff --git a/pkg/detectors/docusign/docusign.go b/pkg/detectors/docusign/docusign.go index b748490b8..7c25d9272 100644 --- a/pkg/detectors/docusign/docusign.go +++ b/pkg/detectors/docusign/docusign.go @@ -14,7 +14,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} type Response struct { AccessToken string `json:"access_token"` diff --git a/pkg/detectors/dotmailer/dotmailer.go b/pkg/detectors/dotmailer/dotmailer.go index b85d71897..a5c81e432 100644 --- a/pkg/detectors/dotmailer/dotmailer.go +++ b/pkg/detectors/dotmailer/dotmailer.go @@ -13,7 +13,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/dovico/dovico.go b/pkg/detectors/dovico/dovico.go index 555f4a7c1..05b1290c3 100644 --- a/pkg/detectors/dovico/dovico.go +++ b/pkg/detectors/dovico/dovico.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/duply/duply.go b/pkg/detectors/duply/duply.go index 3b2d94ccc..1ae732aa0 100644 --- a/pkg/detectors/duply/duply.go +++ b/pkg/detectors/duply/duply.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/dwolla/dwolla.go b/pkg/detectors/dwolla/dwolla.go index 41aa420ff..e558162f4 100644 --- a/pkg/detectors/dwolla/dwolla.go +++ b/pkg/detectors/dwolla/dwolla.go @@ -13,7 +13,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/eagleeyenetworks/eagleeyenetworks.go b/pkg/detectors/eagleeyenetworks/eagleeyenetworks.go index 576030c2f..5a8de5597 100644 --- a/pkg/detectors/eagleeyenetworks/eagleeyenetworks.go +++ b/pkg/detectors/eagleeyenetworks/eagleeyenetworks.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/easyinsight/easyinsight.go b/pkg/detectors/easyinsight/easyinsight.go index dfab594f3..fdaf3858f 100644 --- a/pkg/detectors/easyinsight/easyinsight.go +++ b/pkg/detectors/easyinsight/easyinsight.go @@ -13,7 +13,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/ecostruxureit/ecostruxureit.go b/pkg/detectors/ecostruxureit/ecostruxureit.go index c3d3b9e54..a52deecf3 100644 --- a/pkg/detectors/ecostruxureit/ecostruxureit.go +++ b/pkg/detectors/ecostruxureit/ecostruxureit.go @@ -3,10 +3,11 @@ package ecostruxureit import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/edamam/edamam.go b/pkg/detectors/edamam/edamam.go index 9ed2f5bd3..4a836da93 100644 --- a/pkg/detectors/edamam/edamam.go +++ b/pkg/detectors/edamam/edamam.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/eightxeight/eightxeight.go b/pkg/detectors/eightxeight/eightxeight.go index c2965149b..b7020a5e4 100644 --- a/pkg/detectors/eightxeight/eightxeight.go +++ b/pkg/detectors/eightxeight/eightxeight.go @@ -13,7 +13,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/enablex/enablex.go b/pkg/detectors/enablex/enablex.go index cdfc0d598..bba8301db 100644 --- a/pkg/detectors/enablex/enablex.go +++ b/pkg/detectors/enablex/enablex.go @@ -11,7 +11,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/everhour/everhour.go b/pkg/detectors/everhour/everhour.go index 6e34783b5..4a5e8e2d0 100644 --- a/pkg/detectors/everhour/everhour.go +++ b/pkg/detectors/everhour/everhour.go @@ -2,10 +2,11 @@ package everhour import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/exportsdk/exportsdk.go b/pkg/detectors/exportsdk/exportsdk.go index 71233c87b..70e65438b 100644 --- a/pkg/detectors/exportsdk/exportsdk.go +++ b/pkg/detectors/exportsdk/exportsdk.go @@ -11,7 +11,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/facebookoauth/facebookoauth.go b/pkg/detectors/facebookoauth/facebookoauth.go index fc727e98a..9985f3f7a 100644 --- a/pkg/detectors/facebookoauth/facebookoauth.go +++ b/pkg/detectors/facebookoauth/facebookoauth.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/faceplusplus/faceplusplus.go b/pkg/detectors/faceplusplus/faceplusplus.go index cf17155cb..a88d55f33 100644 --- a/pkg/detectors/faceplusplus/faceplusplus.go +++ b/pkg/detectors/faceplusplus/faceplusplus.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/fibery/fibery.go b/pkg/detectors/fibery/fibery.go index 5f268343a..dbccd03c1 100644 --- a/pkg/detectors/fibery/fibery.go +++ b/pkg/detectors/fibery/fibery.go @@ -13,7 +13,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/flightstats/flightstats.go b/pkg/detectors/flightstats/flightstats.go index b2f100e0e..871abcdb8 100644 --- a/pkg/detectors/flightstats/flightstats.go +++ b/pkg/detectors/flightstats/flightstats.go @@ -13,7 +13,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/flowflu/flowflu.go b/pkg/detectors/flowflu/flowflu.go index 011631053..f8637cfe6 100644 --- a/pkg/detectors/flowflu/flowflu.go +++ b/pkg/detectors/flowflu/flowflu.go @@ -13,7 +13,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/fmfw/fmfw.go b/pkg/detectors/fmfw/fmfw.go index e6514b1c6..d5aa9633d 100644 --- a/pkg/detectors/fmfw/fmfw.go +++ b/pkg/detectors/fmfw/fmfw.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/formbucket/formbucket.go b/pkg/detectors/formbucket/formbucket.go index b3175d8c8..926e7e2d7 100644 --- a/pkg/detectors/formbucket/formbucket.go +++ b/pkg/detectors/formbucket/formbucket.go @@ -3,11 +3,12 @@ package formbucket import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/formsite/formsite.go b/pkg/detectors/formsite/formsite.go index c8ef4dfa8..a69cc2ab2 100644 --- a/pkg/detectors/formsite/formsite.go +++ b/pkg/detectors/formsite/formsite.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/foursquare/foursquare.go b/pkg/detectors/foursquare/foursquare.go index c5b424694..700597053 100644 --- a/pkg/detectors/foursquare/foursquare.go +++ b/pkg/detectors/foursquare/foursquare.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/freshbooks/freshbooks.go b/pkg/detectors/freshbooks/freshbooks.go index 0aabfa8cb..75e65ce4d 100644 --- a/pkg/detectors/freshbooks/freshbooks.go +++ b/pkg/detectors/freshbooks/freshbooks.go @@ -13,7 +13,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/freshdesk/freshdesk.go b/pkg/detectors/freshdesk/freshdesk.go index 35971bfc9..0abdacc30 100644 --- a/pkg/detectors/freshdesk/freshdesk.go +++ b/pkg/detectors/freshdesk/freshdesk.go @@ -13,7 +13,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/front/front.go b/pkg/detectors/front/front.go index 515322b09..303e42f23 100644 --- a/pkg/detectors/front/front.go +++ b/pkg/detectors/front/front.go @@ -3,10 +3,11 @@ package front import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/gcp/gcp.go b/pkg/detectors/gcp/gcp.go index 6a2b3e1da..86900c242 100644 --- a/pkg/detectors/gcp/gcp.go +++ b/pkg/detectors/gcp/gcp.go @@ -19,6 +19,7 @@ type Scanner struct{} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) var _ detectors.CustomFalsePositiveChecker = (*Scanner)(nil) +var _ detectors.MaxSecretSizeProvider = (*Scanner)(nil) var ( keyPat = regexp.MustCompile(`\{[^{]+auth_provider_x509_cert_url[^}]+\}`) @@ -49,6 +50,11 @@ func (s Scanner) Keywords() []string { return []string{"provider_x509"} } +const maxGCPKeySize = 4096 + +// ProvideMaxSecretSize returns the maximum size of a secret that this detector can find. +func (s Scanner) MaxSecretSize() int64 { return maxGCPKeySize } + // FromData will find and optionally verify GCP secrets in a given set of bytes. func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { dataStr := string(data) diff --git a/pkg/detectors/gcpapplicationdefaultcredentials/gcpapplicationdefaultcredentials.go b/pkg/detectors/gcpapplicationdefaultcredentials/gcpapplicationdefaultcredentials.go index debe41fa6..142c8995e 100644 --- a/pkg/detectors/gcpapplicationdefaultcredentials/gcpapplicationdefaultcredentials.go +++ b/pkg/detectors/gcpapplicationdefaultcredentials/gcpapplicationdefaultcredentials.go @@ -11,11 +11,12 @@ import ( regexp "github.com/wasilibs/go-re2" + "golang.org/x/oauth2" + "golang.org/x/oauth2/google" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" - "golang.org/x/oauth2" - "golang.org/x/oauth2/google" ) type Scanner struct { @@ -24,6 +25,7 @@ type Scanner struct { // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) +var _ detectors.MaxSecretSizeProvider = (*Scanner)(nil) var ( defaultClient = common.SaneHttpClient() @@ -44,6 +46,11 @@ func (s Scanner) Keywords() []string { return []string{".apps.googleusercontent.com"} } +const maxGCPADCKeySize = 1024 + +// ProvideMaxSecretSize returns the maximum size of a secret that this detector can find. +func (s Scanner) MaxSecretSize() int64 { return maxGCPADCKeySize } + // FromData will find and optionally verify Gcpapplicationdefaultcredentials secrets in a given set of bytes. func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { dataStr := string(data) diff --git a/pkg/detectors/gcpapplicationdefaultcredentials/gcpapplicationdefaultcredentials_test.go b/pkg/detectors/gcpapplicationdefaultcredentials/gcpapplicationdefaultcredentials_test.go index 93dfdc000..7bed123d1 100644 --- a/pkg/detectors/gcpapplicationdefaultcredentials/gcpapplicationdefaultcredentials_test.go +++ b/pkg/detectors/gcpapplicationdefaultcredentials/gcpapplicationdefaultcredentials_test.go @@ -45,7 +45,7 @@ func TestGcpapplicationdefaultcredentials_Pattern(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { chunkSpecificDetectors := make(map[ahocorasick.DetectorKey]detectors.Detector, 2) - ahoCorasickCore.PopulateMatchingDetectors(test.input, chunkSpecificDetectors) + ahoCorasickCore.FindDetectorMatches(test.input, chunkSpecificDetectors) if len(chunkSpecificDetectors) == 0 { t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return diff --git a/pkg/detectors/gemini/gemini.go b/pkg/detectors/gemini/gemini.go index 6c0828d2b..949cd6fe0 100644 --- a/pkg/detectors/gemini/gemini.go +++ b/pkg/detectors/gemini/gemini.go @@ -19,7 +19,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} const ( baseURL = "https://api.gemini.com" diff --git a/pkg/detectors/gengo/gengo.go b/pkg/detectors/gengo/gengo.go index cea4083cf..2c10adfda 100644 --- a/pkg/detectors/gengo/gengo.go +++ b/pkg/detectors/gengo/gengo.go @@ -19,7 +19,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/geocodio/geocodio.go b/pkg/detectors/geocodio/geocodio.go index 578997e22..fbee186f0 100644 --- a/pkg/detectors/geocodio/geocodio.go +++ b/pkg/detectors/geocodio/geocodio.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/getemails/getemails.go b/pkg/detectors/getemails/getemails.go index 24d53a2cf..694b0eac1 100644 --- a/pkg/detectors/getemails/getemails.go +++ b/pkg/detectors/getemails/getemails.go @@ -11,7 +11,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/getsandbox/getsandbox.go b/pkg/detectors/getsandbox/getsandbox.go index d8a13446e..152b6a738 100644 --- a/pkg/detectors/getsandbox/getsandbox.go +++ b/pkg/detectors/getsandbox/getsandbox.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/github_oauth2/github_oauth2.go b/pkg/detectors/github_oauth2/github_oauth2.go index 7a21053db..c1203433f 100644 --- a/pkg/detectors/github_oauth2/github_oauth2.go +++ b/pkg/detectors/github_oauth2/github_oauth2.go @@ -2,16 +2,21 @@ package github_oauth2 import ( "context" - regexp "github.com/wasilibs/go-re2" "strings" + regexp "github.com/wasilibs/go-re2" + + "golang.org/x/oauth2/clientcredentials" + "golang.org/x/oauth2/github" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" - "golang.org/x/oauth2/clientcredentials" - "golang.org/x/oauth2/github" ) -type Scanner struct{ detectors.EndpointSetter } +type Scanner struct { + detectors.EndpointSetter + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interfaces at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/githubapp/githubapp.go b/pkg/detectors/githubapp/githubapp.go index 8eb48d69f..bbddd92cc 100644 --- a/pkg/detectors/githubapp/githubapp.go +++ b/pkg/detectors/githubapp/githubapp.go @@ -3,18 +3,22 @@ package githubapp import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" "time" + regexp "github.com/wasilibs/go-re2" + "github.com/golang-jwt/jwt" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/gocanvas/gocanvas.go b/pkg/detectors/gocanvas/gocanvas.go index 95cd31cb2..c795debe0 100644 --- a/pkg/detectors/gocanvas/gocanvas.go +++ b/pkg/detectors/gocanvas/gocanvas.go @@ -15,7 +15,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/grafanaserviceaccount/grafanaserviceaccount.go b/pkg/detectors/grafanaserviceaccount/grafanaserviceaccount.go index b450582ac..6df3aaf89 100644 --- a/pkg/detectors/grafanaserviceaccount/grafanaserviceaccount.go +++ b/pkg/detectors/grafanaserviceaccount/grafanaserviceaccount.go @@ -14,6 +14,7 @@ import ( type Scanner struct { client *http.Client + detectors.DefaultMultiPartCredentialProvider } // Ensure the Scanner satisfies the interface at compile time. diff --git a/pkg/detectors/graphcms/graphcms.go b/pkg/detectors/graphcms/graphcms.go index 46fcae8c4..ad5307a08 100644 --- a/pkg/detectors/graphcms/graphcms.go +++ b/pkg/detectors/graphcms/graphcms.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/guru/guru.go b/pkg/detectors/guru/guru.go index 315637852..88cc8dab9 100644 --- a/pkg/detectors/guru/guru.go +++ b/pkg/detectors/guru/guru.go @@ -13,7 +13,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/harvest/harvest.go b/pkg/detectors/harvest/harvest.go index 6f917485a..7078e10b5 100644 --- a/pkg/detectors/harvest/harvest.go +++ b/pkg/detectors/harvest/harvest.go @@ -11,7 +11,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/heroku/heroku.go b/pkg/detectors/heroku/heroku.go index e6ce180e4..6fdee217b 100644 --- a/pkg/detectors/heroku/heroku.go +++ b/pkg/detectors/heroku/heroku.go @@ -3,10 +3,11 @@ package heroku import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/hive/hive.go b/pkg/detectors/hive/hive.go index c12bf1416..0aeec88a7 100644 --- a/pkg/detectors/hive/hive.go +++ b/pkg/detectors/hive/hive.go @@ -11,7 +11,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/hypertrack/hypertrack.go b/pkg/detectors/hypertrack/hypertrack.go index 6a257e546..418ae8937 100644 --- a/pkg/detectors/hypertrack/hypertrack.go +++ b/pkg/detectors/hypertrack/hypertrack.go @@ -11,7 +11,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/instamojo/instamojo.go b/pkg/detectors/instamojo/instamojo.go index 249405f42..82c158b23 100644 --- a/pkg/detectors/instamojo/instamojo.go +++ b/pkg/detectors/instamojo/instamojo.go @@ -3,17 +3,19 @@ package instamojo import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) type Scanner struct { + detectors.DefaultMultiPartCredentialProvider client *http.Client } diff --git a/pkg/detectors/intra42/intra42.go b/pkg/detectors/intra42/intra42.go index 6aa1a5a25..d92a94755 100644 --- a/pkg/detectors/intra42/intra42.go +++ b/pkg/detectors/intra42/intra42.go @@ -17,6 +17,7 @@ import ( type Scanner struct { client *http.Client + detectors.DefaultMultiPartCredentialProvider } // Ensure the Scanner satisfies the interface at compile time diff --git a/pkg/detectors/invoiceocean/invoiceocean.go b/pkg/detectors/invoiceocean/invoiceocean.go index 0669142ff..f95a4ec4c 100644 --- a/pkg/detectors/invoiceocean/invoiceocean.go +++ b/pkg/detectors/invoiceocean/invoiceocean.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/ip2location/ip2location.go b/pkg/detectors/ip2location/ip2location.go index 6dd8ed714..72dc862d2 100644 --- a/pkg/detectors/ip2location/ip2location.go +++ b/pkg/detectors/ip2location/ip2location.go @@ -3,10 +3,11 @@ package ip2location import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/ipinfodb/ipinfodb.go b/pkg/detectors/ipinfodb/ipinfodb.go index 5773f8279..f056faf3f 100644 --- a/pkg/detectors/ipinfodb/ipinfodb.go +++ b/pkg/detectors/ipinfodb/ipinfodb.go @@ -3,11 +3,12 @@ package ipinfodb import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/jdbc/jdbc.go b/pkg/detectors/jdbc/jdbc.go index 4b6cee99f..ef097d5b5 100644 --- a/pkg/detectors/jdbc/jdbc.go +++ b/pkg/detectors/jdbc/jdbc.go @@ -14,6 +14,7 @@ import ( ) type Scanner struct { + detectors.DefaultMultiPartCredentialProvider ignorePatterns []regexp.Regexp } diff --git a/pkg/detectors/jiratoken/v1/jiratoken.go b/pkg/detectors/jiratoken/v1/jiratoken.go index 92e243f34..ae23754e5 100644 --- a/pkg/detectors/jiratoken/v1/jiratoken.go +++ b/pkg/detectors/jiratoken/v1/jiratoken.go @@ -15,6 +15,7 @@ import ( ) type Scanner struct { + detectors.DefaultMultiPartCredentialProvider client *http.Client } diff --git a/pkg/detectors/jiratoken/v2/jiratoken_v2.go b/pkg/detectors/jiratoken/v2/jiratoken_v2.go index c356c16a7..3501bde9b 100644 --- a/pkg/detectors/jiratoken/v2/jiratoken_v2.go +++ b/pkg/detectors/jiratoken/v2/jiratoken_v2.go @@ -16,6 +16,7 @@ import ( type Scanner struct { client *http.Client + detectors.DefaultMultiPartCredentialProvider } // Ensure the Scanner satisfies the interface at compile time. diff --git a/pkg/detectors/kanban/kanban.go b/pkg/detectors/kanban/kanban.go index dabd5561e..d3fb775ea 100644 --- a/pkg/detectors/kanban/kanban.go +++ b/pkg/detectors/kanban/kanban.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/kanbantool/kanbantool.go b/pkg/detectors/kanbantool/kanbantool.go index b2a32f579..a6bb8e23e 100644 --- a/pkg/detectors/kanbantool/kanbantool.go +++ b/pkg/detectors/kanbantool/kanbantool.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/keenio/keenio.go b/pkg/detectors/keenio/keenio.go index 0ff37ba09..d2f76cb0f 100644 --- a/pkg/detectors/keenio/keenio.go +++ b/pkg/detectors/keenio/keenio.go @@ -11,7 +11,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/kraken/kraken.go b/pkg/detectors/kraken/kraken.go index b9cdf814b..d6db8e569 100644 --- a/pkg/detectors/kraken/kraken.go +++ b/pkg/detectors/kraken/kraken.go @@ -19,7 +19,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/kucoin/kucoin.go b/pkg/detectors/kucoin/kucoin.go index 3ca40e1b1..d19329a6e 100644 --- a/pkg/detectors/kucoin/kucoin.go +++ b/pkg/detectors/kucoin/kucoin.go @@ -16,7 +16,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/ldap/ldap.go b/pkg/detectors/ldap/ldap.go index fb8ebad83..ce87f8fc2 100644 --- a/pkg/detectors/ldap/ldap.go +++ b/pkg/detectors/ldap/ldap.go @@ -16,7 +16,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/lemonsqueezy/lemonsqueezy.go b/pkg/detectors/lemonsqueezy/lemonsqueezy.go index 498af18dc..ca6b08a53 100644 --- a/pkg/detectors/lemonsqueezy/lemonsqueezy.go +++ b/pkg/detectors/lemonsqueezy/lemonsqueezy.go @@ -3,10 +3,11 @@ package lemonsqueezy import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/lendflow/lendflow.go b/pkg/detectors/lendflow/lendflow.go index c8777cc85..81f110e7e 100644 --- a/pkg/detectors/lendflow/lendflow.go +++ b/pkg/detectors/lendflow/lendflow.go @@ -3,17 +3,20 @@ package lendflow import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" "time" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/lessannoyingcrm/lessannoyingcrm.go b/pkg/detectors/lessannoyingcrm/lessannoyingcrm.go index 4c475bcb5..04ecf9117 100644 --- a/pkg/detectors/lessannoyingcrm/lessannoyingcrm.go +++ b/pkg/detectors/lessannoyingcrm/lessannoyingcrm.go @@ -3,10 +3,11 @@ package lessannoyingcrm import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/liveagent/liveagent.go b/pkg/detectors/liveagent/liveagent.go index 314845c6b..a4ed7f394 100644 --- a/pkg/detectors/liveagent/liveagent.go +++ b/pkg/detectors/liveagent/liveagent.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/loggly/loggly.go b/pkg/detectors/loggly/loggly.go index 9ed2c60cb..4de7e41d6 100644 --- a/pkg/detectors/loggly/loggly.go +++ b/pkg/detectors/loggly/loggly.go @@ -14,6 +14,7 @@ import ( type Scanner struct { client *http.Client + detectors.DefaultMultiPartCredentialProvider } // Ensure the Scanner satisfies the interface at compile time. diff --git a/pkg/detectors/loginradius/loginradius.go b/pkg/detectors/loginradius/loginradius.go index 0e26b57f0..690b9f84d 100644 --- a/pkg/detectors/loginradius/loginradius.go +++ b/pkg/detectors/loginradius/loginradius.go @@ -2,10 +2,11 @@ package loginradius import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/luno/luno.go b/pkg/detectors/luno/luno.go index af9798b32..1282d3f16 100644 --- a/pkg/detectors/luno/luno.go +++ b/pkg/detectors/luno/luno.go @@ -11,7 +11,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/magicbell/magicbell.go b/pkg/detectors/magicbell/magicbell.go index 65332d0fc..240544504 100644 --- a/pkg/detectors/magicbell/magicbell.go +++ b/pkg/detectors/magicbell/magicbell.go @@ -11,7 +11,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/mailgun/mailgun.go b/pkg/detectors/mailgun/mailgun.go index 991b7f2f0..3d0f9e30a 100644 --- a/pkg/detectors/mailgun/mailgun.go +++ b/pkg/detectors/mailgun/mailgun.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/mailmodo/mailmodo.go b/pkg/detectors/mailmodo/mailmodo.go index 0c8081ca0..58291b02c 100644 --- a/pkg/detectors/mailmodo/mailmodo.go +++ b/pkg/detectors/mailmodo/mailmodo.go @@ -2,10 +2,11 @@ package mailmodo import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/mapbox/mapbox.go b/pkg/detectors/mapbox/mapbox.go index a1341c4e1..8c5a7153f 100644 --- a/pkg/detectors/mapbox/mapbox.go +++ b/pkg/detectors/mapbox/mapbox.go @@ -2,16 +2,19 @@ package mapbox import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/mattermostpersonaltoken/mattermostpersonaltoken.go b/pkg/detectors/mattermostpersonaltoken/mattermostpersonaltoken.go index 80c6627cf..31b72e7d3 100644 --- a/pkg/detectors/mattermostpersonaltoken/mattermostpersonaltoken.go +++ b/pkg/detectors/mattermostpersonaltoken/mattermostpersonaltoken.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/maxmindlicense/v1/maxmindlicense.go b/pkg/detectors/maxmindlicense/v1/maxmindlicense.go index 6a2ac4119..de0e73dff 100644 --- a/pkg/detectors/maxmindlicense/v1/maxmindlicense.go +++ b/pkg/detectors/maxmindlicense/v1/maxmindlicense.go @@ -13,7 +13,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/metaapi/metaapi.go b/pkg/detectors/metaapi/metaapi.go index 4a89aef1c..3f7ee1ade 100644 --- a/pkg/detectors/metaapi/metaapi.go +++ b/pkg/detectors/metaapi/metaapi.go @@ -13,7 +13,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/metabase/metabase.go b/pkg/detectors/metabase/metabase.go index 852e671fc..66fd9dc9a 100644 --- a/pkg/detectors/metabase/metabase.go +++ b/pkg/detectors/metabase/metabase.go @@ -13,7 +13,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/microsoftteamswebhook/microsoftteamswebhook.go b/pkg/detectors/microsoftteamswebhook/microsoftteamswebhook.go index 7175f7fbc..446da7a81 100644 --- a/pkg/detectors/microsoftteamswebhook/microsoftteamswebhook.go +++ b/pkg/detectors/microsoftteamswebhook/microsoftteamswebhook.go @@ -3,12 +3,13 @@ package microsoftteamswebhook import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" "time" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" @@ -16,6 +17,7 @@ import ( type Scanner struct { client *http.Client + detectors.DefaultMultiPartCredentialProvider } // Ensure the Scanner satisfies the interface at compile time. diff --git a/pkg/detectors/mite/mite.go b/pkg/detectors/mite/mite.go index 067d9a470..9d98c76a6 100644 --- a/pkg/detectors/mite/mite.go +++ b/pkg/detectors/mite/mite.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/mixpanel/mixpanel.go b/pkg/detectors/mixpanel/mixpanel.go index 6e9110a3c..b346090bf 100644 --- a/pkg/detectors/mixpanel/mixpanel.go +++ b/pkg/detectors/mixpanel/mixpanel.go @@ -11,7 +11,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/moderation/moderation.go b/pkg/detectors/moderation/moderation.go index 49c230756..726489944 100644 --- a/pkg/detectors/moderation/moderation.go +++ b/pkg/detectors/moderation/moderation.go @@ -3,11 +3,12 @@ package moderation import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" "time" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/monday/monday.go b/pkg/detectors/monday/monday.go index e319fa763..55e649140 100644 --- a/pkg/detectors/monday/monday.go +++ b/pkg/detectors/monday/monday.go @@ -2,10 +2,11 @@ package monday import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/mrticktock/mrticktock.go b/pkg/detectors/mrticktock/mrticktock.go index c2fb5737f..d88b0c222 100644 --- a/pkg/detectors/mrticktock/mrticktock.go +++ b/pkg/detectors/mrticktock/mrticktock.go @@ -13,7 +13,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/multi_part_credential_provider.go b/pkg/detectors/multi_part_credential_provider.go new file mode 100644 index 000000000..b9a0e28e4 --- /dev/null +++ b/pkg/detectors/multi_part_credential_provider.go @@ -0,0 +1,28 @@ +package detectors + +var _ MultiPartCredentialProvider = (*DefaultMultiPartCredentialProvider)(nil) +var _ MultiPartCredentialProvider = (*CustomMultiPartCredentialProvider)(nil) + +type DefaultMultiPartCredentialProvider struct{} + +const defaultMaxCredentialSpan = 1024 + +// MaxCredentialSpan returns the default maximum credential span of 1024 for the +// DefaultMultiPartCredentialProvider. +func (d DefaultMultiPartCredentialProvider) MaxCredentialSpan() int64 { + return defaultMaxCredentialSpan +} + +type CustomMultiPartCredentialProvider struct{ maxCredentialSpan int64 } + +// NewCustomMultiPartCredentialProvider creates a new instance of CustomMultiPartCredentialProvider +// with the specified maximum credential span. +func NewCustomMultiPartCredentialProvider(maxCredentialSpan int64) *CustomMultiPartCredentialProvider { + return &CustomMultiPartCredentialProvider{maxCredentialSpan: maxCredentialSpan} +} + +// MaxCredentialSpan returns the custom maximum credential span specified during the +// creation of the CustomMultiPartCredentialProvider. +func (d CustomMultiPartCredentialProvider) MaxCredentialSpan() int64 { + return d.maxCredentialSpan +} diff --git a/pkg/detectors/multi_part_credential_provider_test.go b/pkg/detectors/multi_part_credential_provider_test.go new file mode 100644 index 000000000..a27ce141b --- /dev/null +++ b/pkg/detectors/multi_part_credential_provider_test.go @@ -0,0 +1,35 @@ +package detectors + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMultiPartCredentialProviders(t *testing.T) { + testCases := []struct { + name string + provider MultiPartCredentialProvider + expectedSpan int64 + }{ + { + name: "DefaultMultiPartCredentialProvider", + provider: DefaultMultiPartCredentialProvider{}, + expectedSpan: defaultMaxCredentialSpan, + }, + { + name: "CustomMultiPartCredentialProvider", + provider: NewCustomMultiPartCredentialProvider(2048), + expectedSpan: 2048, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + span := tc.provider.MaxCredentialSpan() + assert.Equal(t, tc.expectedSpan, span) + }) + } +} diff --git a/pkg/detectors/mux/mux.go b/pkg/detectors/mux/mux.go index ef2679da0..80febdaef 100644 --- a/pkg/detectors/mux/mux.go +++ b/pkg/detectors/mux/mux.go @@ -2,16 +2,19 @@ package mux import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/myfreshworks/myfreshworks.go b/pkg/detectors/myfreshworks/myfreshworks.go index c29801306..4d1c76682 100644 --- a/pkg/detectors/myfreshworks/myfreshworks.go +++ b/pkg/detectors/myfreshworks/myfreshworks.go @@ -16,6 +16,7 @@ import ( type Scanner struct { client *http.Client + detectors.DefaultMultiPartCredentialProvider } // Ensure the Scanner satisfies the interface at compile time. diff --git a/pkg/detectors/nethunt/nethunt.go b/pkg/detectors/nethunt/nethunt.go index 5e5ebdc04..e5c026e0b 100644 --- a/pkg/detectors/nethunt/nethunt.go +++ b/pkg/detectors/nethunt/nethunt.go @@ -11,7 +11,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/neutrinoapi/neutrinoapi.go b/pkg/detectors/neutrinoapi/neutrinoapi.go index d5bc2da97..9da7d71cc 100644 --- a/pkg/detectors/neutrinoapi/neutrinoapi.go +++ b/pkg/detectors/neutrinoapi/neutrinoapi.go @@ -15,7 +15,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/nexmoapikey/nexmoapikey.go b/pkg/detectors/nexmoapikey/nexmoapikey.go index 19bdc4c9f..1ea684d08 100644 --- a/pkg/detectors/nexmoapikey/nexmoapikey.go +++ b/pkg/detectors/nexmoapikey/nexmoapikey.go @@ -11,7 +11,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/ngc/ngc.go b/pkg/detectors/ngc/ngc.go index 8f707cb5d..acfb22a62 100644 --- a/pkg/detectors/ngc/ngc.go +++ b/pkg/detectors/ngc/ngc.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/nozbeteams/nozbeteams.go b/pkg/detectors/nozbeteams/nozbeteams.go index 3282cdc9c..a5d85f376 100644 --- a/pkg/detectors/nozbeteams/nozbeteams.go +++ b/pkg/detectors/nozbeteams/nozbeteams.go @@ -2,10 +2,11 @@ package nozbeteams import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/nutritionix/nutritionix.go b/pkg/detectors/nutritionix/nutritionix.go index 8594761f1..a22ca99a6 100644 --- a/pkg/detectors/nutritionix/nutritionix.go +++ b/pkg/detectors/nutritionix/nutritionix.go @@ -11,7 +11,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/okta/okta.go b/pkg/detectors/okta/okta.go index a3a3d9785..d3e8f2a2b 100644 --- a/pkg/detectors/okta/okta.go +++ b/pkg/detectors/okta/okta.go @@ -14,7 +14,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/onedesk/onedesk.go b/pkg/detectors/onedesk/onedesk.go index 621e78f69..597f16694 100644 --- a/pkg/detectors/onedesk/onedesk.go +++ b/pkg/detectors/onedesk/onedesk.go @@ -3,17 +3,20 @@ package onedesk import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/onelogin/onelogin.go b/pkg/detectors/onelogin/onelogin.go index d77a674a2..16ff03093 100644 --- a/pkg/detectors/onelogin/onelogin.go +++ b/pkg/detectors/onelogin/onelogin.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/onepagecrm/onepagecrm.go b/pkg/detectors/onepagecrm/onepagecrm.go index d6e43e8c5..fe21cbf50 100644 --- a/pkg/detectors/onepagecrm/onepagecrm.go +++ b/pkg/detectors/onepagecrm/onepagecrm.go @@ -11,7 +11,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/onfleet/onfleet_test.go b/pkg/detectors/onfleet/onfleet_test.go index bbae2a885..725fbd89a 100644 --- a/pkg/detectors/onfleet/onfleet_test.go +++ b/pkg/detectors/onfleet/onfleet_test.go @@ -35,9 +35,8 @@ func TestOnfleet_Pattern(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - chunkSpecificDetectors := make(map[ahocorasick.DetectorKey]detectors.Detector, 2) - ahoCorasickCore.PopulateMatchingDetectors(test.input, chunkSpecificDetectors) - if len(chunkSpecificDetectors) == 0 { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } diff --git a/pkg/detectors/openvpn/openvpn.go b/pkg/detectors/openvpn/openvpn.go index ae229809e..96fea975d 100644 --- a/pkg/detectors/openvpn/openvpn.go +++ b/pkg/detectors/openvpn/openvpn.go @@ -4,16 +4,18 @@ import ( "context" b64 "encoding/base64" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) type Scanner struct { + detectors.DefaultMultiPartCredentialProvider client *http.Client } @@ -89,7 +91,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1.SetVerificationError(err, clientSecretRes) } } - + results = append(results, s1) } } diff --git a/pkg/detectors/pagarme/pagarme_test.go b/pkg/detectors/pagarme/pagarme_test.go index a7995a396..595cbd7cd 100644 --- a/pkg/detectors/pagarme/pagarme_test.go +++ b/pkg/detectors/pagarme/pagarme_test.go @@ -35,9 +35,8 @@ func TestPagarme_Pattern(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - chunkSpecificDetectors := make(map[ahocorasick.DetectorKey]detectors.Detector, 2) - ahoCorasickCore.PopulateMatchingDetectors(test.input, chunkSpecificDetectors) - if len(chunkSpecificDetectors) == 0 { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } diff --git a/pkg/detectors/paymongo/paymongo.go b/pkg/detectors/paymongo/paymongo.go index 4ff0ab7b8..ee3d555fc 100644 --- a/pkg/detectors/paymongo/paymongo.go +++ b/pkg/detectors/paymongo/paymongo.go @@ -2,10 +2,11 @@ package paymongo import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/paypaloauth/paypaloauth.go b/pkg/detectors/paypaloauth/paypaloauth.go index c52de5d51..d656f8606 100644 --- a/pkg/detectors/paypaloauth/paypaloauth.go +++ b/pkg/detectors/paypaloauth/paypaloauth.go @@ -13,7 +13,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/pinata/pinata.go b/pkg/detectors/pinata/pinata.go index 5103a2348..7a4865d65 100644 --- a/pkg/detectors/pinata/pinata.go +++ b/pkg/detectors/pinata/pinata.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/plaidkey/plaidkey.go b/pkg/detectors/plaidkey/plaidkey.go index 365ac7daf..c94ff26a7 100644 --- a/pkg/detectors/plaidkey/plaidkey.go +++ b/pkg/detectors/plaidkey/plaidkey.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/planetscale/planetscale.go b/pkg/detectors/planetscale/planetscale.go index 0fa0184a4..d37971a14 100644 --- a/pkg/detectors/planetscale/planetscale.go +++ b/pkg/detectors/planetscale/planetscale.go @@ -12,6 +12,7 @@ import ( ) type Scanner struct { + detectors.DefaultMultiPartCredentialProvider client *http.Client } diff --git a/pkg/detectors/planetscaledb/planetscaledb.go b/pkg/detectors/planetscaledb/planetscaledb.go index 648ab9f4f..08bf01ac1 100644 --- a/pkg/detectors/planetscaledb/planetscaledb.go +++ b/pkg/detectors/planetscaledb/planetscaledb.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/planviewleankit/planviewleankit.go b/pkg/detectors/planviewleankit/planviewleankit.go index 7380716eb..f4547dc19 100644 --- a/pkg/detectors/planviewleankit/planviewleankit.go +++ b/pkg/detectors/planviewleankit/planviewleankit.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/plivo/plivo.go b/pkg/detectors/plivo/plivo.go index 28fd635ab..81fdfde55 100644 --- a/pkg/detectors/plivo/plivo.go +++ b/pkg/detectors/plivo/plivo.go @@ -13,7 +13,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/pollsapi/pollsapi.go b/pkg/detectors/pollsapi/pollsapi.go index 6d305def6..562f73482 100644 --- a/pkg/detectors/pollsapi/pollsapi.go +++ b/pkg/detectors/pollsapi/pollsapi.go @@ -2,10 +2,11 @@ package pollsapi import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/poloniex/poloniex.go b/pkg/detectors/poloniex/poloniex.go index 8c7a0c359..2514c92ad 100644 --- a/pkg/detectors/poloniex/poloniex.go +++ b/pkg/detectors/poloniex/poloniex.go @@ -17,7 +17,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/portainer/portainer.go b/pkg/detectors/portainer/portainer.go index fa93bc65e..aee0be138 100644 --- a/pkg/detectors/portainer/portainer.go +++ b/pkg/detectors/portainer/portainer.go @@ -13,6 +13,7 @@ import ( ) type Scanner struct { + detectors.DefaultMultiPartCredentialProvider client *http.Client } diff --git a/pkg/detectors/portainertoken/portainertoken.go b/pkg/detectors/portainertoken/portainertoken.go index f3925a6ed..54ad63c1d 100644 --- a/pkg/detectors/portainertoken/portainertoken.go +++ b/pkg/detectors/portainertoken/portainertoken.go @@ -14,6 +14,7 @@ import ( type Scanner struct { client *http.Client + detectors.DefaultMultiPartCredentialProvider } // Ensure the Scanner satisfies the interface at compile time. diff --git a/pkg/detectors/postgres/postgres.go b/pkg/detectors/postgres/postgres.go index 4f1a5b8e3..7bb402dcd 100644 --- a/pkg/detectors/postgres/postgres.go +++ b/pkg/detectors/postgres/postgres.go @@ -52,6 +52,7 @@ var ( ) type Scanner struct { + detectors.DefaultMultiPartCredentialProvider detectLoopback bool // Automated tests run against localhost, but we want to ignore those results in the wild } diff --git a/pkg/detectors/privatekey/privatekey.go b/pkg/detectors/privatekey/privatekey.go index d921aee54..715a82a76 100644 --- a/pkg/detectors/privatekey/privatekey.go +++ b/pkg/detectors/privatekey/privatekey.go @@ -9,9 +9,8 @@ import ( "sync" "time" - regexp "github.com/wasilibs/go-re2" - "github.com/AzureAD/microsoft-authentication-library-for-go/apps/errors" + regexp "github.com/wasilibs/go-re2" "golang.org/x/crypto/ssh" "github.com/trufflesecurity/trufflehog/v3/pkg/common" @@ -26,6 +25,7 @@ type Scanner struct { // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) var _ detectors.CustomFalsePositiveChecker = (*Scanner)(nil) +var _ detectors.MaxSecretSizeProvider = (*Scanner)(nil) var ( // TODO: add base64 encoded key support @@ -39,6 +39,11 @@ func (s Scanner) Keywords() []string { return []string{"private key"} } +const maxPrivateKeySize = 4096 + +// ProvideMaxSecretSize returns the maximum size of a secret that this detector can find. +func (s Scanner) MaxSecretSize() int64 { return maxPrivateKeySize } + // FromData will find and optionally verify Privatekey secrets in a given set of bytes. func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { dataStr := string(data) diff --git a/pkg/detectors/prospectcrm/prospectcrm.go b/pkg/detectors/prospectcrm/prospectcrm.go index fcfeb4629..c82cba508 100644 --- a/pkg/detectors/prospectcrm/prospectcrm.go +++ b/pkg/detectors/prospectcrm/prospectcrm.go @@ -3,10 +3,11 @@ package prospectcrm import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/pubnubpublishkey/pubnubpublishkey.go b/pkg/detectors/pubnubpublishkey/pubnubpublishkey.go index 656637799..0c10b6aaf 100644 --- a/pkg/detectors/pubnubpublishkey/pubnubpublishkey.go +++ b/pkg/detectors/pubnubpublishkey/pubnubpublishkey.go @@ -13,6 +13,7 @@ import ( ) type Scanner struct { + detectors.DefaultMultiPartCredentialProvider client *http.Client } diff --git a/pkg/detectors/pusherchannelkey/pusherchannelkey.go b/pkg/detectors/pusherchannelkey/pusherchannelkey.go index 1e56891da..d5bed6666 100644 --- a/pkg/detectors/pusherchannelkey/pusherchannelkey.go +++ b/pkg/detectors/pusherchannelkey/pusherchannelkey.go @@ -18,7 +18,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/ramp/ramp.go b/pkg/detectors/ramp/ramp.go index fb03a0ac2..9100c880b 100644 --- a/pkg/detectors/ramp/ramp.go +++ b/pkg/detectors/ramp/ramp.go @@ -15,6 +15,7 @@ import ( type Scanner struct { client *http.Client + detectors.DefaultMultiPartCredentialProvider } // Ensure the Scanner satisfies the interface at compile time. diff --git a/pkg/detectors/razorpay/razorpay.go b/pkg/detectors/razorpay/razorpay.go index 4f809eda9..9f633f856 100644 --- a/pkg/detectors/razorpay/razorpay.go +++ b/pkg/detectors/razorpay/razorpay.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/rechargepayments/rechargepayments.go b/pkg/detectors/rechargepayments/rechargepayments.go index 53f2a3f21..a37a17be9 100644 --- a/pkg/detectors/rechargepayments/rechargepayments.go +++ b/pkg/detectors/rechargepayments/rechargepayments.go @@ -10,7 +10,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/redis/redis.go b/pkg/detectors/redis/redis.go index 044305d27..69148a6ba 100644 --- a/pkg/detectors/redis/redis.go +++ b/pkg/detectors/redis/redis.go @@ -13,7 +13,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/repairshopr/repairshopr.go b/pkg/detectors/repairshopr/repairshopr.go index c702cbbeb..47976fa9d 100644 --- a/pkg/detectors/repairshopr/repairshopr.go +++ b/pkg/detectors/repairshopr/repairshopr.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/rev/rev.go b/pkg/detectors/rev/rev.go index 8ca7a4979..b376deefb 100644 --- a/pkg/detectors/rev/rev.go +++ b/pkg/detectors/rev/rev.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/revampcrm/revampcrm.go b/pkg/detectors/revampcrm/revampcrm.go index 2c9d2ca84..0a61b65f2 100644 --- a/pkg/detectors/revampcrm/revampcrm.go +++ b/pkg/detectors/revampcrm/revampcrm.go @@ -11,7 +11,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/ringcentral/ringcentral.go b/pkg/detectors/ringcentral/ringcentral.go index f20268656..a71075512 100644 --- a/pkg/detectors/ringcentral/ringcentral.go +++ b/pkg/detectors/ringcentral/ringcentral.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/roaring/roaring.go b/pkg/detectors/roaring/roaring.go index 356d0914a..65f3567f5 100644 --- a/pkg/detectors/roaring/roaring.go +++ b/pkg/detectors/roaring/roaring.go @@ -13,7 +13,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/roninapp/roninapp.go b/pkg/detectors/roninapp/roninapp.go index 020af6450..db817dd8a 100644 --- a/pkg/detectors/roninapp/roninapp.go +++ b/pkg/detectors/roninapp/roninapp.go @@ -14,7 +14,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/rownd/rownd.go b/pkg/detectors/rownd/rownd.go index 67aab70b2..ea79d2399 100644 --- a/pkg/detectors/rownd/rownd.go +++ b/pkg/detectors/rownd/rownd.go @@ -11,7 +11,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/runrunit/runrunit.go b/pkg/detectors/runrunit/runrunit.go index 4df932dbd..fe015f722 100644 --- a/pkg/detectors/runrunit/runrunit.go +++ b/pkg/detectors/runrunit/runrunit.go @@ -11,7 +11,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/salesforce/salesforce.go b/pkg/detectors/salesforce/salesforce.go index de2bb365b..e220ad5ca 100644 --- a/pkg/detectors/salesforce/salesforce.go +++ b/pkg/detectors/salesforce/salesforce.go @@ -18,6 +18,7 @@ const ( type Scanner struct { client *http.Client + detectors.DefaultMultiPartCredentialProvider } // Ensure the Scanner satisfies the interface at compile time. diff --git a/pkg/detectors/salesmate/salesmate.go b/pkg/detectors/salesmate/salesmate.go index 273928a2c..141502d72 100644 --- a/pkg/detectors/salesmate/salesmate.go +++ b/pkg/detectors/salesmate/salesmate.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/satismeterprojectkey/satismeterprojectkey.go b/pkg/detectors/satismeterprojectkey/satismeterprojectkey.go index 01bcdb9d3..a9b6c857e 100644 --- a/pkg/detectors/satismeterprojectkey/satismeterprojectkey.go +++ b/pkg/detectors/satismeterprojectkey/satismeterprojectkey.go @@ -13,7 +13,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/saucelabs/saucelabs.go b/pkg/detectors/saucelabs/saucelabs.go index 9544a0cbd..548b17680 100644 --- a/pkg/detectors/saucelabs/saucelabs.go +++ b/pkg/detectors/saucelabs/saucelabs.go @@ -13,7 +13,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/scalr/scalr.go b/pkg/detectors/scalr/scalr.go index a2a80c39f..025f846fb 100644 --- a/pkg/detectors/scalr/scalr.go +++ b/pkg/detectors/scalr/scalr.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/sendbird/sendbird.go b/pkg/detectors/sendbird/sendbird.go index 51fe9165a..641decf08 100644 --- a/pkg/detectors/sendbird/sendbird.go +++ b/pkg/detectors/sendbird/sendbird.go @@ -15,6 +15,7 @@ import ( type Scanner struct { client *http.Client + detectors.DefaultMultiPartCredentialProvider } // Ensure the Scanner satisfies the interface at compile time. diff --git a/pkg/detectors/sheety/sheety.go b/pkg/detectors/sheety/sheety.go index 5f990d1a5..4e860a272 100644 --- a/pkg/detectors/sheety/sheety.go +++ b/pkg/detectors/sheety/sheety.go @@ -3,16 +3,19 @@ package sheety import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/shopify/shopify.go b/pkg/detectors/shopify/shopify.go index 115d3d4df..7ad1592d1 100644 --- a/pkg/detectors/shopify/shopify.go +++ b/pkg/detectors/shopify/shopify.go @@ -13,7 +13,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/shutterstock/shutterstock.go b/pkg/detectors/shutterstock/shutterstock.go index c097d8202..59a192419 100644 --- a/pkg/detectors/shutterstock/shutterstock.go +++ b/pkg/detectors/shutterstock/shutterstock.go @@ -11,7 +11,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/signable/signable.go b/pkg/detectors/signable/signable.go index 8e13d562a..c2e8fe48a 100644 --- a/pkg/detectors/signable/signable.go +++ b/pkg/detectors/signable/signable.go @@ -15,6 +15,7 @@ import ( type Scanner struct { client *http.Client + detectors.DefaultMultiPartCredentialProvider } // Ensure the Scanner satisfies the interface at compile time. diff --git a/pkg/detectors/signalwire/signalwire.go b/pkg/detectors/signalwire/signalwire.go index a617a867c..743cb5587 100644 --- a/pkg/detectors/signalwire/signalwire.go +++ b/pkg/detectors/signalwire/signalwire.go @@ -13,7 +13,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/sigopt/sigopt.go b/pkg/detectors/sigopt/sigopt.go index 4055f93d2..0e282c548 100644 --- a/pkg/detectors/sigopt/sigopt.go +++ b/pkg/detectors/sigopt/sigopt.go @@ -4,10 +4,11 @@ import ( "context" b64 "encoding/base64" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/sinchmessage/sinchmessage.go b/pkg/detectors/sinchmessage/sinchmessage.go index eb67ecb7c..5277b7cf2 100644 --- a/pkg/detectors/sinchmessage/sinchmessage.go +++ b/pkg/detectors/sinchmessage/sinchmessage.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/sirv/sirv.go b/pkg/detectors/sirv/sirv.go index b6a7531a6..0f5a6657f 100644 --- a/pkg/detectors/sirv/sirv.go +++ b/pkg/detectors/sirv/sirv.go @@ -13,7 +13,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/siteleaf/siteleaf.go b/pkg/detectors/siteleaf/siteleaf.go index a3a3bcabd..f9bf1d0b4 100644 --- a/pkg/detectors/siteleaf/siteleaf.go +++ b/pkg/detectors/siteleaf/siteleaf.go @@ -13,7 +13,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/skybiometry/skybiometry.go b/pkg/detectors/skybiometry/skybiometry.go index 297f5a565..911edb93d 100644 --- a/pkg/detectors/skybiometry/skybiometry.go +++ b/pkg/detectors/skybiometry/skybiometry.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/slack/slack.go b/pkg/detectors/slack/slack.go index e373d9afc..95927eee6 100644 --- a/pkg/detectors/slack/slack.go +++ b/pkg/detectors/slack/slack.go @@ -15,6 +15,7 @@ import ( ) type Scanner struct { + detectors.DefaultMultiPartCredentialProvider client *http.Client } diff --git a/pkg/detectors/slackwebhook/slackwebhook.go b/pkg/detectors/slackwebhook/slackwebhook.go index b038bcb13..d8042114b 100644 --- a/pkg/detectors/slackwebhook/slackwebhook.go +++ b/pkg/detectors/slackwebhook/slackwebhook.go @@ -16,6 +16,7 @@ import ( type Scanner struct { client *http.Client + detectors.DefaultMultiPartCredentialProvider } // Ensure the Scanner satisfies the interface at compile time. diff --git a/pkg/detectors/smartystreets/smartystreets.go b/pkg/detectors/smartystreets/smartystreets.go index 55eb78af8..b3a5bee3d 100644 --- a/pkg/detectors/smartystreets/smartystreets.go +++ b/pkg/detectors/smartystreets/smartystreets.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/smooch/smooch.go b/pkg/detectors/smooch/smooch.go index 491fa7ff9..9e87cb3be 100644 --- a/pkg/detectors/smooch/smooch.go +++ b/pkg/detectors/smooch/smooch.go @@ -13,7 +13,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/snowflake/snowflake.go b/pkg/detectors/snowflake/snowflake.go index a769720d5..a53c72635 100644 --- a/pkg/detectors/snowflake/snowflake.go +++ b/pkg/detectors/snowflake/snowflake.go @@ -17,6 +17,7 @@ import ( ) type Scanner struct { + detectors.DefaultMultiPartCredentialProvider } // Ensure the Scanner satisfies the interface at compile time. diff --git a/pkg/detectors/snykkey/snykkey_test.go b/pkg/detectors/snykkey/snykkey_test.go index 43b41efba..296d2e601 100644 --- a/pkg/detectors/snykkey/snykkey_test.go +++ b/pkg/detectors/snykkey/snykkey_test.go @@ -39,17 +39,17 @@ set PATH=%PATH%;C:\Program Files\nodejs\;C:\Program Files\Git\cmd`, // { // name: "curl example", // `curl --get \ - //-H "Authorization: token ccc9ae71-913f-46bd-9d23-03356323400a" \ - //--data-urlencode "version=2023-03-20" \ - //--data-urlencode "format=cyclonedx1.4%2Bjson" \ - //https://api.snyk.io/rest/orgs/1234/projects/1234/sbom`, + // -H "Authorization: token ccc9ae71-913f-46bd-9d23-03356323400a" \ + // --data-urlencode "version=2023-03-20" \ + // --data-urlencode "format=cyclonedx1.4%2Bjson" \ + // https://api.snyk.io/rest/orgs/1234/projects/1234/sbom`, // }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { chunkSpecificDetectors := make(map[ahocorasick.DetectorKey]detectors.Detector, 2) - ahoCorasickCore.PopulateMatchingDetectors(test.input, chunkSpecificDetectors) + ahoCorasickCore.FindDetectorMatches(test.input, chunkSpecificDetectors) if len(chunkSpecificDetectors) == 0 { t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return diff --git a/pkg/detectors/spotifykey/spotifykey.go b/pkg/detectors/spotifykey/spotifykey.go index c219de884..d3d91d22e 100644 --- a/pkg/detectors/spotifykey/spotifykey.go +++ b/pkg/detectors/spotifykey/spotifykey.go @@ -15,7 +15,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/sqlserver/sqlserver.go b/pkg/detectors/sqlserver/sqlserver.go index 5d7a4ef59..076425903 100644 --- a/pkg/detectors/sqlserver/sqlserver.go +++ b/pkg/detectors/sqlserver/sqlserver.go @@ -9,6 +9,7 @@ import ( mssql "github.com/microsoft/go-mssqldb" "github.com/microsoft/go-mssqldb/msdsn" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" diff --git a/pkg/detectors/squareapp/squareapp.go b/pkg/detectors/squareapp/squareapp.go index 0ec85bf02..f4ad28432 100644 --- a/pkg/detectors/squareapp/squareapp.go +++ b/pkg/detectors/squareapp/squareapp.go @@ -14,7 +14,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/strava/strava.go b/pkg/detectors/strava/strava.go index 42b68d7be..1f8810c4f 100644 --- a/pkg/detectors/strava/strava.go +++ b/pkg/detectors/strava/strava.go @@ -11,7 +11,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/stytch/stytch.go b/pkg/detectors/stytch/stytch.go index ce8024f4f..d03cb347b 100644 --- a/pkg/detectors/stytch/stytch.go +++ b/pkg/detectors/stytch/stytch.go @@ -11,7 +11,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/sugester/sugester.go b/pkg/detectors/sugester/sugester.go index 64054d152..e22be1d9e 100644 --- a/pkg/detectors/sugester/sugester.go +++ b/pkg/detectors/sugester/sugester.go @@ -11,7 +11,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/sumologickey/sumologickey.go b/pkg/detectors/sumologickey/sumologickey.go index b0f6f8fee..28d0b02d9 100644 --- a/pkg/detectors/sumologickey/sumologickey.go +++ b/pkg/detectors/sumologickey/sumologickey.go @@ -13,7 +13,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/supernotesapi/supernotesapi.go b/pkg/detectors/supernotesapi/supernotesapi.go index e94f23930..cb4022a5f 100644 --- a/pkg/detectors/supernotesapi/supernotesapi.go +++ b/pkg/detectors/supernotesapi/supernotesapi.go @@ -2,10 +2,11 @@ package supernotesapi import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/surveyanyplace/surveyanyplace.go b/pkg/detectors/surveyanyplace/surveyanyplace.go index fa6c52765..464ef75db 100644 --- a/pkg/detectors/surveyanyplace/surveyanyplace.go +++ b/pkg/detectors/surveyanyplace/surveyanyplace.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/swell/swell.go b/pkg/detectors/swell/swell.go index c103885b2..15e737857 100644 --- a/pkg/detectors/swell/swell.go +++ b/pkg/detectors/swell/swell.go @@ -11,7 +11,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/tallyfy/tallyfy.go b/pkg/detectors/tallyfy/tallyfy.go index 68f6f6cbc..44b928e51 100644 --- a/pkg/detectors/tallyfy/tallyfy.go +++ b/pkg/detectors/tallyfy/tallyfy.go @@ -3,16 +3,19 @@ package tallyfy import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/teamgate/teamgate.go b/pkg/detectors/teamgate/teamgate.go index 697b8be15..39dffb5cc 100644 --- a/pkg/detectors/teamgate/teamgate.go +++ b/pkg/detectors/teamgate/teamgate.go @@ -11,7 +11,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/testingbot/testingbot.go b/pkg/detectors/testingbot/testingbot.go index 9003c688e..56fb05fdb 100644 --- a/pkg/detectors/testingbot/testingbot.go +++ b/pkg/detectors/testingbot/testingbot.go @@ -11,7 +11,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/textmagic/textmagic.go b/pkg/detectors/textmagic/textmagic.go index bee025d95..b543df318 100644 --- a/pkg/detectors/textmagic/textmagic.go +++ b/pkg/detectors/textmagic/textmagic.go @@ -13,7 +13,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/thinkific/thinkific.go b/pkg/detectors/thinkific/thinkific.go index 66f9459af..d0cb6631d 100644 --- a/pkg/detectors/thinkific/thinkific.go +++ b/pkg/detectors/thinkific/thinkific.go @@ -13,7 +13,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/thousandeyes/thousandeyes.go b/pkg/detectors/thousandeyes/thousandeyes.go index 435c85846..7693e6bc9 100644 --- a/pkg/detectors/thousandeyes/thousandeyes.go +++ b/pkg/detectors/thousandeyes/thousandeyes.go @@ -11,7 +11,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/tineswebhook/tineswebhook.go b/pkg/detectors/tineswebhook/tineswebhook.go index a5e08e9b3..7126e2d7c 100644 --- a/pkg/detectors/tineswebhook/tineswebhook.go +++ b/pkg/detectors/tineswebhook/tineswebhook.go @@ -2,10 +2,11 @@ package tineswebhook import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/tokeet/tokeet.go b/pkg/detectors/tokeet/tokeet.go index 7e7cf8668..a8ac23998 100644 --- a/pkg/detectors/tokeet/tokeet.go +++ b/pkg/detectors/tokeet/tokeet.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/trelloapikey/trelloapikey.go b/pkg/detectors/trelloapikey/trelloapikey.go index 1f2e533fa..d26696d59 100644 --- a/pkg/detectors/trelloapikey/trelloapikey.go +++ b/pkg/detectors/trelloapikey/trelloapikey.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/tru/tru.go b/pkg/detectors/tru/tru.go index 2df42285c..f573a1cff 100644 --- a/pkg/detectors/tru/tru.go +++ b/pkg/detectors/tru/tru.go @@ -13,7 +13,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/trufflehogenterprise/trufflehogenterprise.go b/pkg/detectors/trufflehogenterprise/trufflehogenterprise.go index cc2978e1b..dfd6263b3 100644 --- a/pkg/detectors/trufflehogenterprise/trufflehogenterprise.go +++ b/pkg/detectors/trufflehogenterprise/trufflehogenterprise.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/twilio/twilio.go b/pkg/detectors/twilio/twilio.go index 927a106da..0530d3836 100644 --- a/pkg/detectors/twilio/twilio.go +++ b/pkg/detectors/twilio/twilio.go @@ -14,6 +14,7 @@ import ( ) type Scanner struct { + detectors.DefaultMultiPartCredentialProvider client *http.Client } diff --git a/pkg/detectors/twitch/twitch.go b/pkg/detectors/twitch/twitch.go index 426f89841..994be1ede 100644 --- a/pkg/detectors/twitch/twitch.go +++ b/pkg/detectors/twitch/twitch.go @@ -15,6 +15,7 @@ import ( ) type Scanner struct { + detectors.DefaultMultiPartCredentialProvider client *http.Client } diff --git a/pkg/detectors/twitter/twitter.go b/pkg/detectors/twitter/twitter.go index a043e7517..bcf8c8886 100644 --- a/pkg/detectors/twitter/twitter.go +++ b/pkg/detectors/twitter/twitter.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/typetalk/typetalk.go b/pkg/detectors/typetalk/typetalk.go index 2ad121fe7..b25b5c824 100644 --- a/pkg/detectors/typetalk/typetalk.go +++ b/pkg/detectors/typetalk/typetalk.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/uploadcare/uploadcare.go b/pkg/detectors/uploadcare/uploadcare.go index 6866aa75c..75497eee1 100644 --- a/pkg/detectors/uploadcare/uploadcare.go +++ b/pkg/detectors/uploadcare/uploadcare.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/uptimerobot/uptimerobot.go b/pkg/detectors/uptimerobot/uptimerobot.go index b501e246a..00d3b6df6 100644 --- a/pkg/detectors/uptimerobot/uptimerobot.go +++ b/pkg/detectors/uptimerobot/uptimerobot.go @@ -2,11 +2,12 @@ package uptimerobot import ( "context" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/user/user.go b/pkg/detectors/user/user.go index ee5ec0fa2..7cd928465 100644 --- a/pkg/detectors/user/user.go +++ b/pkg/detectors/user/user.go @@ -3,10 +3,11 @@ package user import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/verifier/verifier.go b/pkg/detectors/verifier/verifier.go index 5bea79f9b..922506c33 100644 --- a/pkg/detectors/verifier/verifier.go +++ b/pkg/detectors/verifier/verifier.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/verimail/verimail.go b/pkg/detectors/verimail/verimail.go index ce90672e9..973dcd440 100644 --- a/pkg/detectors/verimail/verimail.go +++ b/pkg/detectors/verimail/verimail.go @@ -2,10 +2,11 @@ package verimail import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/veriphone/veriphone.go b/pkg/detectors/veriphone/veriphone.go index 16cfb0eab..944d5c333 100644 --- a/pkg/detectors/veriphone/veriphone.go +++ b/pkg/detectors/veriphone/veriphone.go @@ -2,10 +2,11 @@ package veriphone import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/viewneo/viewneo.go b/pkg/detectors/viewneo/viewneo.go index d12fffc69..1f1754355 100644 --- a/pkg/detectors/viewneo/viewneo.go +++ b/pkg/detectors/viewneo/viewneo.go @@ -3,10 +3,11 @@ package viewneo import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" @@ -30,6 +31,11 @@ func (s Scanner) Keywords() []string { return []string{"viewneo"} } +const maxKeySize = 1500 + +// MaxSecretSize returns the maximum size of a secret that this detector can find. +func (s Scanner) MaxSecretSize() int64 { return maxKeySize } + // FromData will find and optionally verify Viewneo secrets in a given set of bytes. func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { dataStr := string(data) diff --git a/pkg/detectors/vouchery/vouchery.go b/pkg/detectors/vouchery/vouchery.go index 2f5d5d296..f143f1898 100644 --- a/pkg/detectors/vouchery/vouchery.go +++ b/pkg/detectors/vouchery/vouchery.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/webex/webex.go b/pkg/detectors/webex/webex.go index fbeecb03e..049ac9fa2 100644 --- a/pkg/detectors/webex/webex.go +++ b/pkg/detectors/webex/webex.go @@ -13,7 +13,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/websitepulse/websitepulse.go b/pkg/detectors/websitepulse/websitepulse.go index 37bd03ba8..8a704c47b 100644 --- a/pkg/detectors/websitepulse/websitepulse.go +++ b/pkg/detectors/websitepulse/websitepulse.go @@ -13,7 +13,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/wepay/wepay.go b/pkg/detectors/wepay/wepay.go index 6e3391c4c..df2300cbd 100644 --- a/pkg/detectors/wepay/wepay.go +++ b/pkg/detectors/wepay/wepay.go @@ -11,7 +11,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/wiz/wiz.go b/pkg/detectors/wiz/wiz.go index c0de255cf..300a6a98f 100644 --- a/pkg/detectors/wiz/wiz.go +++ b/pkg/detectors/wiz/wiz.go @@ -16,6 +16,7 @@ import ( ) type Scanner struct { + detectors.DefaultMultiPartCredentialProvider client *http.Client } diff --git a/pkg/detectors/wiz/wiz_test.go b/pkg/detectors/wiz/wiz_test.go index 2eb7820b5..ae3452cf8 100644 --- a/pkg/detectors/wiz/wiz_test.go +++ b/pkg/detectors/wiz/wiz_test.go @@ -41,9 +41,8 @@ wiz_client_secret = 'lmSlx1fe6yCfwAbDa8pMp9sJDM9rZzDblmSlx1fe6yCfwAbDa8pMp9sJDM9 for _, test := range tests { t.Run(test.name, func(t *testing.T) { - chunkSpecificDetectors := make(map[ahocorasick.DetectorKey]detectors.Detector, 2) - ahoCorasickCore.PopulateMatchingDetectors(test.input, chunkSpecificDetectors) - if len(chunkSpecificDetectors) == 0 { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } diff --git a/pkg/detectors/worldcoinindex/worldcoinindex.go b/pkg/detectors/worldcoinindex/worldcoinindex.go index 287f9aa88..9e8b4e2f2 100644 --- a/pkg/detectors/worldcoinindex/worldcoinindex.go +++ b/pkg/detectors/worldcoinindex/worldcoinindex.go @@ -3,11 +3,12 @@ package worldcoinindex import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/youtubeapikey/youtubeapikey.go b/pkg/detectors/youtubeapikey/youtubeapikey.go index e77ac9201..534ba0cd5 100644 --- a/pkg/detectors/youtubeapikey/youtubeapikey.go +++ b/pkg/detectors/youtubeapikey/youtubeapikey.go @@ -11,7 +11,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/zendeskapi/zendeskapi.go b/pkg/detectors/zendeskapi/zendeskapi.go index 1c0043a9e..913723a5d 100644 --- a/pkg/detectors/zendeskapi/zendeskapi.go +++ b/pkg/detectors/zendeskapi/zendeskapi.go @@ -13,7 +13,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/zipapi/zipapi.go b/pkg/detectors/zipapi/zipapi.go index 8f544a080..5eb877f7c 100644 --- a/pkg/detectors/zipapi/zipapi.go +++ b/pkg/detectors/zipapi/zipapi.go @@ -13,7 +13,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/zipbooks/zipbooks.go b/pkg/detectors/zipbooks/zipbooks.go index 28ada1847..c1651f8e9 100644 --- a/pkg/detectors/zipbooks/zipbooks.go +++ b/pkg/detectors/zipbooks/zipbooks.go @@ -12,7 +12,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct{ + detectors.DefaultMultiPartCredentialProvider +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) diff --git a/pkg/detectors/zulipchat/zulipchat.go b/pkg/detectors/zulipchat/zulipchat.go index 61c70f3d3..c7980d73d 100644 --- a/pkg/detectors/zulipchat/zulipchat.go +++ b/pkg/detectors/zulipchat/zulipchat.go @@ -14,6 +14,7 @@ import ( type Scanner struct { client *http.Client + detectors.DefaultMultiPartCredentialProvider } // Ensure the Scanner satisfies the interface at compile time. diff --git a/pkg/engine/ahocorasick/ahocorasickcore.go b/pkg/engine/ahocorasick/ahocorasickcore.go index ab1f3eeb6..f6a36c764 100644 --- a/pkg/engine/ahocorasick/ahocorasickcore.go +++ b/pkg/engine/ahocorasick/ahocorasickcore.go @@ -1,6 +1,7 @@ package ahocorasick import ( + "bytes" "strings" ahocorasick "github.com/BobuSumisu/aho-corasick" @@ -26,6 +27,68 @@ type DetectorKey struct { // Type returns the detector type of the key. func (k DetectorKey) Type() detectorspb.DetectorType { return k.detectorType } +// spanCalculator is an interface that defines a method for calculating a match span +// in the chunk data. This allows for different strategies to be used without changing the core logic. +type spanCalculator interface { + calculateSpan(startIdx int64, chunkData []byte, detector detectors.Detector) matchSpan +} + +// EntireChunkSpanCalculator is a strategy that calculates the match span to use the entire chunk data. +// This is used when we want to match against the full length of the provided chunk. +type EntireChunkSpanCalculator struct{} + +// calculateSpans returns the match span as the length of the chunk data, +// effectively using the entire chunk for matching. +func (e *EntireChunkSpanCalculator) calculateSpan( + startIdx int64, + chunkData []byte, + _ detectors.Detector, +) matchSpan { + return matchSpan{startOffset: startIdx, endOffset: int64(len(chunkData))} +} + +// maxMatchLengthSpanCalculator is a strategy that calculates match spans based on a default max +// match length or values provided by detectors. This allows for more granular control over the match span. +type maxMatchLengthSpanCalculator struct{ maxMatchLength int64 } + +// newMaxMatchLengthSpanCalculator creates a new instance of maxMatchLengthSpanCalculator with the +// specified max match length. +func newMaxMatchLengthSpanCalculator(maxMatchLength int64) *maxMatchLengthSpanCalculator { + return &maxMatchLengthSpanCalculator{maxMatchLength: maxMatchLength} +} + +// calculateSpans computes the match spans based on the start index and the max match length. +// If the detector provides an override value, it uses that instead of the default max match length. +func (m *maxMatchLengthSpanCalculator) calculateSpan( + startIdx int64, + chunkData []byte, + detector detectors.Detector, +) matchSpan { + maxSize := m.maxMatchLength + + switch d := detector.(type) { + case detectors.MultiPartCredentialProvider: + maxSize = d.MaxCredentialSpan() + case detectors.MaxSecretSizeProvider: + maxSize = d.MaxSecretSize() + default: // Use the default max match length + } + endIdx := startIdx + maxSize + if endIdx > int64(len(chunkData)) { + endIdx = int64(len(chunkData)) + } + + return matchSpan{startOffset: startIdx, endOffset: endIdx} +} + +// AhoCorasickCoreOption is a functional option type for configuring an AhoCorasickCore instance. +type AhoCorasickCoreOption func(*AhoCorasickCore) + +// WithSpanCalculator sets the span calculator for AhoCorasickCore. +func WithSpanCalculator(spanCalculator spanCalculator) AhoCorasickCoreOption { + return func(ac *AhoCorasickCore) { ac.spanCalculator = spanCalculator } +} + // AhoCorasickCore encapsulates the operations and data structures used for keyword matching via the // Aho-Corasick algorithm. It is responsible for constructing and managing the trie for efficient // substring searches, as well as mapping keywords to their associated detectors for rapid lookups. @@ -40,12 +103,13 @@ type AhoCorasickCore struct { // some consuming code a little cleaner.) keywordsToDetectors map[string][]DetectorKey detectorsByKey map[DetectorKey]detectors.Detector + spanCalculator spanCalculator // Strategy for calculating match spans } // NewAhoCorasickCore allocates and initializes a new instance of AhoCorasickCore. It uses the // provided detector slice to create a map from keywords to detectors and build the Aho-Corasick // prefilter trie. -func NewAhoCorasickCore(allDetectors []detectors.Detector) *AhoCorasickCore { +func NewAhoCorasickCore(allDetectors []detectors.Detector, opts ...AhoCorasickCoreOption) *AhoCorasickCore { keywordsToDetectors := make(map[string][]DetectorKey) detectorsByKey := make(map[DetectorKey]detectors.Detector, len(allDetectors)) var keywords []string @@ -59,53 +123,129 @@ func NewAhoCorasickCore(allDetectors []detectors.Detector) *AhoCorasickCore { } } - return &AhoCorasickCore{ + const maxMatchLength int64 = 512 + ac := &AhoCorasickCore{ keywordsToDetectors: keywordsToDetectors, detectorsByKey: detectorsByKey, prefilter: *ahocorasick.NewTrieBuilder().AddStrings(keywords).Build(), + spanCalculator: newMaxMatchLengthSpanCalculator(maxMatchLength), // Default span calculator + } + + for _, opt := range opts { + opt(ac) + } + + return ac +} + +// DetectorMatch represents a detected pattern's metadata in a data chunk. +// It encapsulates the key identifying a specific detector, the detector instance itself, +// the start and end offsets of the matched keyword in the chunk, and the matched portions of the chunk data. +type DetectorMatch struct { + Key DetectorKey + detectors.Detector + matchSpans []matchSpan + + // matches is a slice of byte slices, each representing a matched portion of the chunk data. + matches [][]byte +} + +// MatchSpan represents a single occurrence of a matched keyword in the chunk. +// It contains the start and end byte offsets of the matched keyword within the chunk. +type matchSpan struct { + startOffset int64 + endOffset int64 +} + +// addMatchSpan adds a match span to the DetectorMatch instance. +func (d *DetectorMatch) addMatchSpan(spans ...matchSpan) { + d.matchSpans = append(d.matchSpans, spans...) +} + +// mergeMatches merges overlapping or adjacent matchSpans into a single matchSpan. +// It updates the matchSpans field with the merged spans. +func (d *DetectorMatch) mergeMatches() { + if len(d.matchSpans) <= 1 { + return + } + + merged := make([]matchSpan, 0, len(d.matchSpans)) + current := d.matchSpans[0] + + for i := 1; i < len(d.matchSpans); i++ { + if d.matchSpans[i].startOffset <= current.endOffset { + if d.matchSpans[i].endOffset > current.endOffset { + current.endOffset = d.matchSpans[i].endOffset + } + continue + } + merged = append(merged, current) + current = d.matchSpans[i] + } + + merged = append(merged, current) + d.matchSpans = merged +} + +// extractMatches extracts the matched portions from the chunk data and stores them in the matches field. +func (d *DetectorMatch) extractMatches(chunkData []byte) { + d.matches = make([][]byte, len(d.matchSpans)) + for i, m := range d.matchSpans { + d.matches[i] = chunkData[m.startOffset:m.endOffset] } } -// GetDetectorByKey returns the detector associated with the given key. If no detector is found, it -// returns nil. -func (ac *AhoCorasickCore) GetDetectorByKey(key DetectorKey) detectors.Detector { - return ac.detectorsByKey[key] -} +// Matches returns a slice of byte slices, each representing a matched portion of the chunk data. +func (d *DetectorMatch) Matches() [][]byte { return d.matches } -// DetectorInfo represents a detected pattern's metadata in a data chunk. -// It encapsulates the key identifying a specific detector and the detector instance itself. -type DetectorInfo struct { - Key DetectorKey - detectors.Detector -} +// FindDetectorMatches finds the matching detectors for a given chunk of data using the Aho-Corasick algorithm. +// It returns a slice of DetectorMatch instances, each containing the detector key, detector, +// a slice of matchSpans, and the corresponding matched portions of the chunk data. +// +// Each matchSpan represents a position in the chunk data where a keyword was found, +// along with the corresponding span (start and end positions). +// The span is determined based on the configured spanCalculator strategy. +// Adjacent or overlapping matches are merged to avoid duplicating or overlapping the matched +// portions of the chunk data. +// +// The matches field contains the actual byte slices of the matched portions from the chunk data. +func (ac *AhoCorasickCore) FindDetectorMatches(chunkData []byte) []*DetectorMatch { + matches := ac.prefilter.Match(bytes.ToLower(chunkData)) -// PopulateMatchingDetectors populates the given detector slice with all the detectors matching the -// provided input. This method populates an existing map rather than allocating a new one because -// it will be called once per chunk and that many allocations has a noticeable performance cost. -// It returns a slice of unique 'DetectorInfo' corresponding to the matched detectors. This slice is -// constructed to prevent duplications by utilizing an internal map to track already processed detectors. -func (ac *AhoCorasickCore) PopulateMatchingDetectors(chunkData string, dts map[DetectorKey]detectors.Detector) []DetectorInfo { - matches := ac.prefilter.MatchString(strings.ToLower(chunkData)) + matchCount := len(matches) + if matchCount == 0 { + return nil + } - // Use a map to avoid adding duplicate detectors to the slice. - addedDetectors := make(map[DetectorKey]struct{}) - uniqueDetectors := make([]DetectorInfo, 0, len(matches)) + detectorMatches := make(map[DetectorKey]*DetectorMatch) for _, m := range matches { for _, k := range ac.keywordsToDetectors[m.MatchString()] { - if _, exists := addedDetectors[k]; exists { - continue + if _, exists := detectorMatches[k]; !exists { + detector := ac.detectorsByKey[k] + detectorMatches[k] = &DetectorMatch{ + Key: k, + Detector: detector, + matchSpans: make([]matchSpan, 0), + } } - // Add to the map to track already added detectors. - addedDetectors[k] = struct{}{} - // Add the detector to the map and slice. - detector := ac.detectorsByKey[k] - dts[k] = detector - uniqueDetectors = append(uniqueDetectors, DetectorInfo{Key: k, Detector: detector}) + detectorMatch := detectorMatches[k] + startIdx := m.Pos() + span := ac.spanCalculator.calculateSpan(startIdx, chunkData, detectorMatch.Detector) + detectorMatch.addMatchSpan(span) } } + uniqueDetectors := make([]*DetectorMatch, 0, len(detectorMatches)) + for _, detectorMatch := range detectorMatches { + // Merge overlapping or adjacent match spans. + detectorMatch.mergeMatches() + detectorMatch.extractMatches(chunkData) + + uniqueDetectors = append(uniqueDetectors, detectorMatch) + } + return uniqueDetectors } diff --git a/pkg/engine/ahocorasick/ahocorasickcore_test.go b/pkg/engine/ahocorasick/ahocorasickcore_test.go index 4e5229740..fb8d4be52 100644 --- a/pkg/engine/ahocorasick/ahocorasickcore_test.go +++ b/pkg/engine/ahocorasick/ahocorasickcore_test.go @@ -17,45 +17,57 @@ const TestDetectorType = -1 type testDetectorV1 struct { } -func (d testDetectorV1) FromData(ctx context.Context, verify bool, data []byte) ([]detectors.Result, error) { +func (testDetectorV1) FromData(ctx context.Context, verify bool, data []byte) ([]detectors.Result, error) { return make([]detectors.Result, 0), nil } -func (d testDetectorV1) Keywords() []string { - return []string{"a", "b"} -} +func (testDetectorV1) Keywords() []string { return []string{"a", "b"} } -func (d testDetectorV1) Type() detectorspb.DetectorType { +func (testDetectorV1) Type() detectorspb.DetectorType { return TestDetectorType } -func (d testDetectorV1) Version() int { - return 1 -} +func (testDetectorV1) Version() int { return 1 } type testDetectorV2 struct { } -func (d testDetectorV2) FromData(ctx context.Context, verify bool, data []byte) ([]detectors.Result, error) { +func (testDetectorV2) FromData(ctx context.Context, verify bool, data []byte) ([]detectors.Result, error) { return make([]detectors.Result, 0), nil } -func (d testDetectorV2) Keywords() []string { +func (testDetectorV2) Keywords() []string { return []string{"a"} } -func (d testDetectorV2) Type() detectorspb.DetectorType { +func (testDetectorV2) Type() detectorspb.DetectorType { return TestDetectorType } -func (d testDetectorV2) Version() int { - return 2 +func (testDetectorV2) Version() int { return 2 } + +type testDetectorV3 struct { } +func (testDetectorV3) FromData(ctx context.Context, verify bool, data []byte) ([]detectors.Result, error) { + return make([]detectors.Result, 0), nil +} + +func (testDetectorV3) Keywords() []string { + return []string{"truffle"} +} + +func (testDetectorV3) Type() detectorspb.DetectorType { + return TestDetectorType +} + +func (testDetectorV3) Version() int { return 1 } + var _ detectors.Detector = (*testDetectorV1)(nil) var _ detectors.Detector = (*testDetectorV2)(nil) var _ detectors.Versioner = (*testDetectorV1)(nil) var _ detectors.Versioner = (*testDetectorV2)(nil) +var _ detectors.Versioner = (*testDetectorV3)(nil) func TestAhoCorasickCore_MultipleCustomDetectorsMatchable(t *testing.T) { customDetector1, err := custom_detectors.NewWebhookCustomRegex(&custom_detectorspb.CustomRegex{ @@ -76,11 +88,10 @@ func TestAhoCorasickCore_MultipleCustomDetectorsMatchable(t *testing.T) { ac := NewAhoCorasickCore(allDetectors) - detectorsMap := make(map[DetectorKey]detectors.Detector, 2) - ac.PopulateMatchingDetectors("a", detectorsMap) + dts := ac.FindDetectorMatches([]byte("a")) matchingDetectors := make([]detectors.Detector, 0, 2) - for _, d := range detectorsMap { - matchingDetectors = append(matchingDetectors, d) + for _, d := range dts { + matchingDetectors = append(matchingDetectors, d.Detector) } assert.ElementsMatch(t, allDetectors, matchingDetectors) } @@ -92,11 +103,10 @@ func TestAhoCorasickCore_MultipleDetectorVersionsMatchable(t *testing.T) { ac := NewAhoCorasickCore(allDetectors) - detectorsMap := make(map[DetectorKey]detectors.Detector, 2) - ac.PopulateMatchingDetectors("a", detectorsMap) + dts := ac.FindDetectorMatches([]byte("a")) matchingDetectors := make([]detectors.Detector, 0, 2) - for _, d := range detectorsMap { - matchingDetectors = append(matchingDetectors, d) + for _, d := range dts { + matchingDetectors = append(matchingDetectors, d.Detector) } assert.ElementsMatch(t, allDetectors, matchingDetectors) } @@ -107,11 +117,107 @@ func TestAhoCorasickCore_NoDuplicateDetectorsMatched(t *testing.T) { ac := NewAhoCorasickCore(allDetectors) - detectorsMap := make(map[DetectorKey]detectors.Detector, 2) - ac.PopulateMatchingDetectors("a a b b", detectorsMap) + dts := ac.FindDetectorMatches([]byte("a a b b")) matchingDetectors := make([]detectors.Detector, 0, 2) - for _, d := range detectorsMap { - matchingDetectors = append(matchingDetectors, d) + for _, d := range dts { + matchingDetectors = append(matchingDetectors, d.Detector) } assert.ElementsMatch(t, allDetectors, matchingDetectors) } + +func TestFindDetectorMatches(t *testing.T) { + testCases := []struct { + name string + detectors []detectors.Detector + sampleData string + expectedResult map[DetectorKey][][]int64 + }{ + + { + name: "single matchSpan", + detectors: []detectors.Detector{ + testDetectorV3{}, + }, + sampleData: "This is a sample data containing keyword truffle", + expectedResult: map[DetectorKey][][]int64{ + CreateDetectorKey(testDetectorV3{}): {{41, 48}}, + }, + }, + { + name: "Multiple matches overlapping", + detectors: []detectors.Detector{ + testDetectorV1{}, + }, + sampleData: "This is a sample data containing keyword a", + expectedResult: map[DetectorKey][][]int64{ + CreateDetectorKey(testDetectorV1{}): {{8, 42}}, + }, + }, + { + name: "Multiple matches", + detectors: []detectors.Detector{ + testDetectorV2{}, + }, + sampleData: `This is the first occurrence of the letter a. + Lorem ipsum dolor sit met, consectetur dipiscing elit. Sed uctor, + mgn bibendum bibendum, ugue ugue tincidunt ugue, + eget ultricies ugue ugue id ugue. Meens liquet libero + c libero molestie, nec mlesud ugue ugue eget. Donec + sed ugue. Sed euismod, ugue sit met liqum lcini, + ugue ugue tincidunt ugue, eget ultricies ugue ugue id + ugue. Meens liquet libero c libero molestie, nec + mlesud ugue ugue eget. Donec sed ugue. Sed euismod, + ugue sit met liqum lcini, ugue ugue tincidunt ugue, + eget ultricies ugue ugue id ugue. Meens liquet libero + c libero molestie, nec mlesud ugue ugue eget. This is the second occurrence of the letter a.`, + expectedResult: map[DetectorKey][][]int64{ + CreateDetectorKey(testDetectorV2{}): {{43, 555}, {854, 856}}, + }, + }, + { + name: "No matches", + detectors: []detectors.Detector{ + testDetectorV1{}, + testDetectorV2{}, + }, + sampleData: "xxy yzz lnnope", + expectedResult: map[DetectorKey][][]int64{}, + }, + } + + for _, tc := range testCases { + tc := tc + + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + ac := NewAhoCorasickCore(tc.detectors) + detectorMatches := ac.FindDetectorMatches([]byte(tc.sampleData)) + + // Verify that all matching detectors and their matches are returned. + for _, detectorMatch := range detectorMatches { + assert.Contains(t, tc.expectedResult, detectorMatch.Key, "Expected detector key to be present") + + expectedMatches := tc.expectedResult[detectorMatch.Key] + actualMatches := make([][]int64, len(detectorMatch.matchSpans)) + for i, match := range detectorMatch.matchSpans { + actualMatches[i] = []int64{match.startOffset, match.endOffset} + } + + assert.ElementsMatch(t, expectedMatches, actualMatches, "Expected matches to be returned for the detector") + } + + // Verify that all expected matches are returned for each detector. + for key, expectedMatches := range tc.expectedResult { + var actualMatches [][]int64 + for _, detectorMatch := range detectorMatches { + if detectorMatch.Key == key { + for _, match := range detectorMatch.matchSpans { + actualMatches = append(actualMatches, []int64{match.startOffset, match.endOffset}) + } + } + } + assert.ElementsMatch(t, expectedMatches, actualMatches, "Expected all matches to be returned for the detector") + } + }) + } +} diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go index f4cbb6d2d..a0d674392 100644 --- a/pkg/engine/engine.go +++ b/pkg/engine/engine.go @@ -78,6 +78,9 @@ type Engine struct { logFilteredUnverified bool verificationOverlap bool printAvgDetectorTime bool + // By default, the engine will only scan a subset of the chunk if a detector matches the chunk. + // If this flag is set to true, the engine will scan the entire chunk. + scanEntireChunk bool // ahoCorasickHandler manages the Aho-Corasick trie and related keyword lookups. ahoCorasickCore *ahocorasick.AhoCorasickCore @@ -256,6 +259,11 @@ func filterDetectors(filterFunc func(detectors.Detector) bool, input []detectors return out } +// WithEntireChunkScan sets the flag to configure AhoCorasickCore to scan entire chunks. +func WithEntireChunkScan(enabled bool) Option { + return func(e *Engine) { e.scanEntireChunk = enabled } +} + // HasFoundResults returns true if any results are found. func (e *Engine) HasFoundResults() bool { return atomic.LoadUint32(&e.numFoundResults) > 0 @@ -399,8 +407,14 @@ func (e *Engine) initialize(ctx context.Context, options ...Option) error { } ctx.Logger().V(4).Info("engine initialized") + // Configure the EntireChunkSpanCalculator if the engine is set to scan the entire chunk. + var ahoCOptions []ahocorasick.AhoCorasickCoreOption + if e.scanEntireChunk { + ahoCOptions = append(ahoCOptions, ahocorasick.WithSpanCalculator(new(ahocorasick.EntireChunkSpanCalculator))) + } + ctx.Logger().V(4).Info("setting up aho-corasick core") - e.ahoCorasickCore = ahocorasick.NewAhoCorasickCore(e.detectors) + e.ahoCorasickCore = ahocorasick.NewAhoCorasickCore(e.detectors, ahoCOptions...) ctx.Logger().V(4).Info("set up aho-corasick core") return nil @@ -588,7 +602,7 @@ func (e *Engine) ScanChunk(chunk *sources.Chunk) { // detectableChunk is a decoded chunk that is ready to be scanned by its detector. type detectableChunk struct { - detector detectors.Detector + detector *ahocorasick.DetectorMatch chunk sources.Chunk decoder detectorspb.DecoderType wgDoneFn func() @@ -600,7 +614,7 @@ type detectableChunk struct { type verificationOverlapChunk struct { chunk sources.Chunk decoder detectorspb.DecoderType - detectors []ahocorasick.DetectorInfo + detectors []*ahocorasick.DetectorMatch verificationOverlapWgDoneFn func() } @@ -608,9 +622,6 @@ func (e *Engine) detectorWorker(ctx context.Context) { var wgDetect sync.WaitGroup var wgVerificationOverlap sync.WaitGroup - // Reuse the same map to avoid allocations. - const avgDetectorsPerChunk = 8 - chunkSpecificDetectors := make(map[ahocorasick.DetectorKey]detectors.Detector, avgDetectorsPerChunk) for chunk := range e.ChunksChan() { atomic.AddUint64(&e.metrics.BytesScanned, uint64(len(chunk.Data))) for _, decoder := range e.decoders { @@ -620,8 +631,8 @@ func (e *Engine) detectorWorker(ctx context.Context) { continue } - matchingDetectors := e.ahoCorasickCore.PopulateMatchingDetectors(string(decoded.Chunk.Data), chunkSpecificDetectors) - if len(chunkSpecificDetectors) > 1 && !e.verificationOverlap { + matchingDetectors := e.ahoCorasickCore.FindDetectorMatches(decoded.Chunk.Data) + if len(matchingDetectors) > 1 && !e.verificationOverlap { wgVerificationOverlap.Add(1) e.verificationOverlapChunksChan <- verificationOverlapChunk{ chunk: *decoded.Chunk, @@ -629,14 +640,10 @@ func (e *Engine) detectorWorker(ctx context.Context) { decoder: decoded.DecoderType, verificationOverlapWgDoneFn: wgVerificationOverlap.Done, } - // Empty the map. - for k := range chunkSpecificDetectors { - delete(chunkSpecificDetectors, k) - } continue } - for k, detector := range chunkSpecificDetectors { + for _, detector := range matchingDetectors { decoded.Chunk.Verify = e.verify wgDetect.Add(1) e.detectableChunksChan <- detectableChunk{ @@ -645,9 +652,10 @@ func (e *Engine) detectorWorker(ctx context.Context) { decoder: decoded.DecoderType, wgDoneFn: wgDetect.Done, } - delete(chunkSpecificDetectors, k) } + continue } + atomic.AddUint64(&e.metrics.ChunksScanned, 1) } @@ -707,71 +715,68 @@ func (e *Engine) verificationOverlapWorker(ctx context.Context) { // Reuse the same map and slice to avoid allocations. const avgSecretsPerDetector = 8 - detectorKeysWithResults := make(map[ahocorasick.DetectorKey]struct{}, avgSecretsPerDetector) + detectorKeysWithResults := make(map[ahocorasick.DetectorKey]*ahocorasick.DetectorMatch, avgSecretsPerDetector) chunkSecrets := make(map[chunkSecretKey]struct{}, avgSecretsPerDetector) for chunk := range e.verificationOverlapChunksChan { for _, detector := range chunk.detectors { // DO NOT VERIFY at this stage of the pipeline. - results, err := detector.FromData(ctx, false, chunk.chunk.Data) - if err != nil { - ctx.Logger().Error(err, "error verifying chunk") - } - - if len(results) == 0 { - continue - } - if _, ok := detectorKeysWithResults[detector.Key]; !ok { - detectorKeysWithResults[detector.Key] = struct{}{} - } - - for _, res := range results { - var val []byte - if res.RawV2 != nil { - val = res.RawV2 - } else { - val = res.Raw + matchedBytes := detector.Matches() + for _, match := range matchedBytes { + results, err := detector.FromData(ctx, false, match) + if err != nil { + ctx.Logger().Error(err, "error verifying chunk") } - // Use levenstein distance to determine if the secret is likely the same. - // Ex: - // - postman api key: PMAK-qnwfsLyRSyfCwfpHaQP1UzDhrgpWvHjbYzjpRCMshjt417zWcrzyHUArs7r - // - malicious detector "api key": qnwfsLyRSyfCwfpHaQP1UzDhrgpWvHjbYzjpRCMshjt417zWcrzyHUArs7r - key := chunkSecretKey{secret: string(val), detectorKey: detector.Key} - if _, ok := chunkSecrets[key]; ok { + if len(results) == 0 { continue } + if _, ok := detectorKeysWithResults[detector.Key]; !ok { + detectorKeysWithResults[detector.Key] = detector + } - if likelyDuplicate(ctx, key, chunkSecrets) { - // This indicates that the same secret was found by multiple detectors. - // We should NOT VERIFY this chunk's data. - if e.verificationOverlapTracker != nil { - e.verificationOverlapTracker.increment() + for _, res := range results { + var val []byte + if res.RawV2 != nil { + val = res.RawV2 + } else { + val = res.Raw } - res.SetVerificationError(errOverlap) - e.processResult(ctx, detectableChunk{ - chunk: chunk.chunk, - detector: detector, - decoder: chunk.decoder, - wgDoneFn: wgDetect.Done, - }, res) - // Remove the detector key from the list of detector keys with results. - // This is to ensure that the chunk is not reprocessed with verification enabled - // for this detector. - delete(detectorKeysWithResults, detector.Key) + // Use levenstein distance to determine if the secret is likely the same. + // Ex: + // - postman api key: PMAK-qnwfsLyRSyfCwfpHaQP1UzDhrgpWvHjbYzjpRCMshjt417zWcrzyHUArs7r + // - malicious detector "api key": qnwfsLyRSyfCwfpHaQP1UzDhrgpWvHjbYzjpRCMshjt417zWcrzyHUArs7r + key := chunkSecretKey{secret: string(val), detectorKey: detector.Key} + if _, ok := chunkSecrets[key]; ok { + continue + } + + if likelyDuplicate(ctx, key, chunkSecrets) { + // This indicates that the same secret was found by multiple detectors. + // We should NOT VERIFY this chunk's data. + if e.verificationOverlapTracker != nil { + e.verificationOverlapTracker.increment() + } + res.SetVerificationError(errOverlap) + e.processResult(ctx, detectableChunk{ + chunk: chunk.chunk, + detector: detector, + decoder: chunk.decoder, + wgDoneFn: wgDetect.Done, + }, res) + + // Remove the detector key from the list of detector keys with results. + // This is to ensure that the chunk is not reprocessed with verification enabled + // for this detector. + delete(detectorKeysWithResults, detector.Key) + } + chunkSecrets[key] = struct{}{} } - chunkSecrets[key] = struct{}{} } } - for key := range detectorKeysWithResults { - detector := e.ahoCorasickCore.GetDetectorByKey(key) - if detector == nil { - ctx.Logger().Info("detector not found", "key", key) - continue - } - + for _, detector := range detectorKeysWithResults { wgDetect.Add(1) chunk.chunk.Verify = e.verify e.detectableChunksChan <- detectableChunk{ @@ -811,38 +816,46 @@ func (e *Engine) detectChunk(ctx context.Context, data detectableChunk) { defer common.Recover(ctx) defer cancel() - results, err := data.detector.FromData(ctx, data.chunk.Verify, data.chunk.Data) - if err != nil { - ctx.Logger().Error(err, "error scanning chunk") - } - - if e.printAvgDetectorTime && len(results) > 0 { - elapsed := time.Since(start) - detectorName := results[0].DetectorType.String() - avgTimeI, ok := e.metrics.detectorAvgTime.Load(detectorName) - var avgTime []time.Duration - if ok { - avgTime, ok = avgTimeI.([]time.Duration) - if !ok { - return - } + // To reduce the overhead of regex calls in the detector, + // we limit the amount of data passed to each detector. + // The matches field of the DetectorMatch struct contains the + // relevant portions of the chunk data that were matched. + // This avoids the need for additional regex processing on the entire chunk data. + matchedBytes := data.detector.Matches() + for _, match := range matchedBytes { + results, err := data.detector.FromData(ctx, data.chunk.Verify, match) + if err != nil { + ctx.Logger().Error(err, "error scanning chunk") } - avgTime = append(avgTime, elapsed) - e.metrics.detectorAvgTime.Store(detectorName, avgTime) - } - if e.filterUnverified { - results = detectors.CleanResults(results) - } + if e.printAvgDetectorTime && len(results) > 0 { + elapsed := time.Since(start) + detectorName := results[0].DetectorType.String() + avgTimeI, ok := e.metrics.detectorAvgTime.Load(detectorName) + var avgTime []time.Duration + if ok { + avgTime, ok = avgTimeI.([]time.Duration) + if !ok { + return + } + } + avgTime = append(avgTime, elapsed) + e.metrics.detectorAvgTime.Store(detectorName, avgTime) + } - results = detectors.FilterKnownFalsePositives(ctx, data.detector, results, e.logFilteredUnverified) + if e.filterUnverified { + results = detectors.CleanResults(results) + } - if e.filterEntropy != nil { - results = detectors.FilterResultsWithEntropy(ctx, results, *e.filterEntropy, e.logFilteredUnverified) - } + results = detectors.FilterKnownFalsePositives(ctx, data.detector, results, e.logFilteredUnverified) - for _, res := range results { - e.processResult(ctx, data, res) + if e.filterEntropy != nil { + results = detectors.FilterResultsWithEntropy(ctx, results, *e.filterEntropy, e.logFilteredUnverified) + } + + for _, res := range results { + e.processResult(ctx, data, res) + } } data.wgDoneFn() } diff --git a/pkg/engine/engine_test.go b/pkg/engine/engine_test.go index 506e319a1..67dfccd81 100644 --- a/pkg/engine/engine_test.go +++ b/pkg/engine/engine_test.go @@ -3,6 +3,7 @@ package engine import ( aCtx "context" "fmt" + "math/rand" "net/http" "net/http/httptest" "os" @@ -11,6 +12,7 @@ import ( "time" "github.com/stretchr/testify/assert" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" "github.com/trufflesecurity/trufflehog/v3/pkg/config" @@ -579,11 +581,11 @@ func TestSetLink(t *testing.T) { func TestLikelyDuplicate(t *testing.T) { // Initialize detectors // (not actually calling detector FromData or anything, just using detector struct for key creation) - detectorA := ahocorasick.DetectorInfo{ + detectorA := ahocorasick.DetectorMatch{ Key: ahocorasick.CreateDetectorKey(DefaultDetectors()[0]), Detector: DefaultDetectors()[0], } - detectorB := ahocorasick.DetectorInfo{ + detectorB := ahocorasick.DetectorMatch{ Key: ahocorasick.CreateDetectorKey(DefaultDetectors()[1]), Detector: DefaultDetectors()[1], } @@ -653,3 +655,70 @@ func TestLikelyDuplicate(t *testing.T) { }) } } + +func BenchmarkPopulateMatchingDetectors(b *testing.B) { + allDetectors := DefaultDetectors() + ac := ahocorasick.NewAhoCorasickCore(allDetectors) + + // Generate sample data with keywords from detectors. + dataSize := 1 << 20 // 1 MB + sampleData := generateRandomDataWithKeywords(dataSize, allDetectors) + + smallChunk := 1 << 10 // 1 KB + mediumChunk := 1 << 12 // 4 KB + current := sources.TotalChunkSize + largeChunk := 1 << 14 // 16 KB + xlChunk := 1 << 15 // 32 KB + xxlChunk := 1 << 16 // 64 KB + xxxlChunk := 1 << 18 // 256 KB + chunkSizes := []int{smallChunk, mediumChunk, current, largeChunk, xlChunk, xxlChunk, xxxlChunk} + + for _, chunkSize := range chunkSizes { + b.Run(fmt.Sprintf("ChunkSize_%d", chunkSize), func(b *testing.B) { + b.ReportAllocs() + b.SetBytes(int64(chunkSize)) + + // Create a single chunk of the desired size. + chunk := sampleData[:chunkSize] + + b.ResetTimer() + for i := 0; i < b.N; i++ { + ac.FindDetectorMatches([]byte(chunk)) // Match against the single chunk + } + }) + } +} + +func generateRandomDataWithKeywords(size int, detectors []detectors.Detector) string { + const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + data := make([]byte, size) + + r := rand.New(rand.NewSource(42)) // Seed for reproducibility + + for i := range data { + data[i] = charset[r.Intn(len(charset))] + } + + totalKeywords := 0 + for _, d := range detectors { + totalKeywords += len(d.Keywords()) + } + + // Target keyword density (keywords per character) + // This ensures that the generated data has a reasonable number of keywords and is consistent + // across different data sizes. + keywordDensity := 0.01 + + targetKeywords := int(float64(size) * keywordDensity) + + for i := 0; i < targetKeywords; i++ { + detectorIndex := r.Intn(len(detectors)) + keywordIndex := r.Intn(len(detectors[detectorIndex].Keywords())) + keyword := detectors[detectorIndex].Keywords()[keywordIndex] + + insertPosition := r.Intn(size - len(keyword)) + copy(data[insertPosition:], keyword) + } + + return string(data) +}