Scan all images under a namespace for Docker remote registries (#4514)
* Scan all images under a namespace for Docker remote registries * improvements * Resolved charlie's comment * added logs * formatted error messages
This commit is contained in:
@@ -193,10 +193,12 @@ var (
|
||||
circleCiScan = cli.Command("circleci", "Scan CircleCI")
|
||||
circleCiScanToken = circleCiScan.Flag("token", "CircleCI token. Can also be provided with environment variable").Envar("CIRCLECI_TOKEN").Required().String()
|
||||
|
||||
dockerScan = cli.Command("docker", "Scan Docker Image")
|
||||
dockerScanImages = dockerScan.Flag("image", "Docker image to scan. Use the file:// prefix to point to a local tarball, the docker:// prefix to point to the docker daemon, otherwise an image registry is assumed.").Required().Strings()
|
||||
dockerScanToken = dockerScan.Flag("token", "Docker bearer token. Can also be provided with environment variable").Envar("DOCKER_TOKEN").String()
|
||||
dockerExcludePaths = dockerScan.Flag("exclude-paths", "Comma separated list of paths to exclude from scan").String()
|
||||
dockerScan = cli.Command("docker", "Scan Docker Image")
|
||||
dockerScanImages = dockerScan.Flag("image", "Docker image to scan. Use the file:// prefix to point to a local tarball, the docker:// prefix to point to the docker daemon, otherwise an image registry is assumed.").Strings()
|
||||
dockerScanToken = dockerScan.Flag("token", "Docker bearer token. Can also be provided with environment variable").Envar("DOCKER_TOKEN").String()
|
||||
dockerExcludePaths = dockerScan.Flag("exclude-paths", "Comma separated list of paths to exclude from scan").String()
|
||||
dockerScanNamespace = dockerScan.Flag("namespace", "Docker namespace (organization or user). For non-Docker Hub registries, include the registry address as well (e.g., ghcr.io/namespace or quay.io/namespace).").String()
|
||||
dockerScanRegistryToken = dockerScan.Flag("registry-token", "Optional Docker registry access token. Provide this if you want to include private images within the specified namespace.").String()
|
||||
|
||||
travisCiScan = cli.Command("travisci", "Scan TravisCI")
|
||||
travisCiScanToken = travisCiScan.Flag("token", "TravisCI token. Can also be provided with environment variable").Envar("TRAVISCI_TOKEN").Required().String()
|
||||
@@ -934,11 +936,25 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
|
||||
refs = []sources.JobProgressRef{ref}
|
||||
}
|
||||
case dockerScan.FullCommand():
|
||||
if *dockerScanImages != nil && *dockerScanNamespace != "" {
|
||||
return scanMetrics, fmt.Errorf("invalid config: you cannot specify both images and namespace at the same time")
|
||||
}
|
||||
|
||||
if *dockerScanImages == nil && *dockerScanNamespace == "" {
|
||||
return scanMetrics, fmt.Errorf("invalid config: both images and namespace cannot be empty; one is required")
|
||||
}
|
||||
|
||||
if *dockerScanRegistryToken != "" && *dockerScanNamespace == "" {
|
||||
return scanMetrics, fmt.Errorf("invalid config: registry token can only be used with registry namespace")
|
||||
}
|
||||
|
||||
cfg := sources.DockerConfig{
|
||||
BearerToken: *dockerScanToken,
|
||||
Images: *dockerScanImages,
|
||||
UseDockerKeychain: *dockerScanToken == "",
|
||||
ExcludePaths: strings.Split(*dockerExcludePaths, ","),
|
||||
Namespace: *dockerScanNamespace,
|
||||
RegistryToken: *dockerScanRegistryToken,
|
||||
}
|
||||
if ref, err := eng.ScanDocker(ctx, cfg); err != nil {
|
||||
return scanMetrics, fmt.Errorf("failed to scan Docker: %v", err)
|
||||
|
||||
@@ -15,8 +15,10 @@ import (
|
||||
// ScanDocker scans a given docker connection.
|
||||
func (e *Engine) ScanDocker(ctx context.Context, c sources.DockerConfig) (sources.JobProgressRef, error) {
|
||||
connection := &sourcespb.Docker{
|
||||
Images: c.Images,
|
||||
ExcludePaths: c.ExcludePaths,
|
||||
Images: c.Images,
|
||||
ExcludePaths: c.ExcludePaths,
|
||||
Namespace: c.Namespace,
|
||||
RegistryToken: c.RegistryToken,
|
||||
}
|
||||
|
||||
switch {
|
||||
|
||||
+693
-672
File diff suppressed because it is too large
Load Diff
@@ -1339,6 +1339,10 @@ func (m *Docker) validate(all bool) error {
|
||||
|
||||
var errors []error
|
||||
|
||||
// no validation rules for Namespace
|
||||
|
||||
// no validation rules for RegistryToken
|
||||
|
||||
switch v := m.Credential.(type) {
|
||||
case *Docker_Unauthenticated:
|
||||
if v == nil {
|
||||
|
||||
@@ -32,6 +32,7 @@ Docker is a containerization platform that packages applications and their depen
|
||||
- **Authentication Support**: Multiple authentication methods for private registries
|
||||
- **File Exclusion**: Configure patterns to skip specific files or directories
|
||||
- **Size Limits**: Automatically skips files exceeding 50MB to optimize performance
|
||||
- **Scan All Images Under a Namespace**: Enables automatic discovery and scanning of all container images under a specified namespace (organization or user) in supported registries such as Docker Hub, Quay, and GHCR. Users no longer need to manually list or specify individual image names. The system retrieves all public images within the namespace, and if a valid registry token is provided includes private images as well. This allows for large-scale, automated scanning across all repositories within an organization.
|
||||
|
||||
## Configuration
|
||||
|
||||
@@ -39,7 +40,7 @@ Docker is a containerization platform that packages applications and their depen
|
||||
|
||||
The Docker source supports several image reference formats:
|
||||
|
||||
```go
|
||||
```text
|
||||
// Remote registry (default)
|
||||
"nginx:latest"
|
||||
"myregistry.com/myapp:v1.0.0"
|
||||
@@ -51,6 +52,7 @@ The Docker source supports several image reference formats:
|
||||
// Tarball file
|
||||
"file:///path/to/image.tar"
|
||||
```
|
||||
|
||||
### Authentication Methods
|
||||
|
||||
#### 1. Unauthenticated (Public Images)
|
||||
@@ -159,6 +161,47 @@ docker login quay.io
|
||||
cat ~/.docker/config.json
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Namespace Scanning (This feature is currently in beta version and under testing)
|
||||
|
||||
To scan **all images** under a namespace (organization or user):
|
||||
|
||||
**CLI Usage:**
|
||||
```bash
|
||||
# If no registry prefix is provided, Docker Hub is used by default
|
||||
trufflehog docker --namespace myorg
|
||||
|
||||
# For other registries, include the registry prefix (e.g., quay.io, ghcr.io)
|
||||
trufflehog docker --namespace quay.io/my_namespace
|
||||
```
|
||||
|
||||
To include private images within that namespace:
|
||||
```bash
|
||||
trufflehog docker --namespace myorg --registry-token <access_token>
|
||||
```
|
||||
|
||||
**YAML Configuration:**
|
||||
```yaml
|
||||
sources:
|
||||
- type: docker
|
||||
name: org-scan
|
||||
docker:
|
||||
namespace: myorg
|
||||
registry_token: "ghp_xxxxxxxxxxxxxxxxxxxx"
|
||||
```
|
||||
|
||||
Supported registries:
|
||||
- Docker Hub (`docker.io`)
|
||||
- Quay (`quay.io`)
|
||||
- GitHub Container Registry (`ghcr.io`)
|
||||
|
||||
This mode automatically enumerates all repositories within the specified namespace before scanning.
|
||||
|
||||
Note: According to the GHCR documentation, only GitHub Classic Personal Access Tokens (PATs) are currently supported for accessing container packages - including public ones.
|
||||
Source: [GitHub Roadmap Issue #558](https://github.com/github/roadmap/issues/558)
|
||||
|
||||
---
|
||||
|
||||
### File Exclusion
|
||||
|
||||
@@ -200,6 +243,18 @@ trufflehog docker --image myregistry.com/private-image:latest --exclude-paths **
|
||||
trufflehog docker --image nginx:latest
|
||||
```
|
||||
|
||||
### Scanning All Images Under a Namespace (Beta Version)
|
||||
|
||||
```bash
|
||||
trufflehog docker --namespace trufflesecurity
|
||||
```
|
||||
|
||||
Including private images:
|
||||
|
||||
```bash
|
||||
trufflehog docker --namespace trufflesecurity --registry-token ghp_xxxxxxxxxxxxxxxxxxxx
|
||||
```
|
||||
|
||||
### Scanning Multiple Images
|
||||
|
||||
```bash
|
||||
@@ -215,10 +270,7 @@ trufflehog docker --image docker://myapp:local
|
||||
### Scanning a Tarball
|
||||
|
||||
```bash
|
||||
# First, save an image to a tarball
|
||||
docker save myapp:latest -o myapp.tar
|
||||
|
||||
# Then scan it
|
||||
trufflehog docker --image file:///path/to/myapp.tar
|
||||
```
|
||||
|
||||
@@ -231,15 +283,14 @@ trufflehog docker --image my-registry.io/private-app:v1.0.0
|
||||
|
||||
## Testing Results
|
||||
|
||||
### Integration Test Results
|
||||
|
||||
| Test Case | Status | Command/Configuration | Registry URL | Notes |
|
||||
|-----------|--------|----------------------|--------------|-------|
|
||||
| Scan remote image on DockerHub | ✅ Success | `--image <image_name>` | https://hub.docker.com/ | Public images work without authentication |
|
||||
| Scan specific tag of image on DockerHub | ✅ Success | `--image <image_name>:<tag_name>` | https://hub.docker.com/ | Tag specification working correctly |
|
||||
| Scan all images under namespace | In Progress | `--namespace <namespace>` | DockerHub, Quay, GHCR | Automatically discovers all public images |
|
||||
| Scan remote image on Quay.io | ✅ Success | `--image quay.io/prometheus/prometheus` | https://quay.io/search | Public Quay.io registry supported |
|
||||
| Scan multiple images | ✅ Success | `--image <image_name> --image <image_name>` | Multiple registries | Sequential scanning of multiple images |
|
||||
| Scan remote image on DockerHub with token | ✅ Success | Generate token using username and password | https://hub.docker.com/ | Basic auth with PAT working |
|
||||
| Scan remote image on DockerHub with token | ✅ Success | `--token <token>`(Generate token using username and password) | https://hub.docker.com/ | Authenticated scanning for private repos |
|
||||
| Scan private image on Quay | ⏸️ Halted | N/A | https://quay.io/ | RedHat requires paid account for private repos |
|
||||
| Scan private image on GHCR | ✅ Success | `--image ghcr.io/<image_name>` | https://github.com/packages | GitHub Container Registry |
|
||||
|
||||
@@ -248,23 +299,19 @@ trufflehog docker --image my-registry.io/private-app:v1.0.0
|
||||
### Common Issues
|
||||
|
||||
**Issue**: Authentication failures with private registries
|
||||
|
||||
**Solution**: Ensure credentials are correct and have pull permissions. Use `docker login` first when using Docker Keychain method.
|
||||
**Solution**: Ensure credentials are correct and have pull permissions. Use `docker login` first when using Docker Keychain.
|
||||
|
||||
---
|
||||
|
||||
**Issue**: Out of memory errors with large images
|
||||
|
||||
**Solution**: Reduce concurrency or scan smaller images. Consider increasing available memory.
|
||||
|
||||
---
|
||||
|
||||
**Issue**: Slow scanning performance
|
||||
|
||||
**Solution**: Enable concurrent processing, use local daemon instead of remote registry, or exclude unnecessary directories.
|
||||
|
||||
---
|
||||
|
||||
**Issue**: Files not being scanned
|
||||
|
||||
**Solution**: Check exclude patterns and file size limits. Verify files are under 50MB.
|
||||
|
||||
@@ -62,7 +62,7 @@ func (s *Source) JobID() sources.JobID {
|
||||
}
|
||||
|
||||
// Init initializes the source.
|
||||
func (s *Source) Init(_ context.Context, name string, jobId sources.JobID, sourceId sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
|
||||
func (s *Source) Init(ctx context.Context, name string, jobId sources.JobID, sourceId sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
|
||||
s.name = name
|
||||
s.sourceId = sourceId
|
||||
s.jobId = jobId
|
||||
@@ -119,6 +119,20 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
|
||||
workers := new(errgroup.Group)
|
||||
workers.SetLimit(s.concurrency)
|
||||
|
||||
// if namespace is set and no images are specified, fetch all images in that namespace.
|
||||
registryNamespace := s.conn.GetNamespace()
|
||||
if registryNamespace != "" && len(s.conn.Images) == 0 {
|
||||
start := time.Now()
|
||||
namespaceImages, err := GetNamespaceImages(ctx, registryNamespace, s.conn.GetRegistryToken())
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to list namespace: %s images: %w", registryNamespace, err)
|
||||
}
|
||||
|
||||
dockerListImagesAPIDuration.WithLabelValues(s.name).Observe(time.Since(start).Seconds())
|
||||
|
||||
s.conn.Images = append(s.conn.Images, namespaceImages...)
|
||||
}
|
||||
|
||||
for _, image := range s.conn.GetImages() {
|
||||
if common.IsDone(ctx) {
|
||||
return nil
|
||||
@@ -127,42 +141,42 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
|
||||
imgInfo, err := s.processImage(ctx, image)
|
||||
if err != nil {
|
||||
ctx.Logger().Error(err, "error processing image", "image", image)
|
||||
return nil
|
||||
continue
|
||||
}
|
||||
|
||||
ctx = context.WithValues(ctx, "image", imgInfo.base, "tag", imgInfo.tag)
|
||||
imageCtx := context.WithValues(ctx, "image", imgInfo.base, "tag", imgInfo.tag)
|
||||
|
||||
ctx.Logger().V(2).Info("scanning image history")
|
||||
imageCtx.Logger().V(2).Info("scanning image history")
|
||||
|
||||
layers, err := imgInfo.image.Layers()
|
||||
if err != nil {
|
||||
ctx.Logger().Error(err, "error getting image layers")
|
||||
return nil
|
||||
imageCtx.Logger().Error(err, "error getting image layers")
|
||||
continue
|
||||
}
|
||||
|
||||
// Get history entries and associate them with layers
|
||||
historyEntries, err := getHistoryEntries(ctx, imgInfo, layers)
|
||||
historyEntries, err := getHistoryEntries(imageCtx, imgInfo, layers)
|
||||
if err != nil {
|
||||
ctx.Logger().Error(err, "error getting image history entries")
|
||||
return nil
|
||||
imageCtx.Logger().Error(err, "error getting image history entries")
|
||||
continue
|
||||
}
|
||||
|
||||
// Scan each history entry for secrets in build commands
|
||||
for _, historyEntry := range historyEntries {
|
||||
if err := s.processHistoryEntry(ctx, historyEntry, chunksChan); err != nil {
|
||||
ctx.Logger().Error(err, "error processing history entry")
|
||||
return nil
|
||||
if err := s.processHistoryEntry(imageCtx, historyEntry, chunksChan); err != nil {
|
||||
imageCtx.Logger().Error(err, "error processing history entry")
|
||||
continue
|
||||
}
|
||||
dockerHistoryEntriesScanned.WithLabelValues(s.name).Inc()
|
||||
}
|
||||
|
||||
ctx.Logger().V(2).Info("scanning image layers")
|
||||
imageCtx.Logger().V(2).Info("scanning image layers")
|
||||
|
||||
// Process each layer concurrently
|
||||
for _, layer := range layers {
|
||||
workers.Go(func() error {
|
||||
if err := s.processLayer(ctx, layer, imgInfo, chunksChan); err != nil {
|
||||
ctx.Logger().Error(err, "error processing layer")
|
||||
if err := s.processLayer(imageCtx, layer, imgInfo, chunksChan); err != nil {
|
||||
imageCtx.Logger().Error(err, "error processing layer")
|
||||
return nil
|
||||
}
|
||||
dockerLayersScanned.WithLabelValues(s.name).Inc()
|
||||
@@ -172,8 +186,8 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
|
||||
}
|
||||
|
||||
if err := workers.Wait(); err != nil {
|
||||
ctx.Logger().Error(err, "error processing layers")
|
||||
return nil
|
||||
imageCtx.Logger().Error(err, "error processing layers")
|
||||
continue
|
||||
}
|
||||
|
||||
dockerImagesScanned.WithLabelValues(s.name).Inc()
|
||||
@@ -185,6 +199,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
|
||||
// processImage processes an individual image and prepares it for further processing.
|
||||
// It handles three image source types: remote registry, local daemon, and tarball file.
|
||||
func (s *Source) processImage(ctx context.Context, image string) (imageInfo, error) {
|
||||
ctx.Logger().V(5).Info("Processing individual Image")
|
||||
var (
|
||||
imgInfo imageInfo
|
||||
imageName name.Reference
|
||||
@@ -261,6 +276,7 @@ func (*Source) extractImageNameTagDigest(image string) (imageInfo, name.Referenc
|
||||
// getHistoryEntries collates an image's configuration history together with the
|
||||
// corresponding layer digests for any non-empty layers.
|
||||
func getHistoryEntries(ctx context.Context, imgInfo imageInfo, layers []v1.Layer) ([]historyEntryInfo, error) {
|
||||
ctx.Logger().V(5).Info("Getting history entries")
|
||||
config, err := imgInfo.image.ConfigFile()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -307,6 +323,7 @@ func getHistoryEntries(ctx context.Context, imgInfo imageInfo, layers []v1.Layer
|
||||
// processHistoryEntry processes a history entry from the image configuration metadata.
|
||||
// It scans the CreatedBy field which contains the command used to create that layer.
|
||||
func (s *Source) processHistoryEntry(ctx context.Context, historyInfo historyEntryInfo, chunksChan chan *sources.Chunk) error {
|
||||
ctx.Logger().V(5).Info("Processing history entries")
|
||||
// Create a descriptive identifier for this history entry
|
||||
// There's no file name here, so we use a synthetic path
|
||||
entryPath := fmt.Sprintf("image-metadata:history:%d:created-by", historyInfo.index)
|
||||
@@ -337,6 +354,8 @@ func (s *Source) processHistoryEntry(ctx context.Context, historyInfo historyEnt
|
||||
// processLayer processes an individual layer of an image.
|
||||
// It decompresses the layer and extracts all files for scanning.
|
||||
func (s *Source) processLayer(ctx context.Context, layer v1.Layer, imgInfo imageInfo, chunksChan chan *sources.Chunk) error {
|
||||
ctx.Logger().V(5).Info("Processing layer")
|
||||
|
||||
layerInfo := layerInfo{
|
||||
base: imgInfo.base,
|
||||
tag: imgInfo.tag,
|
||||
@@ -508,6 +527,28 @@ func (s *Source) remoteOpts() ([]remote.Option, error) {
|
||||
return opts, nil
|
||||
}
|
||||
|
||||
func GetNamespaceImages(ctx context.Context, namespace, registryToken string) ([]string, error) {
|
||||
ctx.Logger().V(5).Info("Getting namespace images")
|
||||
|
||||
registry := MakeRegistryFromNamespace(namespace)
|
||||
|
||||
// attach the registry authentication token, if one is available.
|
||||
if registryToken != "" {
|
||||
registry.WithRegistryToken(registryToken)
|
||||
}
|
||||
|
||||
ctx.Logger().Info(fmt.Sprintf("using registry: %s", registry.Name()))
|
||||
|
||||
namespaceImages, err := registry.ListImages(ctx, namespace)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to list namespace images: %w", err)
|
||||
}
|
||||
|
||||
ctx.Logger().Info(fmt.Sprintf("namespace: %s has %d images", namespace, len(namespaceImages)))
|
||||
|
||||
return namespaceImages, nil
|
||||
}
|
||||
|
||||
// baseAndTagFromImage extracts the base name and tag/digest from an image reference string.
|
||||
// It handles both digest-based references (image@sha256:...) and tag-based references (image:tag).
|
||||
func baseAndTagFromImage(image string) (base, tag string, hasDigest bool) {
|
||||
|
||||
@@ -31,4 +31,14 @@ var (
|
||||
Help: "Total number of Docker images scanned.",
|
||||
},
|
||||
[]string{"source_name"})
|
||||
|
||||
dockerListImagesAPIDuration = promauto.NewHistogramVec(
|
||||
prometheus.HistogramOpts{
|
||||
Namespace: common.MetricsNamespace,
|
||||
Subsystem: common.MetricsSubsystem,
|
||||
Name: "docker_list_images_api_duration_seconds",
|
||||
Help: "Duration of Docker list images API calls.",
|
||||
Buckets: prometheus.DefBuckets,
|
||||
},
|
||||
[]string{"source_name"})
|
||||
)
|
||||
|
||||
@@ -0,0 +1,269 @@
|
||||
package docker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// defaultHTTPClient defines a shared HTTP client with timeout for all registry requests.
|
||||
var defaultHTTPClient = &http.Client{Timeout: 10 * time.Second}
|
||||
|
||||
// Image represents a container image or repository entry in a registry API response.
|
||||
type Image struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// Registry is an interface for any Docker/OCI registry implementation that can list all images under a given namespace.
|
||||
type Registry interface {
|
||||
Name() string // return name of the registry
|
||||
WithRegistryToken(registryToken string) // set token for registry
|
||||
// TODO: Handle pagination and rate limits for list images API Call
|
||||
ListImages(ctx context.Context, namespace string) ([]string, error) // list all images
|
||||
}
|
||||
|
||||
// MakeRegistryFromNamespace returns a Registry implementation
|
||||
// based on the namespace prefix (e.g. "ghcr.io/", "quay.io/").
|
||||
// If no known prefix is found, DockerHub is used by default.
|
||||
func MakeRegistryFromNamespace(namespace string) Registry {
|
||||
var registry Registry
|
||||
switch {
|
||||
case strings.HasPrefix(namespace, "quay.io/"): // quay.io/abc123
|
||||
registry = &Quay{}
|
||||
case strings.HasPrefix(namespace, "ghcr.io/"): // ghcr.io/abc123
|
||||
registry = &GHCR{}
|
||||
default: // default is dockerhub
|
||||
registry = &DockerHub{}
|
||||
}
|
||||
|
||||
return registry
|
||||
}
|
||||
|
||||
// === DockerHub registry ===
|
||||
|
||||
// DockerHub implements the Registry interface for Docker Hub.
|
||||
type DockerHub struct {
|
||||
Token string
|
||||
}
|
||||
|
||||
// dockerhubResp models Docker Hub's /v2/namespaces/<ns>/repositories API response.
|
||||
type dockerhubResp struct {
|
||||
Next string `json:"next"`
|
||||
Results []Image `json:"results"`
|
||||
}
|
||||
|
||||
func (d *DockerHub) Name() string {
|
||||
return "Dockerhub"
|
||||
}
|
||||
|
||||
func (d *DockerHub) WithRegistryToken(registryToken string) {
|
||||
d.Token = registryToken
|
||||
}
|
||||
|
||||
// ListImages lists all images under a Docker Hub namespace using Docker Hub's API.
|
||||
func (d *DockerHub) ListImages(ctx context.Context, namespace string) ([]string, error) {
|
||||
url := &url.URL{
|
||||
Scheme: "https",
|
||||
Host: "hub.docker.com",
|
||||
Path: path.Join("v2", "namespaces", namespace, "repositories"),
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url.String(), http.NoBody)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if d.Token != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+d.Token)
|
||||
}
|
||||
|
||||
resp, err := defaultHTTPClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
}()
|
||||
|
||||
responseBodyByte, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
switch resp.StatusCode {
|
||||
case http.StatusOK:
|
||||
var allImages = make([]string, 0)
|
||||
|
||||
var listImagesResp dockerhubResp
|
||||
if err := json.Unmarshal(responseBodyByte, &listImagesResp); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, image := range listImagesResp.Results {
|
||||
allImages = append(allImages, fmt.Sprintf("%s/%s", namespace, image.Name)) // <namespace>/<image_name>
|
||||
}
|
||||
|
||||
return allImages, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("failed to list dockerhub images: unexpected status code: %d", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
// === Red Hat Quay Registry ===
|
||||
|
||||
// Quay implements the Registry interface for Quay.io.
|
||||
type Quay struct {
|
||||
Token string
|
||||
}
|
||||
|
||||
// quayResp models the JSON structure returned by Quay's /api/v1/repository endpoint.
|
||||
type quayResp struct {
|
||||
Repositories []Image `json:"repositories"`
|
||||
}
|
||||
|
||||
func (q *Quay) Name() string {
|
||||
return "Quay.io"
|
||||
}
|
||||
|
||||
func (q *Quay) WithRegistryToken(registryToken string) {
|
||||
q.Token = registryToken
|
||||
}
|
||||
|
||||
// ListImages lists all images under a Quay namespace.
|
||||
func (q *Quay) ListImages(ctx context.Context, namespace string) ([]string, error) {
|
||||
quayNamespace := path.Base(namespace) // quay.io/<namespace> -> namespace
|
||||
url := &url.URL{
|
||||
Scheme: "https",
|
||||
Host: "quay.io",
|
||||
Path: path.Join("api", "v1", "repository"),
|
||||
}
|
||||
|
||||
query := url.Query()
|
||||
query.Set("namespace", quayNamespace)
|
||||
query.Set("public", "true")
|
||||
query.Set("private", "true")
|
||||
url.RawQuery = query.Encode()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url.String(), http.NoBody)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if q.Token != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+q.Token)
|
||||
}
|
||||
|
||||
resp, err := defaultHTTPClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
}()
|
||||
|
||||
responseBodyByte, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
switch resp.StatusCode {
|
||||
case http.StatusOK:
|
||||
var allImages = make([]string, 0)
|
||||
|
||||
var listImagesResp quayResp
|
||||
if err := json.Unmarshal(responseBodyByte, &listImagesResp); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, image := range listImagesResp.Repositories {
|
||||
allImages = append(allImages, fmt.Sprintf("%s/%s", namespace, image.Name)) // quay.io/<namespace>/<image_name>
|
||||
}
|
||||
|
||||
return allImages, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("failed to list quay images: unexpected status code: %d", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
// === GHCR Registry ===
|
||||
|
||||
// GHCR implements the Registry interface for GHCR.io.
|
||||
type GHCR struct {
|
||||
Token string // https://github.com/github/roadmap/issues/558
|
||||
}
|
||||
|
||||
func (g *GHCR) Name() string {
|
||||
return "ghcr.io"
|
||||
}
|
||||
|
||||
func (g *GHCR) WithRegistryToken(registryToken string) {
|
||||
g.Token = registryToken
|
||||
}
|
||||
|
||||
// ListImages lists all images under a Quay namespace.
|
||||
func (g *GHCR) ListImages(ctx context.Context, namespace string) ([]string, error) {
|
||||
ghcrNamespace := path.Base(namespace) // ghcr.io/<namespace> -> namespace
|
||||
|
||||
url := &url.URL{
|
||||
Scheme: "https",
|
||||
Host: "api.github.com",
|
||||
Path: path.Join("users", ghcrNamespace, "packages"),
|
||||
}
|
||||
|
||||
query := url.Query()
|
||||
query.Set("package_type", "container")
|
||||
url.RawQuery = query.Encode()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url.String(), http.NoBody)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// https://stackoverflow.com/questions/72732582/using-github-packages-without-personal-access-token
|
||||
if g.Token != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+g.Token)
|
||||
}
|
||||
|
||||
resp, err := defaultHTTPClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
}()
|
||||
|
||||
responseBodyByte, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
switch resp.StatusCode {
|
||||
case http.StatusOK:
|
||||
var allImages = make([]string, 0)
|
||||
|
||||
var listImagesResp []Image
|
||||
if err := json.Unmarshal(responseBodyByte, &listImagesResp); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, image := range listImagesResp {
|
||||
allImages = append(allImages, fmt.Sprintf("%s/%s", namespace, image.Name)) // ghcr.io/<namespace>/<image_name>
|
||||
}
|
||||
|
||||
return allImages, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("failed to list ghcr images: unexpected status code: %d", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
package docker
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
)
|
||||
|
||||
func TestDockerhubListImages(t *testing.T) {
|
||||
// Dockerhub registry
|
||||
dockerhub := MakeRegistryFromNamespace("trufflesecurity") // no authentication
|
||||
|
||||
dockerImages, err := dockerhub.ListImages(context.Background(), "trufflesecurity") // namespace without any prefix defaults to dockerhub registry
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, len(dockerImages), 7)
|
||||
|
||||
dockerExpectedImages := []string{
|
||||
"trufflesecurity/trufflehog", "trufflesecurity/lint-robot", "trufflesecurity/protos",
|
||||
"trufflesecurity/driftwood", "trufflesecurity/secrets", "trufflesecurity/of-cors", "trufflesecurity/email-graffiti",
|
||||
}
|
||||
slices.Sort(dockerImages)
|
||||
slices.Sort(dockerExpectedImages)
|
||||
assert.Equal(t, dockerImages, dockerExpectedImages)
|
||||
}
|
||||
|
||||
func TestQuayListImages(t *testing.T) {
|
||||
// Quay.io registry
|
||||
quay := MakeRegistryFromNamespace("quay.io/truffledockerman") // no authentication
|
||||
|
||||
quayImages, err := quay.ListImages(context.Background(), "quay.io/truffledockerman")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, len(quayImages), 2)
|
||||
|
||||
expectedQuayImages := []string{"quay.io/truffledockerman/test", "quay.io/truffledockerman/test2"}
|
||||
slices.Sort(quayImages)
|
||||
slices.Sort(expectedQuayImages)
|
||||
assert.Equal(t, quayImages, expectedQuayImages)
|
||||
}
|
||||
|
||||
func TestGHCRListImages(t *testing.T) {
|
||||
secret, err := common.GetTestSecret(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(fmt.Errorf("failed to access secret: %v", err))
|
||||
}
|
||||
|
||||
// For the personal access token test
|
||||
githubToken := secret.MustGetField("GITHUB_PACKAGES_TEST")
|
||||
|
||||
ghcr := MakeRegistryFromNamespace("ghcr.io/mongodb")
|
||||
ghcr.WithRegistryToken(githubToken) // authentication is required for GHCR
|
||||
|
||||
ghcrImages, err := ghcr.ListImages(context.Background(), "ghcr.io/mongodb")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, len(ghcrImages), 1)
|
||||
|
||||
assert.Equal(t, ghcrImages, []string{"ghcr.io/mongodb/kingfisher"})
|
||||
}
|
||||
@@ -230,6 +230,10 @@ type DockerConfig struct {
|
||||
UseDockerKeychain bool
|
||||
// ExcludePaths is a list of paths to exclude from scanning.
|
||||
ExcludePaths []string
|
||||
// Namespace represents a Docker organization or user account.
|
||||
Namespace string
|
||||
// RegistryToken is an optional authentication token used to access private images within the namespace.
|
||||
RegistryToken string
|
||||
}
|
||||
|
||||
// GCSConfig defines the optional configuration for a GCS source.
|
||||
|
||||
@@ -165,6 +165,8 @@ message Docker {
|
||||
}
|
||||
repeated string images = 5;
|
||||
repeated string exclude_paths = 6;
|
||||
string namespace = 7;
|
||||
string registry_token = 8;
|
||||
}
|
||||
|
||||
message ECR {
|
||||
|
||||
Reference in New Issue
Block a user