trufflesecurity/trufflehog
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use new 'local context' (lctx) pattern to avoid stacking duplicate logging keys in our logs and avoid logging paths beyond configured root paths

Browse Source
This commit is contained in:
Charlie Gunyon
2026-03-17 14:34:14 +01:00
parent ba5f4d3f14
commit 6c9c76695a
1 changed files with 51 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/sources/filesystem/filesystem.go
+51 -77
View File
@@ -111,48 +111,50 @@ func (s *Source) canFollowSymlinks() bool {
// Chunks emits chunks of bytes over a channel.
func (s *Source) Chunks(ctx trContext.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
for i, rootPath := range s.paths {
ctx := trContext.WithValues(ctx,
lctx := trContext.WithValues(ctx,
"root_path", rootPath,
)
if common.IsDone(ctx) {
return nil
}
s.SetProgressComplete(i, len(s.paths), fmt.Sprintf("Path: %s", rootPath), "")
cleanPath := filepath.Clean(rootPath)
cleanRootPath := filepath.Clean(rootPath)
ctx = trContext.WithValues(ctx,
"clean_path", cleanPath,
lctx = trContext.WithValues(lctx,
"clean_root_path", cleanRootPath,
)
fileInfo, err := os.Lstat(cleanPath)
fileInfo, err := os.Lstat(cleanRootPath)
if err != nil {
ctx.Logger().Error(err, "unable to get file info")
lctx.Logger().Error(err, "unable to get file info")
continue
}
if fileInfo.Mode()&os.ModeSymlink != 0 {
// if the root path is a symlink we scan the symlink
ctx.Logger().V(5).Info("Root path is a symlink")
lctx.Logger().V(5).Info("Root path is a symlink")
initialDepth := 0
err = s.scanSymlink(ctx, chunksChan, rootPath, initialDepth, cleanPath)
err = s.scanSymlink(ctx, chunksChan, rootPath, initialDepth, cleanRootPath)
s.ClearEncodedResumeInfoFor(rootPath)
} else if fileInfo.IsDir() {
ctx.Logger().V(5).Info("Root path is a dir")
lctx.Logger().V(5).Info("Root path is a dir")
initialDepth := 0
err = s.scanDir(ctx, chunksChan, rootPath, initialDepth, cleanPath)
err = s.scanDir(ctx, chunksChan, rootPath, initialDepth, cleanRootPath)
s.ClearEncodedResumeInfoFor(rootPath)
} else {
if !fileInfo.Mode().IsRegular() {
ctx.Logger().V(2).Info("Root path is a non-regular file; skipping")
lctx.Logger().V(2).Info("Root path is a non-regular file; skipping")
continue
}
ctx.Logger().V(5).Info("Root path is a file")
err = s.scanFile(ctx, chunksChan, cleanPath)
lctx.Logger().V(5).Info("Root path is a file")
err = s.scanFile(ctx, chunksChan, cleanRootPath)
}
if err != nil && !errors.Is(err, io.EOF) {
ctx.Logger().Error(err, "error scanning filesystem")
lctx.Logger().Error(err, "error scanning filesystem")
}
}
@@ -166,23 +168,22 @@ func (s *Source) scanSymlink(
depth int,
path string,
) error {
ctx = trContext.WithValues(ctx,
lctx := trContext.WithValues(ctx,
"root_path", rootPath,
"depth", depth,
"path", path,
"start_depth", depth,
)
ctx.Logger().V(5).Info("scanSymlink")
lctx.Logger().V(5).Info("scanSymlink")
if !s.canFollowSymlinks() {
// If the file or directory is a symlink but the followSymlinks is disable ignore the path
ctx.Logger().V(2).Info("Path is a symlink, but following symlinks is not allowed; skipping")
lctx.Logger().V(2).Info("Path is a symlink, but following symlinks is not allowed; skipping")
return nil
}
depth++
ctx = trContext.WithValues(ctx,
lctx = trContext.WithValues(lctx,
"depth", depth,
)
@@ -195,10 +196,6 @@ func (s *Source) scanSymlink(
cleanPath := filepath.Clean(path)
ctx = trContext.WithValues(ctx,
"clean_path", cleanPath,
)
resolvedPath, err := os.Readlink(cleanPath)
if err != nil {
return fmt.Errorf("readlink error: %w", err)
@@ -208,29 +205,25 @@ func (s *Source) scanSymlink(
resolvedPath = filepath.Join(filepath.Dir(cleanPath), resolvedPath)
}
ctx = trContext.WithValues(ctx,
"resolved_path", resolvedPath,
)
fileInfo, err := os.Lstat(resolvedPath)
if err != nil {
return fmt.Errorf("lstat error: %w", err)
}
if fileInfo.Mode()&os.ModeSymlink != 0 {
ctx.Logger().V(5).Info("Symlink links to another symlink")
lctx.Logger().V(5).Info("Symlink links to another symlink")
return s.scanSymlink(ctx, chunksChan, rootPath, depth, resolvedPath)
}
if fileInfo.IsDir() {
ctx.Logger().V(5).Info("Symlink links to a directory")
lctx.Logger().V(5).Info("Symlink links to a directory")
return s.scanDir(ctx, chunksChan, rootPath, depth, resolvedPath)
}
ctx.Logger().V(5).Info("Symlink links to a file")
lctx.Logger().V(5).Info("Symlink links to a file")
if s.filter != nil && !s.filter.Pass(resolvedPath) {
ctx.Logger().V(2).Info("Symlinked file was filtered out by filter.Pass; skipping")
lctx.Logger().V(2).Info("Symlinked file was filtered out by filter.Pass; skipping")
return nil
}
@@ -239,12 +232,12 @@ func (s *Source) scanSymlink(
resumptionKey := rootPath
if !fileInfo.Mode().Type().IsRegular() {
ctx.Logger().V(2).Info("Symlinked file is a non-regular file; skipping")
lctx.Logger().V(2).Info("Symlinked file is a non-regular file; skipping")
return nil
}
if err := s.scanFile(ctx, chunksChan, resolvedPath); err != nil {
ctx.Logger().Error(err, "error scanning symlinked file")
lctx.Logger().Error(err, "error scanning symlinked file")
}
s.SetEncodedResumeInfoFor(resumptionKey, cleanPath)
@@ -259,18 +252,17 @@ func (s *Source) scanDir(
depth int,
path string,
) error {
ctx = trContext.WithValues(ctx,
lctx := trContext.WithValues(ctx,
"root_path", rootPath,
"depth", depth,
"path", path,
)
ctx.Logger().V(5).Info("scanDir")
lctx.Logger().V(5).Info("scanDir")
// check if the full path is not matching any pattern in include
// FilterRuleSet and matching any exclude FilterRuleSet.
if s.filter != nil && s.filter.ShouldExclude(path) {
ctx.Logger().V(2).Info("Path was filtered out by filter.ShouldExclude; skipping")
lctx.Logger().V(2).Info("Path was filtered out by filter.ShouldExclude; skipping")
return nil
}
@@ -280,10 +272,6 @@ func (s *Source) scanDir(
resumptionKey := rootPath
resumeAfter := s.GetEncodedResumeInfoFor(resumptionKey)
ctx = trContext.WithValues(ctx,
"resume_after", resumeAfter,
)
// Only consider resumption if the resume point is within this directory's subtree.
// Since os.ReadDir returns entries sorted by filename:
// - If we're scanning /root/ccc and the resume point is /root/bbb/file.txt,
@@ -295,19 +283,16 @@ func (s *Source) scanDir(
// Resume point is not in this subtree. Compare paths to determine if we
// should skip this directory (already scanned) or process it (already passed).
if path < resumeAfter {
ctx.Logger().V(5).Info("Resumption: path already seen")
lctx.Logger().V(5).Info("Resumption: path already seen")
// This directory comes before the resume point lexicographically,
// meaning it was already fully scanned. Skip it entirely.
return nil
}
ctx.Logger().V(5).Info("Resumption: complete")
lctx.Logger().V(5).Info("Resumption: complete")
// This directory comes after the resume point, so we've already passed
// the resume point. Process this directory normally.
resumeAfter = ""
ctx = trContext.WithValues(ctx,
"resume_after", resumeAfter,
)
}
entries, err := os.ReadDir(path)
@@ -320,14 +305,10 @@ func (s *Source) scanDir(
for _, entry := range entries {
entryPath := filepath.Join(path, entry.Name())
ctx := trContext.WithValues(ctx,
"entry_path", entryPath,
"resume_after", resumeAfter,
)
if s.filter != nil && !s.filter.Pass(entryPath) {
if !entry.IsDir() && entry.Type()&os.ModeSymlink == 0 {
ctx.Logger().V(2).Info("File entry was filtered by filter.Pass; skipping")
lctx.Logger().V(2).Info("File entry was filtered by filter.Pass; skipping")
continue
}
}
@@ -340,7 +321,7 @@ func (s *Source) scanDir(
if resumeAfter != "" {
// If this entry is the resume point, stop skipping.
if entryPath == resumeAfter {
ctx.Logger().V(5).Info("Resumption: resume point already seen")
lctx.Logger().V(5).Info("Resumption: resume point already seen")
resumeAfter = ""
continue // Skip the resume point itself since it was already processed.
}
@@ -349,7 +330,7 @@ func (s *Source) scanDir(
if entry.IsDir() && strings.HasPrefix(resumeAfter, entryPath+string(filepath.Separator)) {
// Recurse into this directory to find the resume point.
if err := s.scanDir(ctx, chunksChan, rootPath, depth, entryPath); err != nil {
ctx.Logger().Error(err, "error scanning directory")
lctx.Logger().Error(err, "error scanning directory")
}
// After recursing, clear local resumeAfter. The child scanDir will have
// handled resumption within its subtree, and subsequent entries in this
@@ -363,24 +344,24 @@ func (s *Source) scanDir(
}
if entry.Type()&os.ModeSymlink != 0 {
ctx.Logger().V(5).Info("Entry is a symlink")
lctx.Logger().V(5).Info("Entry is a symlink")
if err := s.scanSymlink(ctx, chunksChan, rootPath, depth, entryPath); err != nil {
ctx.Logger().Error(err, "error scanning symlink")
lctx.Logger().Error(err, "error scanning symlink")
}
} else if entry.IsDir() {
ctx.Logger().V(5).Info("Entry is a directory")
lctx.Logger().V(5).Info("Entry is a directory")
if err := s.scanDir(ctx, chunksChan, rootPath, depth, entryPath); err != nil {
ctx.Logger().Error(err, "error scanning directory")
lctx.Logger().Error(err, "error scanning directory")
}
} else {
if !entry.Type().IsRegular() {
continue
}
ctx.Logger().V(5).Info("Entry is a file")
lctx.Logger().V(5).Info("Entry is a file")
workerPool.Go(func() error {
if err := s.scanFile(ctx, chunksChan, entryPath); err != nil {
ctx.Logger().Error(err, "error scanning file")
lctx.Logger().Error(err, "error scanning file")
}
s.SetEncodedResumeInfoFor(resumptionKey, entryPath)
@@ -396,10 +377,6 @@ func (s *Source) scanDir(
}
func (s *Source) scanFile(ctx trContext.Context, chunksChan chan *sources.Chunk, path string) error {
ctx = trContext.WithValues(ctx,
"path", path,
)
ctx.Logger().V(5).Info("scanFile")
// Check if file is binary and should be skipped
@@ -437,12 +414,12 @@ func (s *Source) scanFile(ctx trContext.Context, chunksChan chan *sources.Chunk,
// filepath or a directory.
func (s *Source) Enumerate(ctx trContext.Context, reporter sources.UnitReporter) error {
for _, rootPath := range s.paths {
ctx := trContext.WithValues(ctx,
lctx := trContext.WithValues(ctx,
"root_path", rootPath,
)
if _, err := os.Lstat(filepath.Clean(rootPath)); err != nil {
ctx.Logger().Error(err, "unable to stat path")
lctx.Logger().Error(err, "unable to stat path")
if unitErrErr := reporter.UnitErr(ctx, err); unitErrErr != nil {
return unitErrErr
@@ -464,14 +441,11 @@ func (s *Source) Enumerate(ctx trContext.Context, reporter sources.UnitReporter)
func (s *Source) ChunkUnit(ctx trContext.Context, unit sources.SourceUnit, reporter sources.ChunkReporter) error {
rootPath, _ := unit.SourceUnitID()
ctx = trContext.WithValues(ctx,
"root_path", rootPath,
)
cleanPath := filepath.Clean(rootPath)
ctx = trContext.WithValues(ctx,
"clean_path", cleanPath,
lctx := trContext.WithValues(ctx,
"root_path", rootPath,
"clean_root_path", cleanPath,
)
fileInfo, err := os.Lstat(cleanPath)
@@ -485,12 +459,12 @@ func (s *Source) ChunkUnit(ctx trContext.Context, unit sources.SourceUnit, repor
defer close(ch)
if fileInfo.Mode()&os.ModeSymlink != 0 {
// if the root path is a symlink we scan the symlink
ctx.Logger().V(5).Info("Root unit is a symlink")
lctx.Logger().V(5).Info("Root unit is a symlink")
initialDepth := 0
scanErr = s.scanSymlink(ctx, ch, rootPath, initialDepth, cleanPath)
s.ClearEncodedResumeInfoFor(rootPath)
} else if fileInfo.IsDir() {
ctx.Logger().V(5).Info("Root unit is a directory")
lctx.Logger().V(5).Info("Root unit is a directory")
initialDepth := 0
// TODO: Finer grain error tracking of individual chunks.
scanErr = s.scanDir(ctx, ch, rootPath, initialDepth, cleanPath)
@@ -499,11 +473,11 @@ func (s *Source) ChunkUnit(ctx trContext.Context, unit sources.SourceUnit, repor
// TODO: Finer grain error tracking of individual
// chunks (in the case of archives).
if !fileInfo.Mode().IsRegular() {
ctx.Logger().V(2).Info("Root unit is a non-regular file; skipping")
lctx.Logger().V(2).Info("Root unit is a non-regular file; skipping")
return
}
ctx.Logger().V(5).Info("Root unit is a file")
lctx.Logger().V(5).Info("Root unit is a file")
scanErr = s.scanFile(ctx, ch, cleanPath)
}
}()
@@ -519,7 +493,7 @@ func (s *Source) ChunkUnit(ctx trContext.Context, unit sources.SourceUnit, repor
}
if scanErr != nil && !errors.Is(scanErr, io.EOF) {
ctx.Logger().Error(scanErr, "error scanning filesystem")
lctx.Logger().Error(scanErr, "error scanning filesystem")
return reporter.ChunkErr(ctx, scanErr)
}