From 6c9c76695a11ecf179341b5bdd0d7356d15da3dc Mon Sep 17 00:00:00 2001 From: Charlie Gunyon Date: Tue, 17 Mar 2026 14:34:14 +0100 Subject: [PATCH] Use new 'local context' (lctx) pattern to avoid stacking duplicate logging keys in our logs and avoid logging paths beyond configured root paths --- pkg/sources/filesystem/filesystem.go | 128 +++++++++++---------------- 1 file changed, 51 insertions(+), 77 deletions(-) diff --git a/pkg/sources/filesystem/filesystem.go b/pkg/sources/filesystem/filesystem.go index 7ba3f0919..b56d3bc67 100644 --- a/pkg/sources/filesystem/filesystem.go +++ b/pkg/sources/filesystem/filesystem.go @@ -111,48 +111,50 @@ func (s *Source) canFollowSymlinks() bool { // Chunks emits chunks of bytes over a channel. func (s *Source) Chunks(ctx trContext.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error { for i, rootPath := range s.paths { - ctx := trContext.WithValues(ctx, + lctx := trContext.WithValues(ctx, "root_path", rootPath, ) + if common.IsDone(ctx) { return nil } + s.SetProgressComplete(i, len(s.paths), fmt.Sprintf("Path: %s", rootPath), "") - cleanPath := filepath.Clean(rootPath) + cleanRootPath := filepath.Clean(rootPath) - ctx = trContext.WithValues(ctx, - "clean_path", cleanPath, + lctx = trContext.WithValues(lctx, + "clean_root_path", cleanRootPath, ) - fileInfo, err := os.Lstat(cleanPath) + fileInfo, err := os.Lstat(cleanRootPath) if err != nil { - ctx.Logger().Error(err, "unable to get file info") + lctx.Logger().Error(err, "unable to get file info") continue } if fileInfo.Mode()&os.ModeSymlink != 0 { // if the root path is a symlink we scan the symlink - ctx.Logger().V(5).Info("Root path is a symlink") + lctx.Logger().V(5).Info("Root path is a symlink") initialDepth := 0 - err = s.scanSymlink(ctx, chunksChan, rootPath, initialDepth, cleanPath) + err = s.scanSymlink(ctx, chunksChan, rootPath, initialDepth, cleanRootPath) s.ClearEncodedResumeInfoFor(rootPath) } else if fileInfo.IsDir() { - ctx.Logger().V(5).Info("Root path is a dir") + lctx.Logger().V(5).Info("Root path is a dir") initialDepth := 0 - err = s.scanDir(ctx, chunksChan, rootPath, initialDepth, cleanPath) + err = s.scanDir(ctx, chunksChan, rootPath, initialDepth, cleanRootPath) s.ClearEncodedResumeInfoFor(rootPath) } else { if !fileInfo.Mode().IsRegular() { - ctx.Logger().V(2).Info("Root path is a non-regular file; skipping") + lctx.Logger().V(2).Info("Root path is a non-regular file; skipping") continue } - ctx.Logger().V(5).Info("Root path is a file") - err = s.scanFile(ctx, chunksChan, cleanPath) + lctx.Logger().V(5).Info("Root path is a file") + err = s.scanFile(ctx, chunksChan, cleanRootPath) } if err != nil && !errors.Is(err, io.EOF) { - ctx.Logger().Error(err, "error scanning filesystem") + lctx.Logger().Error(err, "error scanning filesystem") } } @@ -166,23 +168,22 @@ func (s *Source) scanSymlink( depth int, path string, ) error { - ctx = trContext.WithValues(ctx, + lctx := trContext.WithValues(ctx, "root_path", rootPath, - "depth", depth, - "path", path, + "start_depth", depth, ) - ctx.Logger().V(5).Info("scanSymlink") + lctx.Logger().V(5).Info("scanSymlink") if !s.canFollowSymlinks() { // If the file or directory is a symlink but the followSymlinks is disable ignore the path - ctx.Logger().V(2).Info("Path is a symlink, but following symlinks is not allowed; skipping") + lctx.Logger().V(2).Info("Path is a symlink, but following symlinks is not allowed; skipping") return nil } depth++ - ctx = trContext.WithValues(ctx, + lctx = trContext.WithValues(lctx, "depth", depth, ) @@ -195,10 +196,6 @@ func (s *Source) scanSymlink( cleanPath := filepath.Clean(path) - ctx = trContext.WithValues(ctx, - "clean_path", cleanPath, - ) - resolvedPath, err := os.Readlink(cleanPath) if err != nil { return fmt.Errorf("readlink error: %w", err) @@ -208,29 +205,25 @@ func (s *Source) scanSymlink( resolvedPath = filepath.Join(filepath.Dir(cleanPath), resolvedPath) } - ctx = trContext.WithValues(ctx, - "resolved_path", resolvedPath, - ) - fileInfo, err := os.Lstat(resolvedPath) if err != nil { return fmt.Errorf("lstat error: %w", err) } if fileInfo.Mode()&os.ModeSymlink != 0 { - ctx.Logger().V(5).Info("Symlink links to another symlink") + lctx.Logger().V(5).Info("Symlink links to another symlink") return s.scanSymlink(ctx, chunksChan, rootPath, depth, resolvedPath) } if fileInfo.IsDir() { - ctx.Logger().V(5).Info("Symlink links to a directory") + lctx.Logger().V(5).Info("Symlink links to a directory") return s.scanDir(ctx, chunksChan, rootPath, depth, resolvedPath) } - ctx.Logger().V(5).Info("Symlink links to a file") + lctx.Logger().V(5).Info("Symlink links to a file") if s.filter != nil && !s.filter.Pass(resolvedPath) { - ctx.Logger().V(2).Info("Symlinked file was filtered out by filter.Pass; skipping") + lctx.Logger().V(2).Info("Symlinked file was filtered out by filter.Pass; skipping") return nil } @@ -239,12 +232,12 @@ func (s *Source) scanSymlink( resumptionKey := rootPath if !fileInfo.Mode().Type().IsRegular() { - ctx.Logger().V(2).Info("Symlinked file is a non-regular file; skipping") + lctx.Logger().V(2).Info("Symlinked file is a non-regular file; skipping") return nil } if err := s.scanFile(ctx, chunksChan, resolvedPath); err != nil { - ctx.Logger().Error(err, "error scanning symlinked file") + lctx.Logger().Error(err, "error scanning symlinked file") } s.SetEncodedResumeInfoFor(resumptionKey, cleanPath) @@ -259,18 +252,17 @@ func (s *Source) scanDir( depth int, path string, ) error { - ctx = trContext.WithValues(ctx, + lctx := trContext.WithValues(ctx, "root_path", rootPath, "depth", depth, - "path", path, ) - ctx.Logger().V(5).Info("scanDir") + lctx.Logger().V(5).Info("scanDir") // check if the full path is not matching any pattern in include // FilterRuleSet and matching any exclude FilterRuleSet. if s.filter != nil && s.filter.ShouldExclude(path) { - ctx.Logger().V(2).Info("Path was filtered out by filter.ShouldExclude; skipping") + lctx.Logger().V(2).Info("Path was filtered out by filter.ShouldExclude; skipping") return nil } @@ -280,10 +272,6 @@ func (s *Source) scanDir( resumptionKey := rootPath resumeAfter := s.GetEncodedResumeInfoFor(resumptionKey) - ctx = trContext.WithValues(ctx, - "resume_after", resumeAfter, - ) - // Only consider resumption if the resume point is within this directory's subtree. // Since os.ReadDir returns entries sorted by filename: // - If we're scanning /root/ccc and the resume point is /root/bbb/file.txt, @@ -295,19 +283,16 @@ func (s *Source) scanDir( // Resume point is not in this subtree. Compare paths to determine if we // should skip this directory (already scanned) or process it (already passed). if path < resumeAfter { - ctx.Logger().V(5).Info("Resumption: path already seen") + lctx.Logger().V(5).Info("Resumption: path already seen") // This directory comes before the resume point lexicographically, // meaning it was already fully scanned. Skip it entirely. return nil } - ctx.Logger().V(5).Info("Resumption: complete") + lctx.Logger().V(5).Info("Resumption: complete") // This directory comes after the resume point, so we've already passed // the resume point. Process this directory normally. resumeAfter = "" - ctx = trContext.WithValues(ctx, - "resume_after", resumeAfter, - ) } entries, err := os.ReadDir(path) @@ -320,14 +305,10 @@ func (s *Source) scanDir( for _, entry := range entries { entryPath := filepath.Join(path, entry.Name()) - ctx := trContext.WithValues(ctx, - "entry_path", entryPath, - "resume_after", resumeAfter, - ) if s.filter != nil && !s.filter.Pass(entryPath) { if !entry.IsDir() && entry.Type()&os.ModeSymlink == 0 { - ctx.Logger().V(2).Info("File entry was filtered by filter.Pass; skipping") + lctx.Logger().V(2).Info("File entry was filtered by filter.Pass; skipping") continue } } @@ -340,7 +321,7 @@ func (s *Source) scanDir( if resumeAfter != "" { // If this entry is the resume point, stop skipping. if entryPath == resumeAfter { - ctx.Logger().V(5).Info("Resumption: resume point already seen") + lctx.Logger().V(5).Info("Resumption: resume point already seen") resumeAfter = "" continue // Skip the resume point itself since it was already processed. } @@ -349,7 +330,7 @@ func (s *Source) scanDir( if entry.IsDir() && strings.HasPrefix(resumeAfter, entryPath+string(filepath.Separator)) { // Recurse into this directory to find the resume point. if err := s.scanDir(ctx, chunksChan, rootPath, depth, entryPath); err != nil { - ctx.Logger().Error(err, "error scanning directory") + lctx.Logger().Error(err, "error scanning directory") } // After recursing, clear local resumeAfter. The child scanDir will have // handled resumption within its subtree, and subsequent entries in this @@ -363,24 +344,24 @@ func (s *Source) scanDir( } if entry.Type()&os.ModeSymlink != 0 { - ctx.Logger().V(5).Info("Entry is a symlink") + lctx.Logger().V(5).Info("Entry is a symlink") if err := s.scanSymlink(ctx, chunksChan, rootPath, depth, entryPath); err != nil { - ctx.Logger().Error(err, "error scanning symlink") + lctx.Logger().Error(err, "error scanning symlink") } } else if entry.IsDir() { - ctx.Logger().V(5).Info("Entry is a directory") + lctx.Logger().V(5).Info("Entry is a directory") if err := s.scanDir(ctx, chunksChan, rootPath, depth, entryPath); err != nil { - ctx.Logger().Error(err, "error scanning directory") + lctx.Logger().Error(err, "error scanning directory") } } else { if !entry.Type().IsRegular() { continue } - ctx.Logger().V(5).Info("Entry is a file") + lctx.Logger().V(5).Info("Entry is a file") workerPool.Go(func() error { if err := s.scanFile(ctx, chunksChan, entryPath); err != nil { - ctx.Logger().Error(err, "error scanning file") + lctx.Logger().Error(err, "error scanning file") } s.SetEncodedResumeInfoFor(resumptionKey, entryPath) @@ -396,10 +377,6 @@ func (s *Source) scanDir( } func (s *Source) scanFile(ctx trContext.Context, chunksChan chan *sources.Chunk, path string) error { - ctx = trContext.WithValues(ctx, - "path", path, - ) - ctx.Logger().V(5).Info("scanFile") // Check if file is binary and should be skipped @@ -437,12 +414,12 @@ func (s *Source) scanFile(ctx trContext.Context, chunksChan chan *sources.Chunk, // filepath or a directory. func (s *Source) Enumerate(ctx trContext.Context, reporter sources.UnitReporter) error { for _, rootPath := range s.paths { - ctx := trContext.WithValues(ctx, + lctx := trContext.WithValues(ctx, "root_path", rootPath, ) if _, err := os.Lstat(filepath.Clean(rootPath)); err != nil { - ctx.Logger().Error(err, "unable to stat path") + lctx.Logger().Error(err, "unable to stat path") if unitErrErr := reporter.UnitErr(ctx, err); unitErrErr != nil { return unitErrErr @@ -464,14 +441,11 @@ func (s *Source) Enumerate(ctx trContext.Context, reporter sources.UnitReporter) func (s *Source) ChunkUnit(ctx trContext.Context, unit sources.SourceUnit, reporter sources.ChunkReporter) error { rootPath, _ := unit.SourceUnitID() - ctx = trContext.WithValues(ctx, - "root_path", rootPath, - ) - cleanPath := filepath.Clean(rootPath) - ctx = trContext.WithValues(ctx, - "clean_path", cleanPath, + lctx := trContext.WithValues(ctx, + "root_path", rootPath, + "clean_root_path", cleanPath, ) fileInfo, err := os.Lstat(cleanPath) @@ -485,12 +459,12 @@ func (s *Source) ChunkUnit(ctx trContext.Context, unit sources.SourceUnit, repor defer close(ch) if fileInfo.Mode()&os.ModeSymlink != 0 { // if the root path is a symlink we scan the symlink - ctx.Logger().V(5).Info("Root unit is a symlink") + lctx.Logger().V(5).Info("Root unit is a symlink") initialDepth := 0 scanErr = s.scanSymlink(ctx, ch, rootPath, initialDepth, cleanPath) s.ClearEncodedResumeInfoFor(rootPath) } else if fileInfo.IsDir() { - ctx.Logger().V(5).Info("Root unit is a directory") + lctx.Logger().V(5).Info("Root unit is a directory") initialDepth := 0 // TODO: Finer grain error tracking of individual chunks. scanErr = s.scanDir(ctx, ch, rootPath, initialDepth, cleanPath) @@ -499,11 +473,11 @@ func (s *Source) ChunkUnit(ctx trContext.Context, unit sources.SourceUnit, repor // TODO: Finer grain error tracking of individual // chunks (in the case of archives). if !fileInfo.Mode().IsRegular() { - ctx.Logger().V(2).Info("Root unit is a non-regular file; skipping") + lctx.Logger().V(2).Info("Root unit is a non-regular file; skipping") return } - ctx.Logger().V(5).Info("Root unit is a file") + lctx.Logger().V(5).Info("Root unit is a file") scanErr = s.scanFile(ctx, ch, cleanPath) } }() @@ -519,7 +493,7 @@ func (s *Source) ChunkUnit(ctx trContext.Context, unit sources.SourceUnit, repor } if scanErr != nil && !errors.Is(scanErr, io.EOF) { - ctx.Logger().Error(scanErr, "error scanning filesystem") + lctx.Logger().Error(scanErr, "error scanning filesystem") return reporter.ChunkErr(ctx, scanErr) }