aboutsummaryrefslogtreecommitdiff
path: root/internal/scan
diff options
context:
space:
mode:
Diffstat (limited to 'internal/scan')
-rw-r--r--internal/scan/scan.go233
-rw-r--r--internal/scan/scan_test.go263
2 files changed, 496 insertions, 0 deletions
diff --git a/internal/scan/scan.go b/internal/scan/scan.go
new file mode 100644
index 0000000..e87b741
--- /dev/null
+++ b/internal/scan/scan.go
@@ -0,0 +1,233 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+// Package scan walks a directory tree and reports the files krino will
+// consider sorting, and why the rest were skipped.
+package scan
+
+import (
+ "errors"
+ "fmt"
+ "io/fs"
+ "os"
+ "path/filepath"
+ "sort"
+ "time"
+
+ "krino/internal/ignore"
+)
+
+// File is a regular file found by Walk.
+type File struct {
+ Path string // absolute, cleaned
+ Rel string // slash-separated, relative to the root
+ Name string // base name
+ Size int64
+ ModTime time.Time
+ Mode fs.FileMode
+}
+
+// Reason is why an entry was not returned as a File.
+type Reason int
+
+const (
+ Ignored Reason = iota // matched an ignore pattern
+ Busy // a sibling NAME<busy-suffix> exists
+ TooNew // modified less than MinAge ago
+ Symlink // a symbolic link (never followed)
+ NotRegular // fifo, socket, device
+ Unreadable // a directory that could not be read
+)
+
+// String names a Reason the way it should read in a report.
+func (r Reason) String() string {
+ switch r {
+ case Ignored:
+ return "ignored"
+ case Busy:
+ return "busy"
+ case TooNew:
+ return "too new"
+ case Symlink:
+ return "symlink"
+ case NotRegular:
+ return "not a regular file"
+ case Unreadable:
+ return "unreadable"
+ default:
+ return fmt.Sprintf("Reason(%d)", int(r))
+ }
+}
+
+// Skipped is one entry Walk did not return as a File, and why.
+type Skipped struct {
+ Rel string
+ Reason Reason
+}
+
+// Options controls how Walk traverses a directory.
+type Options struct {
+ Recursive bool
+ MaxDepth int // 0: unlimited; 1: the root's own entries only
+ Ignore *ignore.Matcher // nil: nothing ignored
+ Exclude []string // absolute directories never entered
+ Busy []string // suffixes, e.g. ".part"
+ MinAge time.Duration
+ Now time.Time
+}
+
+// Result is everything Walk found under a root.
+type Result struct {
+ Files []File // sorted by Rel
+ Skipped []Skipped // sorted by Rel
+}
+
+// Walk lists the files under root that krino will consider, and reports why
+// the rest were skipped. root must exist, be a directory, and be readable;
+// an unreadable subdirectory found during the walk is reported as
+// Unreadable and does not abort the scan.
+func Walk(root string, opt Options) (*Result, error) {
+ root, err := filepath.Abs(root)
+ if err != nil {
+ return nil, err
+ }
+ info, err := os.Stat(root)
+ if err != nil {
+ return nil, err
+ }
+ if !info.IsDir() {
+ return nil, fmt.Errorf("%s is not a directory", root)
+ }
+ entries, err := os.ReadDir(root)
+ if err != nil {
+ return nil, err
+ }
+
+ exclude := make(map[string]bool, len(opt.Exclude))
+ for _, e := range opt.Exclude {
+ exclude[filepath.Clean(e)] = true
+ }
+
+ w := &walker{root: root, opt: opt, exclude: exclude}
+ if err := w.walk(root, "", 1, entries); err != nil {
+ return nil, err
+ }
+
+ sort.Slice(w.result.Files, func(i, j int) bool { return w.result.Files[i].Rel < w.result.Files[j].Rel })
+ sort.Slice(w.result.Skipped, func(i, j int) bool { return w.result.Skipped[i].Rel < w.result.Skipped[j].Rel })
+ return &w.result, nil
+}
+
+// walker accumulates the Result across recursive calls.
+type walker struct {
+ opt Options
+ root string
+ exclude map[string]bool
+ result Result
+}
+
+// walk applies the skip checks to entries, the already-read contents of dir
+// (at relDir relative to the root, dir's own entries at depth). A
+// subdirectory it recurses into is read here, right before recursing, so a
+// ReadDir failure on it can be reported as Unreadable and skipped without
+// aborting the rest of the walk; only a failure reading dir itself (passed
+// in by the caller) would need to propagate, and only Walk's own read of
+// the root works that way.
+func (w *walker) walk(dir, relDir string, depth int, entries []os.DirEntry) error {
+ // The set of names in this directory, built once, for the busy check.
+ names := make(map[string]bool, len(entries))
+ for _, e := range entries {
+ names[e.Name()] = true
+ }
+
+ for _, e := range entries {
+ name := e.Name()
+ rel := name
+ if relDir != "" {
+ rel = relDir + "/" + name
+ }
+ path := filepath.Join(dir, name)
+
+ // A symlink is never followed, whatever it points to; DirEntry's
+ // Type is Lstat-like and does not resolve it.
+ if e.Type()&fs.ModeSymlink != 0 {
+ w.result.Skipped = append(w.result.Skipped, Skipped{Rel: rel, Reason: Symlink})
+ continue
+ }
+
+ if e.IsDir() {
+ // Directories themselves are not reported, with two
+ // exceptions: the symlink case above, and an ignored
+ // directory (C2) — reported once for the directory itself,
+ // not for each file inside it, since pruning it without
+ // descending is the whole point; without this, its contents
+ // would appear in no count and in no -v listing at all.
+ if !w.opt.Recursive {
+ continue
+ }
+ if w.opt.Ignore != nil && w.opt.Ignore.Match(rel, true) {
+ w.result.Skipped = append(w.result.Skipped, Skipped{Rel: rel, Reason: Ignored})
+ continue
+ }
+ if w.exclude[filepath.Clean(path)] {
+ continue
+ }
+ if w.opt.MaxDepth > 0 && depth+1 > w.opt.MaxDepth {
+ continue
+ }
+ subEntries, err := os.ReadDir(path)
+ if err != nil {
+ w.result.Skipped = append(w.result.Skipped, Skipped{Rel: rel, Reason: Unreadable})
+ continue
+ }
+ if err := w.walk(path, rel, depth+1, subEntries); err != nil {
+ return err
+ }
+ continue
+ }
+
+ info, err := e.Info()
+ if err != nil {
+ if errors.Is(err, fs.ErrNotExist) {
+ continue // vanished mid-walk (a download finishing, say): silently skipped
+ }
+ w.result.Skipped = append(w.result.Skipped, Skipped{Rel: rel, Reason: Unreadable})
+ continue
+ }
+ if !info.Mode().IsRegular() {
+ w.result.Skipped = append(w.result.Skipped, Skipped{Rel: rel, Reason: NotRegular})
+ continue
+ }
+ if w.opt.Ignore != nil && w.opt.Ignore.Match(rel, false) {
+ w.result.Skipped = append(w.result.Skipped, Skipped{Rel: rel, Reason: Ignored})
+ continue
+ }
+ if busy := w.isBusy(name, names); busy {
+ w.result.Skipped = append(w.result.Skipped, Skipped{Rel: rel, Reason: Busy})
+ continue
+ }
+ if w.opt.Now.Sub(info.ModTime()) < w.opt.MinAge {
+ w.result.Skipped = append(w.result.Skipped, Skipped{Rel: rel, Reason: TooNew})
+ continue
+ }
+ w.result.Files = append(w.result.Files, File{
+ Path: path,
+ Rel: rel,
+ Name: name,
+ Size: info.Size(),
+ ModTime: info.ModTime(),
+ Mode: info.Mode(),
+ })
+ }
+ return nil
+}
+
+// isBusy reports whether name+suffix, for any configured Busy suffix, is
+// among names — the sibling of an in-progress download.
+func (w *walker) isBusy(name string, names map[string]bool) bool {
+ for _, suffix := range w.opt.Busy {
+ if names[name+suffix] {
+ return true
+ }
+ }
+ return false
+}
diff --git a/internal/scan/scan_test.go b/internal/scan/scan_test.go
new file mode 100644
index 0000000..94f922b
--- /dev/null
+++ b/internal/scan/scan_test.go
@@ -0,0 +1,263 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package scan
+
+import (
+ "errors"
+ "fmt"
+ "io/fs"
+ "os"
+ "path/filepath"
+ "reflect"
+ "syscall"
+ "testing"
+ "time"
+
+ "krino/internal/ignore"
+)
+
+var now = time.Date(2026, 9, 11, 12, 0, 0, 0, time.UTC)
+
+// tree creates files (with an mtime one hour before now) and returns the root.
+func tree(t *testing.T, files ...string) string {
+ t.Helper()
+ root := t.TempDir()
+ for _, f := range files {
+ p := filepath.Join(root, f)
+ if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(p, []byte(f), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ old := now.Add(-time.Hour)
+ if err := os.Chtimes(p, old, old); err != nil {
+ t.Fatal(err)
+ }
+ }
+ return root
+}
+
+func rels(r *Result) []string {
+ var out []string
+ for _, f := range r.Files {
+ out = append(out, f.Rel)
+ }
+ return out
+}
+
+func skipped(r *Result) map[string]Reason {
+ m := map[string]Reason{}
+ for _, s := range r.Skipped {
+ m[s.Rel] = s.Reason
+ }
+ return m
+}
+
+func TestTopLevelOnly(t *testing.T) {
+ root := tree(t, "a.pdf", "b.txt", "sub/c.txt")
+ r, err := Walk(root, Options{Now: now})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if want := []string{"a.pdf", "b.txt"}; !reflect.DeepEqual(rels(r), want) {
+ t.Fatalf("files = %v, want %v", rels(r), want)
+ }
+ f := r.Files[0]
+ if f.Name != "a.pdf" || f.Size != int64(len("a.pdf")) || f.Path != filepath.Join(root, "a.pdf") || !f.ModTime.Equal(now.Add(-time.Hour)) {
+ t.Fatalf("file = %+v", f)
+ }
+}
+
+func TestRecursiveDepthExcludeIgnore(t *testing.T) {
+ root := tree(t, "a.txt", "one/b.txt", "one/two/c.txt", "Work/filed.pdf", "skip/x.txt", "keep/y.log")
+ m, _ := ignore.New([]string{"skip/", "*.log"})
+ r, err := Walk(root, Options{Recursive: true, Ignore: m, Exclude: []string{filepath.Join(root, "Work")}, Now: now})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if want := []string{"a.txt", "one/b.txt", "one/two/c.txt"}; !reflect.DeepEqual(rels(r), want) {
+ t.Fatalf("files = %v, want %v", rels(r), want)
+ }
+ if got := skipped(r); got["keep/y.log"] != Ignored {
+ t.Fatalf("skipped = %v", got)
+ }
+ r, _ = Walk(root, Options{Recursive: true, MaxDepth: 2, Now: now})
+ for _, rel := range rels(r) {
+ if rel == "one/two/c.txt" {
+ t.Fatalf("MaxDepth 2 reached depth 3: %v", rels(r))
+ }
+ }
+}
+
+func TestBusyTooNewSymlinkFifo(t *testing.T) {
+ root := tree(t, "movie.mkv", "movie.mkv.aria2", "doc.pdf", "doc.pdf.part", "plain.txt")
+ fresh := filepath.Join(root, "fresh.txt")
+ if err := os.WriteFile(fresh, nil, 0o644); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Chtimes(fresh, now.Add(-30*time.Second), now.Add(-30*time.Second)); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Symlink(filepath.Join(root, "plain.txt"), filepath.Join(root, "link.txt")); err != nil {
+ t.Fatal(err)
+ }
+ fifo := filepath.Join(root, "pipe")
+ haveFifo := syscall.Mkfifo(fifo, 0o644) == nil
+ m, _ := ignore.New([]string{"*.part", "*.aria2"})
+ r, err := Walk(root, Options{Ignore: m, Busy: []string{".part", ".aria2"}, MinAge: 2 * time.Minute, Now: now})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if want := []string{"plain.txt"}; !reflect.DeepEqual(rels(r), want) {
+ t.Fatalf("files = %v, want %v", rels(r), want)
+ }
+ want := map[string]Reason{
+ "movie.mkv": Busy, "doc.pdf": Busy, "movie.mkv.aria2": Ignored, "doc.pdf.part": Ignored,
+ "fresh.txt": TooNew, "link.txt": Symlink,
+ }
+ if haveFifo {
+ want["pipe"] = NotRegular
+ }
+ if got := skipped(r); !reflect.DeepEqual(got, want) {
+ t.Fatalf("skipped = %v, want %v", got, want)
+ }
+}
+
+func TestSymlinkedDirNotFollowed(t *testing.T) {
+ root := tree(t, "real/x.txt")
+ if err := os.Symlink(filepath.Join(root, "real"), filepath.Join(root, "alias")); err != nil {
+ t.Fatal(err)
+ }
+ r, err := Walk(root, Options{Recursive: true, Now: now})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if want := []string{"real/x.txt"}; !reflect.DeepEqual(rels(r), want) {
+ t.Fatalf("files = %v, want %v", rels(r), want)
+ }
+ if skipped(r)["alias"] != Symlink {
+ t.Fatalf("skipped = %v", skipped(r))
+ }
+}
+
+func TestReasonString(t *testing.T) {
+ want := map[Reason]string{Ignored: "ignored", Busy: "busy", TooNew: "too new", Symlink: "symlink", NotRegular: "not a regular file", Unreadable: "unreadable"}
+ for r, s := range want {
+ if r.String() != s {
+ t.Errorf("%d = %q, want %q", r, r.String(), s)
+ }
+ }
+}
+
+func TestUnreadableSubdir(t *testing.T) {
+ if os.Geteuid() == 0 {
+ t.Skip("running as root: permissions are not enforced")
+ }
+ root := tree(t, "a.txt", "locked/secret.txt", "one/b.txt")
+ locked := filepath.Join(root, "locked")
+ if err := os.Chmod(locked, 0o000); err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() {
+ if err := os.Chmod(locked, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ })
+ r, err := Walk(root, Options{Recursive: true, Now: now})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if want := []string{"a.txt", "one/b.txt"}; !reflect.DeepEqual(rels(r), want) {
+ t.Fatalf("files = %v, want %v", rels(r), want)
+ }
+ if got := skipped(r)["locked"]; got != Unreadable {
+ t.Fatalf("skipped[locked] = %v, want Unreadable", got)
+ }
+}
+
+func TestRootMustBeDirectory(t *testing.T) {
+ root := tree(t, "f")
+ if _, err := Walk(filepath.Join(root, "f"), Options{Now: now}); err == nil {
+ t.Fatal("walking a file succeeded")
+ }
+ if _, err := Walk(filepath.Join(root, "missing"), Options{Now: now}); err == nil {
+ t.Fatal("walking a missing path succeeded")
+ }
+}
+
+// fakeDirEntry is an os.DirEntry whose Info() returns a canned result,
+// simulating a race scan_test cannot otherwise provoke portably: a file
+// renamed or removed between being listed by ReadDir and having Info()
+// called on it (A2), or some other Info() failure.
+type fakeDirEntry struct {
+ name string
+ info fs.FileInfo
+ infoErr error
+}
+
+func (f fakeDirEntry) Name() string { return f.name }
+func (f fakeDirEntry) IsDir() bool { return false }
+func (f fakeDirEntry) Type() fs.FileMode { return 0 }
+func (f fakeDirEntry) Info() (fs.FileInfo, error) { return f.info, f.infoErr }
+
+// TestFileInfoFailureMidWalk: A2. A vanished file's Info() failure
+// (fs.ErrNotExist) is skipped silently, with no trace in Skipped; any
+// other Info() failure is reported as Unreadable; and the walk continues
+// to later entries in either case rather than aborting.
+func TestFileInfoFailureMidWalk(t *testing.T) {
+ root := t.TempDir()
+ okPath := filepath.Join(root, "ok.txt")
+ if err := os.WriteFile(okPath, []byte("ok"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ old := now.Add(-time.Hour)
+ if err := os.Chtimes(okPath, old, old); err != nil {
+ t.Fatal(err)
+ }
+ okInfo, err := os.Lstat(okPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ entries := []os.DirEntry{
+ fakeDirEntry{name: "vanished.txt", infoErr: fmt.Errorf("stat vanished.txt: %w", fs.ErrNotExist)},
+ fakeDirEntry{name: "denied.txt", infoErr: errors.New("stat denied.txt: permission denied")},
+ fakeDirEntry{name: "ok.txt", info: okInfo},
+ }
+
+ w := &walker{root: root, opt: Options{Now: now}}
+ if err := w.walk(root, "", 1, entries); err != nil {
+ t.Fatalf("walk aborted: %v", err)
+ }
+
+ if want := []string{"ok.txt"}; !reflect.DeepEqual(rels(&w.result), want) {
+ t.Fatalf("files = %v, want %v", rels(&w.result), want)
+ }
+ got := skipped(&w.result)
+ if _, vanished := got["vanished.txt"]; vanished {
+ t.Errorf("vanished file recorded as skipped: %v", got)
+ }
+ if got["denied.txt"] != Unreadable {
+ t.Errorf("skipped[denied.txt] = %v, want Unreadable", got["denied.txt"])
+ }
+}
+
+// TestIgnoredDirectoryReportedOnce: C2. An ignored directory with files
+// inside it is reported once, for the directory itself, not once per file
+// — pruning it without descending is the point — so its contents are not
+// simply invisible to every count and to -v.
+func TestIgnoredDirectoryReportedOnce(t *testing.T) {
+ root := tree(t, "keep.txt", "skip/a.txt", "skip/b.txt", "skip/c.txt")
+ m, _ := ignore.New([]string{"skip/"})
+ r, err := Walk(root, Options{Recursive: true, Ignore: m, Now: now})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if want := []string{"keep.txt"}; !reflect.DeepEqual(rels(r), want) {
+ t.Fatalf("files = %v, want %v", rels(r), want)
+ }
+ if want := (map[string]Reason{"skip": Ignored}); !reflect.DeepEqual(skipped(r), want) {
+ t.Fatalf("skipped = %v, want %v (the directory once, not its three files)", skipped(r), want)
+ }
+}
generated by cgit v1.3 (git 2.53.0) at 2026-09-19 23:46:52 +0000