// SPDX-License-Identifier: GPL-3.0-or-later package dup import ( "bytes" "errors" "fmt" "io/fs" "os" "path/filepath" "strings" "testing" "time" "krino/internal/scan" ) var base = time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC) // put writes content at dir/name with an mtime `age` hours after base. func put(t *testing.T, dir, name string, content []byte, hours int) scan.File { t.Helper() p := filepath.Join(dir, name) if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { t.Fatal(err) } if err := os.WriteFile(p, content, 0o644); err != nil { t.Fatal(err) } mt := base.Add(time.Duration(hours) * time.Hour) if err := os.Chtimes(p, mt, mt); err != nil { t.Fatal(err) } return scan.File{Path: p, Rel: name, Name: filepath.Base(name), Size: int64(len(content)), ModTime: mt} } func lookup(t *testing.T, x *Index, f scan.File) (string, bool) { t.Helper() orig, dup, err := x.Lookup(f.Path) if err != nil { t.Fatal(err) } return orig, dup } func TestDuplicatesInScan(t *testing.T) { d := t.TempDir() a := put(t, d, "report.pdf", []byte("same content"), 1) b := put(t, d, "report (1).pdf", []byte("same content"), 5) c := put(t, d, "other.pdf", []byte("diff content"), 0) // same size, different bytes e1 := put(t, d, "empty1", nil, 0) e2 := put(t, d, "empty2", nil, 1) x, errs := NewIndex([]scan.File{a, b, c, e1, e2}, nil) if len(errs) > 0 { t.Fatal(errs) } if orig, dup := lookup(t, x, b); !dup || orig != a.Path { t.Errorf("copy: dup=%v orig=%s, want dup of %s", dup, orig, a.Path) } if _, dup := lookup(t, x, a); dup { t.Error("the original reported as a duplicate") } if _, dup := lookup(t, x, c); dup { t.Error("same size, different content reported as a duplicate") } if _, dup := lookup(t, x, e2); dup { t.Error("empty files reported as duplicates") } } func TestExtraDirHoldsTheOriginal(t *testing.T) { scanned, filed := t.TempDir(), t.TempDir() dl := put(t, scanned, "invoice.pdf", []byte("invoice 42"), 0) // older than the filed copy put(t, filed, "2026/invoice-42.pdf", []byte("invoice 42"), 9) x, errs := NewIndex([]scan.File{dl}, []string{filed, filepath.Join(filed, "missing")}) if len(errs) != 1 { t.Errorf("want one error for the missing extra dir, got %v", errs) } if orig, dup := lookup(t, x, dl); !dup || orig != filepath.Join(filed, "2026/invoice-42.pdf") { t.Errorf("dup=%v orig=%s, want the filed copy as original", dup, orig) } } func TestTieBreaks(t *testing.T) { d := t.TempDir() long := put(t, d, "longer-name.txt", []byte("x"), 0) short := put(t, d, "b.txt", []byte("x"), 0) same := put(t, d, "a.txt", []byte("x"), 0) x, _ := NewIndex([]scan.File{long, short, same}, nil) for _, f := range []scan.File{long, short} { if orig, dup := lookup(t, x, f); !dup || orig != same.Path { t.Errorf("%s: dup=%v orig=%s, want %s (shortest name, then path order)", f.Name, dup, orig, same.Path) } } } func TestPartialHashCollisionResolvedByFullHash(t *testing.T) { d := t.TempDir() head, tail := bytes.Repeat([]byte("h"), 70<<10), bytes.Repeat([]byte("t"), 70<<10) one := append(append(append([]byte{}, head...), []byte("MIDDLE-ONE")...), tail...) two := append(append(append([]byte{}, head...), []byte("MIDDLE-TWO")...), tail...) a := put(t, d, "a.bin", one, 0) b := put(t, d, "b.bin", two, 1) x, _ := NewIndex([]scan.File{a, b}, nil) if _, dup := lookup(t, x, b); dup { t.Error("files differing only in the middle reported as duplicates") } } func TestLookupUnknownPath(t *testing.T) { x, _ := NewIndex(nil, nil) if _, _, err := x.Lookup("/nowhere"); err == nil { t.Fatal("no error for a path that was not scanned") } } // TestTieBreakNameBeforePath: same-length names, same mtime, in directories // that sort in the opposite order from the names — the base name decides, // not the full path (spec §5.5's flat chain, not a path-only fallback). func TestTieBreakNameBeforePath(t *testing.T) { d := t.TempDir() catInZzz := put(t, d, "zzz/cat.txt", []byte("x"), 0) dogInAaa := put(t, d, "aaa/dog.txt", []byte("x"), 0) x, _ := NewIndex([]scan.File{catInZzz, dogInAaa}, nil) for _, f := range []scan.File{catInZzz, dogInAaa} { if orig, dup := lookup(t, x, f); orig != catInZzz.Path || dup != (f.Path != catInZzz.Path) { t.Errorf("%s: orig=%s dup=%v, want %s (name sorts before path)", f.Rel, orig, dup, catInZzz.Path) } } } // TestTieBreakExtraVsExtra: two extra directories hold identical copies; // the one under the lexically later directory is older and must still win // on ModTime — extra-vs-extra ties are not resolved by path alone. func TestTieBreakExtraVsExtra(t *testing.T) { common := t.TempDir() aaa, zzz := filepath.Join(common, "aaa"), filepath.Join(common, "zzz") newer := put(t, aaa, "copy.txt", []byte("invoice 42"), 5) // lexically first, newer older := put(t, zzz, "copy.txt", []byte("invoice 42"), 0) // lexically last, older scanned := t.TempDir() dl := put(t, scanned, "download.txt", []byte("invoice 42"), 3) x, errs := NewIndex([]scan.File{dl}, []string{aaa, zzz}) if len(errs) != 0 { t.Fatal(errs) } if orig, dup := lookup(t, x, dl); !dup || orig != older.Path { t.Errorf("dup=%v orig=%s, want %s (older extra copy, despite sorting after %s)", dup, orig, older.Path, newer.Path) } } // TestUnreadableSubdirReported: an unreadable subdirectory under an extra // directory is reported as its own error, and the rest of that extra // directory is still indexed. func TestUnreadableSubdirReported(t *testing.T) { if os.Geteuid() == 0 { t.Skip("permissions are not enforced running as root") } filed := t.TempDir() blocked := filepath.Join(filed, "blocked") if err := os.MkdirAll(blocked, 0o755); err != nil { t.Fatal(err) } put(t, blocked, "secret.pdf", []byte("secret 42"), 0) put(t, filed, "visible.pdf", []byte("visible content"), 0) if err := os.Chmod(blocked, 0o000); err != nil { t.Fatal(err) } t.Cleanup(func() { if err := os.Chmod(blocked, 0o755); err != nil { t.Fatal(err) } }) scanned := t.TempDir() dl := put(t, scanned, "visible.pdf", []byte("visible content"), 1) x, errs := NewIndex([]scan.File{dl}, []string{filed}) if len(errs) != 1 || !strings.Contains(errs[0].Error(), blocked) { t.Fatalf("want one error naming %s, got %v", blocked, errs) } if orig, dup := lookup(t, x, dl); !dup || orig != filepath.Join(filed, "visible.pdf") { t.Errorf("dup=%v orig=%s, want the filed copy (visible.pdf still indexed despite the unreadable sibling)", dup, orig) } } // TestUnreadableCandidateSkipped: three files share a size; one of them // (not the subject of either Lookup call) is unreadable. A1: the other two // are still reported as a duplicate pair, and the unreadable one is // recorded exactly once as a candidate error, not returned as a Lookup // error. func TestUnreadableCandidateSkipped(t *testing.T) { if os.Geteuid() == 0 { t.Skip("permissions are not enforced running as root") } d := t.TempDir() a := put(t, d, "a.bin", []byte("same content"), 0) b := put(t, d, "b.bin", []byte("same content"), 1) c := put(t, d, "c.bin", []byte("diff content"), 2) // same size, different bytes if err := os.Chmod(c.Path, 0o000); err != nil { t.Fatal(err) } t.Cleanup(func() { os.Chmod(c.Path, 0o644) }) x, errs := NewIndex([]scan.File{a, b, c}, nil) if len(errs) != 0 { t.Fatalf("NewIndex errors: %v", errs) } if orig, dup := lookup(t, x, b); !dup || orig != a.Path { t.Errorf("a/b duplicate pair broken by unreadable sibling: dup=%v orig=%s", dup, orig) } if orig, dup := lookup(t, x, a); dup { t.Errorf("a reported as a duplicate: orig=%s", orig) } cerrs := x.Errors() if len(cerrs) != 1 { t.Fatalf("got %d candidate errors, want 1: %v", len(cerrs), cerrs) } if cerrs[0].Path != c.Path { t.Errorf("candidate error names %q, want %q", cerrs[0].Path, c.Path) } } // TestVanishedCandidateSkippedSilently: a candidate that vanishes between // being indexed and being hashed is dropped with no error recorded at all. func TestVanishedCandidateSkippedSilently(t *testing.T) { d := t.TempDir() a := put(t, d, "a.bin", []byte("same content"), 0) b := put(t, d, "b.bin", []byte("same content"), 1) c := put(t, d, "c.bin", []byte("diff content"), 2) x, errs := NewIndex([]scan.File{a, b, c}, nil) if len(errs) != 0 { t.Fatalf("NewIndex errors: %v", errs) } if err := os.Remove(c.Path); err != nil { t.Fatal(err) } if orig, dup := lookup(t, x, b); !dup || orig != a.Path { t.Errorf("a/b duplicate pair broken by vanished sibling: dup=%v orig=%s", dup, orig) } if got := x.Errors(); len(got) != 0 { t.Errorf("vanished candidate recorded as an error: %v", got) } } // TestLookupFailsWhenSubjectUnreadable: Lookup still fails outright when // the file it was asked about (not some other candidate) cannot be read. func TestLookupFailsWhenSubjectUnreadable(t *testing.T) { if os.Geteuid() == 0 { t.Skip("permissions are not enforced running as root") } d := t.TempDir() a := put(t, d, "a.bin", []byte("same content"), 0) b := put(t, d, "b.bin", []byte("same content"), 1) if err := os.Chmod(a.Path, 0o000); err != nil { t.Fatal(err) } t.Cleanup(func() { os.Chmod(a.Path, 0o644) }) x, _ := NewIndex([]scan.File{a, b}, nil) if _, _, err := x.Lookup(a.Path); err == nil { t.Fatal("no error looking up an unreadable subject") } } // TestSameContentEqualSizeDifferentContent: two files of identical size but // different bytes must not be reported as the same content — the partial // hash (not just the size check) has to separate them. func TestSameContentEqualSizeDifferentContent(t *testing.T) { d := t.TempDir() a := filepath.Join(d, "a.bin") b := filepath.Join(d, "b.bin") one := []byte("acme-invoice-01") two := []byte("acme-invoice-02") if len(one) != len(two) { t.Fatal("fixture bug: files must be the same size") } if err := os.WriteFile(a, one, 0o644); err != nil { t.Fatal(err) } if err := os.WriteFile(b, two, 0o644); err != nil { t.Fatal(err) } if ok, err := SameContent(a, b); err != nil || ok { t.Errorf("SameContent(a, b) = %v, %v; want false, nil", ok, err) } } // fakeDirEntry is an fs.DirEntry whose Info() returns a canned result, for // exercising addEntry's Info()-failure handling directly (A3) — a real // filepath.WalkDir gives no hook to inject a stat failure deterministically // and without root. type fakeDirEntry struct { name string info fs.FileInfo infoErr error } func (f fakeDirEntry) Name() string { return f.name } func (f fakeDirEntry) IsDir() bool { return false } func (f fakeDirEntry) Type() fs.FileMode { return 0 } func (f fakeDirEntry) Info() (fs.FileInfo, error) { return f.info, f.infoErr } // TestAddEntryInfoFailure: A3. A vanished entry's Info() failure // (fs.ErrNotExist) is dropped with no error recorded; any other Info() // failure is dropped too, but recorded in errs, naming the entry. func TestAddEntryInfoFailure(t *testing.T) { x := &Index{bySize: make(map[int64][]int), scanned: make(map[string]int)} var errs []error x.addEntry("/extra/vanished.txt", fakeDirEntry{ name: "vanished.txt", infoErr: fmt.Errorf("stat vanished.txt: %w", fs.ErrNotExist), }, &errs) if len(errs) != 0 { t.Fatalf("vanished entry recorded an error: %v", errs) } if len(x.candidates) != 0 { t.Fatalf("vanished entry was indexed: %v", x.candidates) } x.addEntry("/extra/denied.txt", fakeDirEntry{ name: "denied.txt", infoErr: errors.New("permission denied"), }, &errs) if len(errs) != 1 || !strings.Contains(errs[0].Error(), "/extra/denied.txt") { t.Fatalf("want one error naming /extra/denied.txt, got %v", errs) } if len(x.candidates) != 0 { t.Fatalf("denied entry was indexed: %v", x.candidates) } } // TestExtraDirSymlinkNotFollowed: A4. An extra directory that is itself a // symlink to a real directory is not silently treated as empty: // filepath.WalkDir Lstats its root, so without a check for this the walk // would report no error and index nothing, misleading the user into // thinking an archive was consulted when it never was. func TestExtraDirSymlinkNotFollowed(t *testing.T) { real := t.TempDir() put(t, real, "invoice.pdf", []byte("invoice 42"), 0) link := filepath.Join(t.TempDir(), "link") if err := os.Symlink(real, link); err != nil { t.Fatal(err) } scanned := t.TempDir() dl := put(t, scanned, "download.pdf", []byte("invoice 42"), 1) x, errs := NewIndex([]scan.File{dl}, []string{link}) if len(errs) != 1 || !strings.Contains(errs[0].Error(), link) || !strings.Contains(errs[0].Error(), "symlink") { t.Fatalf("want one error naming %s as a symlink, got %v", link, errs) } if orig, dup := lookup(t, x, dl); dup { t.Errorf("symlinked extra dir was indexed despite the error: dup=%v orig=%s", dup, orig) } } // linked builds a scan.File for a hardlink of an already-put file: same // inode, so same content and metadata by construction, under a new name. func linked(t *testing.T, dir, name string, target scan.File) scan.File { t.Helper() p := filepath.Join(dir, name) if err := os.Link(target.Path, p); err != nil { t.Fatal(err) } fi, err := os.Stat(p) if err != nil { t.Fatal(err) } return scan.File{Path: p, Rel: name, Name: name, Size: fi.Size(), ModTime: fi.ModTime()} } // TestHardlinksAreNotDuplicatesOfEachOther: spec §5.5 groups duplicate // candidates by size and hash, with no inode check, so two hardlinked names // - one inode, byte-identical by construction - were judged a duplicate // pair. A rule of (when (duplicate)) (delete) would then remove a name the // user relies on even though nothing was ever actually copied. os.SameFile // must stop a file being judged a duplicate of itself under another name. func TestHardlinksAreNotDuplicatesOfEachOther(t *testing.T) { d := t.TempDir() a := put(t, d, "a.pdf", []byte("same content"), 0) b := linked(t, d, "b.pdf", a) x, errs := NewIndex([]scan.File{a, b}, nil) if len(errs) > 0 { t.Fatal(errs) } if orig, dup := lookup(t, x, a); dup { t.Errorf("a.pdf reported as a duplicate of its own hardlink b.pdf (orig=%s)", orig) } if orig, dup := lookup(t, x, b); dup { t.Errorf("b.pdf reported as a duplicate of its own hardlink a.pdf (orig=%s)", orig) } } // TestHardlinksAreStillDuplicatesOfASeparateIdenticalFile is the direction // a naive "same size+hash means never a duplicate" fix would get wrong: // a.pdf and b.pdf are hardlinks of one inode, but c.pdf is a genuinely // separate, byte-identical copy under an extra (duplicate "DIR") directory, // so spec §5.5 prefers it as the original. Deleting a.pdf and b.pdf then // leaves the content intact in c.pdf - they really are duplicates, of // c.pdf, and must still be reported as such. func TestHardlinksAreStillDuplicatesOfASeparateIdenticalFile(t *testing.T) { scanned, filed := t.TempDir(), t.TempDir() a := put(t, scanned, "a.pdf", []byte("same content"), 0) b := linked(t, scanned, "b.pdf", a) put(t, filed, "c.pdf", []byte("same content"), 0) // extra-dir copy: preferred as the original regardless of mtime x, errs := NewIndex([]scan.File{a, b}, []string{filed}) if len(errs) > 0 { t.Fatal(errs) } cPath := filepath.Join(filed, "c.pdf") if orig, dup := lookup(t, x, a); !dup || orig != cPath { t.Errorf("a.pdf: dup=%v orig=%s, want a real duplicate of %s", dup, orig, cPath) } if orig, dup := lookup(t, x, b); !dup || orig != cPath { t.Errorf("b.pdf: dup=%v orig=%s, want a real duplicate of %s", dup, orig, cPath) } } // TestHardlinkUnderExtraDirWithNoOtherCopyIsNotADuplicate: a candidate is // never a duplicate of a candidate that is the same file, and extra- // directory candidates are candidates - the rule is not scanned-vs-scanned // only. This is the shape (duplicate "DIR") exists for: (duplicate // "~/backup") means "the backup already holds a copy, so the local name can // go", but if ~/backup/a.pdf is a hardlink of ~/dl/a.pdf, the backup holds // no copy at all, just the same file under a second name - judging the // scanned file a duplicate would delete the only copy while the user // believes it is backed up. // // Lookup's os.SameFile check compares the elected original with the file // asked about whether the original was scanned or found under an extra // directory, so this case needs no special handling; the test pins it as // its own named case. func TestHardlinkUnderExtraDirWithNoOtherCopyIsNotADuplicate(t *testing.T) { scanned, backup := t.TempDir(), t.TempDir() a := put(t, scanned, "a.pdf", []byte("same content"), 0) backupPath := filepath.Join(backup, "a.pdf") if err := os.Link(a.Path, backupPath); err != nil { t.Fatal(err) } x, errs := NewIndex([]scan.File{a}, []string{backup}) if len(errs) > 0 { t.Fatal(errs) } if orig, dup := lookup(t, x, a); dup { t.Errorf("a.pdf reported as a duplicate of its own hardlink under the extra dir (orig=%s)", orig) } } // TestThreeWayHardlinksAreNotDuplicatesOfEachOther is the direct // generalisation of TestHardlinksAreNotDuplicatesOfEachOther to N names for // one inode: three names, one inode, no other copy anywhere - none of them // is a duplicate of either of the others. No special-casing for N > 2: all // three elect the same original, and Lookup's SameFile check finds each // name to be that original's own file. func TestThreeWayHardlinksAreNotDuplicatesOfEachOther(t *testing.T) { d := t.TempDir() a := put(t, d, "a.pdf", []byte("same content"), 0) b := linked(t, d, "b.pdf", a) c := linked(t, d, "c.pdf", a) x, errs := NewIndex([]scan.File{a, b, c}, nil) if len(errs) > 0 { t.Fatal(errs) } for _, f := range []scan.File{a, b, c} { if orig, dup := lookup(t, x, f); dup { t.Errorf("%s reported as a duplicate (orig=%s)", f.Name, orig) } } } // dupVerdicts looks up every file in fs and returns the paths reported as // not a duplicate, and the originals the duplicates were reported against. func dupVerdicts(t *testing.T, x *Index, fs ...scan.File) (kept []string, origs map[string]string) { t.Helper() origs = make(map[string]string) for _, f := range fs { orig, dup := lookup(t, x, f) if dup { origs[f.Path] = orig } else { kept = append(kept, f.Path) } } return kept, origs } // TestExtraDirIsTheScannedRoot: (duplicate "DIR") where DIR is the scanned // root itself, so every scanned file is also indexed as an extra-directory // candidate under the same path. Two identical files are one duplicate and // one original, never two duplicates of each other: a (delete) rule must // leave one copy. func TestExtraDirIsTheScannedRoot(t *testing.T) { d := t.TempDir() a := put(t, d, "a.pdf", []byte("same content"), 0) // older: the original b := put(t, d, "b.pdf", []byte("same content"), 5) x, errs := NewIndex([]scan.File{a, b}, []string{d}) if len(errs) > 0 { t.Fatal(errs) } kept, origs := dupVerdicts(t, x, a, b) if len(kept) != 1 || kept[0] != a.Path { t.Errorf("kept = %v, want exactly %s (duplicates: %v)", kept, a.Path, origs) } if origs[b.Path] != a.Path { t.Errorf("b.pdf: original = %q, want %s", origs[b.Path], a.Path) } } // TestExtraDirInsideRecursiveRoot: DIR is a subdirectory of a recursively // scanned root, the "the archive already holds a copy, so the loose one can // go" shape. The copy under DIR is the original even though it is newer; // only the copy outside DIR is a duplicate. func TestExtraDirInsideRecursiveRoot(t *testing.T) { d := t.TempDir() archived := put(t, d, "Archive/x.pdf", []byte("same content"), 5) loose := put(t, d, "x-copy.pdf", []byte("same content"), 0) // older, but not under DIR x, errs := NewIndex([]scan.File{archived, loose}, []string{filepath.Join(d, "Archive")}) if len(errs) > 0 { t.Fatal(errs) } kept, origs := dupVerdicts(t, x, archived, loose) if len(kept) != 1 || kept[0] != archived.Path { t.Errorf("kept = %v, want exactly %s (duplicates: %v)", kept, archived.Path, origs) } if origs[loose.Path] != archived.Path { t.Errorf("x-copy.pdf: original = %q, want %s", origs[loose.Path], archived.Path) } } // TestExtraDirIsAnAncestorOfTheRoot: DIR contains the scanned root, as // (duplicate "~") would for a root under the home directory. func TestExtraDirIsAnAncestorOfTheRoot(t *testing.T) { parent := t.TempDir() root := filepath.Join(parent, "dl") a := put(t, root, "a.pdf", []byte("same content"), 0) // older: the original b := put(t, root, "b.pdf", []byte("same content"), 5) x, errs := NewIndex([]scan.File{a, b}, []string{parent}) if len(errs) > 0 { t.Fatal(errs) } kept, origs := dupVerdicts(t, x, a, b) if len(kept) != 1 || kept[0] != a.Path { t.Errorf("kept = %v, want exactly %s (duplicates: %v)", kept, a.Path, origs) } if origs[b.Path] != a.Path { t.Errorf("b.pdf: original = %q, want %s", origs[b.Path], a.Path) } } // TestThreeCopiesWithOverlappingExtraDirKeepOne: two copies under DIR and a // third outside it, all scanned. Exactly one of the three is not a // duplicate, it lies under DIR, and every duplicate names that same file as // its original. func TestThreeCopiesWithOverlappingExtraDirKeepOne(t *testing.T) { d := t.TempDir() x1 := put(t, d, "Archive/x1.pdf", []byte("same content"), 3) x2 := put(t, d, "Archive/x2.pdf", []byte("same content"), 4) loose := put(t, d, "x-copy.pdf", []byte("same content"), 0) x, errs := NewIndex([]scan.File{x1, x2, loose}, []string{filepath.Join(d, "Archive")}) if len(errs) > 0 { t.Fatal(errs) } kept, origs := dupVerdicts(t, x, x1, x2, loose) if len(kept) != 1 || kept[0] != x1.Path { t.Fatalf("kept = %v, want exactly %s (duplicates: %v)", kept, x1.Path, origs) } for p, orig := range origs { if orig != x1.Path { t.Errorf("%s: original = %q, want %s", p, orig, x1.Path) } } }