aboutsummaryrefslogtreecommitdiff
path: root/internal/dup
diff options
context:
space:
mode:
Diffstat (limited to 'internal/dup')
-rw-r--r--internal/dup/dup.go50
-rw-r--r--internal/dup/dup_test.go226
2 files changed, 266 insertions, 10 deletions
diff --git a/internal/dup/dup.go b/internal/dup/dup.go
index e3c7424..32bcfb0 100644
--- a/internal/dup/dup.go
+++ b/internal/dup/dup.go
@@ -174,18 +174,47 @@ func (x *Index) Lookup(path string) (original string, dup bool, err error) {
return path, false, nil
}
orig := x.candidates[x.original(identical)].path
- return orig, orig != path, nil
+ if orig == path {
+ return orig, false, nil
+ }
+ // Spec §5.5: two names for one file are never duplicates of each other.
+ // The other name may be a hardlink, or path itself indexed a second time
+ // under a DIR that overlaps the scanned tree. identicalTo gives every
+ // member of the content class the same set, so every lookup elects the
+ // same original, and no name for that original's file is reported as a
+ // duplicate: its content always keeps at least one name. Portable:
+ // os.SameFile, never a Stat_t.Dev/Ino read (that field's type differs
+ // across freebsd/openbsd, which `make ci` vets).
+ origInfo, err := os.Lstat(orig)
+ if err != nil {
+ return "", false, err
+ }
+ pathInfo, err := os.Lstat(path)
+ if err != nil {
+ return "", false, err
+ }
+ if os.SameFile(origInfo, pathInfo) {
+ return orig, false, nil
+ }
+ return orig, true, nil
}
// identicalTo returns the indexes in group (which all share idx's size,
// idx included) whose content matches candidates[idx]: same partial hash,
-// then, only for those that collide, the same full hash. idx is the file
-// Lookup was asked about; a failure hashing it propagates, since Lookup can
-// answer nothing without it. A failure hashing any other candidate in group
-// only removes that candidate from consideration: a vanished candidate
-// (errors.Is fs.ErrNotExist) is dropped silently, any other failure is
-// recorded on the Index (see recordCandidateError) so the caller can warn
-// about it once matching is done.
+// then, only for those that collide, the same full hash. Every candidate
+// with identical bytes is included, whatever its path or inode: a hardlink
+// of idx, and idx's own path indexed a second time under an overlapping
+// extra directory, are both members. That keeps the set the same whichever
+// member Lookup was asked about, so every member elects the same original;
+// Lookup, not this function, decides that a name for the elected original's
+// own file is not a duplicate of it.
+//
+// idx is the file Lookup was asked about; a failure hashing it propagates,
+// since Lookup can answer nothing without it. A failure hashing any other
+// candidate in group only removes that candidate from consideration: a
+// vanished candidate (errors.Is fs.ErrNotExist) is dropped silently, any
+// other failure is recorded on the Index (see recordCandidateError) so the
+// caller can warn about it once matching is done.
func (x *Index) identicalTo(idx int, group []int) ([]int, error) {
idxPartial, err := x.partialHash(x.candidates[idx].path)
if err != nil {
@@ -219,9 +248,10 @@ func (x *Index) identicalTo(idx int, group []int) ([]int, error) {
x.recordCandidateError(x.candidates[j].path, err)
continue
}
- if jFull == idxFull {
- same = append(same, j)
+ if jFull != idxFull {
+ continue
}
+ same = append(same, j)
}
return same, nil
}
diff --git a/internal/dup/dup_test.go b/internal/dup/dup_test.go
index 5d2621c..ed7c1a3 100644
--- a/internal/dup/dup_test.go
+++ b/internal/dup/dup_test.go
@@ -352,3 +352,229 @@ func TestExtraDirSymlinkNotFollowed(t *testing.T) {
t.Errorf("symlinked extra dir was indexed despite the error: dup=%v orig=%s", dup, orig)
}
}
+
+// linked builds a scan.File for a hardlink of an already-put file: same
+// inode, so same content and metadata by construction, under a new name.
+func linked(t *testing.T, dir, name string, target scan.File) scan.File {
+ t.Helper()
+ p := filepath.Join(dir, name)
+ if err := os.Link(target.Path, p); err != nil {
+ t.Fatal(err)
+ }
+ fi, err := os.Stat(p)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return scan.File{Path: p, Rel: name, Name: name, Size: fi.Size(), ModTime: fi.ModTime()}
+}
+
+// TestHardlinksAreNotDuplicatesOfEachOther is R10 (plan 5 Task 2, added to
+// the task outside the brief): spec §5.5 groups duplicate candidates by size
+// and hash, with no inode check, so two hardlinked names - one inode, byte-
+// identical by construction - were judged a duplicate pair. A rule of
+// (when (duplicate)) (delete) would then remove a name the user relies on
+// even though nothing was ever actually copied. os.SameFile must stop a
+// file being judged a duplicate of itself under another name.
+func TestHardlinksAreNotDuplicatesOfEachOther(t *testing.T) {
+ d := t.TempDir()
+ a := put(t, d, "a.pdf", []byte("same content"), 0)
+ b := linked(t, d, "b.pdf", a)
+
+ x, errs := NewIndex([]scan.File{a, b}, nil)
+ if len(errs) > 0 {
+ t.Fatal(errs)
+ }
+ if orig, dup := lookup(t, x, a); dup {
+ t.Errorf("a.pdf reported as a duplicate of its own hardlink b.pdf (orig=%s)", orig)
+ }
+ if orig, dup := lookup(t, x, b); dup {
+ t.Errorf("b.pdf reported as a duplicate of its own hardlink a.pdf (orig=%s)", orig)
+ }
+}
+
+// TestHardlinksAreStillDuplicatesOfASeparateIdenticalFile is R10's other
+// direction, and the one a naive "same size+hash means never a duplicate"
+// fix would get wrong: a.pdf and b.pdf are hardlinks of one inode, but
+// c.pdf is a genuinely separate, byte-identical copy under an extra
+// (duplicate "DIR") directory, so spec §5.5 prefers it as the original.
+// Deleting a.pdf and b.pdf then leaves the content intact in c.pdf - they
+// really are duplicates, of c.pdf, and must still be reported as such.
+func TestHardlinksAreStillDuplicatesOfASeparateIdenticalFile(t *testing.T) {
+ scanned, filed := t.TempDir(), t.TempDir()
+ a := put(t, scanned, "a.pdf", []byte("same content"), 0)
+ b := linked(t, scanned, "b.pdf", a)
+ put(t, filed, "c.pdf", []byte("same content"), 0) // extra-dir copy: preferred as the original regardless of mtime
+
+ x, errs := NewIndex([]scan.File{a, b}, []string{filed})
+ if len(errs) > 0 {
+ t.Fatal(errs)
+ }
+ cPath := filepath.Join(filed, "c.pdf")
+ if orig, dup := lookup(t, x, a); !dup || orig != cPath {
+ t.Errorf("a.pdf: dup=%v orig=%s, want a real duplicate of %s", dup, orig, cPath)
+ }
+ if orig, dup := lookup(t, x, b); !dup || orig != cPath {
+ t.Errorf("b.pdf: dup=%v orig=%s, want a real duplicate of %s", dup, orig, cPath)
+ }
+}
+
+// TestHardlinkUnderExtraDirWithNoOtherCopyIsNotADuplicate is R10 extended
+// to extra-directory candidates: a candidate is never a duplicate of a
+// candidate that is the same file, and extra-directory candidates are
+// candidates - the rule is not scanned-vs-scanned only. This is the shape
+// (duplicate "DIR") exists for: (duplicate "~/backup") means "the backup
+// already holds a copy, so the local name can go", but if ~/backup/a.pdf is
+// a hardlink of ~/dl/a.pdf, the backup holds no copy at all, just the same
+// file under a second name - judging the scanned file a duplicate would
+// delete the only copy while the user believes it is backed up.
+//
+// Lookup's os.SameFile check compares the elected original with the file
+// asked about whether the original was scanned or found under an extra
+// directory, so this case needs no special handling; the test pins it as
+// its own named case.
+func TestHardlinkUnderExtraDirWithNoOtherCopyIsNotADuplicate(t *testing.T) {
+ scanned, backup := t.TempDir(), t.TempDir()
+ a := put(t, scanned, "a.pdf", []byte("same content"), 0)
+ backupPath := filepath.Join(backup, "a.pdf")
+ if err := os.Link(a.Path, backupPath); err != nil {
+ t.Fatal(err)
+ }
+
+ x, errs := NewIndex([]scan.File{a}, []string{backup})
+ if len(errs) > 0 {
+ t.Fatal(errs)
+ }
+ if orig, dup := lookup(t, x, a); dup {
+ t.Errorf("a.pdf reported as a duplicate of its own hardlink under the extra dir (orig=%s)", orig)
+ }
+}
+
+// TestThreeWayHardlinksAreNotDuplicatesOfEachOther is the direct
+// generalisation of TestHardlinksAreNotDuplicatesOfEachOther to N names for
+// one inode: three names, one inode, no other copy anywhere - none of them
+// is a duplicate of either of the others. No special-casing for N > 2: all
+// three elect the same original, and Lookup's SameFile check finds each
+// name to be that original's own file.
+func TestThreeWayHardlinksAreNotDuplicatesOfEachOther(t *testing.T) {
+ d := t.TempDir()
+ a := put(t, d, "a.pdf", []byte("same content"), 0)
+ b := linked(t, d, "b.pdf", a)
+ c := linked(t, d, "c.pdf", a)
+
+ x, errs := NewIndex([]scan.File{a, b, c}, nil)
+ if len(errs) > 0 {
+ t.Fatal(errs)
+ }
+ for _, f := range []scan.File{a, b, c} {
+ if orig, dup := lookup(t, x, f); dup {
+ t.Errorf("%s reported as a duplicate (orig=%s)", f.Name, orig)
+ }
+ }
+}
+
+// dupVerdicts looks up every file in fs and returns the paths reported as
+// not a duplicate, and the originals the duplicates were reported against.
+func dupVerdicts(t *testing.T, x *Index, fs ...scan.File) (kept []string, origs map[string]string) {
+ t.Helper()
+ origs = make(map[string]string)
+ for _, f := range fs {
+ orig, dup := lookup(t, x, f)
+ if dup {
+ origs[f.Path] = orig
+ } else {
+ kept = append(kept, f.Path)
+ }
+ }
+ return kept, origs
+}
+
+// TestExtraDirIsTheScannedRoot: (duplicate "DIR") where DIR is the scanned
+// root itself, so every scanned file is also indexed as an extra-directory
+// candidate under the same path. Two identical files are one duplicate and
+// one original, never two duplicates of each other: a (delete) rule must
+// leave one copy.
+func TestExtraDirIsTheScannedRoot(t *testing.T) {
+ d := t.TempDir()
+ a := put(t, d, "a.pdf", []byte("same content"), 0) // older: the original
+ b := put(t, d, "b.pdf", []byte("same content"), 5)
+
+ x, errs := NewIndex([]scan.File{a, b}, []string{d})
+ if len(errs) > 0 {
+ t.Fatal(errs)
+ }
+ kept, origs := dupVerdicts(t, x, a, b)
+ if len(kept) != 1 || kept[0] != a.Path {
+ t.Errorf("kept = %v, want exactly %s (duplicates: %v)", kept, a.Path, origs)
+ }
+ if origs[b.Path] != a.Path {
+ t.Errorf("b.pdf: original = %q, want %s", origs[b.Path], a.Path)
+ }
+}
+
+// TestExtraDirInsideRecursiveRoot: DIR is a subdirectory of a recursively
+// scanned root, the "the archive already holds a copy, so the loose one can
+// go" shape. The copy under DIR is the original even though it is newer;
+// only the copy outside DIR is a duplicate.
+func TestExtraDirInsideRecursiveRoot(t *testing.T) {
+ d := t.TempDir()
+ archived := put(t, d, "Archive/x.pdf", []byte("same content"), 5)
+ loose := put(t, d, "x-copy.pdf", []byte("same content"), 0) // older, but not under DIR
+
+ x, errs := NewIndex([]scan.File{archived, loose}, []string{filepath.Join(d, "Archive")})
+ if len(errs) > 0 {
+ t.Fatal(errs)
+ }
+ kept, origs := dupVerdicts(t, x, archived, loose)
+ if len(kept) != 1 || kept[0] != archived.Path {
+ t.Errorf("kept = %v, want exactly %s (duplicates: %v)", kept, archived.Path, origs)
+ }
+ if origs[loose.Path] != archived.Path {
+ t.Errorf("x-copy.pdf: original = %q, want %s", origs[loose.Path], archived.Path)
+ }
+}
+
+// TestExtraDirIsAnAncestorOfTheRoot: DIR contains the scanned root, as
+// (duplicate "~") would for a root under the home directory.
+func TestExtraDirIsAnAncestorOfTheRoot(t *testing.T) {
+ parent := t.TempDir()
+ root := filepath.Join(parent, "dl")
+ a := put(t, root, "a.pdf", []byte("same content"), 0) // older: the original
+ b := put(t, root, "b.pdf", []byte("same content"), 5)
+
+ x, errs := NewIndex([]scan.File{a, b}, []string{parent})
+ if len(errs) > 0 {
+ t.Fatal(errs)
+ }
+ kept, origs := dupVerdicts(t, x, a, b)
+ if len(kept) != 1 || kept[0] != a.Path {
+ t.Errorf("kept = %v, want exactly %s (duplicates: %v)", kept, a.Path, origs)
+ }
+ if origs[b.Path] != a.Path {
+ t.Errorf("b.pdf: original = %q, want %s", origs[b.Path], a.Path)
+ }
+}
+
+// TestThreeCopiesWithOverlappingExtraDirKeepOne: two copies under DIR and a
+// third outside it, all scanned. Exactly one of the three is not a
+// duplicate, it lies under DIR, and every duplicate names that same file as
+// its original.
+func TestThreeCopiesWithOverlappingExtraDirKeepOne(t *testing.T) {
+ d := t.TempDir()
+ x1 := put(t, d, "Archive/x1.pdf", []byte("same content"), 3)
+ x2 := put(t, d, "Archive/x2.pdf", []byte("same content"), 4)
+ loose := put(t, d, "x-copy.pdf", []byte("same content"), 0)
+
+ x, errs := NewIndex([]scan.File{x1, x2, loose}, []string{filepath.Join(d, "Archive")})
+ if len(errs) > 0 {
+ t.Fatal(errs)
+ }
+ kept, origs := dupVerdicts(t, x, x1, x2, loose)
+ if len(kept) != 1 || kept[0] != x1.Path {
+ t.Fatalf("kept = %v, want exactly %s (duplicates: %v)", kept, x1.Path, origs)
+ }
+ for p, orig := range origs {
+ if orig != x1.Path {
+ t.Errorf("%s: original = %q, want %s", p, orig, x1.Path)
+ }
+ }
+}