Report every copy of a duplicated block in duplicate-code (#11680)

``Symilar._compute_sims`` discarded a pair whenever one of its chunks was
already in a group, so the third and later copies of a block were never
reported. Pairs that share a chunk with existing groups now merge into
them. Because a merged group can hold several chunks of one file, the
message counts distinct files.

Closes #11679

Co-authored-by: Pierre Sassoulas <pierre.sassoulas@gmail.com>

Author: soodoku

doc/whatsnew/fragments/11679.bugfix

@@ -0,0 +1,6 @@
+``duplicate-code`` now reports every copy of a duplicated block instead of only the
+first two. Each message lists all the copies, so there are fewer messages and the
+number of duplicated lines in the statistics goes up. The message counts distinct
+files, so a group can list several locations in one file.
+
+Closes #11679

pylint/checkers/symilar.py

@@ -230,6 +230,11 @@ def hash_lineset(
     return LineSetHashResult(hash_to_index, index_to_lines)
 
 
+def _count_files(couples: set[LinesChunkLimits_T]) -> int:
+    """Count the files in a group, which can hold several chunks of one file."""
+    return len({lineset for lineset, _, _ in couples})
+
+
 def remove_successive(all_couples: CplIndexToCplLines_T) -> None:
     """Removes all successive entries in the dictionary in argument.
 
@@ -398,22 +403,19 @@ def _compute_sims(self) -> list[tuple[int, set[LinesChunkLimits_T]]]:
             start_line_2 = commonality.snd_file_start
             end_line_2 = commonality.snd_file_end
 
-            duplicate = no_duplicates[num]
-            couples: set[LinesChunkLimits_T]
-            for couples in duplicate:
-                if (lineset1, start_line_1, end_line_1) in couples or (
-                    lineset2,
-                    start_line_2,
-                    end_line_2,
-                ) in couples:
-                    break
-            else:
-                duplicate.append(
-                    {
-                        (lineset1, start_line_1, end_line_1),
-                        (lineset2, start_line_2, end_line_2),
-                    }
-                )
+            chunk_1 = (lineset1, start_line_1, end_line_1)
+            chunk_2 = (lineset2, start_line_2, end_line_2)
+            # A pair that shares a chunk with existing groups joins them, so a
+            # block copied into three files is one group of three, not one pair.
+            merged = {chunk_1, chunk_2}
+            kept: list[set[LinesChunkLimits_T]] = []
+            for couples in no_duplicates[num]:
+                if chunk_1 in couples or chunk_2 in couples:
+                    merged |= couples
+                else:
+                    kept.append(couples)
+            kept.append(merged)
+            no_duplicates[num] = kept
         sims: list[tuple[int, set[LinesChunkLimits_T]]] = []
         ensembles: list[set[LinesChunkLimits_T]]
         for num, ensembles in no_duplicates.items():
@@ -436,7 +438,7 @@ def _get_similarity_report(
         report: str = ""
         duplicated_line_number: int = 0
         for number, couples in similarities:
-            report += f"\n{number} similar lines in {len(couples)} files\n"
+            report += f"\n{number} similar lines in {_count_files(couples)} files\n"
             couples_l = sorted(couples)
             line_set = start_line = end_line = None
             for line_set, start_line, end_line in couples_l:
@@ -866,7 +868,7 @@ def close(self) -> None:
                 for line in lineset.real_lines[start_line:end_line]:
                     msg.append(line.rstrip())
 
-            self.add_message("R0801", args=(len(couples), "\n".join(msg)))
+            self.add_message("R0801", args=(_count_files(couples), "\n".join(msg)))
             duplicated += num * (len(couples) - 1)
         stats.nb_duplicated_lines += int(duplicated)
         stats.percent_duplicated_lines += float(total and duplicated * 100.0 / total)

tests/checkers/unittest_symilar.py

@@ -2,6 +2,7 @@
 # For details: https://github.com/pylint-dev/pylint/blob/main/LICENSE
 # Copyright (c) https://github.com/pylint-dev/pylint/blob/main/CONTRIBUTORS.txt
 
+from collections.abc import Generator
 from contextlib import redirect_stdout
 from io import BytesIO, StringIO
 from pathlib import Path
@@ -574,3 +575,47 @@ def test_hash_bucket_product_limit_fallback(
     assert "15 similar lines in 2 files" in out
     assert f"=={file_a}:[0:15]" in out
     assert f"=={file_b}:[0:15]" in out
+
+
+def test_three_identical_files_are_reported_together(tmp_path: Path) -> None:
+    block = "def f(a):\n    x = a + 1\n    y = x * 2\n    z = y - 3\n    w = z / 4\n    return w\n"
+    paths = []
+    for name in ("a.py", "b.py", "c.py"):
+        path = tmp_path / name
+        path.write_text(block, encoding="utf-8")
+        paths.append(str(path))
+    output = StringIO()
+    with redirect_stdout(output), pytest.raises(SystemExit) as ex:
+        symilar.Run(paths)
+    assert ex.value.code == 0
+    out = output.getvalue()
+    assert "6 similar lines in 3 files" in out
+    for path in paths:
+        assert f"=={path}:[0:6]" in out
+    assert "duplicates=12" in out
+
+
+def test_compute_sims_merges_groups_joined_by_a_later_pair(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Pairs (a, b), (c, d) then (b, c) end up as one group of four chunks."""
+    linesets = [LineSet(name, ["x = 1\n"]) for name in "abcd"]
+    a, b, c, d = linesets
+
+    def fake_iter_sims(_: Symilar) -> Generator[symilar.Commonality]:
+        for first, second in ((a, b), (c, d), (b, c)):
+            yield symilar.Commonality(
+                cmn_lines_nb=5,
+                fst_lset=first,
+                fst_file_start=symilar.LineNumber(0),
+                fst_file_end=symilar.LineNumber(5),
+                snd_lset=second,
+                snd_file_start=symilar.LineNumber(0),
+                snd_file_end=symilar.LineNumber(5),
+            )
+
+    monkeypatch.setattr(Symilar, "_iter_sims", fake_iter_sims)
+    sims = Symilar()._compute_sims()
+    assert len(sims) == 1
+    assert sims[0][0] == 5
+    assert {lineset for lineset, _, _ in sims[0][1]} == set(linesets)

tests/test_similar.py

@@ -85,7 +85,7 @@ def test_min_similarity_lines_zero_disables_checker(self) -> None:
     def test_duplicate_code_raw_strings_all(self) -> None:
         """Test similar lines in 3 similar files."""
         path = join(DATA, "raw_strings_all")
-        expected_output = "Similar lines in 2 files"
+        expected_output = "Similar lines in 3 files"
         self._test_output(
             [
                 path,
@@ -101,7 +101,7 @@ def test_duplicate_code_raw_strings_all(self) -> None:
     @pytest.mark.needs_two_cores
     def test_duplicate_code_parallel(self) -> None:
         path = join(DATA, "raw_strings_all")
-        expected_output = "Similar lines in 2 files"
+        expected_output = "Similar lines in 3 files"
         self._test_output(
             [
                 path,