"""Localization metrics for agent diagnoses, as used for the paper's results. Given ground-truth ``buggy_blocks`` (from ``blocks.json``) and an agent's predicted ``buggy_blocks``, compute precision and coverage at four granularities plus two coverage variants. Line ranges are scored as half-open ``[start_line, end_line)``. - file: distinct files. Paths are normalized, and a path matches another that contains it (e.g. an absolute path matches its repository-relative form). - function: distinct functions. Names are canonicalized to lowercase ``class.method`` (argument lists, Go receivers, ``::``/``#``/``$`` delimiters, and Java ```` are handled), and a bare method matches its qualified form. - line: sets of line numbers per matched file; precision and coverage are the overlap divided by predicted and ground-truth lines. - block: a predicted block hits a ground-truth block in the same file whose range overlaps it within a 5-line tolerance. Function names are not used. ``block_coverage`` is the paper's main metric. - strict_block_coverage: fraction of ground-truth blocks overlapped by a predicted block in the same file, without tolerance. - mention_coverage: fraction of ground-truth blocks whose function appears anywhere in ``diagnosis_text``. The paper passes the raw ``diagnosis.json`` text. Generic method names (``run``, ``start``, ...) must appear as ``class.method``. Precision is 0 when nothing is predicted; coverage is 0 when there is no ground truth. """ from __future__ import annotations import re from typing import Any from distdebug_bench.models import BuggyBlock BLOCK_TOLERANCE = 5 _GRANULARITIES = ("file", "function", "line", "block") _FUNC_DELIM_RE = re.compile(r"[.#:$]+") # Go receivers: func (r *Type) Method(...) and func (r Type) Method(...) _GO_PTR_RECEIVER_RE = re.compile(r"^func\s*\(\s*\w+\s+\*(\w+)\s*(?:,[^)]*)?\)\s+(\w+)", re.I) _GO_VAL_RECEIVER_RE = re.compile(r"^func\s*\(\s*\w+\s+(\w+)\s*(?:,[^)]*)?\)\s+(\w+)", re.I) # Method names too generic to count as a mention on their own. _GENERIC_METHOD_NAMES = frozenset( """run start stop init initialize close open get set put add remove delete execute process handle create build load save update check reset clear flush connect disconnect read write send receive parse format convert merge split join copy move apply notify register unregister submit schedule cancel abort commit rollback acquire release lock unlock wait signal invoke call dispatch publish subscribe consume produce encode decode validate verify test main""".split() ) # --------------------------------------------------------------------------- # # Matching # --------------------------------------------------------------------------- # def _normalize_path(path: str) -> str: if not path: return "" normalized = path.strip().replace("\\", "/").lstrip("./") while "//" in normalized: normalized = normalized.replace("//", "/") return normalized def _files_equivalent(a: str, b: str) -> bool: a, b = _normalize_path(a), _normalize_path(b) return bool(a and b) and (a == b or a in b or b in a) def _find_equivalent_file(path: str, candidates: set[str]) -> str | None: """Exact match first; otherwise the longest candidate that contains or is contained.""" norm = _normalize_path(path) containing = [] for candidate in sorted(candidates): if _normalize_path(candidate) == norm: return candidate if _files_equivalent(path, candidate): containing.append(candidate) if containing: return max(containing, key=lambda c: len(_normalize_path(c))) return None def canonicalize_function_name(name: str) -> str: """Lowercase ``class.method`` (or bare ``method``) form of a function label.""" if not name: return "" name = name.strip() for pattern in (_GO_PTR_RECEIVER_RE, _GO_VAL_RECEIVER_RE): if m := pattern.match(name): return f"{m.group(1).lower()}.{m.group(2).lower()}" if name.startswith("func "): if m := re.match(r"^func\s+(\w+)", name): return m.group(1).lower() if m := re.search(r"\)\s+(\w+)", name): return m.group(1).lower() paren = name.find("(") if paren != -1 and name[:paren].strip(): name = name[:paren].strip() parts = [p for p in _FUNC_DELIM_RE.sub(".", name).split(".") if p] if not parts: return "" if parts[-1].lower() == "" and len(parts) >= 2: parts = parts[:-1] + [parts[-2]] if len(parts) >= 2: return f"{parts[-2].lower()}.{parts[-1].lower()}" return parts[-1].lower() def _functions_equivalent(a: str, b: str) -> bool: a, b = canonicalize_function_name(a), canonicalize_function_name(b) if not (a and b): return False return a == b or a.endswith(f".{b}") or b.endswith(f".{a}") def _count_matches(truths: set[str], predictions: set[str], equivalent) -> int: """Greedy one-to-one matching; each ground-truth entry is used at most once.""" matched: set[str] = set() for prediction in sorted(predictions): for truth in sorted(truths): if truth not in matched and equivalent(prediction, truth): matched.add(truth) break return len(matched) def _ratio(numerator: float, denominator: float) -> float: return numerator / denominator if denominator else 0.0 # --------------------------------------------------------------------------- # # Metrics # --------------------------------------------------------------------------- # def _file_and_function_scores( ground_truth: list[BuggyBlock], predicted: list[BuggyBlock] ) -> dict[str, float]: gt_files = {b.file for b in ground_truth if b.file} pred_files = {b.file for b in predicted if b.file} gt_funcs = {b.function for b in ground_truth if b.function} pred_funcs = {b.function for b in predicted if b.function} files = _count_matches(gt_files, pred_files, _files_equivalent) funcs = _count_matches(gt_funcs, pred_funcs, _functions_equivalent) return { "file_precision": _ratio(files, len(pred_files)), "file_coverage": _ratio(files, len(gt_files)), "function_precision": _ratio(funcs, len(pred_funcs)), "function_coverage": _ratio(funcs, len(gt_funcs)), } def _lines_by_file(blocks: list[BuggyBlock]) -> dict[str, set[int]]: out: dict[str, set[int]] = {} for b in blocks: out.setdefault(b.file, set()).update(range(b.start_line, b.end_line)) return out def _line_scores(ground_truth: list[BuggyBlock], predicted: list[BuggyBlock]) -> dict[str, float]: gt_lines = _lines_by_file(ground_truth) pred_lines = _lines_by_file(predicted) overlap = 0 for file, lines in pred_lines.items(): match = _find_equivalent_file(file, set(gt_lines)) if match is not None: overlap += len(lines & gt_lines[match]) return { "line_precision": _ratio(overlap, sum(len(v) for v in pred_lines.values())), "line_coverage": _ratio(overlap, sum(len(v) for v in gt_lines.values())), } def _ranges_by_file(blocks: list[BuggyBlock]) -> dict[str, list[tuple[int, int]]]: out: dict[str, list[tuple[int, int]]] = {} for b in blocks: out.setdefault(b.file, []).append((b.start_line, b.end_line)) return out def _near(a: tuple[int, int], b: tuple[int, int]) -> bool: return a[0] - BLOCK_TOLERANCE < b[1] and a[1] + BLOCK_TOLERANCE > b[0] def _hits(source: dict[str, list[tuple[int, int]]], target: dict[str, list[tuple[int, int]]]) -> int: """Ranges in ``source`` that are near a range of the equivalent file in ``target``.""" hits = 0 for file, ranges in source.items(): match = _find_equivalent_file(file, set(target)) if match is not None: hits += sum(any(_near(r, t) for t in target[match]) for r in ranges) return hits def _block_scores(ground_truth: list[BuggyBlock], predicted: list[BuggyBlock]) -> dict[str, float]: gt, pred = _ranges_by_file(ground_truth), _ranges_by_file(predicted) return { "block_precision": _ratio(_hits(pred, gt), len(predicted)), "block_coverage": _ratio(_hits(gt, pred), len(ground_truth)), } def _strict_block_coverage(ground_truth: list[BuggyBlock], predicted: list[BuggyBlock]) -> float: covered = sum( any( _files_equivalent(p.file, g.file) and max(p.start_line, g.start_line) < min(p.end_line, g.end_line) for p in predicted ) for g in ground_truth ) return _ratio(covered, len(ground_truth)) def _mention_coverage(ground_truth: list[BuggyBlock], diagnosis_text: str) -> float: if not ground_truth or not diagnosis_text: return 0.0 text = diagnosis_text.lower() mentioned = 0 for block in ground_truth: norm = canonicalize_function_name(block.function) if not norm: continue leaf = norm.split(".")[-1] token = leaf if leaf and leaf not in _GENERIC_METHOD_NAMES else norm mentioned += token in text return mentioned / len(ground_truth) def compute_correctness_report( ground_truth: list[BuggyBlock], predicted: list[BuggyBlock], diagnosis_text: str = "", ) -> dict[str, Any]: """Score a diagnosis against the ground truth. See the module docstring. Pass the raw text of the agent's ``diagnosis.json`` as ``diagnosis_text`` to compute ``mention_coverage`` as in the paper. """ return { "num_ground_truth": len(ground_truth), "num_predicted": len(predicted), **_file_and_function_scores(ground_truth, predicted), **_line_scores(ground_truth, predicted), **_block_scores(ground_truth, predicted), "mention_coverage": _mention_coverage(ground_truth, diagnosis_text), "strict_block_coverage": _strict_block_coverage(ground_truth, predicted), } def format_report(report: dict[str, Any]) -> str: """Pretty-print a correctness report for terminal output.""" lines = [ f" ground-truth blocks: {report['num_ground_truth']}", f" predicted blocks: {report['num_predicted']}", ] for g in _GRANULARITIES: lines.append( f" {g:<9} precision={report[f'{g}_precision']:.2%} " f"coverage={report[f'{g}_coverage']:.2%}" ) lines.append(f" strict block coverage={report['strict_block_coverage']:.2%}") lines.append(f" mention coverage={report['mention_coverage']:.2%}") return "\n".join(lines)