"""Coordinate-space handling for GUI grounding evaluation. This is the module where silent failures live. Every model reports clicks in a different space, and a mismatch produces a plausible-looking wrong coordinate rather than an exception, so bugs here quietly depress scores instead of crashing. Everything is therefore explicit: a model's output space is declared up front, and conversion back to original-image pixels goes through one path. """ from __future__ import annotations from dataclasses import dataclass from enum import Enum class CoordSpace(str, Enum): """The space a model emits click coordinates in. ABSOLUTE_IMAGE coordinates in pixels of the image the model was shown. If we resized before sending, these are in resized pixels and must be scaled back. NORMALIZED coordinates in [0, 1], relative to the image shown. FIXED_GRID coordinates in a model-specific virtual grid (e.g. Qwen-VL family's 1000x1000), independent of the image sent. """ ABSOLUTE_IMAGE = "absolute_image" NORMALIZED = "normalized" FIXED_GRID = "fixed_grid" @dataclass(frozen=True) class ImageTransform: """Records how an original image was transformed before being sent. A crop is applied first, then a resize. `crop_box` is in original-image pixels; `sent_size` is the size the model actually saw. """ orig_size: tuple[int, int] sent_size: tuple[int, int] crop_box: tuple[int, int, int, int] | None = None @property def crop_origin(self) -> tuple[int, int]: return (0, 0) if self.crop_box is None else self.crop_box[:2] @property def crop_size(self) -> tuple[int, int]: if self.crop_box is None: return self.orig_size x1, y1, x2, y2 = self.crop_box return (x2 - x1, y2 - y1) @property def scale(self) -> tuple[float, float]: """Multipliers taking sent-image pixels back to crop-region pixels.""" cw, ch = self.crop_size sw, sh = self.sent_size if sw <= 0 or sh <= 0: raise ValueError(f"sent_size must be positive, got {self.sent_size}") return (cw / sw, ch / sh) def with_sent_size(self, sent_size: tuple[int, int]) -> ImageTransform: """Same crop, but the image was ultimately sent at `sent_size`. Composes a second pure resize on top of this transform. Needed when the model's own preprocessing resizes the image again after ours — the composite is just this transform with the final size substituted, since a resize-after-resize collapses into one ratio. """ return ImageTransform( orig_size=self.orig_size, sent_size=sent_size, crop_box=self.crop_box ) def to_original_point( raw_x: float, raw_y: float, space: CoordSpace, transform: ImageTransform, grid: tuple[int, int] | None = None, ) -> tuple[float, float]: """Map a model's raw output to a point in original-image pixels. `grid` is required for FIXED_GRID and gives the model's virtual extent (e.g. (1000, 1000)). """ sw, sh = transform.sent_size if space is CoordSpace.NORMALIZED: sent_x, sent_y = raw_x * sw, raw_y * sh elif space is CoordSpace.FIXED_GRID: if grid is None: raise ValueError("grid is required for CoordSpace.FIXED_GRID") gw, gh = grid sent_x, sent_y = raw_x / gw * sw, raw_y / gh * sh elif space is CoordSpace.ABSOLUTE_IMAGE: sent_x, sent_y = raw_x, raw_y else: # pragma: no cover - exhaustive over the enum raise ValueError(f"unhandled coordinate space: {space}") scale_x, scale_y = transform.scale off_x, off_y = transform.crop_origin return (sent_x * scale_x + off_x, sent_y * scale_y + off_y) def to_original_box( raw_box: tuple[float, float, float, float], space: CoordSpace, transform: ImageTransform, grid: tuple[int, int] | None = None, ) -> tuple[float, float, float, float]: """Map a model's raw box to original-image pixels, by mapping both corners. Mapping corners is exact here because every transform in this codebase is an axis-aligned crop followed by an axis-aligned resize; neither rotates, so a box stays a box. Re-sorted afterwards anyway, since a model can emit corners in either order and a negative-width box would silently break area comparisons downstream. """ x1, y1, x2, y2 = raw_box ox1, oy1 = to_original_point(x1, y1, space, transform, grid=grid) ox2, oy2 = to_original_point(x2, y2, space, transform, grid=grid) return (min(ox1, ox2), min(oy1, oy2), max(ox1, ox2), max(oy1, oy2)) def point_in_box(x: float, y: float, box: tuple[float, float, float, float]) -> bool: """ScreenSpot-style correctness: is the predicted point inside the target? Bounds are inclusive. `box` is (x1, y1, x2, y2) in original-image pixels. """ x1, y1, x2, y2 = box return x1 <= x <= x2 and y1 <= y <= y2 def point_in_image( x: float, y: float, image_size: tuple[int, int], tolerance: float = 1.0 ) -> bool: """Whether a predicted point is physically on the screen at all. Models occasionally emit coordinates like (-144, -144), which cannot be a click target. These score as incorrect either way, but they are worth distinguishing from a near-miss: an off-image prediction points at a prompt or format problem, whereas a near-miss points at a grounding limitation. Conflating the two hides which one you have. """ w, h = image_size return -tolerance <= x <= w + tolerance and -tolerance <= y <= h + tolerance def box_area_fraction( box: tuple[float, float, float, float], image_size: tuple[int, int] ) -> float: """Target area as a fraction of the screenshot. The headline difficulty signal for ScreenSpot-Pro, whose targets average 0.07% of the screen against 2.01% for the original ScreenSpot. Used to bucket results by target size. """ x1, y1, x2, y2 = box w, h = image_size if w <= 0 or h <= 0: raise ValueError(f"image_size must be positive, got {image_size}") return (max(0.0, x2 - x1) * max(0.0, y2 - y1)) / (w * h) # Bucket edges in percent of screen area: the measured quartiles of ScreenSpot-Pro # (n=1581, computed 2026-08-24), so each bucket holds ~25% of the set and the # per-bucket accuracies are comparable. # # p25 = 0.0143% p50 = 0.0355% p75 = 0.0805% # min = 0.0017% mean = 0.0651% max = 4.7288% # # The mean matches the 0.07% figure reported in the ScreenSpot-Pro paper, which # is the check that the bbox parsing is right. For reference, the original # ScreenSpot averages 2.01% — roughly 30x larger, and above this set's maximum. # # Report these edges alongside any per-bucket number: they are dataset-specific, # so bucket labels are not comparable across benchmarks. # # CONFOUND, measured on the same run: target size and ui_type are strongly # correlated in this dataset. # # bucket text icon # tiny 75 316 # small 169 231 # medium 354 42 # large 379 15 # # So "accuracy drops on small targets" and "accuracy drops on icons" are largely # the same observation here. Attributing a size effect to size alone is # unsupported — condition on ui_type (or report the 2x4 grid) before claiming one. SIZE_BUCKET_EDGES = (0.0143, 0.0355, 0.0805) SIZE_BUCKET_LABELS = ("tiny", "small", "medium", "large") def size_bucket(area_fraction: float) -> str: """Bucket a target by area fraction, for per-size breakdowns.""" pct = area_fraction * 100.0 for edge, label in zip(SIZE_BUCKET_EDGES, SIZE_BUCKET_LABELS): if pct < edge: return label return SIZE_BUCKET_LABELS[-1]