#!/usr/bin/env python3 """Build the Coral (Edge TPU) format of the ``yunet-2023mar`` face detector (D710). python3 scripts/build-yunet-edgetpu.py --work --frames \ --tf-python [--python ] \ [--compiler docker|lima:] produces, under ``/stage/faceDetection/yunet/tflite/``: camstack-yunet-2023mar_int8.tflite full-integer CPU graph (the compiler's input; what every co-compiled set is built from) camstack-yunet-2023mar_edgetpu.tflite the same graph, Edge TPU compiled alone plus the evidence D710 asks for in ``/report.json`` (op mapping, on-chip memory alone and co-compiled with the root SSD, parity) and the fixture ``packages/addon-pipeline/python/postprocessors/testdata/yunet_int8_outputs.json``. Upload with ``hf upload camstack/camstack-models /stage/faceDetection/yunet faceDetection/yunet``. SOURCE ------ The published ``camstack-yunet-2023mar.onnx`` (built by build-yunet-model.py, pinned by sha256 below). Its input contract is baked in: RGB float [0,1] NCHW in, ``Mul(255)`` + ``Gather([2,1,0])`` to the upstream BGR 0..255 (D645). THE GRAPH EDIT -------------- A Gather is not an Edge TPU op, and the compiler maps nothing after the first op it cannot map. The bake is therefore FOLDED into the first Conv — ``W'[:, bgr[c]] += 255 * W[:, c]`` — which is the same function (the verify stage proves it: max |diff| ~1e-5 on random input). The contract does not change: the int8 graph's input is uint8 with scale 1/255 and zero point 0, so the raw pixel bytes the edgetpu preprocess feeds ARE the RGB [0,1] tensor, and the entry still declares no ``inputNormalization``. The outputs keep their names (``cls_8`` … ``kps_32``): ``postprocessors/yunet.py`` matches by name only, and the edgetpu predict closure keys a name-matching postprocessor's outputs by tensor name (``edgetpu_output_keys``). QUANTISATION ------------ Full-integer post-training quantisation (onnx2tf → TFLiteConverter, per-channel weights, uint8 input, int8 outputs) calibrated on every third frame of ``--frames``; the other two thirds are the parity set. The frames are the operator's: padded person crops (+15% of the box, the detail plane's convention), as the face step sees them. They are NOT kept or published — the int8 CPU graph is, so a recompile or a co-compiled set starts from that file and never needs them again. 640 is the only admitted size: at 480 recall falls to ~0.82 of the reference and at 320 to ~0.50, and the loss is resolution, not quantisation (the fp32 graph at 320 loses the same faces). GATES (exit 1 on a miss), at the step gate (the model's ``defaultConfidence`` 0.85) against the ONNX reference fed the same letterboxed frame: - the emitted face set is identical on >= 95% of the parity crops; - every face only one side emits has the other side's score for the same box within 0.05 of the gate (a straddle, not a different detection); - landmarks (face-align depends on them): mean point error <= 2% of the face box diagonal. STAGES, each in its own interpreter: onnxruntime, LiteRT and TensorFlow are never loaded into one process (mixing ML runtimes deadlocks or crashes on this Mac). The parent imports onnx, numpy and Pillow only. """ from __future__ import annotations import argparse import base64 import json import shutil import subprocess import sys import zlib from hashlib import sha256 from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) import edgetpu_toolchain as tc # noqa: E402 _REPO = Path(__file__).resolve().parents[1] _PY = _REPO / "packages" / "addon-pipeline" / "python" _FIXTURE = _PY / "postprocessors" / "testdata" / "yunet_int8_outputs.json" _FIXTURE_IMAGE = _REPO / "packages" / "addon-pipeline" / "assets" / "reference-images" / "3-persons.jpg" _HF = "https://huggingface.co/camstack/camstack-models/resolve/main/faceDetection/yunet" SRC_ONNX = (f"{_HF}/onnx/camstack-yunet-2023mar.onnx", "camstack-yunet-2023mar.onnx", "55df63fd5a5128903ee2b6dd4315aa82979be9a1d6cf83cc154b74cd4bd2c42f", 233_080) FOLDER = "faceDetection/yunet" NAME = "camstack-yunet-2023mar" SIZE = 640 OUTPUTS = [f"{k}_{s}" for k in ("cls", "obj", "bbox", "kps") for s in (8, 16, 32)] STEP_GATE = 0.85 # model-catalogs.ts yunet-2023mar.defaultConfidence (D662) LOW_FLOOR = 0.3 # decode floor: a box just under the gate on one side is still seen PAD = 0.15 GATE_SET_AGREEMENT = 0.95 GATE_STRADDLE = 0.05 GATE_LANDMARK_DIAG = 0.02 FOLD_MAX_ABS = 1e-4 # ── graph edit (parent: onnx + numpy) ───────────────────────────────────────── def _fold(src: Path, dst: Path) -> None: import numpy as np import onnx from onnx import numpy_helper as nh m = onnx.load(str(src)) g = m.graph mul, gat, conv = list(g.node)[:3] if (mul.op_type, gat.op_type, conv.op_type) != ("Mul", "Gather", "Conv"): raise SystemExit("unexpected graph head: the published bake (Mul, Gather, Conv) is not first") inits = {i.name: i for i in g.initializer} scale = float(nh.to_array(inits[mul.input[1]]).reshape(-1)[0]) bgr = nh.to_array(inits[gat.input[1]]).astype(int).tolist() w = nh.to_array(inits[conv.input[1]]) folded = np.zeros_like(w) for c in range(3): folded[:, bgr[c]] += w[:, c] * scale inits[conv.input[1]].CopyFrom(nh.from_array(folded.astype(np.float32), conv.input[1])) conv.input[0] = g.input[0].name g.node.remove(mul) g.node.remove(gat) for name in (mul.input[1], gat.input[1]): g.initializer.remove(inits[name]) onnx.checker.check_model(m) onnx.save(m, str(dst)) # ── stages (own interpreter each) ───────────────────────────────────────────── def _letterbox(path: str, size: int): """Byte-for-byte the pool's ``letterbox_image`` (pad 114), uint8 HWC.""" import numpy as np from PIL import Image img = Image.open(path).convert("RGB") w, h = img.size scale = size / max(w, h) nw, nh_ = int(w * scale), int(h * scale) px, py = (size - nw) // 2, (size - nh_) // 2 if (w, h) != (size, size): canvas = Image.new("RGB", (size, size), (114, 114, 114)) canvas.paste(img if (w, h) == (nw, nh_) else img.resize((nw, nh_), Image.BILINEAR), (px, py)) img, pad = canvas, (px, py) else: scale, pad = 1.0, (0, 0) return np.asarray(img, dtype=np.uint8), scale, pad, (w, h) def _stage_calib(index: Path, out: Path, sample: Path) -> None: import numpy as np from PIL import Image rows = [e for e in json.loads(index.read_text()) if e["split"] == "calib"] np.save(out, np.stack([_letterbox(e["path"], SIZE)[0] for e in rows]).astype(np.float32) / 255.0) # onnx2tf's own output check wants a 20x128x128x3 sample next to it; give # it ours instead of letting it download one. np.save(sample, np.stack([np.asarray(Image.open(e["path"]).convert("RGB").resize((128, 128)), np.float32) / 255.0 for e in (rows * 20)[:20]]).astype(np.float32)) def _stage_rename(src: Path, dst: Path) -> None: """Signature order = ONNX output order; give each output its public name.""" from tensorflow.lite.tools import flatbuffer_utils as fu m = fu.read_model(str(src)) sg, sig = m.subgraphs[0], m.signatureDefs[0] by_key = {(o.name.decode() if isinstance(o.name, bytes) else o.name): o.tensorIndex for o in sig.outputs} order = sorted(by_key, key=lambda k: int(k.rsplit("_", 1)[1])) if len(order) != len(OUTPUTS): raise SystemExit(f"expected {len(OUTPUTS)} outputs, got {order}") for key, name in zip(order, OUTPUTS): sg.tensors[by_key[key]].name = name.encode() sg.tensors[sig.inputs[0].tensorIndex].name = b"image" fu.write_model(m, str(dst)) def _decode(pred: dict, w: int, h: int, scale: float, pad, floor: float) -> list: sys.path.insert(0, str(_PY)) from postprocessors.yunet import postprocess_yunet cfg = {"confidence": floor, "inputSize": SIZE, "nmsIouThreshold": 0.45} return postprocess_yunet(pred, cfg, w, h, scale, pad)["detections"] def _ort_predictor(model: Path): import numpy as np import onnxruntime as ort s = ort.InferenceSession(str(model), providers=["CPUExecutionProvider"]) names = [o.name for o in s.get_outputs()] return lambda x: dict(zip(names, s.run(None, {"image": (x.astype(np.float32) / 255.0).transpose(2, 0, 1)[None]}))) def _litert_predictor(model: Path, raw_sink: dict | None = None): """The edgetpu predict closure's contract: uint8 in, every output dequantised.""" import numpy as np from ai_edge_litert.interpreter import Interpreter sys.path.insert(0, str(_PY)) from edgetpu_dequant import dequant_edgetpu_output it = Interpreter(str(model)) it.allocate_tensors() ind, outs = it.get_input_details()[0], it.get_output_details() def pred(x): it.set_tensor(ind["index"], x[None]) it.invoke() r = {} for d in outs: raw = it.get_tensor(d["index"]) q = d["quantization"] if raw_sink is not None: raw_sink[d["name"]] = {"shape": [int(v) for v in raw.shape], "dtype": np.dtype(d["dtype"]).name, "scale": float(q[0]), "zeroPoint": int(q[1]), "zlibB64": base64.b64encode(zlib.compress(raw.tobytes(), 9)).decode()} r[d["name"]] = dequant_edgetpu_output(raw, dtype_name=np.dtype(d["dtype"]).name, scale=float(q[0]), zero_point=int(q[1])) return r return pred def _stage_run(runtime: str, model: Path, index: Path, out: Path) -> None: pred = _ort_predictor(model) if runtime == "ort" else _litert_predictor(model) res = {} for e in json.loads(index.read_text()): if e["split"] != "eval": continue x, scale, pad, (w, h) = _letterbox(e["path"], SIZE) res[e["name"]] = _decode(pred(x), w, h, scale, pad, LOW_FLOOR) out.write_text(json.dumps(res)) def _stage_fold_check(published: Path, folded: Path) -> None: import numpy as np import onnxruntime as ort a = ort.InferenceSession(str(published), providers=["CPUExecutionProvider"]) b = ort.InferenceSession(str(folded), providers=["CPUExecutionProvider"]) rng = np.random.default_rng(0) worst = 0.0 for _ in range(4): x = rng.random((1, 3, SIZE, SIZE), dtype=np.float32) worst = max(worst, max(float(np.abs(p - q).max()) for p, q in zip(a.run(None, {"image": x}), b.run(None, {"image": x})))) print(f" fold: max |diff| vs the published graph = {worst:.2e}") if worst > FOLD_MAX_ABS: raise SystemExit("fold is not exact") def _stage_fixture(onnx_model: Path, int8_model: Path, out: Path) -> None: raw: dict = {} x, scale, pad, (w, h) = _letterbox(str(_FIXTURE_IMAGE), SIZE) cand = _decode(_litert_predictor(int8_model, raw)(x), w, h, scale, pad, STEP_GATE) out.write_text(json.dumps({"frame": {"w": w, "h": h, "scale": scale, "pad": list(pad)}, "outputs": raw, "int8Detections": cand})) def _stage_fixture_ref(onnx_model: Path, fixture: Path) -> None: data = json.loads(fixture.read_text()) x, scale, pad, (w, h) = _letterbox(str(_FIXTURE_IMAGE), SIZE) data["onnxDetections"] = _decode(_ort_predictor(onnx_model)(x), w, h, scale, pad, STEP_GATE) fixture.write_text(json.dumps(data)) def _run_stage(python: str, name: str, *args: str) -> None: subprocess.run([python, __file__, "--stage", name, *args], check=True) # ── parity ─────────────────────────────────────────────────────────────────── def _iou(a, b) -> float: ix = max(0.0, min(a[2], b[2]) - max(a[0], b[0])) iy = max(0.0, min(a[3], b[3]) - max(a[1], b[1])) inter = ix * iy union = (a[2] - a[0]) * (a[3] - a[1]) + (b[2] - b[0]) * (b[3] - b[1]) - inter return inter / union if union > 0 else 0.0 def _parity(ref: dict, cand: dict) -> dict: same_set, matched, n_ref, n_cand, lm, ious, one_sided = 0, 0, 0, 0, [], [], [] for k in ref: r_all, c_all = ref[k], cand[k] R = [d for d in r_all if d["score"] >= STEP_GATE] C = [d for d in c_all if d["score"] >= STEP_GATE] n_ref += len(R) n_cand += len(C) used: set[int] = set() here = 0 for r in R: best = max(((_iou(r["bbox"], c["bbox"]), j) for j, c in enumerate(C) if j not in used), default=(0.0, -1)) if best[0] >= 0.5: used.add(best[1]) here += 1 ious.append(best[0]) c = C[best[1]] diag = ((r["bbox"][2] - r["bbox"][0]) ** 2 + (r["bbox"][3] - r["bbox"][1]) ** 2) ** 0.5 d = [((p["x"] - q["x"]) ** 2 + (p["y"] - q["y"]) ** 2) ** 0.5 for p, q in zip(r["landmarks"], c["landmarks"])] lm.append(sum(d) / len(d) / diag) else: other = max([c["score"] for c in c_all if _iou(r["bbox"], c["bbox"]) >= 0.5], default=0.0) one_sided.append({"crop": k, "side": "reference", "score": r["score"], "other": other}) for j, c in enumerate(C): if j not in used: other = max([r["score"] for r in r_all if _iou(r["bbox"], c["bbox"]) >= 0.5], default=0.0) one_sided.append({"crop": k, "side": "int8", "score": c["score"], "other": other}) matched += here same_set += int(len(R) == len(C) == here) lm_sorted = sorted(lm) return { "crops": len(ref), "reference_faces": n_ref, "int8_faces": n_cand, "matched": matched, "recall_of_reference": round(matched / max(1, n_ref), 4), "same_emitted_set": same_set, "same_emitted_set_rate": round(same_set / max(1, len(ref)), 4), "box_iou_mean": round(sum(ious) / max(1, len(ious)), 4), "landmark_err_over_diag_mean": round(sum(lm) / max(1, len(lm)), 4), "landmark_err_over_diag_p95": round(lm_sorted[int(0.95 * (len(lm_sorted) - 1))], 4) if lm_sorted else None, "one_sided": one_sided, } def main() -> None: sys.stdout.reconfigure(line_buffering=True) ap = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0]) ap.add_argument("--work", type=Path) ap.add_argument("--frames", type=Path, action="append", default=[], help="directory of padded person crops (jpg/png): every 3rd calibrates, the rest is the parity set") ap.add_argument("--compiler", default="docker", help="docker | lima:") ap.add_argument("--python", default=sys.executable, help="interpreter with onnxruntime + ai-edge-litert + pillow") ap.add_argument("--tf-python", help="interpreter with tensorflow + onnx2tf (onnx2tf on PATH next to it)") ap.add_argument("--stage", help=argparse.SUPPRESS) ap.add_argument("stage_args", nargs="*", help=argparse.SUPPRESS) a = ap.parse_args() if a.stage: fn = {"calib": _stage_calib, "rename": _stage_rename, "run": None, "fold-check": _stage_fold_check, "fixture": _stage_fixture, "fixture-ref": _stage_fixture_ref}[a.stage] if a.stage == "run": _stage_run(a.stage_args[0], *(Path(p) for p in a.stage_args[1:])) else: fn(*(Path(p) for p in a.stage_args)) return if a.work is None or not a.frames or a.tf_python is None: ap.error("--work, --frames and --tf-python are required") work = a.work.absolute() # not resolve(): /tmp/lima must stay /tmp/lima for the VM work.mkdir(parents=True, exist_ok=True) print("1/6 fetch + fold the bake into the first Conv") published = tc.fetch(SRC_ONNX, work) folded = work / "yunet-folded.onnx" _fold(published, folded) _run_stage(a.python, "fold-check", str(published), str(folded)) print("2/6 frames") paths = sorted(str(p) for d in a.frames for p in d.iterdir() if p.suffix.lower() in (".jpg", ".jpeg", ".png")) index = [{"name": Path(p).stem, "path": p, "split": "calib" if i % 3 == 0 else "eval"} for i, p in enumerate(paths)] (work / "frames.json").write_text(json.dumps(index)) print(f" {len(index)} frames: {sum(e['split'] == 'calib' for e in index)} calibrate, the rest is the parity set") _run_stage(a.python, "calib", str(work / "frames.json"), str(work / "calib.npy"), str(work / "calibration_image_sample_data_20x128x128x3_float32.npy")) print("3/6 full-integer quantisation (onnx2tf)") tf_dir = work / "tf" shutil.rmtree(tf_dir, ignore_errors=True) onnx2tf = Path(a.tf_python).parent / "onnx2tf" subprocess.run([str(onnx2tf), "-i", str(folded), "-o", str(tf_dir), "-oiqt", "-qt", "per-channel", "-cind", "image", str(work / "calib.npy"), "[[[[0,0,0]]]]", "[[[[1,1,1]]]]", "-iqd", "uint8", "-oqd", "int8", "-n"], check=True, cwd=work, stdout=(work / "onnx2tf.log").open("w"), stderr=subprocess.STDOUT) int8 = work / f"{NAME}_int8.tflite" _run_stage(a.tf_python, "rename", str(tf_dir / "yunet-folded_full_integer_quant.tflite"), str(int8)) print("4/6 compile: alone, and co-compiled after the root SSD") ssd = tc.fetch(tc.SRC_SSD_QUANT, work) tc.compile_jobs(a.compiler, work, [int8, ssd], [("compiled", [int8.name]), ("cocompiled", [ssd.name, int8.name])]) alone = tc.parse_compiler_output((work / "compiled" / "compiler.txt").read_text()) co = tc.parse_compiler_output((work / "cocompiled" / "compiler.txt").read_text()) for label, rows in (("alone", alone), ("SSD + YuNet", co)): for r in rows: print(f" {label:12s} {r['model']:40s} ops {r['ops_tpu']}/{r['ops_total']} on TPU, " f"on-chip {tc.mib(r['on_chip_bytes'])}, streamed {tc.mib(r['streamed_bytes'])}") if alone[0]["ops_cpu"] != 0: raise SystemExit("not every op maps to the Edge TPU") print("5/6 parity at the step gate") _run_stage(a.python, "run", "ort", str(published), str(work / "frames.json"), str(work / "ref.json")) _run_stage(a.python, "run", "litert", str(int8), str(work / "frames.json"), str(work / "int8.json")) parity = _parity(json.loads((work / "ref.json").read_text()), json.loads((work / "int8.json").read_text())) print(json.dumps({k: v for k, v in parity.items() if k != "one_sided"}, indent=1)) for row in parity["one_sided"]: print(f" one-sided {row}") straddles = all(abs(r["other"] - STEP_GATE) <= GATE_STRADDLE for r in parity["one_sided"]) if (parity["same_emitted_set_rate"] < GATE_SET_AGREEMENT or not straddles or parity["landmark_err_over_diag_mean"] > GATE_LANDMARK_DIAG): raise SystemExit("parity gate FAILED") print("6/6 fixture + stage") _run_stage(a.python, "fixture", str(published), str(int8), str(_FIXTURE)) _run_stage(a.python, "fixture-ref", str(published), str(_FIXTURE)) fx = json.loads(_FIXTURE.read_text()) fx["producedBy"] = ("scripts/build-yunet-edgetpu.py: raw outputs of the int8 graph the Edge TPU file is compiled " "from, on reference-images/3-persons.jpg letterboxed to 640; onnxDetections = the ONNX " "reference at the step gate") _FIXTURE.write_text(json.dumps(fx)) dest = work / "stage" / FOLDER / "tflite" dest.mkdir(parents=True, exist_ok=True) shutil.copyfile(int8, dest / int8.name) shutil.copyfile(work / "compiled" / f"{int8.stem}_edgetpu.tflite", dest / f"{NAME}_edgetpu.tflite") files = {p.name: {"bytes": p.stat().st_size, "sha256": sha256(p.read_bytes()).hexdigest()} for p in dest.iterdir()} (work / "report.json").write_text(json.dumps({"files": files, "alone": alone, "cocompiled_after_ssd": co, "parity": parity}, indent=1)) for name, meta in files.items(): print(f" {dest / name} {meta['bytes']} bytes sha256 {meta['sha256']}") print(f" hf upload camstack/camstack-models {work / 'stage' / FOLDER} {FOLDER}") if __name__ == "__main__": main()