#!/usr/bin/env python3 """Deploy/stop real MicroServices and check the first post-Stop DNS response. Use an empty, dedicated Linux/KVM QA API with Pipe Auth already connected. The image must serve HTTP on the supplied port and health path. The caller template must provide a serial root shell and /etc/firecrab/busybox. """ import argparse import importlib.util import ipaddress import json from pathlib import Path import re import subprocess import time from urllib.parse import urlsplit from urllib.request import ProxyHandler, Request, build_opener import uuid def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--api", default="http://127.0.0.1:15480") parser.add_argument("--image", required=True) parser.add_argument("--caller-template", required=True) parser.add_argument("--port", type=int, default=80) parser.add_argument("--health-path", default="/") parser.add_argument("--subnet", default="172.29.48.0/24") parser.add_argument("--uplink", default="qa-uplink") parser.add_argument("--cycles", type=int, default=2) parser.add_argument("--dns-deadline", type=float, default=1.0) parser.add_argument("--output", type=Path, required=True) args = parser.parse_args() if args.cycles < 1 or args.dns_deadline <= 0: parser.error("cycles and DNS deadline must be positive") address = urlsplit(args.api) if address.scheme != "http" or address.hostname != "127.0.0.1": parser.error("use a dedicated loopback QA API") opener = build_opener(ProxyHandler({})) report = {"status": "running", "checks": [], "dns": [], "cycles": args.cycles} services, networks, callers = [], [], [] tag = "qa480-" + uuid.uuid4().hex[:8] def save(): args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(report, indent=2)) def request(path, method="GET", body=None): data = json.dumps(body).encode() if body is not None else (b"" if method == "POST" else None) req = Request(args.api.rstrip("/") + path, data=data, method=method, headers={"Content-Type": "application/json"}) with opener.open(req, timeout=30) as response: raw = response.read() return json.loads(raw) if raw else None def until(check, seconds=180): deadline = time.monotonic() + seconds while time.monotonic() < deadline: result = check() if result: return result time.sleep(0.05) raise AssertionError("QA condition timed out") def passed(name, **evidence): report["checks"].append({"name": name, **evidence}) save() print("PASS " + name, flush=True) def deployment(service, phase): def ready(): state = request(f"/api/micro-services/{service['id']}/deployment") assert state["phase"] != "failed", state.get("error") return state if state["phase"] == phase else None return until(ready) module = importlib.util.spec_from_file_location("qa_console", Path(__file__).with_name("ci-qa-vm-pause.py")) console_module = importlib.util.module_from_spec(module) module.loader.exec_module(console_module) class Console(console_module.Console): def command(self, text): token = uuid.uuid4().hex[:12] self.text = "" self.send("echo BEGIN" + token + "; " + text + "; echo END" + token + "$?\n") until(lambda: (self.read(0.01), re.search(r"\nEND" + token + r"\d+\r?\n", self.text))[1], 30) output = self.text.split("\r\nBEGIN" + token + "\r\n")[-1] with args.output.with_suffix(".console.txt").open("a") as file: file.write(output) return output for path in ["/api/vms", "/api/micro-services", "/api/micro-networks"]: assert request(path) == [], "refusing a non-empty QA API: " + path save() console = None try: network = request("/api/micro-networks", "POST", {"name": tag, "subnetCidr": args.subnet, "internetEnabled": False, "uplink": args.uplink}) networks.append(network["id"]) gateway = str(next(ipaddress.ip_network(args.subnet).hosts())) def gateway_mac(): links = json.loads(subprocess.check_output(["ip", "-j", "address", "show"], text=True)) return next(link["address"] for link in links if any(item.get("local") == gateway for item in link.get("addr_info", []))) initial_mac = gateway_mac() report["gatewayMac"] = initial_mac caller = request("/api/vms", "POST", {"name": tag + "-caller", "template": args.caller_template, "cpu": 1, "ram": 128, "diskGb": 1, "microNetworkId": network["id"]}) callers.append(caller["id"]) request(f"/api/vms/{caller['id']}/start", "POST") until(lambda: request(f"/api/vms/{caller['id']}")["state"] == "running") assert gateway_mac() == initial_mac, "caller TAP changed gateway MAC" console = Console(address.port, caller["id"]) console.send("\n") def prompt(): console.read(0.05) if "login:" in console.text and "# " not in console.text: console.send("root\n") return "# " in console.text until(prompt, 30) def create_service(suffix, replicas): service = request("/api/micro-services", "POST", {"name": tag + suffix, "image": args.image, "port": args.port, "healthPath": args.health_path, "ram": 128, "cpu": 1, "replicas": replicas, "startupTimeout": 90, "drainTimeout": 1}) services.append(service) return service def deploy(service): request(f"/api/micro-services/{service['id']}/deployment", "POST", {"microNetworkId": network["id"]}) result = deployment(service, "running") assert result["internalRouteOwned"] and all(i["healthy"] for i in result["instances"]) assert all(i["hostPort"] == 0 for i in result["instances"]) assert gateway_mac() == initial_mac, "deployment changed gateway MAC" return result keep = create_service("-keep", 1) deploy(keep) target = create_service("-target", 2) for cycle in range(args.cycles): current = deploy(target) host = target["name"] + ".internal" dns = console.command("/etc/firecrab/busybox nslookup " + host) assert gateway in dns and "NXDOMAIN" not in dns http = console.command("/etc/firecrab/busybox wget -T 2 -S -O /tmp/qa480-http http://" + host + args.health_path) assert "200 OK" in http, http neighbor = console.command("/etc/firecrab/busybox ip neigh show " + gateway) assert initial_mac in neighbor.lower(), neighbor passed(f"cycle {cycle + 1}: real deployment registers DNS and HTTP route", replicas=len(current["instances"]), digest=current["digest"], gatewayMac=gateway_mac()) request(f"/api/micro-services/{target['id']}/stop", "POST") stopped = deployment(target, "stopped") assert not stopped["internalRouteOwned"] and stopped["instances"] == [] assert gateway_mac() == initial_mac, "Stop changed gateway MAC" for query in ["", "-type=A ", "-type=AAAA "]: start = time.monotonic() output = console.command("/etc/firecrab/busybox nslookup " + query + host) elapsed = time.monotonic() - start report["dns"].append({"cycle": cycle + 1, "query": query.strip() or "A+AAAA", "seconds": elapsed, "nxdomain": "NXDOMAIN" in output}) save() assert "NXDOMAIN" in output and elapsed < args.dns_deadline, (elapsed, output) survivor = console.command("/etc/firecrab/busybox wget -T 2 -S -O /tmp/qa480-keep http://" + keep["name"] + ".internal" + args.health_path) assert "200 OK" in survivor, survivor passed(f"cycle {cycle + 1}: first DNS queries meet deadline and other service stays reachable", gatewayMac=gateway_mac()) report["status"] = "passed" finally: failures = [] for service in reversed(services): try: state = request(f"/api/micro-services/{service['id']}/deployment") if state and state["phase"] != "stopped": request(f"/api/micro-services/{service['id']}/stop", "POST") deployment(service, "stopped") request(f"/api/micro-services/{service['id']}", "DELETE") except Exception as error: failures.append(str(error)) if console is not None: console.socket.close() for identifier in callers: try: request(f"/api/vms/{identifier}/stop", "POST") request(f"/api/vms/{identifier}", "DELETE") except Exception as error: failures.append(str(error)) for identifier in networks: try: request(f"/api/micro-networks/{identifier}", "DELETE") except Exception as error: failures.append(str(error)) report["cleanupErrors"] = failures if report["status"] != "passed" or failures: report["status"] = "failed" save() assert not failures, failures if __name__ == "__main__": main()