name: Reproduce NeMo Guardrails source with Presidio on: workflow_dispatch: inputs: source_ref: description: NeMo tag, branch or commit; blank tests this repository's commit required: false type: string source_repository: description: Repository containing the selected NeMo source required: false type: string pull_request: paths: - .github/workflows/nemo-source-reproduction.yml permissions: contents: read jobs: reproduce: runs-on: ubuntu-latest timeout-minutes: 60 env: # Names this run on the results wall. Change them if your fork measures something else. WALL_PRODUCT: NeMo Guardrails WALL_CONFIGURATION: Presidio-backed output detection rail WALL_LICENSE: Apache-2.0 WALL_PROJECT_URL: https://github.com/NVIDIA/NeMo-Guardrails SOURCE_REPOSITORY: ${{ inputs.source_repository || (github.repository == 'ninadphalak/LLM-Shield-Proxy' && 'NVIDIA/NeMo-Guardrails' || github.repository) }} SOURCE_SELECTOR: ${{ inputs.source_ref || (github.repository == 'ninadphalak/LLM-Shield-Proxy' && 'v0.24.0' || github.sha) }} SOURCE_IMAGE: nemo-source-reproduction:${{ github.run_id }} CONFORMANCE_TARGET_API_KEY: sk-dummy-not-used V2_GATEWAY_TOKEN: sk-dummy-not-used steps: - name: Require a ref for another repository id: ref_check if: inputs.source_repository != '' && inputs.source_ref == '' run: | echo 'Choose an explicit tag or commit when testing another repository.' exit 2 - name: Check out selected NeMo source id: checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: repository: ${{ env.SOURCE_REPOSITORY }} ref: ${{ env.SOURCE_SELECTOR }} path: proxy-source persist-credentials: false - name: Check out the pinned response instrument uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: repository: ninadphalak/LLM-Shield-Proxy ref: 74d744a72adb053ff4f4025bdd676bba0e5be03d # benchmark-v0.2.1 path: benchmark-instrument persist-credentials: false - name: Set up Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 with: python-version: '3.12' - name: Build the selected source with pinned local detector dependencies id: build run: | set -euo pipefail python -m pip install 'pii-leak-benchmark[validate]==0.2.1' out="$RUNNER_TEMP/nemo-source-reproduction" mkdir -p "$out" git -C proxy-source rev-parse HEAD > "$out/source-commit.txt" python -m pip freeze > "$out/response-harness-packages.txt" python --version > "$out/python-version.txt" 2>&1 uname -a > "$out/runner.txt" docker version --format '{{.Server.Version}}' > "$out/docker-version.txt" docker build --file - --tag "$SOURCE_IMAGE" proxy-source <<'DOCKERFILE' FROM python:3.11-slim@sha256:da047cb8f9d1d98e5c070f5300ba9f7274e33b8fc0e5be5ed88740aed1b95ba9 COPY . /source RUN pip install --no-cache-dir "/source[server,sdd]" \ "presidio-analyzer==2.2.364" "presidio-anonymizer==2.2.364" "spacy==3.8.16" \ && python -m spacy download en_core_web_lg \ && python -c "import importlib.metadata as m; assert m.version('en-core-web-lg') == '3.8.0'" RUN cp -a /source/examples /usr/local/lib/python3.11/site-packages/examples WORKDIR /config ENTRYPOINT ["nemoguardrails", "server", "--config", "/config"] DOCKERFILE docker image inspect "$SOURCE_IMAGE" --format '{{.Id}}' > "$out/image-id.txt" docker run --rm --entrypoint python "$SOURCE_IMAGE" -m pip freeze > "$out/proxy-packages.txt" python - <<'PY' import json import os from pathlib import Path root = Path(os.environ['RUNNER_TEMP']) / 'nemo-source-reproduction' identity = { 'schema': 'pii-leak-benchmark/source-build/v1', 'source_repository': os.environ['SOURCE_REPOSITORY'], 'source_selector': os.environ['SOURCE_SELECTOR'], 'source_commit': (root / 'source-commit.txt').read_text(encoding='utf-8').strip(), 'product': os.environ['WALL_PRODUCT'], 'configuration': os.environ['WALL_CONFIGURATION'], 'license': os.environ['WALL_LICENSE'], 'project_url': os.environ['WALL_PROJECT_URL'], 'base_image': 'python:3.11-slim@sha256:da047cb8f9d1d98e5c070f5300ba9f7274e33b8fc0e5be5ed88740aed1b95ba9', 'detector': 'in-process Presidio 2.2.364; en_core_web_lg 3.8.0', } (root / 'source-identity.json').write_text( json.dumps(identity, indent=2, sort_keys=True) + '\n', encoding='utf-8' ) PY - name: Write exact output-rail configurations run: | set -euo pipefail python - <<'PY' import os import socket from pathlib import Path root = Path(os.environ['RUNNER_TEMP']) / 'nemo-source-reproduction' with socket.socket() as listener: listener.bind(('127.0.0.1', 0)) gateway_port = listener.getsockname()[1] with open(os.environ['GITHUB_ENV'], 'a', encoding='utf-8') as env: env.write(f'NEMO_PORT={gateway_port}\n') for name, capture_port in (('operator', 8765), ('response', 8799)): config_root = root / name config_root.mkdir(exist_ok=True) config = f'''models: - type: main engine: openai model: capture parameters: base_url: http://127.0.0.1:{capture_port}/v1 api_key: sk-dummy-not-used rails: config: sensitive_data_detection: output: entities: - EMAIL_ADDRESS - US_SSN - CREDIT_CARD - PHONE_NUMBER output: flows: - detect sensitive data on output streaming: enabled: true stream_first: false context_size: 50 ''' (config_root / 'config.yml').write_text(config, encoding='utf-8') PY - name: Measure request-path behavior id: operator continue-on-error: true uses: ninadphalak/LLM-Shield-Proxy@29a9932b085e5b063b8c366c3e6bfca2c4af4b06 # operator action; the recipe writes the submission with: target-base-url: http://127.0.0.1:${{ env.NEMO_PORT }}/v1 start-command: >- docker run --rm --network host --name nemo-operator-$GITHUB_RUN_ID -e OPENAI_API_KEY=sk-dummy-not-used -v "$RUNNER_TEMP/nemo-source-reproduction/operator:/config:ro" "$SOURCE_IMAGE" --port "$NEMO_PORT" --default-config-id config target-model: config target-version: ${{ env.SOURCE_SELECTOR }} duty: anonymize artifact-name: '' submission-section: 'false' - name: Stage the operator result if: always() && steps.operator.outputs.report-directory != '' env: OPERATOR_REPORT_DIRECTORY: ${{ steps.operator.outputs.report-directory }} run: | set -euo pipefail out="$RUNNER_TEMP/nemo-source-reproduction" for name in current.json summary.md; do if [ -f "$OPERATOR_REPORT_DIRECTORY/$name" ]; then cp "$OPERATOR_REPORT_DIRECTORY/$name" "$out/$name" fi done - name: Measure the 32-case midpoint response profile id: response continue-on-error: true working-directory: benchmark-instrument run: | set -euo pipefail name="nemo-response-$GITHUB_RUN_ID" docker run --rm --network host --name "$name" \ -e OPENAI_API_KEY=sk-dummy-not-used \ -v "$RUNNER_TEMP/nemo-source-reproduction/response:/config:ro" \ "$SOURCE_IMAGE" --port "$NEMO_PORT" --default-config-id config \ > "$RUNNER_TEMP/nemo-source-proxy.log" 2>&1 & docker_pid=$! trap 'docker rm -f "$name" > /dev/null 2>&1 || true; wait "$docker_pid" 2>/dev/null || true' EXIT for _ in $(seq 1 90); do if curl --max-time 1 --silent -o /dev/null "http://127.0.0.1:$NEMO_PORT/v1/models"; then break; fi sleep 2 done curl --max-time 3 --silent -o /dev/null "http://127.0.0.1:$NEMO_PORT/v1/models" python -m pii_leak_benchmark.v2_emitter \ --validate --oracle midpoint --seed a1b2c3d4e5f60001 \ --only nemo-guardrails-0.24.0 \ --gateway-url "http://127.0.0.1:$NEMO_PORT/v1/chat/completions" \ --upstream-port 8799 --model config \ --out "$RUNNER_TEMP/nemo-source-reproduction" \ > "$RUNNER_TEMP/nemo-source-instrument.log" 2>&1 test -s "$RUNNER_TEMP/nemo-source-reproduction/nemo-guardrails-0.24.0.json" mv "$RUNNER_TEMP/nemo-source-reproduction/nemo-guardrails-0.24.0.json" \ "$RUNNER_TEMP/nemo-source-reproduction/nemo-source.json" - name: Verify publishable evidence id: verify if: always() continue-on-error: true env: BENCHMARK_PYTHON: ${{ steps.operator.outputs.python-path }} run: | set -euo pipefail export RESPONSE_PYTHON="$(command -v python)" out="$RUNNER_TEMP/nemo-source-reproduction" "$BENCHMARK_PYTHON" -m pip freeze > "$out/operator-packages.txt" "$BENCHMARK_PYTHON" - <<'PY' import json import os import subprocess from base64 import b64encode, urlsafe_b64encode from html import escape from pathlib import Path from urllib.parse import quote, quote_plus from pii_leak_benchmark.operator_profile import seeded_fixture root = Path(os.environ['RUNNER_TEMP']) / 'nemo-source-reproduction' reports = ('current.json', 'summary.md', 'nemo-source.json', 'source-identity.json') provenance = ('source-commit.txt', 'image-id.txt', 'operator/config.yml', 'response/config.yml', 'response-harness-packages.txt', 'proxy-packages.txt', 'operator-packages.txt', 'python-version.txt', 'runner.txt', 'docker-version.txt') missing = [name for name in reports + provenance if not (root / name).is_file() or not (root / name).stat().st_size] if missing: raise SystemExit('Incomplete evidence: missing or empty ' + ', '.join(missing)) operator = json.loads((root / 'current.json').read_text(encoding='utf-8')) contract = operator.get('contract') or {} if (operator.get('schema') != 'pii-leak-benchmark/operator-run/v1' or operator.get('verdict') not in ('CLEAN', 'LEAK') or contract.get('profile') not in ('pii-v1', 'pii-secrets-v1') or not isinstance(contract.get('seed'), str)): raise SystemExit('Incomplete evidence: operator profile was not measured.') fixture, marker = seeded_fixture(contract['seed'], contract['profile'] == 'pii-secrets-v1') response = subprocess.run( [os.environ['RESPONSE_PYTHON'], '-c', 'import json; from pii_leak_benchmark.v2_emitter import build_segments; ' 's = build_segments("a1b2c3d4e5f60001"); ' 'print(json.dumps(list(s.echo.values()) + list(s.injection.values())))'], capture_output=True, check=True, text=True, ) values = tuple(fixture.values()) + (marker,) + tuple(json.loads(response.stdout)) needles = set(values) for value in values: encoded = value.encode('utf-8') needles.update((quote(value, safe=''), quote_plus(value), escape(value, quote=True), b64encode(encoded).decode(), urlsafe_b64encode(encoded).decode(), encoded.hex(), encoded.hex().upper())) def strings(node): if isinstance(node, str): yield node elif isinstance(node, dict): for key, item in node.items(): yield from strings(key) yield from strings(item) elif isinstance(node, list): for item in node: yield from strings(item) for name in reports: content = (root / name).read_text(encoding='utf-8') fields = (content,) if name.endswith('.md') else strings(json.loads(content)) if any(needle in field for field in fields for needle in needles): raise SystemExit('Unsafe evidence: a generated value reached a report.') report = json.loads((root / 'nemo-source.json').read_text(encoding='utf-8')) if (report.get('schema') != 'llm-shield.streaming-privacy-http-profile/v2.0.0' or (report.get('fixture') or {}).get('values_published') is not False or (report.get('corpus') or {}).get('values_published') is not False or (report.get('metrics') or {}).get('cases_scored') != 32 or ((report.get('capture') or {}).get('self_probe') or {}).get('recorded') is not True): raise SystemExit('Incomplete evidence: response profile contract failed.') (root / 'verification.txt').write_text( 'Required reports complete; generated fixture values absent.\n', encoding='utf-8' ) PY - name: Upload verified source evidence id: upload if: always() && steps.verify.outcome == 'success' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: source-reproduction path: | ${{ runner.temp }}/nemo-source-reproduction/current.json ${{ runner.temp }}/nemo-source-reproduction/summary.md ${{ runner.temp }}/nemo-source-reproduction/nemo-source.json ${{ runner.temp }}/nemo-source-reproduction/source-identity.json ${{ runner.temp }}/nemo-source-reproduction/source-commit.txt ${{ runner.temp }}/nemo-source-reproduction/image-id.txt ${{ runner.temp }}/nemo-source-reproduction/operator/config.yml ${{ runner.temp }}/nemo-source-reproduction/response/config.yml ${{ runner.temp }}/nemo-source-reproduction/response-harness-packages.txt ${{ runner.temp }}/nemo-source-reproduction/proxy-packages.txt ${{ runner.temp }}/nemo-source-reproduction/operator-packages.txt ${{ runner.temp }}/nemo-source-reproduction/python-version.txt ${{ runner.temp }}/nemo-source-reproduction/runner.txt ${{ runner.temp }}/nemo-source-reproduction/docker-version.txt ${{ runner.temp }}/nemo-source-reproduction/verification.txt if-no-files-found: error retention-days: 90 - name: Report the result and how to submit it id: result if: always() continue-on-error: true env: REPORT_DIR: ${{ runner.temp }}/nemo-source-reproduction VERIFY_OUTCOME: ${{ steps.verify.outcome }} RESPONSE_OUTCOME: ${{ steps.response.outcome }} UPLOAD_OUTCOME: ${{ steps.upload.outcome }} REF_CHECK_OUTCOME: ${{ steps.ref_check.outcome }} CHECKOUT_OUTCOME: ${{ steps.checkout.outcome }} BUILD_OUTCOME: ${{ steps.build.outcome }} SOURCE_REF: ${{ inputs.source_ref }} STEP_OUTCOMES: request=${{ steps.operator.outcome }}, response=${{ steps.response.outcome }}, verification=${{ steps.verify.outcome }}, upload=${{ steps.upload.outcome }} run: | python - <<'PY' >> "$GITHUB_STEP_SUMMARY" import json import os from pathlib import Path from urllib.parse import urlencode root = Path(os.environ['REPORT_DIR']) run_url = '{}/{}/actions/runs/{}'.format( os.environ['GITHUB_SERVER_URL'], os.environ['GITHUB_REPOSITORY'], os.environ['GITHUB_RUN_ID']) product = os.environ['WALL_PRODUCT'] status = 'incomplete' # A verdict needs a complete run: verified and uploaded evidence, a passing response # step, and at least one applicable case in every condition of every non-control # response report. Anything less is incomplete, whatever it did observe, because the # summary calls a measured result complete and offers it to the wall. if (os.environ['VERIFY_OUTCOME'] == 'success' and (root / 'verification.txt').is_file() and (root / 'current.json').is_file()): operator = json.loads((root / 'current.json').read_text(encoding='utf-8')) leaked = sorted(name for name, state in (operator.get('entities') or {}).items() if state == 'leak') table, response_leak, measured_reports, unmeasured = [], False, 0, False for path in sorted(root.glob('*.json')): try: metrics = json.loads(path.read_text(encoding='utf-8')).get('metrics') or {} except (ValueError, AttributeError): continue rates = metrics.get('leak_rate') if isinstance(metrics, dict) else None if not isinstance(rates, dict): continue counts = metrics.get('cases_by_condition') or {} # The Shield recipe's redaction-off arm is a control: it is meant to leak. control = path.stem.endswith('-off') cells = [] for key in ('single_chunk', 'adversarial'): rate, total = rates.get(key), counts.get(key) known = isinstance(rate, (int, float)) and isinstance(total, int) and total > 0 cells.append(f'{round(rate * total)} of {total} leaked' if known else 'not measured') response_leak = response_leak or (known and not control and rate > 0) unmeasured = unmeasured or (not known and not control) scored, inconclusive = metrics.get('cases_scored'), metrics.get('cases_inconclusive', 0) applicable = scored - inconclusive if isinstance(scored, int) and isinstance(inconclusive, int) else 0 if not control: measured_reports += 1 unmeasured = unmeasured or applicable <= 0 label = path.stem + (' (control, redaction off)' if control else '') table.append(f"| {label} | {cells[0]} | {cells[1]} | {metrics.get('cases_inconclusive', 0)} |") complete = (os.environ['RESPONSE_OUTCOME'] == 'success' and os.environ['UPLOAD_OUTCOME'] == 'success' and measured_reports > 0 and not unmeasured and operator.get('verdict') in ('CLEAN', 'LEAK')) status = 'incomplete' if not complete else 'leak' if leaked or response_leak else 'clean' reason = '' if status == 'incomplete': # Name the first stage that failed and what to do about it, so an incomplete run # tells its operator more than "open the logs". # The ref is operator input: collapse whitespace so a newline in it cannot add # lines to GITHUB_OUTPUT (such as status=clean) or to the error annotation. source_ref = ' '.join(os.environ.get('SOURCE_REF', '').split()) where = f'`{source_ref}`' if source_ref else 'the latest commit of the branch you ran' new_run = 'Start a new run with **Run workflow**; **Re-run jobs** reuses the same inputs.' rerun = ('This is usually the runner or a download, not the proxy: click **Re-run jobs**, ' 'then **Re-run all jobs**. If it fails the same way twice, ' '[open an issue](https://github.com/ninadphalak/LLM-Shield-Proxy/issues) ' 'with this run\'s link.') stages = ( ('REF_CHECK_OUTCOME', 'A source repository was named without a tag, branch or commit.', 'Type the tag, branch or commit to build in the first box. ' + new_run), ('CHECKOUT_OUTCOME', f'The source to build, {where}, could not be fetched.', 'Check its spelling. A fork copies only its default branch, so a release tag may exist ' 'only in the original repository. Type your fork\'s default branch, such as `main`, or a ' 'commit your fork has. ' + new_run), ('BUILD_OUTCOME', f'The source to build, {where}, did not build.', 'Open the failed step, marked with a red cross in the job\'s step list, for the error. ' 'A download or network error usually passes on a rerun; a compile error means that ' 'commit does not build, so pick another. ' + rerun), ('STARTUP_OUTCOME', 'The proxy was built but did not start.', 'The step "Check proxy startup before fixtures" prints its log. If the log points at ' 'the code, that commit does not start; pick another. ' + rerun), ('RESPONSE_OUTCOME', 'The response-path measurement did not finish.', rerun), ('VERIFY_OUTCOME', 'The evidence did not pass verification.', rerun), ('UPLOAD_OUTCOME', 'The verified evidence could not be uploaded.', rerun), ) failed = next((stage for stage in stages if os.environ.get(stage[0]) == 'failure'), None) reason, todo = (failed[1], failed[2]) if failed else ( 'No step failed outright, but the evidence was not complete enough to count.', rerun) print('## Result: INCOMPLETE, do not submit') print() print('This run did not produce a verified measurement, so nothing was uploaded and the ' 'results wall has nothing to read. It says nothing about the gateway.') print() print(f'**What went wrong:** {reason}') print() print(f'**What to do:** {todo}') print() print(f"Step outcomes: {os.environ['STEP_OUTCOMES']}.") else: print('## Result: MEASURED LEAK' if status == 'leak' else '## Result: MEASURED CLEAN') print() print('This run is a complete, verified measurement. ' + ('The job is red because the gateway leaked, not because the run broke. ' 'A leak is a result worth publishing.' if status == 'leak' else 'No leak in any case this run measured.')) print() print('Request path: ' + ('the provider received raw ' + ', '.join(leaked) + '.' if leaked else 'no raw value reached the provider.')) print() print('| Response report | Value in one chunk | Value split across two chunks | Inconclusive |') print('| :--- | :--- | :--- | ---: |') print('\n'.join(table)) print() # One issue is one row. A run that measured several configurations at once sets # SUBMISSION_HOLD_REASON, and its summary says so in place of the submission link. hold = ' '.join(os.environ.get('SUBMISSION_HOLD_REASON', '').split()) if hold: print('### Not offered to the results wall') print() print(hold) else: identity = json.loads((root / 'source-identity.json').read_text(encoding='utf-8')) commit, selector = identity['source_commit'], identity['source_selector'] where = f'commit {commit[:12]}' if selector in ('', commit) else f'{selector} (commit {commit[:12]})' sections = { 'Gateway': product, 'Version and configuration': f"{where}, {os.environ['WALL_CONFIGURATION']}", 'Project link': os.environ['WALL_PROJECT_URL'], 'CI run link': run_url, 'How it reads the stream': 'not stated', 'License': os.environ['WALL_LICENSE'], 'Outcome': status.upper(), } body = '\n\n'.join(f'### {heading}\n\n{value}' for heading, value in sections.items()) + '\n' query = urlencode({'title': f'Result: {product}', 'labels': 'conformance-result', 'body': body}) print('### Put this run on the results wall') print() print(f'**[Submit this run to the results wall](https://github.com/ninadphalak/LLM-Shield-Proxy/issues/new?{query})**: ' 'opens a prefilled issue. Press **Create**; nothing needs typing.') print() print('Or, from any terminal signed in to the GitHub CLI:') print() print('```') print(f'gh issue create --repo ninadphalak/LLM-Shield-Proxy --title "Result: {product}" --body "{run_url}"') print('```') print() print('Within a few minutes a bot answers on that issue. It either links your published row ' 'or says exactly what stopped it. The numbers on the wall are read from this run\'s ' 'artifact, not from the issue.') with open(os.environ['GITHUB_OUTPUT'], 'a', encoding='utf-8') as output: output.write(f'status={status}\n') output.write(f'reason={reason}\n') PY - name: Fail unless the measurement is complete and clean if: always() && steps.result.outputs.status != 'clean' env: RESULT: ${{ steps.result.outputs.status }} REASON: ${{ steps.result.outputs.reason }} run: | if [ "${RESULT:-incomplete}" = incomplete ]; then echo "::error title=Incomplete, do not submit::${REASON:-The run did not finish.} The run's summary says what to do next." else echo "Result: $RESULT. This is a complete measurement: the red mark means a value leaked, not that the run broke. The job summary has the submission link." fi exit 1