# Generated by serving_tasks/_manage/taskctl.py. Do not hand-edit. kind: serving_task_reference schema: serverbench.task_reference/1 task_id: qwen3_5_9b_standard_2xL40 correctness: rule: replay_fidelity_and_held_out_answers_vs_plain_checkpoint reference_arm: sglang answer_sample_size: 20 max_absolute_accuracy_drop: 0.2 max_token_regret: 4.0 eos_audit: true arms: - framework: sglang serve_extra: --context-length 1048576 --json-model-override-args '{"text_config":{"max_position_embeddings":1048576,"rope_parameters":{"rope_type":"yarn","factor":4.0,"original_max_position_embeddings":262144,"mrope_interleaved":true,"mrope_section":[11,11,10],"rope_theta":10000000,"partial_rotary_factor":0.25}}}' --mem-fraction-static 0.90 --max-running-requests {concurrency} - framework: vllm serve_extra: --max-model-len 1048576 --hf-overrides '{"text_config":{"max_position_embeddings":1048576,"rope_parameters":{"rope_type":"yarn","factor":4.0,"original_max_position_embeddings":262144,"mrope_interleaved":true,"mrope_section":[11,11,10],"rope_theta":10000000,"partial_rotary_factor":0.25}}}' --gpu-memory-utilization 0.90 --max-num-seqs {concurrency} - framework: vllm-omni serve_extra: --max-model-len 1048576 --hf-overrides '{"text_config":{"max_position_embeddings":1048576,"rope_parameters":{"rope_type":"yarn","factor":4.0,"original_max_position_embeddings":262144,"mrope_interleaved":true,"mrope_section":[11,11,10],"rope_theta":10000000,"partial_rotary_factor":0.25}}}' --gpu-memory-utilization 0.90 --max-num-seqs {concurrency} - framework: tensorrt-llm serve_extra: --max_seq_len 1048576 --max_batch_size {concurrency} # sglang_settings: generated from arms[sglang].serve_extra by `taskctl render`; # change the launch line, not this block. sglang_settings: image: lmsysorg/sglang:v0.5.19 context-length: 1048576 json-model-override-args: '{"text_config":{"max_position_embeddings":1048576,"rope_parameters":{"rope_type":"yarn","factor":4.0,"original_max_position_embeddings":262144,"mrope_interleaved":true,"mrope_section":[11,11,10],"rope_theta":10000000,"partial_rotary_factor":0.25}}}' mem-fraction-static: 0.9 max-running-requests: '{concurrency}' concurrency_search: arm: sglang image: lmsysorg/sglang:v0.5.19 window_seconds: 1800 selection_rule: max_gross_dollars_per_dollar_over_p90_e2e_seconds adopted: 1 points: - label: search-c1 offered: 1 random_seed: 8304790408129311260 requests: 90.0 ttft_p90_ms: 3958.5 tpot_p90_ms: 20.68 e2e_p90_s: 55.03 output_tokens_per_s: 41.25 gross_dollars_per_dollar: 2.5042 replay_fidelity: true speculative_decoding: false - label: search-c2 offered: 2 random_seed: 7480629484020748344 requests: 89.0 ttft_p90_ms: 3899.1 tpot_p90_ms: 20.93 e2e_p90_s: 53.3 output_tokens_per_s: 51.54 gross_dollars_per_dollar: 1.404 replay_fidelity: true speculative_decoding: false - label: search-c4 offered: 4 random_seed: 2526591610271297258 requests: 116.0 ttft_p90_ms: 3611.4 tpot_p90_ms: 29.57 e2e_p90_s: 77.27 output_tokens_per_s: 63.85 gross_dollars_per_dollar: 2.0415 replay_fidelity: true speculative_decoding: false - label: search-c8 offered: 8 random_seed: 6832514070830097198 requests: 166.0 ttft_p90_ms: 3139.7 tpot_p90_ms: 33.89 e2e_p90_s: 74.09 output_tokens_per_s: 87.77 gross_dollars_per_dollar: 2.988 replay_fidelity: true speculative_decoding: false - label: search-c16 offered: 16 random_seed: 3567178020008677088 requests: 113.0 ttft_p90_ms: 252037.8 tpot_p90_ms: 203.9 e2e_p90_s: 317.67 output_tokens_per_s: 30.94 gross_dollars_per_dollar: 1.5224 replay_fidelity: true speculative_decoding: false - label: search-c32 offered: 32 random_seed: 3313083194911214464 requests: 66.0 ttft_p90_ms: 553071.3 tpot_p90_ms: 404.94 e2e_p90_s: 728.05 output_tokens_per_s: 35.23 gross_dollars_per_dollar: 0.9286 replay_fidelity: true speculative_decoding: false - label: search-c64 offered: 64 random_seed: 2388795154496059312 requests: 44.0 ttft_p90_ms: 1299016.8 tpot_p90_ms: 344.88 e2e_p90_s: 1425.32 output_tokens_per_s: 29.96 gross_dollars_per_dollar: 0.9262 replay_fidelity: true speculative_decoding: false slo_derivation: rule: the worst (highest) over 5 random seeds of SGLang's per-run p90 TTFT and p90 TPOT, at the searched operating point sglang_worst_p90_ttft_ms: 4100.2848398999995 sglang_worst_p90_tpot_ms: 23.497353185237426 multiplier: 1.0 p90_ttft_ms: 4100.2848398999995 p90_tpot_ms: 23.497353185237426 runs: - label: slo-c1-s1 offered: 1 random_seed: 6169357301623660236 requests: 106.0 ttft_p90_ms: 2680.1 tpot_p90_ms: 19.5 e2e_p90_s: 54.74 output_tokens_per_s: 52.24 gross_dollars_per_dollar: 1.6744 replay_fidelity: true speculative_decoding: false - label: slo-c1-s2 offered: 1 random_seed: 7722860180808051792 requests: 89.0 ttft_p90_ms: 3959.6 tpot_p90_ms: 20.81 e2e_p90_s: 54.35 output_tokens_per_s: 39.32 gross_dollars_per_dollar: 2.503 replay_fidelity: true speculative_decoding: false - label: slo-c1-s3 offered: 1 random_seed: 1857954363362789946 requests: 98.0 ttft_p90_ms: 4100.3 tpot_p90_ms: 23.5 e2e_p90_s: 53.43 output_tokens_per_s: 53.9 gross_dollars_per_dollar: 1.3396 replay_fidelity: true speculative_decoding: false - label: slo-c1-s4 offered: 1 random_seed: 4626124851300198492 requests: 103.0 ttft_p90_ms: 2943.4 tpot_p90_ms: 19.38 e2e_p90_s: 55.32 output_tokens_per_s: 53.5 gross_dollars_per_dollar: 1.6499 replay_fidelity: true speculative_decoding: false - label: slo-c1-s5 offered: 1 random_seed: 199655142063407422 requests: 103.0 ttft_p90_ms: 2913.0 tpot_p90_ms: 19.19 e2e_p90_s: 55.41 output_tokens_per_s: 53.5 gross_dollars_per_dollar: 1.651 replay_fidelity: true speculative_decoding: false