sglang serve

sglang serve

```
usage: sglang serve -h
--model-path MODEL_PATH
--tokenizer-path TOKENIZER_PATH
--tokenizer-mode auto slow
--tokenizer-backend huggingface fastokens
--tokenizer-worker-num TOKENIZER_WORKER_NUM
--detokenizer-worker-num DETOKENIZER_WORKER_NUM
--skip-tokenizer-init
--load-format auto pt safetensors npcache dummy sharded_state gguf bitsandbytes mistral layered flash_rl remote remote_instance fastsafetensors private runai_streamer
--model-loader-extra-config MODEL_LOADER_EXTRA_CONFIG
--trust-remote-code
--context-length CONTEXT_LENGTH
--is-embedding
--enable-multimodal
--revision REVISION
--model-impl MODEL_IMPL
--model-config-parser MODEL_CONFIG_PARSER
--json-model-override-args JSON_MODEL_OVERRIDE_ARGS
--host HOST
--port PORT
--fastapi-root-path FASTAPI_ROOT_PATH
--grpc-mode
--skip-server-warmup
--warmups WARMUPS
--enable-http2
--ssl-keyfile SSL_KEYFILE
--ssl-certfile SSL_CERTFILE
--ssl-ca-certs SSL_CA_CERTS
--ssl-keyfile-password SSL_KEYFILE_PASSWORD
--enable-ssl-refresh
--dtype auto half float16 bfloat16 float float32
--quantization awq fp8 mxfp8 gptq marlin gptq_marlin awq_marlin bitsandbytes gguf modelopt modelopt_fp8 modelopt_fp4 nvfp4_online modelopt_mixed petit_nvfp4 w8a8_int8 w8a8_fp8 moe_wna16 qoq w4afp8 mxfp4 auto-round compressed-tensors modelslim quark quark_int4fp8_moe quark_mxfp4 mlx_q4 mlx_q8 unquant
--quantization-param-path QUANTIZATION_PARAM_PATH
--kv-cache-dtype auto fp8_e5m2 fp8_e4m3 bf16 bfloat16 fp4_e2m1
--enable-fp32-lm-head
--modelopt-quant MODELOPT_QUANT
--modelopt-checkpoint-restore-path MODELOPT_CHECKPOINT_RESTORE_PATH
--modelopt-checkpoint-save-path MODELOPT_CHECKPOINT_SAVE_PATH
--modelopt-export-path MODELOPT_EXPORT_PATH
--quantize-and-serve
--rl-quant-profile RL_QUANT_PROFILE
--mem-fraction-static MEM_FRACTION_STATIC
--max-running-requests MAX_RUNNING_REQUESTS
--max-queued-requests MAX_QUEUED_REQUESTS
--max-total-tokens MAX_TOTAL_TOKENS
--chunked-prefill-size CHUNKED_PREFILL_SIZE
--enable-dynamic-chunking
--max-prefill-tokens MAX_PREFILL_TOKENS
--prefill-max-requests PREFILL_MAX_REQUESTS
--schedule-policy lpm random fcfs dfs-weight lof priority routing-key
--enable-priority-scheduling
--disable-priority-preemption
--default-priority-value DEFAULT_PRIORITY_VALUE
--abort-on-priority-when-disabled
--schedule-low-priority-values-first
--priority-scheduling-preemption-threshold PRIORITY_SCHEDULING_PREEMPTION_THRESHOLD
--schedule-conservativeness SCHEDULE_CONSERVATIVENESS
--page-size PAGE_SIZE
--swa-full-tokens-ratio SWA_FULL_TOKENS_RATIO
--disable-hybrid-swa-memory
--radix-eviction-policy lru lfu slru priority
--prefill-only-disable-kv-cache
--disable-radix-cache
--disable-chunked-prefix-cache
--disable-overlap-schedule
--num-continuous-decode-steps NUM_CONTINUOUS_DECODE_STEPS
--scheduler-recv-interval SCHEDULER_RECV_INTERVAL
--enable-mixed-chunk
--device DEVICE
--base-gpu-id BASE_GPU_ID
--gpu-id-step GPU_ID_STEP
--random-seed RANDOM_SEED
--watchdog-timeout WATCHDOG_TIMEOUT
--soft-watchdog-timeout SOFT_WATCHDOG_TIMEOUT
--sleep-on-idle
--use-ray
--numa-node NUMA_NODE
--gc-threshold GC_THRESHOLD
--nccl-port NCCL_PORT
--dist-timeout DIST_TIMEOUT
--dist-init-addr DIST_INIT_ADDR
--nnodes NNODES
--node-rank NODE_RANK
--tp-size TP_SIZE
--pp-size PP_SIZE
--pp-max-micro-batch-size PP_MAX_MICRO_BATCH_SIZE
--pp-async-batch-depth PP_ASYNC_BATCH_DEPTH
--dp-size DP_SIZE
--load-balance-method auto round_robin follow_bootstrap_room total_requests total_tokens
--attn-cp-size ATTN_CP_SIZE
--moe-dp-size MOE_DP_SIZE
--enable-prefill-cp
--cp-strategy zigzag interleave
--enable-dp-attention
--enable-dp-attention-local-control-broadcast
--enable-dp-lm-head
--enable-attn-tp-input-scattered
--disable-attn-tp-gather
--enable-p2p-check
--stream-interval STREAM_INTERVAL
--batch-notify-size BATCH_NOTIFY_SIZE
--stream-response-default-include-usage
--incremental-streaming-output
--enable-streaming-session
--constrained-json-whitespace-pattern CONSTRAINED_JSON_WHITESPACE_PATTERN
--constrained-json-disable-any-whitespace
--log-level LOG_LEVEL
--log-level-http LOG_LEVEL_HTTP
--log-requests
--log-requests-level 0 1 2 3
--log-requests-format text json
--log-requests-target LOG_REQUESTS_TARGET
--uvicorn-access-log-exclude-prefixes UVICORN_ACCESS_LOG_EXCLUDE_PREFIXES
--crash-dump-folder CRASH_DUMP_FOLDER
--show-time-cost
--enable-metrics
--grpc-http-sidecar-port GRPC_HTTP_SIDECAR_PORT
--enable-mfu-metrics
--enable-metrics-for-all-schedulers
--load-snapshot-publish-interval LOAD_SNAPSHOT_PUBLISH_INTERVAL
--tokenizer-metrics-custom-labels-header TOKENIZER_METRICS_CUSTOM_LABELS_HEADER
--tokenizer-metrics-allowed-custom-labels TOKENIZER_METRICS_ALLOWED_CUSTOM_LABELS
--extra-metric-labels EXTRA_METRIC_LABELS
--bucket-time-to-first-token BUCKET_TIME_TO_FIRST_TOKEN
--bucket-inter-token-latency BUCKET_INTER_TOKEN_LATENCY
--bucket-e2e-request-latency BUCKET_E2E_REQUEST_LATENCY
--prompt-tokens-buckets PROMPT_TOKENS_BUCKETS
--generation-tokens-buckets GENERATION_TOKENS_BUCKETS
--gc-warning-threshold-secs GC_WARNING_THRESHOLD_SECS
--decode-log-interval DECODE_LOG_INTERVAL
--enable-request-time-stats-logging
--kv-events-config KV_EVENTS_CONFIG
--enable-forward-pass-metrics
--enable-trace
--trace-modules TRACE_MODULES
--otlp-traces-endpoint OTLP_TRACES_ENDPOINT
--export-metrics-to-file
--export-metrics-to-file-dir EXPORT_METRICS_TO_FILE_DIR
--api-key API_KEY
--admin-api-key ADMIN_API_KEY
--served-model-name SERVED_MODEL_NAME
--weight-version WEIGHT_VERSION
--chat-template CHAT_TEMPLATE
--hf-chat-template-name HF_CHAT_TEMPLATE_NAME
--completion-template COMPLETION_TEMPLATE
--file-storage-path FILE_STORAGE_PATH
--enable-cache-report
--strip-thinking-cache
--enable-strict-thinking
--tool-server TOOL_SERVER
--sampling-defaults openai model
--asr-max-buffer-seconds ASR_MAX_BUFFER_SECONDS
--asr-max-concurrent-sessions ASR_MAX_CONCURRENT_SESSIONS
--preferred-sampling-params PREFERRED_SAMPLING_PARAMS
--allow-auto-truncate
--enable-prefill-delayer
--prefill-delayer-max-delay-passes PREFILL_DELAYER_MAX_DELAY_PASSES
--prefill-delayer-token-usage-low-watermark PREFILL_DELAYER_TOKEN_USAGE_LOW_WATERMARK
--prefill-delayer-forward-passes-buckets PREFILL_DELAYER_FORWARD_PASSES_BUCKETS
--prefill-delayer-wait-seconds-buckets PREFILL_DELAYER_WAIT_SECONDS_BUCKETS
--prefill-delayer-queue-min-ratio PREFILL_DELAYER_QUEUE_MIN_RATIO
--prefill-delayer-max-delay-ms PREFILL_DELAYER_MAX_DELAY_MS
--enable-lora
--enable-lora-overlap-loading
--max-lora-rank MAX_LORA_RANK
--lora-target-modules q_proj k_proj v_proj o_proj q_a_proj kv_a_proj_with_mqa q_b_proj kv_b_proj wq_b wk weights_proj gate_proj up_proj down_proj qkv_proj gate_up_proj embed_tokens lm_head all
--lora-paths LORA_PATHS
--max-loaded-loras MAX_LOADED_LORAS
--max-loras-per-batch MAX_LORAS_PER_BATCH
--lora-eviction-policy lru fifo
--lora-backend triton csgmv ascend torch_native
--max-lora-chunk-size 16 32 64 128
--experts-shared-outer-loras
--no-experts-shared-outer-loras
--lora-use-virtual-experts
--lora-strict-loading
```

--no-lora-strict-loading
--lora-drain-wait-threshold LORA_DRAIN_WAIT_THRESHOLD
--attention-backend triton torch_native flex_attention dsa nsa dsv4 compressed cutlass_mla fa3 fa4 flashinfer flashmla trtllm_mla cutedsl_mla tokenspeed_mla trtllm_mha dual_chunk_flash_attn aiter wave intel_amx ascend intel_xpu
--decode-attention-backend triton torch_native flex_attention dsa nsa dsv4 compressed cutlass_mla fa3 fa4 flashinfer flashmla trtllm_mla cutedsl_mla tokenspeed_mla trtllm_mha dual_chunk_flash_attn aiter wave intel_amx ascend intel_xpu
--prefill-attention-backend triton torch_native flex_attention dsa nsa dsv4 compressed cutlass_mla fa3 fa4 flashinfer flashmla trtllm_mla cutedsl_mla tokenspeed_mla trtllm_mha dual_chunk_flash_attn aiter wave intel_amx ascend intel_xpu
--sampling-backend ascend pytorch flashinfer
--grammar-backend xgrammar outlines llguidance none
--radix-cache-backend RADIX_CACHE_BACKEND
--mm-attention-backend sdpa fa3 fa4 triton_attn ascend_attn aiter_attn flashinfer_cudnn amx_attn xpu_attn
--fp8-gemm-backend auto deep_gemm flashinfer_trtllm flashinfer_cutlass flashinfer_deepgemm cutlass triton aiter
--fp4-gemm-backend auto cutlass flashinfer_cudnn flashinfer_cutedsl flashinfer_cutlass flashinfer_trtllm marlin
--dsa-prefill-backend flashmla_sparse flashmla_kv flashmla_auto fa3 tilelang aiter trtllm
--dsa-decode-backend flashmla_sparse flashmla_kv flashmla_auto fa3 tilelang aiter trtllm
--dsa-topk-backend sgl-kernel torch flashinfer
--disable-flashinfer-autotune
--mamba-backend triton flashinfer
--speculative-algorithm SPECULATIVE_ALGORITHM
--speculative-draft-model-path SPECULATIVE_DRAFT_MODEL_PATH
--speculative-draft-model-revision SPECULATIVE_DRAFT_MODEL_REVISION
--speculative-draft-load-format auto pt safetensors npcache dummy sharded_state gguf bitsandbytes mistral layered flash_rl remote remote_instance fastsafetensors private runai_streamer
--speculative-num-steps SPECULATIVE_NUM_STEPS
--speculative-eagle-topk SPECULATIVE_EAGLE_TOPK
--speculative-num-draft-tokens SPECULATIVE_NUM_DRAFT_TOKENS
--speculative-dflash-block-size SPECULATIVE_DFLASH_BLOCK_SIZE
--speculative-accept-threshold-single SPECULATIVE_ACCEPT_THRESHOLD_SINGLE
--speculative-accept-threshold-acc SPECULATIVE_ACCEPT_THRESHOLD_ACC
--speculative-use-rejection-sampling
--speculative-token-map SPECULATIVE_TOKEN_MAP
--speculative-attention-mode prefill decode
--speculative-draft-attention-backend SPECULATIVE_DRAFT_ATTENTION_BACKEND
--speculative-draft-window-size SPECULATIVE_DRAFT_WINDOW_SIZE
--speculative-moe-runner-backend auto deep_gemm triton triton_kernel flashinfer_trtllm experimental_sgl_trtllm flashinfer_trtllm_routed flashinfer_cutlass flashinfer_mxfp4 flashinfer_cutedsl cutlass aiter marlin
--speculative-moe-a2a-backend none deepep mooncake nixl mori ascend_fuseep flashinfer megamoe
--speculative-draft-model-quantization awq fp8 mxfp8 gptq marlin gptq_marlin awq_marlin bitsandbytes gguf modelopt modelopt_fp8 modelopt_fp4 nvfp4_online modelopt_mixed petit_nvfp4 w8a8_int8 w8a8_fp8 moe_wna16 qoq w4afp8 mxfp4 auto-round compressed-tensors modelslim quark quark_int4fp8_moe quark_mxfp4 mlx_q4 mlx_q8 unquant
--speculative-skip-dp-mlp-sync
--enable-multi-layer-eagle
--speculative-adaptive
--speculative-adaptive-config SPECULATIVE_ADAPTIVE_CONFIG
--speculative-ngram-min-bfs-breadth SPECULATIVE_NGRAM_MIN_BFS_BREADTH
--speculative-ngram-max-bfs-breadth SPECULATIVE_NGRAM_MAX_BFS_BREADTH
--speculative-ngram-match-type BFS PROB
--speculative-ngram-max-trie-depth SPECULATIVE_NGRAM_MAX_TRIE_DEPTH
--speculative-ngram-capacity SPECULATIVE_NGRAM_CAPACITY
--speculative-ngram-external-corpus-path SPECULATIVE_NGRAM_EXTERNAL_CORPUS_PATH
--speculative-ngram-external-sam-budget SPECULATIVE_NGRAM_EXTERNAL_SAM_BUDGET
--speculative-ngram-external-corpus-max-tokens SPECULATIVE_NGRAM_EXTERNAL_CORPUS_MAX_TOKENS
--ep-size EP_SIZE
--moe-a2a-backend none deepep mooncake nixl mori ascend_fuseep flashinfer megamoe
--moe-runner-backend auto deep_gemm triton triton_kernel flashinfer_trtllm experimental_sgl_trtllm flashinfer_trtllm_routed flashinfer_cutlass flashinfer_mxfp4 flashinfer_cutedsl cutlass aiter marlin
--flashinfer-mxfp4-moe-precision default bf16
--deepep-mode auto normal low_latency
--deepep-dispatcher-output-dtype auto bf16 fp8 int8 nvfp4
--ep-num-redundant-experts EP_NUM_REDUNDANT_EXPERTS
--ep-dispatch-algorithm static dynamic fake lp
--init-expert-location INIT_EXPERT_LOCATION
--enable-eplb
--eplb-algorithm EPLB_ALGORITHM

--eplb-rebalance-num-iterations EPLB_REBALANCE_NUM_ITERATIONS
--eplb-rebalance-layers-per-chunk EPLB_REBALANCE_LAYERS_PER_CHUNK
--eplb-min-rebalancing-utilization-threshold EPLB_MIN_REBALANCING_UTILIZATION_THRESHOLD
--expert-distribution-recorder-mode stat stat_approx per_pass per_token
--expert-distribution-recorder-buffer-size EXPERT_DISTRIBUTION_RECORDER_BUFFER_SIZE
--enable-expert-distribution-metrics
--deepep-config DEEPEP_CONFIG
--moe-dense-tp-size MOE_DENSE_TP_SIZE
--elastic-ep-backend none mooncake nixl
--enable-elastic-expert-backup
--mooncake-ib-device MOONCAKE_IB_DEVICE
--enable-deepep-waterfill
--elastic-ep-rejoin
--disable-flashinfer-cutlass-moe-fp4-allgather
--disable-shared-experts-fusion
--enforce-shared-experts-fusion
--max-mamba-cache-size MAX_MAMBA_CACHE_SIZE
--mamba-ssm-dtype float32 bfloat16 float16
--mamba-full-memory-ratio MAMBA_FULL_MEMORY_RATIO
--mamba-radix-cache-strategy auto no_buffer extra_buffer extra_buffer_lazy
--mamba-track-interval MAMBA_TRACK_INTERVAL
--enable-int8-mamba-checkpoint
--int8-mamba-ckpt-size INT8_MAMBA_CKPT_SIZE
--linear-attn-backend triton cutedsl flashinfer
--linear-attn-decode-backend triton cutedsl flashinfer
--linear-attn-prefill-backend triton cutedsl flashinfer
--enable-hierarchical-cache
--hicache-ratio HICACHE_RATIO
--hicache-size HICACHE_SIZE
--hicache-write-policy write_back write_through write_through_selective
--hicache-io-backend direct kernel kernel_ascend
--hicache-mem-layout layer_first page_first page_first_direct page_first_kv_split page_head
--hicache-storage-backend file mooncake hf3fs nixl aibrix dynamic eic simm
--hicache-storage-prefetch-policy best_effort wait_complete timeout
--hicache-storage-backend-extra-config HICACHE_STORAGE_BACKEND_EXTRA_CONFIG
--enable-hisparse
--hisparse-config HISPARSE_CONFIG
--enable-lmcache
--lmcache-config-file LMCACHE_CONFIG_FILE
--kt-weight-path KT_WEIGHT_PATH
--kt-method KT_METHOD
--kt-cpuinfer KT_CPUINFER
--kt-threadpool-count KT_THREADPOOL_COUNT
--kt-num-gpu-experts KT_NUM_GPU_EXPERTS
--kt-max-deferred-experts-per-token KT_MAX_DEFERRED_EXPERTS_PER_TOKEN
--dllm-algorithm DLLM_ALGORITHM
--dllm-algorithm-config DLLM_ALGORITHM_CONFIG
--cpu-offload-gb CPU_OFFLOAD_GB
--offload-group-size OFFLOAD_GROUP_SIZE
--offload-num-in-group OFFLOAD_NUM_IN_GROUP
--offload-prefetch-step OFFLOAD_PREFETCH_STEP
--offload-mode OFFLOAD_MODE
--cuda-graph-config CUDA_GRAPH_CONFIG
--cuda-graph-backend-decode full breakable tc_piecewise disabled
--cuda-graph-backend-prefill full breakable tc_piecewise disabled
--cuda-graph-max-bs-decode CUDA_GRAPH_MAX_BS_DECODE
--cuda-graph-max-bs-prefill CUDA_GRAPH_MAX_BS_PREFILL
--cuda-graph-bs-decode CUDA_GRAPH_BS_DECODE
--cuda-graph-bs-prefill CUDA_GRAPH_BS_PREFILL
--cuda-graph-tc-compiler eager inductor
--disable-prefill-cuda-graph
--disable-decode-cuda-graph
--disable-cuda-graph-padding
--enable-profile-cuda-graph
--enable-cudagraph-gc
--debug-cuda-graph
--enable-layerwise-nvtx-marker
--enable-nccl-nvls
--enable-symm-mem
--triton-attention-reduce-in-fp32
--triton-attention-num-kv-splits TRITON_ATTENTION_NUM_KV_SPLITS
--triton-attention-split-tile-size TRITON_ATTENTION_SPLIT_TILE_SIZE
--flashinfer-mla-disable-ragged
--enable-fused-qk-norm-rope
--enable-precise-embedding-interpolation
--enable-fused-moe-sum-all-reduce
--enable-deepseek-v4-fp4-indexer
--disable-custom-all-reduce
--enable-mscclpp
--enable-torch-symm-mem
--pre-warm-nccl
--enable-quant-communications
--enforce-disable-flashinfer-allreduce-fusion
--flashinfer-allreduce-fusion-backend auto trtllm mnnvl
--enable-aiter-allreduce-fusion
--enable-two-batch-overlap
--enable-single-batch-overlap
--tbo-token-distribution-threshold TBO_TOKEN_DISTRIBUTION_THRESHOLD
--enable-torch-compile
--enable-torch-compile-debug-mode
--torch-compile-max-bs TORCH_COMPILE_MAX_BS
--torchao-config TORCHAO_CONFIG
--enable-memory-saver
--enable-weights-cpu-backup
--enable-draft-weights-cpu-backup
--enable-custom-logit-processor
--enable-return-hidden-states

--enable-return-routed-experts
--enable-return-indexer-topk
--disable-outlines-disk-cache
--enable-mis
--enable-deterministic-inference
--rl-on-policy-target fsdp
--kv-canary none log raise
--kv-canary-sweep-interval KV_CANARY_SWEEP_INTERVAL
--enable-dynamic-batch-tokenizer
--dynamic-batch-tokenizer-batch-size DYNAMIC_BATCH_TOKENIZER_BATCH_SIZE
--dynamic-batch-tokenizer-batch-timeout DYNAMIC_BATCH_TOKENIZER_BATCH_TIMEOUT
--enable-tokenizer-batch-encode
--disable-tokenizer-batch-decode
--debug-tensor-dump-output-folder DEBUG_TENSOR_DUMP_OUTPUT_FOLDER
--debug-tensor-dump-layers DEBUG_TENSOR_DUMP_LAYERS
--debug-tensor-dump-input-file DEBUG_TENSOR_DUMP_INPUT_FILE
--disaggregation-mode null prefill decode
--disaggregation-transfer-backend mooncake nixl ascend fake mori mooncake_tcp
--disaggregation-bootstrap-port DISAGGREGATION_BOOTSTRAP_PORT
--disaggregation-ib-device DISAGGREGATION_IB_DEVICE
--disaggregation-decode-enable-radix-cache
--disaggregation-decode-enable-offload-kvcache
--num-reserved-decode-tokens NUM_RESERVED_DECODE_TOKENS
--disaggregation-decode-extra-slots DISAGGREGATION_DECODE_EXTRA_SLOTS
--disaggregation-decode-polling-interval DISAGGREGATION_DECODE_POLLING_INTERVAL
--optimistic-prefill-retries OPTIMISTIC_PREFILL_RETRIES
--encoder-only
--language-only
--encoder-transfer-backend zmq_to_scheduler zmq_to_tokenizer mooncake
--encoder-urls ENCODER_URLS
--encoder-bootstrap-port ENCODER_BOOTSTRAP_PORT
--encoder-register-urls ENCODER_REGISTER_URLS
--enable-adaptive-dispatch-to-encoder
--enable-pdmux
--pdmux-config-path PDMUX_CONFIG_PATH
--sm-group-num SM_GROUP_NUM
--custom-weight-loader CUSTOM_WEIGHT_LOADER
--weight-loader-disable-mmap
--weight-loader-prefetch-checkpoints
--weight-loader-prefetch-num-threads WEIGHT_LOADER_PREFETCH_NUM_THREADS
--weight-loader-drop-cache-after-load
--remote-instance-weight-loader-seed-instance-ip REMOTE_INSTANCE_WEIGHT_LOADER_SEED_INSTANCE_IP
--remote-instance-weight-loader-seed-instance-service-port REMOTE_INSTANCE_WEIGHT_LOADER_SEED_INSTANCE_SERVICE_PORT
--remote-instance-weight-loader-send-weights-group-ports REMOTE_INSTANCE_WEIGHT_LOADER_SEND_WEIGHTS_GROUP_PORTS
--remote-instance-weight-loader-backend transfer_engine nccl modelexpress
--remote-instance-weight-loader-start-seed-via-transfer-engine
--engine-info-bootstrap-port ENGINE_INFO_BOOTSTRAP_PORT
--modelexpress-config MODELEXPRESS_CONFIG
--download-dir DOWNLOAD_DIR
--model-checksum MODEL_CHECKSUM
--delete-ckpt-after-loading
--decrypted-config-file DECRYPTED_CONFIG_FILE
--decrypted-draft-config-file DECRYPTED_DRAFT_CONFIG_FILE
--checkpoint-engine-wait-weights-before-ready
--enable-broadcast-mm-inputs-process
--enable-prefix-mm-cache
--mm-enable-dp-encoder
--mm-process-config MM_PROCESS_CONFIG
--limit-mm-data-per-request LIMIT_MM_DATA_PER_REQUEST
--enable-mm-global-cache
--disable-fast-image-processor
--keep-mm-feature-on-device
--forward-hooks FORWARD_HOOKS
--msprobe-dump-config MSPROBE_DUMP_CONFIG
--reasoning-parser auto apertus2509 deepseek-r1 deepseek-v3 deepseek-v4 glm45 hunyuan gpt-oss kimi kimi_k2 mimo poolside_v1 qwen3 qwen3-thinking minimax minimax-append-think step3 step3p5 mistral nemotron_3 interns1 gemma4 cohere_command4
--tool-call-parser auto apertus2509 cohere_command4 deepseekv3 deepseekv31 deepseekv32 deepseekv4 glm glm45 glm47 gpt-oss kimi_k2 lfm2 llama3 mimo minicpm5 mistral poolside_v1 pythonic qwen qwen25 qwen3_coder step3 step3p5 minimax-m2 trinity interns1 hermes hunyuan gigachat3 gemma4
--kv-canary-real-data none partial all
--config CONFIG
--stream-output
--prefill-round-robin-balance
--collect-tokens-histogram
--nsa-prefill-backend flashmla_sparse flashmla_kv flashmla_auto fa3 tilelang aiter trtllm
--nsa-decode-backend flashmla_sparse flashmla_kv flashmla_auto fa3 tilelang aiter trtllm
--mamba-scheduler-strategy MAMBA_RADIX_CACHE_STRATEGY
--cuda-graph-max-bs CUDA_GRAPH_MAX_BS_DECODE
--cuda-graph-bs CUDA_GRAPH_BS_DECODE
--disable-cuda-graph
--enable-breakable-cuda-graph
--disable-piecewise-cuda-graph
--enforce-piecewise-cuda-graph
--piecewise-cuda-graph-tokens CUDA_GRAPH_BS_PREFILL
--piecewise-cuda-graph-compiler eager inductor
--piecewise-cuda-graph-max-tokens CUDA_GRAPH_MAX_BS_PREFILL
--enable-dsa-prefill-context-parallel
--enable-nsa-prefill-context-parallel
--enable-prefill-context-parallel
--dsa-prefill-cp-mode in-seq-split round-robin-split
--nsa-prefill-cp-mode in-seq-split round-robin-split
--prefill-cp-mode in-seq-split
--enable-flashinfer-allreduce-fusion