Download model: https://huggingface.co/neroued/Qwen3.8-27B-nvfp4-NInfer/tree/main export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/lib64 export CUDA_HOME=/usr/local/cuda/ export PATH=$PATH:/usr/local/cuda/bin sudo apt install libavformat-dev libavcodec-dev libavutil-dev libswscale-dev sudo apt update sudo apt install libcurl4-openssl-dev git clone https://github.com/Neroued/ninfer cd ninfer-master cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release cmake --build build -j ./build/apps/ninfer-serve /data/G/LLM/Qwen3.8-27B-nvfp4-NInfer_neroued/qwen3_8_27b_nvfp4.ninfer \ --model-id qwen3.8-27b \ --port 8099 \ --host 0.0.0.0 \ --max-context 262144 \ --kv-capacity 262144 \ --max-concurrency 2 \ --kv-dtype fp8 \ --device-state-slots 2 \ --host-state-slots 8 \ --host-kv-mib 8192 \ --spec mtp --draft-tokens 3 \ --lm-head-draft \ --preserve-thinking \ --api-key 123456 prefill 3.75k tok/s (18,760 tok) | decode 14.0 tok/s (70 tok) prefill 1.48k tok/s (7,402 tok) | decode 111.0 tok/s (555 tok) decode 154.4 tok/s (772 tok) decode 137.0 tok/s (685 tok) 30297MiB / 32607MiB