NVIDIA BF16 Hugging Face checkpoint -> 

Qualcomm Llama 3.1 8B PyTorch wrapper -> 

Fixed-shape ONNX graphs -> 

AIMET-ONNX W4A16 calibration -> 

model.encodings + model.data + ONNX graphs -> 

Qualcomm AI Hub Workbench
  - split into five model parts
  - compile prompt processor (sequence length 128)
  - compile token generator (sequence length 1)
  - link shared-weight binaries for QCS9075 -> 

Five-bin Genie bundle -> 

QAIRT 2.45 + Genie + QnnHtp -> 

Hexagon v73 on the physical IQ-9075 EVK

mkdir -p ~/repos-native/qai-nemotron
cd ~/repos-native/qai-nemotron

C:\Users\<YOUR_WINDOWS_USER>\.wslconfig

[wsl2]
memory=176GB
swap=96GB

wsl --shutdown

free -h
swapon --show
grep -E 'MemTotal|SwapTotal' /proc/meminfo

df -h ~

conda create -n qai-qcom310 python=3.10 pip -y
conda activate qai-qcom310

unset PYTHONPATH
export PYTHONNOUSERSITE=1

python -m pip install --upgrade pip setuptools wheel

which python
python -V

/home/<user>/miniconda3/envs/qai-qcom310/bin/python
Python 3.10.x

python -m pip install \
  torch==2.7.1 \
  torchvision==0.22.1 \
  --index-url https://download.pytorch.org/whl/cu128

python - <<'PY'
import torch

print('torch:', torch.__version__)
print('CUDA runtime:', torch.version.cuda)
print('CUDA available:', torch.cuda.is_available())
print('GPU:', torch.cuda.get_device_name(0))
print('capability:', torch.cuda.get_device_capability(0))

x = torch.randn((1024, 1024), device='cuda')
y = x @ x
torch.cuda.synchronize()
print('CUDA matmul passed:', float(y[0, 0]))
PY

cat > /tmp/qai-qcom310-constraints.txt <<'EOF'
numpy==1.26.4
torch==2.7.1
torchvision==0.22.1
transformers==4.45.0
onnx==1.18.0
onnxsim==0.5.0
EOF

python -m pip install \
  -c /tmp/qai-qcom310-constraints.txt \
  numpy==1.26.4 \
  qai-hub-models==0.56.0

python -m pip install \
  -c /tmp/qai-qcom310-constraints.txt \
  transformers==4.45.0 \
  sentencepiece==0.2.1 \
  psutil==6.1.1 \
  onnx==1.18.0 \
  onnxsim==0.5.0

python -m pip install \
  'https://github.com/quic/aimet/releases/download/2.26.0/aimet_onnx-2.26.0+cu121-cp310-cp310-manylinux_2_34_x86_64.whl'

python -m pip uninstall -y onnxruntime onnxruntime-gpu
python -m pip install --no-deps onnxruntime-gpu==1.23.2

RuntimeError: Unable to run function PtrToInt64
libpython3.10.so.1.0: cannot open shared object file

export LD_LIBRARY_PATH="$CONDA_PREFIX/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"

mkdir -p "$CONDA_PREFIX/etc/conda/activate.d"

cat > "$CONDA_PREFIX/etc/conda/activate.d/qai-libpython.sh" <<'EOF'
export LD_LIBRARY_PATH="$CONDA_PREFIX/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
EOF

python - <<'PY'
import ctypes
lib = ctypes.CDLL('libpython3.10.so.1.0', mode=ctypes.RTLD_GLOBAL)
print('Loaded:', lib._name)
PY

python - <<'PY'
import sys
from importlib.metadata import PackageNotFoundError, version
import torch
import onnxruntime as ort
import aimet_onnx

packages = [
    'qai-hub-models', 'aimet-onnx', 'onnx',
    'onnxruntime-gpu', 'transformers', 'torch',
    'torchvision', 'numpy'
]

print('Python:', sys.version)
for package in packages:
    try:
        print(f'{package:22} {version(package)}')
    except PackageNotFoundError:
        print(f'{package:22} not installed')

print('GPU:', torch.cuda.get_device_name(0))
print('Capability:', torch.cuda.get_device_capability(0))
print('ORT providers:', ort.get_available_providers())
print('AIMET:', aimet_onnx.__file__)
PY

Python                 3.10.20
qai-hub-models         0.56.0
aimet-onnx             2.26.0+cu121
onnx                   1.18.0
onnxruntime-gpu        1.23.2
transformers           4.45.0
torch                  2.7.1+cu128
torchvision            0.22.1+cu128

python -m pip freeze > qai-qcom310-successful-freeze.txt

python - <<'PY'
import gc
import numpy as np
from onnx import TensorProto, helper, numpy_helper
from aimet_onnx.quantsim import QuantizationSimModel

input_info = helper.make_tensor_value_info('input', TensorProto.FLOAT, [1, 4])
output_info = helper.make_tensor_value_info('output', TensorProto.FLOAT, [1, 4])
weight = numpy_helper.from_array(np.eye(4, dtype=np.float32), name='weight')
node = helper.make_node('MatMul', ['input', 'weight'], ['output'])
graph = helper.make_graph([node], 'aimet_test', [input_info], [output_info], [weight])
model = helper.make_model(graph, opset_imports=[helper.make_opsetid('', 13)])
model.ir_version = 9

sim = QuantizationSimModel(
    model=model,
    providers=['CUDAExecutionProvider', 'CPUExecutionProvider'],
)

print('QuantSim providers:', sim.session.get_providers())
print('AIMET libpymo test passed')
del sim
gc.collect()
PY

QuantSim providers: ['CUDAExecutionProvider', 'CPUExecutionProvider']
AIMET libpymo test passed

python -m pip install 'huggingface_hub[cli]'
hf auth login

cd ~/repos-native/qai-nemotron
conda activate qai-qcom310
export LD_LIBRARY_PATH="$CONDA_PREFIX/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"

rm -rf nemotron_validation_ckpt_py310_aimet226

PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
python -m qai_hub_models.models.llama_v3_1_8b_instruct.quantize \
  --checkpoint nvidia/Llama-3.1-Nemotron-Nano-8B-v1 \
  --context-length 512 \
  --calibration-sequence-length 128 \
  --num-samples 1 \
  --output-dir nemotron_validation_ckpt_py310_aimet226

TracerWarning: Converting a tensor to a Python boolean...

The target quantizers could not be found. MatMul exception rule does not apply...

Token indices sequence length is longer than the specified maximum...

python -m qai_hub_models.models.llama_v3_1_8b_instruct.demo \
  --checkpoint nemotron_validation_ckpt_py310_aimet226 \
  --context-length 512 \
  --sequence-length 128 \
  --max-output-tokens 16 \
  --seed 42 \
  --prompt 'Explain gravity in one short English sentence.'

Switching from sequence_length=128 to sequence_length=1

FULL_CKPT=nemotron_nano_w4a16_cl4096_py310_aimet226
rm -rf "$FULL_CKPT"

PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
/usr/bin/time -v \
python -m qai_hub_models.models.llama_v3_1_8b_instruct.quantize \
  --checkpoint nvidia/Llama-3.1-Nemotron-Nano-8B-v1 \
  --context-length 4096 \
  --calibration-sequence-length 2048 \
  --num-samples 20 \
  --output-dir "$FULL_CKPT" \
  2>&1 | tee quantize_cl4096_aimet226.log

Quantization completed successfully.
Elapsed time: 44:58.61
Maximum resident set size: 182062912 KiB
Exit status: 0

model.encodings
model.data
model_seqlen2048_cl4096.onnx
model_seqlen128_cl4096.onnx
model_seqlen1_cl4096.onnx
config.json
tokenizer.json
tokenizer_config.json
args.json

CKPT=nemotron_nano_w4a16_cl4096_py310_aimet226

python -m qai_hub_models.models.llama_v3_1_8b_instruct.demo \
  --checkpoint "$CKPT" \
  --context-length 4096 \
  --sequence-length 128 \
  --max-output-tokens 32 \
  --seed 42 \
  --prompt 'Explain gravity in one short English sentence.'

Gravity is the force that causes things to attractively accelerate towards...

qai-hub configure --api_token YOUR_API_TOKEN

CKPT=nemotron_nano_w4a16_cl4096_py310_aimet226
OUT=nemotron_iq9075_genie

rm -rf "$OUT"

python -m qai_hub_models.models.llama_v3_1_8b_instruct.export \
  --checkpoint "$CKPT" \
  --device 'Dragonwing IQ-9075 EVK' \
  --context-length 4096 \
  --sequence-length 128,1 \
  --model-cache-mode disable \
  --skip-profiling \
  --skip-inferencing \
  --output-dir "$OUT"

nemotron_iq9075_genie/
  llama_v3_1_8b_instruct-genie-w4a16-qualcomm_qcs9075/

...part_1_of_5.bin
...part_2_of_5.bin
...part_3_of_5.bin
...part_4_of_5.bin
...part_5_of_5.bin
genie_config.json
htp_backend_ext_config.json
tokenizer.json
tool-versions.yaml

cat "$OUT"/*/tool-versions.yaml

qairt: 2.45.0.260326154327

BUNDLE="$OUT/llama_v3_1_8b_instruct-genie-w4a16-qualcomm_qcs9075"

rsync -avh --progress \
  "$BUNDLE/" \
  ubuntu@EVK_IP:~/nemotron_genie/

sudo apt-get update
sudo apt-get install -y curl ca-certificates unzip rsync

QAIRT_VER='2.45.0.260326'
QAIRT_ZIP="/tmp/v${QAIRT_VER}.zip"
QAIRT_URL="https://softwarecenter.qualcomm.com/api/download/software/sdks/Qualcomm_AI_Runtime_Community/All/${QAIRT_VER}/v${QAIRT_VER}.zip"

curl -fL --retry 3 "$QAIRT_URL" -o "$QAIRT_ZIP"

TMP_UNZIP="$(mktemp -d)"
unzip -q "$QAIRT_ZIP" -d "$TMP_UNZIP"

sudo mkdir -p "/opt/qairt/${QAIRT_VER}"

if [ -d "$TMP_UNZIP/qairt/${QAIRT_VER}" ]; then
  sudo rsync -a "$TMP_UNZIP/qairt/${QAIRT_VER}/" "/opt/qairt/${QAIRT_VER}/"
else
  sudo rsync -a "$TMP_UNZIP/" "/opt/qairt/${QAIRT_VER}/"
fi

sudo ln -sfn "/opt/qairt/${QAIRT_VER}" /opt/qairt/current
sudo chmod -R a+rX "/opt/qairt/${QAIRT_VER}"

rm -rf "$TMP_UNZIP" "$QAIRT_ZIP"

cat > "$HOME/qairt-env.sh" <<'EOF'
#!/usr/bin/env bash

export QAIRT_HOME='/opt/qairt/current'
export QAIRT_SDK_ROOT="$QAIRT_HOME"
export QNN_SDK_ROOT="$QAIRT_HOME"

export QAIRT_TARGET='aarch64-oe-linux-gcc11.2'
export PRODUCT_SOC='9075'
export DSP_ARCH='73'

# Use one QAIRT/QNN installation only.
export PATH="$QAIRT_HOME/bin/$QAIRT_TARGET:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
export LD_LIBRARY_PATH="$QAIRT_HOME/lib/$QAIRT_TARGET:/usr/lib/aarch64-linux-gnu:/lib/aarch64-linux-gnu"
export ADSP_LIBRARY_PATH="$QAIRT_HOME/lib/hexagon-v73/unsigned"
EOF

chmod +x "$HOME/qairt-env.sh"
source "$HOME/qairt-env.sh"
hash -r

printf 'QAIRT_HOME=%s\n' "$QAIRT_HOME"
printf 'LD_LIBRARY_PATH=%s\n' "$LD_LIBRARY_PATH"
printf 'ADSP_LIBRARY_PATH=%s\n' "$ADSP_LIBRARY_PATH"

type -a genie-t2t-run
readlink -f "$(command -v genie-t2t-run)"

Failed to create device: 14001
Device Creation failure

sudo apt-get update
sudo apt-get install -y software-properties-common acl

if ! grep -Rqs 'ubuntu-qcom-iot/qcom-ppa' \
    /etc/apt/sources.list /etc/apt/sources.list.d 2>/dev/null; then
  sudo add-apt-repository -y ppa:ubuntu-qcom-iot/qcom-ppa
fi

sudo apt-get update
sudo apt-get install -y \
  qcom-fastrpc1 \
  qcom-fastrpc-dev \
  qcom-libdmabufheap-dev

sudo ldconfig

sudo systemd-sysusers
getent group fastrpc || sudo groupadd --system fastrpc
sudo usermod -aG fastrpc ubuntu
sudo reboot

source "$HOME/qairt-env.sh"

id
ls -l /dev/fastrpc-cdsp
ldconfig -p | grep libcdsprpc
systemctl --no-pager --full status cdsprpcd

ubuntu is a member of fastrpc
/dev/fastrpc-cdsp is group-accessible
libcdsprpc.so resolves from /lib/aarch64-linux-gnu
cdsprpcd is active

qnn-platform-validator --backend dsp --coreVersion
qnn-platform-validator --backend dsp --testBackend

Backend Hardware  : Supported
Backend Libraries : Found
Core Version      : Hexagon Architecture V73
Unit Test         : Passed

cd ~/nemotron_genie

cat > prompt.txt <<'EOF'
<|begin_of_text|><|start_header_id|>system<|end_header_id|>

detailed thinking off<|eot_id|><|start_header_id|>user<|end_header_id|>

Explain gravity in one short English sentence.<|eot_id|><|start_header_id|>assistant<|end_header_id|>

EOF

detailed thinking on

source "$HOME/qairt-env.sh"
cd ~/nemotron_genie

set -o pipefail

genie-t2t-run \
  -c genie_config.json \
  --prompt_file prompt.txt \
  --profile profile.txt \
  2>&1 | tee genie-run.log

echo "exit code: ${PIPESTATUS[0]}"

Using libGenie.so version 1.17.0
[INFO] "Using create From Binary"
[INFO] "Allocated total size = 306545152 across 10 buffers"
[BEGIN]: Gravity is the force that pulls objects toward each other...
[END]
exit code: 0