使用 AI Hub 和原生 QNN 在 NPU 上运行实时 YOLO 深度估计

USB camera
  ↓
Python live app
  ↓
320×320 NHWC float32 RGB input.raw
  ↓
persistent native C++ QNN server
  ↓
AI Hub quantized + compiled QNN context
  ↓
Hexagon NPU / HTP backend
  ↓
1×1×320×320 float32 depth output.raw
  ↓
Python unletterbox + colorize + overlay
  ↓
live RGB | DEPTH | OVERLAY window

sudo apt update
sudo apt install -y python3.12-venv build-essential cmake pkg-config \
  qairt-headers libqnn-dev

qnn-platform-validator \
  --backend dsp \
  --coreVersion \
  --libVersion \
  --testBackend \
  --targetPath /tmp/qnnval

Core Version of the backend DSP: Hexagon Architecture V75
Unit Test on the backend DSP: Passed.
QNN is supported for backend DSP on the device.

mkdir -p ~/yolo-depth-run
cd ~/yolo-depth-run

python3.12 -m venv ~/yolo-depth-venv
source ~/yolo-depth-venv/bin/activate

pip install --upgrade pip
pip install ultralytics opencv-python-headless onnx onnxslim onnxruntime-qnn qai-hub numpy

source ~/yolo-depth-venv/bin/activate
qai-hub configure --api_token "$QAI_HUB_API_TOKEN"

cd ~/yolo-depth-run
source ~/yolo-depth-venv/bin/activate

python - <<'PY'
from ultralytics import YOLO
model = YOLO("yolo26n-depth.pt")
model.export(format="onnx", imgsz=320, device="cpu", simplify=True)
PY

~/yolo-depth-run/yolo26n-depth.onnx

cd ~/yolo-depth-run
source ~/yolo-depth-venv/bin/activate

python - <<'PY'
import onnx
src = "yolo26n-depth.onnx"
dst = "yolo26n-depth-aihub-clean.onnx"
model = onnx.load(src)
outputs = {o.name for o in model.graph.output}
kept = [v for v in model.graph.value_info if v.name not in outputs]
removed = [v.name for v in model.graph.value_info if v.name in outputs]
del model.graph.value_info[:]
model.graph.value_info.extend(kept)
onnx.checker.check_model(model)
onnx.save(model, dst)
print("removed duplicate value_info:", removed)
print("wrote", dst)
PY

import argparse, time
import cv2
import numpy as np


def preprocess(frame, size=320):
    h, w = frame.shape[:2]
    scale = min(size / h, size / w)
    nh, nw = int(round(h * scale)), int(round(w * scale))
    resized = cv2.resize(frame, (nw, nh), interpolation=cv2.INTER_LINEAR)
    canvas = np.full((size, size, 3), 114, dtype=np.uint8)
    top = (size - nh) // 2
    left = (size - nw) // 2
    canvas[top:top+nh, left:left+nw] = resized
    rgb = cv2.cvtColor(canvas, cv2.COLOR_BGR2RGB)
    return (rgb.astype(np.float32) / 255.0)[None, ...]


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--camera", default="0")
    ap.add_argument("--frames", type=int, default=64)
    ap.add_argument("--size", type=int, default=320)
    ap.add_argument("--skip", type=int, default=3)
    ap.add_argument("--out", default="aihub_calib_images.npz")
    args = ap.parse_args()

    cam = int(args.camera) if args.camera.isdigit() else args.camera
    cap = cv2.VideoCapture(cam, cv2.CAP_V4L2)
    if not cap.isOpened():
        raise SystemExit(f"Could not open camera {cam!r}")

    cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640)
    cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480)
    cap.set(cv2.CAP_PROP_BUFFERSIZE, 1)

    arrays = []
    i = 0
    while len(arrays) < args.frames:
        ok, frame = cap.read()
        if ok and i % args.skip == 0:
            arrays.append(preprocess(frame, args.size))
            print(f"{len(arrays)}/{args.frames}")
        i += 1
        time.sleep(0.03)
    cap.release()

    arr = np.concatenate(arrays, axis=0).astype(np.float32)
    np.savez_compressed(args.out, images=arr)
    print("saved", args.out, arr.shape, arr.dtype, float(arr.min()), float(arr.max()))


if __name__ == "__main__":
    main()

cd ~/yolo-depth-run
source ~/yolo-depth-venv/bin/activate
python make_aihub_calib.py --camera 0 --frames 64 --out aihub_calib_images.npz

python make_aihub_calib.py --camera 0 --frames 300 --out aihub_calib_images_300.npz

from pathlib import Path
import numpy as np
import qai_hub as hub

client = hub.Client()
cal = np.load("aihub_calib_images.npz")["images"].astype("float32")

# Calibration entries are keyed by ONNX input name.
# This ONNX input is NHWC: images [1,320,320,3].
calibration_data = {"images": [cal[i:i+1] for i in range(cal.shape[0])]}
print("calibration samples", len(calibration_data["images"]), calibration_data["images"][0].shape)

job = client.submit_quantize_job(
    model="yolo26n-depth-aihub-clean.onnx",
    calibration_data=calibration_data,
    weights_dtype=hub.QuantizeDtype.INT8,
    activations_dtype=hub.QuantizeDtype.INT16,
    name="yolo-depth-w8a16-qcs8275",
)
print("job_id", job.job_id)
print("job_url", getattr(job, "url", None) or getattr(job, "_url", None))
job.wait()
print("status", job.get_status())

result = job.get_target_model()
out_dir = Path("aihub_quantized")
out_dir.mkdir(exist_ok=True)
if hasattr(result, "download"):
    print("downloaded", result.download(str(out_dir / "yolo-depth-w8a16.onnx")))
elif hasattr(result, "download_model"):
    print("downloaded", result.download_model(str(out_dir / "yolo-depth-w8a16.onnx")))
else:
    print("target model", result)

cd ~/yolo-depth-run
source ~/yolo-depth-venv/bin/activate
python submit_aihub_quant.py

~/yolo-depth-run/aihub_compiled/yolo26n-depth-aihub-qcs8275-qnn-dlc.dlc

cd ~/yolo-depth-run
mkdir -p aihub_dlc_test/input aihub_dlc_test/out

# input/images.raw must be one preprocessed [1,320,320,3] float32 tensor.
# Use your own preprocessing script or the live app's letterbox function.

echo "images:=aihub_dlc_test/input/images.raw" > aihub_dlc_test/input_list.txt

qnn-net-run \
  --backend /usr/lib/libQnnHtp.so \
  --dlc_path aihub_compiled/yolo26n-depth-aihub-qcs8275-qnn-dlc.dlc \
  --input_list aihub_dlc_test/input_list.txt \
  --output_dir aihub_dlc_test/out \
  --log_level warn

cd ~/yolo-depth-run
mkdir -p qnn_context

qnn-context-binary-generator \
  --backend /usr/lib/libQnnHtp.so \
  --dlc_path aihub_compiled/yolo26n-depth-aihub-qcs8275-qnn-dlc.dlc \
  --binary_file yolo26n-depth-aihub-qcs8275-context.bin \
  --output_dir qnn_context \
  --log_level warn 2>&1 | tee qnn_context/generate_context.log

~/yolo-depth-run/qnn_context/yolo26n-depth-aihub-qcs8275-context.bin.bin

READY <input_bytes> <output_bytes>
RUN <input.raw> <output.raw>
OK <inference_ms> <output_bytes>
QUIT

cd ~/yolo-depth-run
mkdir -p qnn_app

CXX ?= g++
CXXFLAGS ?= -std=c++17 -O3 -Wall -Wextra -I/usr/include -I/usr/include/QNN
LDFLAGS ?= -ldl

all: qnn_dlc_runner

qnn_dlc_runner: qnn_dlc_runner.cpp
	$(CXX) $(CXXFLAGS) $< -o $@ $(LDFLAGS)

clean:
	rm -f qnn_dlc_runner

cd ~/yolo-depth-run/qnn_app
make -j$(nproc)

cd ~/yolo-depth-run/qnn_app

./qnn_dlc_runner \
  --dlc ../qnn_context/yolo26n-depth-aihub-qcs8275-context.bin.bin \
  --input ../aihub_dlc_test/input/images.raw \
  --output qnn_app_output.raw \
  --warmup 20 \
  --loops 500

graph: graph_ymndtmzg inputs=1 outputs=1
input: images bytes=1228800 dtype=0x232
output: output_0 bytes=409600 dtype=0x232
loops=500 warmup=20
avg_ms=16.1 p50_ms=16.1 p90_ms=16.6 p99_ms=17.4 fps=62.0
wrote=qnn_app_output.raw bytes=409600

cd ~/yolo-depth-run/qnn_app
python3 - <<'PY'
import numpy as np
app = np.fromfile("qnn_app_output.raw", dtype=np.float32)
ref = np.fromfile("../aihub_dlc_test/out/Result_0/output_0.raw", dtype=np.float32)
print("app", app.shape, app.min(), app.mean(), app.max())
print("ref", ref.shape, ref.min(), ref.mean(), ref.max())
print("max_abs_diff", np.max(np.abs(app-ref)))
print("mean_abs_diff", np.mean(np.abs(app-ref)))
PY

proc.stdin.write(f"RUN {in_raw} {out_raw}\n")
proc.stdin.flush()
line = proc.stdout.readline().strip()
if not line.startswith("OK "):
    raise RuntimeError(line)
qnn_ms = float(line.split()[1])

depth_320 = np.fromfile(out_raw, dtype=np.float32).reshape(1, 1, 320, 320)[0, 0]
depth = unletterbox_depth(depth_320, meta)
color = colorize_depth(depth, cmap="spectral", mode="disparity")
overlay = cv2.addWeighted(frame, 0.45, color, 0.55, 0)
cv2.imshow("AI Hub QNN Native Live Depth", np.hstack([frame, color, overlay]))

cd ~/yolo-depth-run
source ~/yolo-depth-venv/bin/activate
python live_aihub_qnn_native.py --camera 0 --display-width 1280 --font-scale 1.2

ls /dev/video*
python live_aihub_qnn_native.py --camera /dev/video0

output0_q -> DequantizeLinear -> output0

np.zeros((1, 320, 320, 3), dtype=np.float32).tofile(in_raw)

bad live path:
  Python frame → spawn qnn-net-run → load model → run once → exit

good live path:
  start native runner once → load context once → Python sends RUN commands per frame