pycoral Benchmark¶

A self-contained app for running pycoral inference benchmarks. No repository cloning or PYTHONPATH configuration is needed. Supports execution on Edge TPU, board CPU, and development PC CPU.

Prerequisites¶

CMake 3.16 or later
Internet connection (only for the initial model download)
Board execution: pycoral / tflite-runtime installed on Coral Dev Board (pre-installed on Mendel Linux)
Dev PC execution: Python 3 + venv with ai-edge-litert (successor to tflite-runtime) and numpy installed

Design¶

Why self-contained¶

Running benchmarks from the pycoral repository requires cloning the repo and configuring PYTHONPATH. There are also conflicts with the system pycoral package. This app consolidates benchmark scripts and model files in apps/pycoral-benchmark/, eliminating these issues.

Conditional imports¶

pycoral is only imported in Edge TPU mode. CPU mode uses only ai_edge_litert (dev PC) or tflite_runtime (board), so CPU benchmarks can run on development PCs without pycoral installed.

Benchmark management with ctest¶

Benchmarks are registered with enable_testing() + add_test() and classified by labels. The ctest filter feature allows selective execution of Edge TPU / CPU benchmarks. To add future benchmarks, simply add add_test() + set_tests_properties().

CMakeLists.txt¶

apps/pycoral-benchmark/CMakeLists.txt:

cmake_minimum_required(VERSION 3.16)
project(pycoral-benchmark NONE)

# --- Download test models from google-coral/test_data ---
set(MODEL_DIR "${CMAKE_CURRENT_BINARY_DIR}/models")
set(MODEL_BASE_URL "https://github.com/google-coral/test_data/raw/master")

set(MODELS
    inception_v1_224_quant
    inception_v4_299_quant
    mobilenet_v1_1.0_224_quant
    mobilenet_v2_1.0_224_quant
    ssd_mobilenet_v1_coco_quant_postprocess
    ssd_mobilenet_v2_face_quant_postprocess
)

file(MAKE_DIRECTORY ${MODEL_DIR})

foreach(model ${MODELS})
    # CPU version
    if(NOT EXISTS "${MODEL_DIR}/${model}.tflite")
        message(STATUS "Downloading ${model}.tflite ...")
        file(DOWNLOAD
            "${MODEL_BASE_URL}/${model}.tflite"
            "${MODEL_DIR}/${model}.tflite"
            SHOW_PROGRESS
        )
    endif()

    # Edge TPU version
    if(NOT EXISTS "${MODEL_DIR}/${model}_edgetpu.tflite")
        message(STATUS "Downloading ${model}_edgetpu.tflite ...")
        file(DOWNLOAD
            "${MODEL_BASE_URL}/${model}_edgetpu.tflite"
            "${MODEL_DIR}/${model}_edgetpu.tflite"
            SHOW_PROGRESS
        )
    endif()
endforeach()

# --- Deploy ---
include(${CMAKE_CURRENT_LIST_DIR}/../../cmake/coral-deploy.cmake)

set(DEPLOY_DIR /home/mendel/work/pycoral-benchmark)

# Collect deploy file mappings (flat: all files go to DEPLOY_DIR root)
set(DEPLOY_FILES
    "${CMAKE_CURRENT_SOURCE_DIR}/src/inference_benchmark.py:inference_benchmark.py"
)
foreach(model ${MODELS})
    list(APPEND DEPLOY_FILES
        "${MODEL_DIR}/${model}.tflite:${model}.tflite"
        "${MODEL_DIR}/${model}_edgetpu.tflite:${model}_edgetpu.tflite"
    )
endforeach()

coral_add_deploy_target(
    DEPLOY_DIR ${DEPLOY_DIR}
    FILES ${DEPLOY_FILES}
)

# --- Tests (ctest) ---
enable_testing()

add_test(NAME inference-edgetpu
    COMMAND ${_CORAL_SCRIPTS_DIR}/run.sh
            "cd ${DEPLOY_DIR} && python3 inference_benchmark.py --device edgetpu"
)
set_tests_properties(inference-edgetpu PROPERTIES LABELS "edgetpu;inference")

add_test(NAME inference-cpu
    COMMAND ${_CORAL_SCRIPTS_DIR}/run.sh
            "cd ${DEPLOY_DIR} && python3 inference_benchmark.py --device cpu"
)
set_tests_properties(inference-cpu PROPERTIES LABELS "cpu;inference")

project(pycoral-benchmark NONE) skips compiler detection, so no toolchain file is needed for configure. file(DOWNLOAD ...) automatically downloads 12 models (6 models x CPU/TPU) from google-coral/test_data.

Model Download¶

Run from the repository root directory. No toolchain file is needed.

cmake -B build/pycoral-benchmark -S apps/pycoral-benchmark

Downloaded models are saved to build/pycoral-benchmark/models/.

Target Models¶

Base Name	CPU Version	TPU Version
inception_v1_224_quant	`.tflite`	`_edgetpu.tflite`
inception_v4_299_quant	`.tflite`	`_edgetpu.tflite`
mobilenet_v1_1.0_224_quant	`.tflite`	`_edgetpu.tflite`
mobilenet_v2_1.0_224_quant	`.tflite`	`_edgetpu.tflite`
ssd_mobilenet_v1_coco_quant_postprocess	`.tflite`	`_edgetpu.tflite`
ssd_mobilenet_v2_face_quant_postprocess	`.tflite`	`_edgetpu.tflite`

Deploy and Run¶

Deploy to Board¶

cmake --build build/pycoral-benchmark --target deploy

Scripts and model files are transferred to /home/mendel/work/pycoral-benchmark/ on the board.

Run Benchmarks (ctest)¶

# Run all benchmarks
ctest --test-dir build/pycoral-benchmark -V

# Edge TPU only
ctest --test-dir build/pycoral-benchmark -V -L edgetpu

# CPU only
ctest --test-dir build/pycoral-benchmark -V -L cpu

# inference only (filter by name)
ctest --test-dir build/pycoral-benchmark -V -R inference

Running on Development PC¶

CPU benchmarks can be run via ctest in a venv environment.

source .venv/bin/activate
pip install -r requirements.txt
ctest --test-dir build/pycoral-benchmark -V -L local

Direct execution is also possible:

python3 apps/pycoral-benchmark/src/inference_benchmark.py \
    --device cpu --model-dir build/pycoral-benchmark/models

CMake Targets / ctest List¶

Configuration¶

cmake -B build/pycoral-benchmark -S apps/pycoral-benchmark

Target List¶

Target	Command	Description
`deploy`	`cmake --build build/pycoral-benchmark --target deploy`	Transfer scripts + models to board

ctest List¶

Test Name	Labels	Description
`inference-edgetpu`	`edgetpu`, `inference`	Edge TPU inference benchmark
`inference-cpu`	`cpu`, `inference`	Board CPU inference benchmark
`inference-cpu-local`	`cpu`, `inference`, `local`	Dev PC CPU inference benchmark
`softmax-regression`	`cpu`, `softmax-regression`	Softmax regression training benchmark
`imprinting`	`edgetpu`, `imprinting`	Weight imprinting training benchmark
`online-imprinting`	`edgetpu`, `imprinting`	Online imprinting inference benchmark

Connection Settings¶

Customize connection parameters via CMake cache variables:

Variable	Default	Description
`CORAL_IP`	(empty = auto-detect via mdt)	Coral Dev Board IP address

Benchmark Results¶

InferenceSoftmax RegressionImprinting

Edge TPU (Board, 200 iterations)¶

Model	Mean (ms)	Std (ms)	Min (ms)	Max (ms)
inception_v1_224_quant_edgetpu	5.65	0.42	4.74	6.70
inception_v4_299_quant_edgetpu	101.37	0.13	100.79	101.67
mobilenet_v1_1.0_224_quant_edgetpu	3.15	0.27	2.68	4.05
mobilenet_v2_1.0_224_quant_edgetpu	3.39	0.23	3.21	4.28
ssd_mobilenet_v1_coco_quant_postprocess_edgetpu	11.44	0.28	10.51	11.74
ssd_mobilenet_v2_face_quant_postprocess_edgetpu	8.07	0.66	6.38	9.02

CPU (Board, 20 iterations)¶

Model	Mean (ms)	Std (ms)	Min (ms)	Max (ms)
inception_v1_224_quant	376.13	1.57	375.60	382.97
inception_v4_299_quant	2973.37	0.28	2972.86	2973.91
mobilenet_v1_1.0_224_quant	167.42	1.75	166.87	175.04
mobilenet_v2_1.0_224_quant	126.85	0.08	126.72	127.07
ssd_mobilenet_v1_coco_quant_postprocess	350.32	0.09	350.17	350.51
ssd_mobilenet_v2_face_quant_postprocess	288.89	0.09	288.70	289.04

CPU (Dev PC, 20 iterations)¶

Reference specs: AMD Ryzen 7 8840U (16 threads) / 16 GB RAM

Model	Mean (ms)	Std (ms)	Min (ms)	Max (ms)
inception_v1_224_quant	21.81	0.41	21.52	23.36
inception_v4_299_quant	159.80	0.54	159.01	160.62
mobilenet_v1_1.0_224_quant	8.26	0.14	8.14	8.57
mobilenet_v2_1.0_224_quant	5.49	0.06	5.45	5.70
ssd_mobilenet_v1_coco_quant_postprocess	17.84	0.12	17.73	18.24
ssd_mobilenet_v2_face_quant_postprocess	13.94	0.02	13.88	14.00

Softmax Regression Training (Board CPU, 500 SGD iterations)¶

Classes	Features	Time (ms)	Accuracy
4	256	1035.28	25.39%
16	256	1504.50	7.81%
4	1024	4994.52	23.83%
16	1024	6260.45	5.08%

Accuracy is low because training uses random synthetic data (this is expected behavior).

Weight Imprinting Training (Board Edge TPU, 10 categories x 20 images)¶

Model	Training time (ms)
mobilenet_v1_1.0_224_l2norm_quant_edgetpu	1057.06

Online Imprinting Inference (Board Edge TPU, 10 categories x 20 images)¶

After training each category, the model is rebuilt and one inference is performed. Total inference time is reported.

Model	Inference time (ms)
mobilenet_v1_1.0_224_l2norm_quant_edgetpu	248.89