From c1f41098988a797fd562782d1e3961fbb0f291f5 Mon Sep 17 00:00:00 2001 From: Neo Zhang Date: Fri, 7 Aug 2026 13:18:47 +0800 Subject: [PATCH] sycl : update guide Q&A and script for device setting (#26442) --- docs/backend/SYCL.md | 42 +++++++++++++++++++++++++++++++++ examples/sycl/start-svr.sh | 21 +++++++++++------ examples/sycl/test.sh | 16 +++++++++---- examples/sycl/win-start-svr.bat | 28 ++++++++++++++++++---- examples/sycl/win-test.bat | 29 +++++++++++++++++++---- 5 files changed, 115 insertions(+), 21 deletions(-) diff --git a/docs/backend/SYCL.md b/docs/backend/SYCL.md index 814e541e1a..73f89a7063 100644 --- a/docs/backend/SYCL.md +++ b/docs/backend/SYCL.md @@ -449,6 +449,8 @@ Or use 1 SYCL GPUs: [0] with Max compute units:512 ``` +User can use the device management in [docs/multi-gpu.md](https://github.com/ggml-org/llama.cpp/blob/master/docs/multi-gpu.md), like parameter `--device SYCL0,SYCL1` to assign one or more devices. + ## Windows ### Install GPU driver @@ -763,6 +765,7 @@ Or use 1 SYCL GPUs: [0] with Max compute units:512 ``` +User can use the device management in [docs/multi-gpu.md](https://github.com/ggml-org/llama.cpp/blob/master/docs/multi-gpu.md), like parameter `--device SYCL0,SYCL1` to assign one or more devices. ## Environment Variable @@ -895,6 +898,45 @@ Pass these via `CXXFLAGS` or add a one-off `#define` to enable a flag on the spo set UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS=1 ``` +- When I set `SYCL_CACHE_PERSISTENT=1` in running time, I meet crash. + + `SYCL_CACHE_PERSISTENT=1` is not recommended by llama.cpp SYCL backend. + When cache is enabled, SYCL runtime will try to cache and reuse JIT-compiled binaries. + + We find some AI will tell user this cmd to speed up SYCL backend. It only speeds up the startup to skip the JIT process, instead of running speed. + + It will bring negative impact when the SYCL binary file is changed frequently in your running environment. The new & old codes mix will lead to crash. + + Compare to the benefit, it has brought more failed cases. + If you are not familiar with the SYCL compiler principle of JIT and AOT, please don't use it. + + To restore, you need to remove the local cache: `~/.cache/libsycl_cache/` and execute `unset SYCL_CACHE_PERSISTENT` in running time. + +- How to use iGPU and dGPU in same time? + + 1. Detect the devices in your running time. + ``` + source /opt/intel/oneapi/setvars.sh + ./build/bin/llama-server --list-devices + + or + ./build/bin/llama-cli --list-devices + ./build/bin/llama-bench --list-devices + ./build/bin/llama-completion --list-devices + + Available devices: + SYCL0: Intel(R) Arc(TM) A770 Graphics (15473 MiB, 15473 MiB free) + SYCL1: Intel(R) UHD Graphics 770 (59675 MiB, 44986 MiB free) + ``` + + The dGPU will be in the head of this list and iGPU will be the end. + If not all GPUs are listed, please check the env var: ONEAPI_DEVICE_SELECTOR and unset it. + + 2. Set the iGPU and dGPU + + Set the iGPU and dGPU by `./build/bin/llama-server --device SYCL0,SYCL1,SYCLxxx`. + + ### **GitHub contribution**: Please add the `[SYCL]` prefix/tag in issues/PRs titles to help the SYCL contributors to check/address them without delay. diff --git a/examples/sycl/start-svr.sh b/examples/sycl/start-svr.sh index ce31ec51d2..49177ba2dc 100755 --- a/examples/sycl/start-svr.sh +++ b/examples/sycl/start-svr.sh @@ -12,6 +12,7 @@ This script processes files with specified options. Options: -h, --help Display this help message and exit. + -d, --device Set SYCL devices (default: SYCL0). -c, --context Set context length. Bigger need more memory. -p, --promote Prompt to start generation with. -m, --model Full model file path. @@ -41,10 +42,16 @@ MODEL_FILE=../models/Qwen3.5-4B-Q4_0.gguf NGL=99 CONTEXT=4096 GGML_SYCL_DEVICE=-1 +SYCL_DEVICES="SYCL0" SPLIT_MODE=layer LOG_VERBOSE=3 while [[ $# -gt 0 ]]; do case "$1" in + -d|--device) + SYCL_DEVICES="$2" + shift + shift + ;; -c|--context) CONTEXT=$2 # Shift twice to consume both the option flag and its value @@ -95,8 +102,6 @@ while [[ $# -gt 0 ]]; do esac done - - source /opt/intel/oneapi/setvars.sh #export GGML_SYCL_DEBUG=1 @@ -107,17 +112,19 @@ source /opt/intel/oneapi/setvars.sh export UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS=1 echo "UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS=${UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS}" +echo "ONEAPI_DEVICE_SELECTOR=${ONEAPI_DEVICE_SELECTOR}" + + if [ $GGML_SYCL_DEVICE -ne -1 ]; then echo "Use $GGML_SYCL_DEVICE as main GPU" #use signle GPU only GPUS_SETTING="-mg $GGML_SYCL_DEVICE -sm ${SPLIT_MODE}" - echo "ONEAPI_DEVICE_SELECTOR=${ONEAPI_DEVICE_SELECTOR}" else - echo "Use all Intel GPUs, including iGPU & dGPU" + echo "Use Intel GPUs: ${SYCL_DEVICES}" GPUS_SETTING="-sm ${SPLIT_MODE}" - fi +fi -echo "run cmd: ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE} --mmap --host 0.0.0.0 --port 8000" -ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE} --mmap --host 0.0.0.0 --port 8000 +echo "run cmd: ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE} --device ${SYCL_DEVICES} --mmap --host 0.0.0.0 --port 8000" +ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE} --device ${SYCL_DEVICES} --mmap --host 0.0.0.0 --port 8000 diff --git a/examples/sycl/test.sh b/examples/sycl/test.sh index 116047cd2e..b9498f49b7 100755 --- a/examples/sycl/test.sh +++ b/examples/sycl/test.sh @@ -12,6 +12,7 @@ This script processes files with specified options. Options: -h, --help Display this help message and exit. + -d, --device Set SYCL devices (default: SYCL0). -c, --context Set context length. Bigger need more memory. -p, --promote Prompt to start generation with. -m, --model Full model file path. @@ -42,10 +43,16 @@ MODEL_FILE=../models/llama-2-7b.Q4_0.gguf NGL=99 CONTEXT=4096 GGML_SYCL_DEVICE=-1 +SYCL_DEVICES="SYCL0" SPLIT_MODE=layer LOG_VERBOSE=3 while [[ $# -gt 0 ]]; do case "$1" in + -d|--device) + SYCL_DEVICES="$2" + shift + shift + ;; -c|--context) CONTEXT=$2 # Shift twice to consume both the option flag and its value @@ -115,16 +122,17 @@ source /opt/intel/oneapi/setvars.sh export UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS=1 echo "UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS=${UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS}" +echo "ONEAPI_DEVICE_SELECTOR=${ONEAPI_DEVICE_SELECTOR}" + if [ $GGML_SYCL_DEVICE -ne -1 ]; then echo "Use $GGML_SYCL_DEVICE as main GPU" #use signle GPU only GPUS_SETTING="-mg $GGML_SYCL_DEVICE -sm ${SPLIT_MODE}" - echo "ONEAPI_DEVICE_SELECTOR=${ONEAPI_DEVICE_SELECTOR}" else - echo "Use all Intel GPUs, including iGPU & dGPU" + echo "Use Intel GPUs: ${SYCL_DEVICES}" GPUS_SETTING="-sm ${SPLIT_MODE}" fi -echo "run cmd: ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -no-cnv -p "${INPUT_PROMPT}" -n 200 -e -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE} --mmap " -ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -no-cnv -p "${INPUT_PROMPT}" -n 200 -e -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE} --mmap +echo "run cmd: ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -no-cnv -p "${INPUT_PROMPT}" -n 200 -e -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE} --device ${SYCL_DEVICES} --mmap " +ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -no-cnv -p "${INPUT_PROMPT}" -n 200 -e -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE} --device ${SYCL_DEVICES} --mmap diff --git a/examples/sycl/win-start-svr.bat b/examples/sycl/win-start-svr.bat index 13b5159e00..8077105893 100644 --- a/examples/sycl/win-start-svr.bat +++ b/examples/sycl/win-start-svr.bat @@ -13,6 +13,7 @@ set "MODEL_FILE=..\models\Qwen3.5-4B-Q4_0.gguf" set "NGL=99" set "CONTEXT=4096" set "GGML_SYCL_DEVICE=-1" +set "SYCL_DEVICES=SYCL0" set "SPLIT_MODE=layer" set "LOG_VERBOSE=3" @@ -36,6 +37,21 @@ if /I "%~1"=="--context" ( goto parse_args ) +if /I "%~1"=="-d" ( + if "%~2"=="" goto missing_value + set "SYCL_DEVICES=%~2" + shift + shift + goto parse_args +) +if /I "%~1"=="--device" ( + if "%~2"=="" goto missing_value + set "SYCL_DEVICES=%~2" + shift + shift + goto parse_args +) + if /I "%~1"=="-m" ( if "%~2"=="" goto missing_value set "MODEL_FILE=%~2" @@ -130,6 +146,7 @@ echo This script processes files with specified options. echo. echo Options: echo -h, --help Display this help message and exit. +echo -d, --device ^ Set SYCL devices (default: SYCL0). echo -c, --context ^ Set context length. Bigger need more memory. echo -m, --model ^ Full model file path. echo -mg,--main-gpu ^ Set main GPU ID (0 - n) for single GPU mode. @@ -160,19 +177,20 @@ REM Support malloc device memory more than 4GB. set "UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS=1" echo UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS=%UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS% +echo ONEAPI_DEVICE_SELECTOR=%ONEAPI_DEVICE_SELECTOR% + if not "%GGML_SYCL_DEVICE%"=="-1" ( echo Use %GGML_SYCL_DEVICE% as main GPU REM Use single GPU only. set "GPUS_SETTING=-mg %GGML_SYCL_DEVICE% -sm %SPLIT_MODE%" - echo ONEAPI_DEVICE_SELECTOR=%ONEAPI_DEVICE_SELECTOR% -) else ( - echo Use all Intel GPUs, including iGPU ^& dGPU + ) else ( + echo Use Intel GPUs: %SYCL_DEVICES% set "GPUS_SETTING=-sm %SPLIT_MODE%" ) -echo run cmd: ZES_ENABLE_SYSMAN=1 %BIN_FILE% -m "%MODEL_FILE%" -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --mmap --host 0.0.0.0 --port 8000 +echo run cmd: ZES_ENABLE_SYSMAN=1 %BIN_FILE% -m "%MODEL_FILE%" -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --device %SYCL_DEVICES% --mmap --host 0.0.0.0 --port 8000 set "ZES_ENABLE_SYSMAN=1" -%BIN_FILE% -m "%MODEL_FILE%" -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --mmap --host 0.0.0.0 --port 8000 +%BIN_FILE% -m "%MODEL_FILE%" -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --device "%SYCL_DEVICES%" --mmap --host 0.0.0.0 --port 8000 endlocal diff --git a/examples/sycl/win-test.bat b/examples/sycl/win-test.bat index 39640908b0..cc6da44138 100644 --- a/examples/sycl/win-test.bat +++ b/examples/sycl/win-test.bat @@ -19,6 +19,7 @@ set "MODEL_FILE=..\models\llama-2-7b.Q4_0.gguf" set "NGL=99" set "CONTEXT=4096" set "GGML_SYCL_DEVICE=-1" +set "SYCL_DEVICES=SYCL0" set "SPLIT_MODE=layer" set "LOG_VERBOSE=3" @@ -42,6 +43,21 @@ if /I "%~1"=="--context" ( goto parse_args ) +if /I "%~1"=="-d" ( + if "%~2"=="" goto missing_value + set "SYCL_DEVICES=%~2" + shift + shift + goto parse_args +) +if /I "%~1"=="--device" ( + if "%~2"=="" goto missing_value + set "SYCL_DEVICES=%~2" + shift + shift + goto parse_args +) + if /I "%~1"=="-p" ( if "%~2"=="" goto missing_value set "INPUT_PROMPT=%~2" @@ -151,6 +167,7 @@ echo This script processes files with specified options. echo. echo Options: echo -h, --help Display this help message and exit. +echo -d, --device ^ Set SYCL devices (default: SYCL0). echo -c, --context ^ Set context length. Bigger need more memory. echo -p, --promote ^ Prompt to start generation with. echo -m, --model ^ Full model file path. @@ -182,19 +199,21 @@ REM Support malloc device memory more than 4GB. set "UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS=1" echo UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS=%UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS% +echo ONEAPI_DEVICE_SELECTOR=%ONEAPI_DEVICE_SELECTOR% + if not "%GGML_SYCL_DEVICE%"=="-1" ( echo Use %GGML_SYCL_DEVICE% as main GPU REM Use single GPU only. set "GPUS_SETTING=-mg %GGML_SYCL_DEVICE% -sm %SPLIT_MODE%" - echo ONEAPI_DEVICE_SELECTOR=%ONEAPI_DEVICE_SELECTOR% -) else ( - echo Use all Intel GPUs, including iGPU ^& dGPU + ) +else ( + echo Use Intel GPUs: %SYCL_DEVICES% set "GPUS_SETTING=-sm %SPLIT_MODE%" ) -echo run cmd: ZES_ENABLE_SYSMAN=1 %BIN_FILE% -m %MODEL_FILE% -no-cnv -p "%INPUT_PROMPT%" -n 200 -e -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --mmap +echo run cmd: ZES_ENABLE_SYSMAN=1 %BIN_FILE% -m %MODEL_FILE% -no-cnv -p "%INPUT_PROMPT%" -n 200 -e -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --device %SYCL_DEVICES% --mmap set "ZES_ENABLE_SYSMAN=1" -%BIN_FILE% -m "%MODEL_FILE%" -no-cnv -p "%INPUT_PROMPT%" -n 200 -e -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --mmap +%BIN_FILE% -m "%MODEL_FILE%" -no-cnv -p "%INPUT_PROMPT%" -n 200 -e -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --device "%SYCL_DEVICES%" --mmap endlocal