LLM-Inference/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-08-12 22:31:11 +04:00
Code Issues Packages Projects Releases Wiki Activity

tests : clean-up server test, use tests.sh in ci (#26886)

Browse Source 
* tests : remove fetch_server_test_models.py

* ci : use tests.sh wrapper of pytest
This commit is contained in:
Georgi Gerganov
2026-08-11 09:07:13 +03:00
committed by GitHub
parent 4801e3c567
commit b3df57286c
5 changed files with 17 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/server-sanitize.yml
+2 -2
View File
@@ -110,7 +110,7 @@ jobs:
source .venv/bin/activate
cd tools/server/tests
export ${{ matrix.extra_args }}
pytest -v -x -m "not slow"
./tests.sh
- name: Slow tests
id: server_integration_tests_slow
@@ -119,4 +119,4 @@ jobs:
source .venv/bin/activate
cd tools/server/tests
export ${{ matrix.extra_args }}
SLOW_TESTS=1 pytest -v -x
SLOW_TESTS=1 ./tests.sh
.github/workflows/server-self-hosted.yml
+9 -9
View File
@@ -72,7 +72,7 @@ jobs:
run: |
cd tools/server/tests
source venv/bin/activate
pytest -v -x -m "not slow"
./tests.sh
- name: Tests (GPUx1, backend-sampling)
id: server_integration_tests_backend_sampling
@@ -81,7 +81,7 @@ jobs:
cd tools/server/tests
source venv/bin/activate
export LLAMA_ARG_BACKEND_SAMPLING=1
pytest -v -x -m "not slow"
./tests.sh
- name: Tests (GPUx2)
id: server_integration_tests_gpu2
@@ -90,7 +90,7 @@ jobs:
cd tools/server/tests
source venv/bin/activate
export GGML_METAL_DEVICES=2
pytest -v -x -m "not slow"
./tests.sh
- name: Tests (GPUx2, backend-sampling)
id: server_integration_tests_gpu2_backend_sampling
@@ -99,7 +99,7 @@ jobs:
cd tools/server/tests
source venv/bin/activate
export GGML_METAL_DEVICES=2 LLAMA_ARG_BACKEND_SAMPLING=1
pytest -v -x -m "not slow"
./tests.sh
server-cuda:
runs-on: [self-hosted, llama-server, Linux, NVIDIA]
@@ -132,7 +132,7 @@ jobs:
run: |
cd tools/server/tests
source venv/bin/activate
pytest -v -x -m "not slow"
./tests.sh
- name: Tests (GPUx1, backend-sampling)
id: server_integration_tests_backend_sampling
@@ -141,7 +141,7 @@ jobs:
cd tools/server/tests
source venv/bin/activate
export LLAMA_ARG_BACKEND_SAMPLING=1
pytest -v -x -m "not slow"
./tests.sh
- name: Tests (GPUx2)
id: server_integration_tests_gpu2
@@ -150,7 +150,7 @@ jobs:
cd tools/server/tests
source venv/bin/activate
export GGML_CUDA_DEVICES=2
pytest -v -x -m "not slow"
./tests.sh
- name: Tests (GPUx2, backend-sampling)
id: server_integration_tests_gpu2_backend_sampling
@@ -159,7 +159,7 @@ jobs:
cd tools/server/tests
source venv/bin/activate
export GGML_CUDA_DEVICES=2 LLAMA_ARG_BACKEND_SAMPLING=1
pytest -v -x -m "not slow"
./tests.sh
server-kleidiai:
runs-on: ah-ubuntu_22_04-c8g_8x
@@ -219,4 +219,4 @@ jobs:
run: |
cd tools/server/tests
source venv/bin/activate
pytest -v -x -m "not slow"
./tests.sh
.github/workflows/server.yml
+6 -6
View File
@@ -104,21 +104,21 @@ jobs:
id: server_integration_tests
run: |
cd tools/server/tests
pytest -v -x -m "not slow"
./tests.sh
- name: Slow tests
id: server_integration_tests_slow
if: ${{ github.event.schedule || github.event.inputs.slow_tests == 'true' }}
run: |
cd tools/server/tests
SLOW_TESTS=1 pytest -v -x
SLOW_TESTS=1 ./tests.sh
- name: Tests (Backend sampling)
id: server_integration_tests_backend_sampling
run: |
cd tools/server/tests
export LLAMA_ARG_BACKEND_SAMPLING=1
pytest -v -x -m "not slow"
./tests.sh
- name: Slow tests (Backend sampling)
id: server_integration_tests_slow_backend_sampling
@@ -126,7 +126,7 @@ jobs:
run: |
cd tools/server/tests
export LLAMA_ARG_BACKEND_SAMPLING=1
SLOW_TESTS=1 pytest -v -x
SLOW_TESTS=1 ./tests.sh
windows:
runs-on: windows-2025
@@ -170,7 +170,7 @@ jobs:
run: |
cd tools/server/tests
$env:PYTHONIOENCODING = ":replace"
pytest -v -x -m "not slow"
./tests.sh
- name: Slow tests
id: server_integration_tests_slow
@@ -178,4 +178,4 @@ jobs:
run: |
cd tools/server/tests
$env:SLOW_TESTS = "1"
pytest -v -x
./tests.sh
scripts/fetch_server_test_models.py
-105
View File
@@ -1,105 +0,0 @@
#!/usr/bin/env python
'''
This script fetches all the models used in the server tests.
This is useful for slow tests that use larger models, to avoid them timing out on the model downloads.
It is meant to be run from the root of the repository.
Example:
python scripts/fetch_server_test_models.py
( cd tools/server/tests && ./tests.sh -v -x -m slow )
'''
import ast
import glob
import logging
import os
from typing import Generator
from pydantic import BaseModel
from typing import Optional
import subprocess
class HuggingFaceModel(BaseModel):
hf_repo: str
hf_file: Optional[str] = None
class Config:
frozen = True
def collect_hf_model_test_parameters(test_file) -> Generator[HuggingFaceModel, None, None]:
try:
with open(test_file) as f:
tree = ast.parse(f.read())
except Exception as e:
logging.error(f'collect_hf_model_test_parameters failed on {test_file}: {e}')
return
for node in ast.walk(tree):
if isinstance(node, ast.FunctionDef):
for dec in node.decorator_list:
if isinstance(dec, ast.Call) and isinstance(dec.func, ast.Attribute) and dec.func.attr == 'parametrize':
param_names = ast.literal_eval(dec.args[0]).split(",")
if "hf_repo" not in param_names:
continue
raw_param_values = dec.args[1]
if not isinstance(raw_param_values, ast.List):
logging.warning(f'Skipping non-list parametrize entry at {test_file}:{node.lineno}')
continue
hf_repo_idx = param_names.index("hf_repo")
hf_file_idx = param_names.index("hf_file") if "hf_file" in param_names else None
for t in raw_param_values.elts:
if not isinstance(t, ast.Tuple):
logging.warning(f'Skipping non-tuple parametrize entry at {test_file}:{node.lineno}')
continue
yield HuggingFaceModel(
hf_repo=ast.literal_eval(t.elts[hf_repo_idx]),
hf_file=ast.literal_eval(t.elts[hf_file_idx]) if hf_file_idx is not None else None)
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
models = sorted(list(set([
model
for test_file in glob.glob('tools/server/tests/unit/test_*.py')
for model in collect_hf_model_test_parameters(test_file)
])), key=lambda m: (m.hf_repo, m.hf_file))
logging.info(f'Found {len(models)} models in parameterized tests:')
for m in models:
logging.info(f' - {m.hf_repo} / {m.hf_file}')
cli_path = os.environ.get(
'LLAMA_CLI_BIN_PATH',
os.path.join(
os.path.dirname(__file__),
'../build/bin/Release/llama-cli.exe' if os.name == 'nt' else '../build/bin/llama-cli'))
for m in models:
if '<' in m.hf_repo or (m.hf_file is not None and '<' in m.hf_file):
continue
if m.hf_file is not None and '-of-' in m.hf_file:
logging.warning(f'Skipping model at {m.hf_repo} / {m.hf_file} because it is a split file')
continue
logging.info(f'Using llama-cli to ensure model {m.hf_repo}/{m.hf_file} was fetched')
cmd = [
cli_path,
'-hfr', m.hf_repo,
*([] if m.hf_file is None else ['-hff', m.hf_file]),
'-n', '1',
'-p', 'Hey',
'--no-warmup',
'--log-disable',
'-st']
if m.hf_file != 'tinyllamas/stories260K.gguf' and 'Mistral-Nemo' not in m.hf_repo:
cmd += ('-fa', 'on')
try:
subprocess.check_call(cmd)
except subprocess.CalledProcessError:
logging.error(f'Failed to fetch model at {m.hf_repo} / {m.hf_file} with command:\n {" ".join(cmd)}')
exit(1)
tools/server/tests/tests.sh
-5
View File
@@ -6,11 +6,6 @@ cd $SCRIPT_DIR
set -eu
if [[ "${SLOW_TESTS:-0}" == 1 ]]; then
# Slow tests for tool calls need quite a few models ahead of time to avoid timing out.
python $SCRIPT_DIR/../../../scripts/fetch_server_test_models.py
fi
if [ $# -lt 1 ]
then
if [[ "${SLOW_TESTS:-0}" == 1 ]]; then