NixOS/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

python3Packages.exllamav3: 0.0.43 -> 1.1.0; tabbyapi: 0-unstable-2026-06-27 -> 0-unstable-2026-07-18 (#543569)

Browse Source
This commit is contained in:
isabel
2026-07-19 19:30:15 +00:00
committed by GitHub
parent 88f00c40d1 b88fa50389
commit 194379ec34
6 changed files with 94 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nixos/modules/services/web-apps/tabbyapi.nix
+88 -9
View File
@@ -34,7 +34,7 @@ in
host = lib.mkOption {
type = lib.types.str;
default = "127.0.0.1";
description = "The IP to host on. Use 0.0.0.0 to expose on all adapters.";
description = "The IP to host on. Use 0.0.0.0 to expose on all network adapters.";
};
port = lib.mkOption {
@@ -46,13 +46,22 @@ in
disable_auth = lib.mkOption {
type = lib.types.bool;
default = false;
description = "Disable HTTP token authentication. WARNING: Vulnerable if exposed.";
description = ''
Disable HTTP token authentication with requests.
WARNING: This will make your instance vulnerable! Only turn this on if you are ONLY connecting from localhost.
'';
};
disable_fetch_requests = lib.mkOption {
type = lib.types.bool;
default = false;
description = "Disable fetching external content in response to requests, such as images from URLs.";
};
api_servers = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ "OAI" ];
description = "Select API servers to enable. Options: OAI, Kobold.";
description = "Select API servers to enable. Possible values: OAI, Kobold.";
};
};
@@ -66,7 +75,18 @@ in
log_requests = lib.mkOption {
type = lib.types.bool;
default = false;
description = "Enable request logging. Only use for debug.";
description = "Enable request logging. NOTE: Only use this for debugging!";
};
log_chat_completion_requests = lib.mkOption {
type = lib.types.bool;
default = false;
description = ''
Write every /v1/chat/completions request to logs/debug/ as JSON.
PRIVACY WARNING: Enabling this creates a comprehensive request log, including the
full message history and generation parameters. API keys are redacted, but prompts
and user-provided content are preserved for bug-report reproduction.
'';
};
};
@@ -111,35 +131,94 @@ in
'';
};
inline_model_loading = lib.mkOption {
type = lib.types.bool;
default = false;
description = ''
Allow direct loading of models from a completion or chat completion request.
This method of loading is strict by default; enable dummy models to add
exceptions for invalid model names.
'';
};
model_name = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = "The initial model to load on startup. Must exist in model_dir.";
description = ''
An initial model to load. Make sure the model is located in the model directory!
REQUIRED: This must be filled out to load a model on startup.
'';
example = "Qwen3_5-9B";
};
max_seq_len = lib.mkOption {
type = lib.types.nullOr lib.types.int;
default = null;
description = "Max sequence length. Set null to use model defaults.";
description = ''
Max sequence length (default: min(max_position_embeddings, cache_size)).
Set to -1 to fetch from the model's config.json.
'';
};
cache_mode = lib.mkOption {
type = lib.types.str;
default = "FP16";
description = "Cache mode for VRAM savings. ExLlamaV2: FP16, Q8, Q6, Q4. ExLlamaV3: specific pair string (e.g., '8,8').";
description = ''
Enable different cache modes for VRAM savings.
Specify the pair k_bits,v_bits where k_bits and v_bits are integers from 2-8 (e.g. '8,8').
The legacy values 'FP16', 'Q8', 'Q6', 'Q4' are also accepted.
'';
};
tensor_parallel = lib.mkOption {
type = lib.types.bool;
default = false;
description = ''
Load model with tensor parallelism.
Falls back to autosplit if GPU split isn't provided. This ignores the gpu_split_auto value.
'';
};
gpu_split_auto = lib.mkOption {
type = lib.types.bool;
default = true;
description = "Automatically allocate resources to GPUs.";
description = "Automatically allocate resources to GPUs. Not parsed for single GPU users.";
};
dummy_model_names = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ "gpt-3.5-turbo" ];
description = "List of fake model names sent via the /v1/models endpoint.";
description = ''
A list of fake model names that are sent via the /v1/models endpoint.
Also used as bypasses for strict mode if inline_model_loading is true.
'';
};
vision = lib.mkOption {
type = lib.types.bool;
default = false;
description = "Enables vision support if the model supports it.";
};
reasoning = lib.mkOption {
type = lib.types.bool;
default = false;
description = ''
Enable reasoning parser.
Do NOT enable this if the model is not a reasoning model (e.g. deepseek-r1 series).
'';
};
reasoning_start_token = lib.mkOption {
type = lib.types.str;
default = "<think>";
description = "The start token for reasoning content.";
};
reasoning_end_token = lib.mkOption {
type = lib.types.str;
default = "</think>";
description = "The end token for reasoning content.";
};
};
};
pkgs/by-name/ta/tabbyapi/package.nix
+3 -4
View File
@@ -7,14 +7,14 @@
}:
python3Packages.buildPythonApplication {
pname = "tabbyapi";
version = "0-unstable-2026-06-27";
version = "0-unstable-2026-07-18";
pyproject = true;
src = fetchFromGitHub {
owner = "theroyallab";
repo = "tabbyAPI";
rev = "3cf468c28362c28be1c8fc731ce1ccaf7b2206d0";
hash = "sha256-s97YFyij2/oYlClmV2laDrCkkoK4uVZgRsn5WwftLag=";
rev = "0158fb48d76546a6475d1d63f6cd5b90932d1d11";
hash = "sha256-Bkpx3MyZg7Np5zXAsq8mgxdAFsHUZhy2NZ93XSLbJgk=";
};
build-system = with python3Packages; [
@@ -57,7 +57,6 @@ python3Packages.buildPythonApplication {
numpy
setuptools
exllamav2
exllamav3
]
++ lib.optionals stdenv.hostPlatform.isLinux [
pkgs/development/python-modules/exllamav2/default.nix
-108
View File
@@ -1,108 +0,0 @@
{
lib,
buildPythonPackage,
fetchFromGitHub,
# build-system
setuptools,
torch,
cudaPackages,
# nativeBuildInputs
pybind11,
# dependencies
fastparquet,
flash-attn,
ninja,
numpy,
pandas,
pillow,
pygments,
regex,
rich,
safetensors,
tokenizers,
websockets,
}:
buildPythonPackage.override { inherit (torch) stdenv; } (finalAttrs: {
pname = "exllamav2";
version = "0.3.2";
pyproject = true;
__structuredAttrs = true;
src = fetchFromGitHub {
owner = "turboderp-org";
repo = "exllamav2";
tag = "v${finalAttrs.version}";
hash = "sha256-WbpbANenOuy6F0qAKVKAmolHjgRKfPxSVud8FZG1TXw=";
};
build-system = [
setuptools
torch
];
nativeBuildInputs = [
ninja
];
preConfigure = ''
export MAX_JOBS="$NIX_BUILD_CORES"
export NVCC_THREADS=2
'';
buildInputs = [
pybind11
]
++ lib.optionals torch.cudaSupport [
cudaPackages.cuda_cudart # cuda_runtime.h
cudaPackages.libcublas # cublas_v2.h
cudaPackages.libcurand # curand_kernel.h
cudaPackages.libcusolver # cusolverDn.h
cudaPackages.libcusparse # cusparse.h
];
env = lib.optionalAttrs torch.cudaSupport {
CUDA_HOME = lib.getDev cudaPackages.cuda_nvcc;
TORCH_CUDA_ARCH_LIST = lib.concatStringsSep ";" torch.cudaCapabilities;
};
dependencies = [
fastparquet
flash-attn
ninja
numpy
pandas
pillow
pygments
regex
rich
safetensors
tokenizers
torch
websockets
];
pythonImportsCheck = [ "exllamav2" ];
# Tests require GPU hardware and external model files
doCheck = false;
meta = {
homepage = "https://github.com/turboderp-org/exllamav2";
description = "Inference library for running LLMs locally on modern consumer-class GPUs";
changelog = "https://github.com/turboderp-org/exllamav2/releases/tag/${finalAttrs.src.tag}";
license = lib.licenses.mit;
platforms = [
"x86_64-windows"
"x86_64-linux"
];
# Package requires CUDA or ROCm for functionality
# ROCm support is partially implemented but untested
broken = !torch.cudaSupport;
maintainers = with lib.maintainers; [ BatteredBunny ];
};
})
pkgs/development/python-modules/exllamav3/default.nix
+2 -6
View File
@@ -7,7 +7,6 @@
setuptools,
flash-attn,
flash-linear-attention,
formatron,
kbnf,
@@ -22,21 +21,20 @@
tokenizers,
torch,
typing-extensions,
xformers,
}:
let
newerThanTuring = lib.filter (version: lib.versionOlder "7.9" version) torch.cudaCapabilities;
in
buildPythonPackage.override { inherit (torch) stdenv; } (finalAttrs: {
pname = "exllamav3";
version = "0.0.43";
version = "1.1.0";
pyproject = true;
src = fetchFromGitHub {
owner = "turboderp-org";
repo = "exllamav3";
tag = "v${finalAttrs.version}";
hash = "sha256-68v8ptvtOzRTnnRXrgU0emqmbCO0pECidgJ36bwm8/s=";
hash = "sha256-JlZt1UuTMmjaQWhiQZxzbHK3WgYYoBjP9PMEizEfsLY=";
};
pythonRelaxDeps = [
@@ -60,7 +58,6 @@ buildPythonPackage.override { inherit (torch) stdenv; } (finalAttrs: {
];
dependencies = [
flash-attn
flash-linear-attention
formatron
kbnf
@@ -74,7 +71,6 @@ buildPythonPackage.override { inherit (torch) stdenv; } (finalAttrs: {
tokenizers
torch
typing-extensions
xformers
];
env = lib.optionalAttrs torch.cudaSupport {
pkgs/top-level/python-aliases.nix
+1
View File
@@ -219,6 +219,7 @@ mapAliases {
esig = throw "'esig' has been removed as it was broken and unmaintained upstream"; # Added 2026-05-27
et_xmlfile = throw "'et_xmlfile' has been renamed to/replaced by 'et-xmlfile'"; # Converted to throw 2025-10-29
ev3dev2 = throw "'ev3dev2' has been renamed to/replaced by 'python-ev3dev2'"; # Converted to throw 2025-10-29
exllamav2 = throw "'exllamav2' was removed because it is archived upstream and support was dropped in tabbyapi"; # Added 2026-07-19
eyeD3 = throw "'eyeD3' has been renamed to/replaced by 'eyed3'"; # Converted to throw 2025-10-29
f3d_egl = lib.warnOnInstantiate "'f3d' now build with egl support by default, so `f3d_egl` is deprecated, consider using 'f3d' instead." f3d; # added 2025-07-18
Fabric = throw "'Fabric' has been renamed to/replaced by 'fabric'"; # Converted to throw 2025-10-29
pkgs/top-level/python-packages.nix
-2
View File
@@ -5662,8 +5662,6 @@ self: super: with self; {
exiv2 = callPackage ../development/python-modules/exiv2 { inherit (pkgs) exiv2; };
exllamav2 = callPackage ../development/python-modules/exllamav2 { };
exllamav3 = callPackage ../development/python-modules/exllamav3 { };
expandvars = callPackage ../development/python-modules/expandvars { };