From 2e2d99cfd27d8bab08b6ee36836e20979ba48445 Mon Sep 17 00:00:00 2001 From: shivamkumard-ctrl Date: Mon, 10 Aug 2026 14:16:44 +0530 Subject: [PATCH] ci: Add support for CUDA 13.4 ARM64 builds for Windows (#26650) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ci: Add support for CUDA 13.4 ARM64 builds for Windows Added an architecture-specific CUDA 13.4 Windows build entry targeting ARM64. Added a CMake configuration to enable ARM64 CUDA cross-compilation from an x64 Windows environment using the x64-hosted CUDA and MSVC toolchain while linking against the ARM64 CUDA import libraries to produce ggml-cuda.dll. Validated the self-hosted Windows x64 workflow, including toolkit acquisition, CMake configuration, ARM64 CUDA cross-compilation, and packaging. Runtime validation was performed separately on a native ARM64 RTX Spark system using TinyLlama 1.1B Q4_K_M to verify the generated binaries. The ARM64 CUDA job builds only the ggml-cuda.dll backend (LLAMA_BUILD_SERVER=OFF). The release consists of two packages: the main ARM64 release package, which combines the existing ARM64 CPU outputs with ggml-cuda.dll, and a separate runtime package containing the required CUDA runtime libraries (cudart64_13.dll, cublas64_13.dll, and cublasLt64_13.dll). The CUDA 13.4 setup uses NVIDIA Developer Preview component archives instead of the GA component downloads used by the existing CUDA setups and will require updates once CUDA 13.4 reaches GA. * ci: cleans up to align with x64 CUDA setup - Moves CUDA-specific CMake options into matrix defines. - Keeps the CUB 3DOT2 option only for CUDA 12.4. - Removes runtime argument construction and the unnecessary server option. - Aligns ARM64 CUDA runtime packaging with the existing robocopy approach. - Generalizes the ARM64 release label from CUDA 13.4 to CUDA 13. * ci: Set CUDA job name as version-architecture pair * mark as preview Co-authored-by: Georgi Gerganov --------- Co-authored-by: Sigbjørn Skjæret Co-authored-by: Georgi Gerganov --- .github/actions/windows-setup-cuda/action.yml | 27 +++++++++++ .github/workflows/release.yml | 46 +++++++++++++------ cmake/arm64-windows-msvc-cuda.cmake | 26 +++++++++++ 3 files changed, 86 insertions(+), 13 deletions(-) create mode 100644 cmake/arm64-windows-msvc-cuda.cmake diff --git a/.github/actions/windows-setup-cuda/action.yml b/.github/actions/windows-setup-cuda/action.yml index 43c63ce44f..31250eda1b 100644 --- a/.github/actions/windows-setup-cuda/action.yml +++ b/.github/actions/windows-setup-cuda/action.yml @@ -4,6 +4,10 @@ inputs: cuda_version: description: "CUDA toolkit version" required: true + cuda_arch: + description: "CUDA target architecture" + required: false + default: "x64" runs: using: "composite" @@ -127,3 +131,26 @@ runs: echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 echo "CUDA_PATH_V13_3=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + + - name: Install Cuda Toolkit 13.4 for ARM64 + if: ${{ inputs.cuda_version == '13.4' && inputs.cuda_arch == 'arm64' }} + shell: pwsh + run: | + mkdir -p "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" + choco install unzip -y + curl -O "https://packages.nvidia.com/bin-archive/pool/windows-x86_64/5B515474-7E78-11F1-8656-C51E4F4B317F/cccl-windows-x86_64-13.3.4.1.2-archive.zip" + curl -O "https://packages.nvidia.com/bin-archive/pool/windows-x86_64/5B515474-7E78-11F1-8656-C51E4F4B317F/cuda_crt-windows-x86_64-13.4.46-archive.zip" + curl -O "https://packages.nvidia.com/bin-archive/pool/windows-x86_64/5B515474-7E78-11F1-8656-C51E4F4B317F/cuda_nvcc-windows-x86_64-13.4.46-archive.zip" + curl -O "https://packages.nvidia.com/bin-archive/pool/windows-x86_64/5B515474-7E78-11F1-8656-C51E4F4B317F/libnvvm-windows-x86_64-13.4.46-archive.zip" + curl -O "https://packages.nvidia.com/bin-archive/pool/windows-arm64/5B515474-7E78-11F1-8656-C51E4F4B317F/cuda_cudart-windows-arm64-13.4.46-archive.zip" + curl -O "https://packages.nvidia.com/bin-archive/pool/windows-arm64/5B515474-7E78-11F1-8656-C51E4F4B317F/libcublas-windows-arm64-13.7.0.10-archive.zip" + unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cccl-windows-x86_64-13.3.4.1.2-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_crt-windows-x86_64-13.4.46-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_nvcc-windows-x86_64-13.4.46-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libnvvm-windows-x86_64-13.4.46-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_cudart-windows-arm64-13.4.46-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libcublas-windows-arm64-13.7.0.10-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y + echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + echo "CUDA_PATH_V13_4=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 968d2d4b7e..c668930b0c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -848,6 +848,7 @@ jobs: name: llama-bin-win-${{ matrix.backend }}-${{ matrix.arch }}.zip windows-cuda: + name: windows-cuda (${{ matrix.cuda }}, ${{ matrix.arch }}) needs: [check-release] if: ${{ needs.check-release.outputs.should_release == 'true' }} @@ -858,7 +859,16 @@ jobs: strategy: matrix: - cuda: ['12.4', '13.3'] + include: + - cuda: '12.4' + arch: x64 + defines: '-DGGML_CUDA_CUB_3DOT2=ON' + - cuda: '13.3' + arch: x64 + defines: '' + - cuda: '13.4' + arch: arm64 + defines: '-DCMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-msvc-cuda.cmake' steps: - name: Clone @@ -876,6 +886,7 @@ jobs: uses: ./.github/actions/windows-setup-cuda with: cuda_version: ${{ matrix.cuda }} + cuda_arch: ${{ matrix.arch }} - name: Install Ninja id: install_ninja @@ -885,54 +896,62 @@ jobs: - name: ccache uses: ggml-org/ccache-action@v1.2.21 with: - key: release-windows-2022-x64-cuda-${{ matrix.cuda }} + key: release-windows-2022-${{ matrix.arch }}-cuda-${{ matrix.cuda }} - name: Build id: cmake_build shell: cmd # TODO: Remove GGML_CUDA_CUB_3DOT2 flag once CCCL 3.2 is bundled within CTK and that CTK version is used in this project run: | - call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 + call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ matrix.arch == 'x64' && 'x64' || 'amd64_arm64' }} cmake -S . -B build -G "Ninja Multi-Config" ^ -DGGML_BACKEND_DL=ON ^ -DGGML_NATIVE=OFF ^ -DGGML_CPU=OFF ^ -DGGML_CUDA=ON ^ - -DLLAMA_BUILD_BORINGSSL=ON ^ - -DGGML_CUDA_CUB_3DOT2=ON + -DLLAMA_BUILD_BORINGSSL=ON ${{ matrix.defines }} set /A NINJA_JOBS=%NUMBER_OF_PROCESSORS%-1 cmake --build build --config Release -j %NINJA_JOBS% --target ggml-cuda - name: ccache-clear uses: ./.github/actions/ccache-clear with: - key: release-windows-2022-x64-cuda-${{ matrix.cuda }} + key: release-windows-2022-${{ matrix.arch }}-cuda-${{ matrix.cuda }} - name: Pack artifacts id: pack_artifacts run: | - 7z a -snl llama-bin-win-cuda-${{ matrix.cuda }}-x64.zip .\build\bin\Release\ggml-cuda.dll + 7z a -snl llama-bin-win-cuda-${{ matrix.cuda }}-${{ matrix.arch }}.zip .\build\bin\Release\ggml-cuda.dll - name: Upload artifacts uses: actions/upload-artifact@v6 with: - path: llama-bin-win-cuda-${{ matrix.cuda }}-x64.zip - name: llama-bin-win-cuda-${{ matrix.cuda }}-x64.zip + path: llama-bin-win-cuda-${{ matrix.cuda }}-${{ matrix.arch }}.zip + name: llama-bin-win-cuda-${{ matrix.cuda }}-${{ matrix.arch }}.zip - - name: Copy and pack Cuda runtime + - name: Copy and pack Cuda runtime (x64) + if: ${{ matrix.arch == 'x64' }} run: | echo "Cuda install location: ${{ env.CUDA_PATH }}" $dst='.\build\bin\cudart\' robocopy "${{env.CUDA_PATH}}\bin" $dst cudart64_*.dll cublas64_*.dll cublasLt64_*.dll robocopy "${{env.CUDA_PATH}}\lib" $dst cudart64_*.dll cublas64_*.dll cublasLt64_*.dll robocopy "${{env.CUDA_PATH}}\bin\x64" $dst cudart64_*.dll cublas64_*.dll cublasLt64_*.dll - 7z a cudart-llama-bin-win-cuda-${{ matrix.cuda }}-x64.zip $dst\* + 7z a cudart-llama-bin-win-cuda-${{ matrix.cuda }}-${{ matrix.arch }}.zip $dst\* + + - name: Copy and pack Cuda runtime (ARM64) + if: ${{ matrix.arch == 'arm64' }} + run: | + echo "Cuda install location: ${{ env.CUDA_PATH }}" + $dst='.\build\bin\cudart\' + robocopy "${{env.CUDA_PATH}}\bin\arm64" $dst cudart64_*.dll cublas64_*.dll cublasLt64_*.dll + 7z a cudart-llama-bin-win-cuda-${{ matrix.cuda }}-${{ matrix.arch }}.zip $dst\* - name: Upload Cuda runtime uses: actions/upload-artifact@v6 with: - path: cudart-llama-bin-win-cuda-${{ matrix.cuda }}-x64.zip - name: cudart-llama-bin-win-cuda-${{ matrix.cuda }}-x64.zip + path: cudart-llama-bin-win-cuda-${{ matrix.cuda }}-${{ matrix.arch }}.zip + name: cudart-llama-bin-win-cuda-${{ matrix.cuda }}-${{ matrix.arch }}.zip windows-sycl: needs: [check-release] @@ -1679,6 +1698,7 @@ jobs: - [Windows arm64 (OpenCL Adreno)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-opencl-adreno-arm64.zip) - [Windows x64 (CUDA 12)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-12.4-x64.zip) - [CUDA 12.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-12.4-x64.zip) - [Windows x64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.3-x64.zip) - [CUDA 13.3 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.3-x64.zip) + - [Windows arm64 (CUDA 13) (preview)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.4-arm64.zip) - [CUDA 13.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.4-arm64.zip) - [Windows x64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-vulkan-x64.zip) - [Windows x64 (OpenVINO)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-openvino-${{ needs.windows-openvino.outputs.openvino_version }}-x64.zip) - [Windows x64 (SYCL)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-sycl-x64.zip) diff --git a/cmake/arm64-windows-msvc-cuda.cmake b/cmake/arm64-windows-msvc-cuda.cmake new file mode 100644 index 0000000000..370f2b3d21 --- /dev/null +++ b/cmake/arm64-windows-msvc-cuda.cmake @@ -0,0 +1,26 @@ +# Used to cross-compile ggml-cuda for Windows ARM64 on an x64 Windows host. +set( CMAKE_SYSTEM_NAME Windows ) +set( CMAKE_SYSTEM_PROCESSOR arm64 ) + +if ( DEFINED CUDAToolkit_ROOT ) + file( TO_CMAKE_PATH "${CUDAToolkit_ROOT}" CUDA_ROOT ) +elseif ( DEFINED ENV{CUDA_PATH} ) + file( TO_CMAKE_PATH "$ENV{CUDA_PATH}" CUDA_ROOT ) +else() + message( FATAL_ERROR "Set CUDAToolkit_ROOT or CUDA_PATH to a Windows CUDA Toolkit with ARM64 target libraries" ) +endif() + +if ( DEFINED ENV{VCToolsInstallDir} ) + file( TO_CMAKE_PATH "$ENV{VCToolsInstallDir}" MSVC_TOOLS_ROOT ) + set( CMAKE_CUDA_HOST_COMPILER "${MSVC_TOOLS_ROOT}/bin/Hostx64/arm64/cl.exe" CACHE FILEPATH "" ) +endif() + +set( CMAKE_CUDA_COMPILER "${CUDA_ROOT}/bin/nvcc.exe" CACHE FILEPATH "" ) +set( CMAKE_CUDA_FLAGS_INIT "-target-dir=arm64" ) + +# FindCUDAToolkit selects lib/x64 from the host architecture on Windows. +set( CUDA_CUDART "${CUDA_ROOT}/lib/arm64/cudart.lib" CACHE FILEPATH "" ) +set( CUDA_cudart_LIBRARY "${CUDA_ROOT}/lib/arm64/cudart.lib" CACHE FILEPATH "" ) +set( CUDA_cublas_LIBRARY "${CUDA_ROOT}/lib/arm64/cublas.lib" CACHE FILEPATH "" ) +set( CUDA_cublasLt_LIBRARY "${CUDA_ROOT}/lib/arm64/cublasLt.lib" CACHE FILEPATH "" ) +set( CUDA_cuda_driver_LIBRARY "${CUDA_ROOT}/lib/arm64/cuda.lib" CACHE FILEPATH "" )