diff --git a/ggml/src/ggml-metal/ggml-metal-device.h b/ggml/src/ggml-metal/ggml-metal-device.h index d0956df506..91b841b67b 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.h +++ b/ggml/src/ggml-metal/ggml-metal-device.h @@ -213,7 +213,7 @@ typedef void * ggml_metal_rset_t; // a collection of residency sets (non-owning) typedef struct ggml_metal_rsets * ggml_metal_rsets_t; -ggml_metal_rsets_t ggml_metal_rsets_init(void); +ggml_metal_rsets_t ggml_metal_rsets_init(ggml_metal_device_t dev); void ggml_metal_rsets_free(ggml_metal_rsets_t rsets); // diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m index 4edd77c6f2..7d2a686850 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.m +++ b/ggml/src/ggml-metal/ggml-metal-device.m @@ -557,7 +557,32 @@ struct ggml_metal_rsets { dispatch_group_t d_group; }; -ggml_metal_rsets_t ggml_metal_rsets_init(void) { +#if defined(GGML_METAL_HAS_RESIDENCY_SETS) +static void ggml_metal_dummy_work(ggml_metal_device_t dev) { + if (dev->mtl_queue == nil) { + return; + } + + @autoreleasepool { + // perform a minimal dummy operation on the GPU + id buf = [dev->mtl_device newBufferWithLength:1 options:MTLResourceStorageModePrivate]; + id cmd_buf = [dev->mtl_queue commandBuffer]; + + { + id encoder = [cmd_buf blitCommandEncoder]; + + [encoder fillBuffer:buf range:NSMakeRange(0, 1) value:0]; + + [encoder endEncoding]; + } + + [cmd_buf commit]; + [buf release]; + } +} +#endif + +ggml_metal_rsets_t ggml_metal_rsets_init(ggml_metal_device_t dev) { ggml_metal_rsets_t res = calloc(1, sizeof(struct ggml_metal_rsets)); res->lock = [[NSLock alloc] init]; @@ -610,6 +635,15 @@ ggml_metal_rsets_t ggml_metal_rsets_init(void) { #endif }); +#if defined(GGML_METAL_HAS_RESIDENCY_SETS) + if (@available(macOS 15.0, iOS 18.0, tvOS 18.0, visionOS 2.0, *)) { + // workaround for residency set memory not being released if no GPU operation occurs + // https://developer.apple.com/forums/thread/839089 + // https://github.com/ggml-org/llama.cpp/issues/25937 + ggml_metal_dummy_work(dev); + } +#endif + return res; } @@ -864,7 +898,7 @@ ggml_metal_device_t ggml_metal_device_init(int device) { } if (dev->props.use_residency_sets) { - dev->rsets = ggml_metal_rsets_init(); + dev->rsets = ggml_metal_rsets_init(dev); } else { dev->rsets = nil; } @@ -1484,6 +1518,7 @@ static void ggml_metal_buffer_rset_free(ggml_metal_buffer_t buf) { if (buf->rset) { [buf->rset endResidency]; [buf->rset removeAllAllocations]; + [buf->rset commit]; [buf->rset release]; } } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7a93b19a07..805b744726 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -278,6 +278,9 @@ set_tests_properties(test-state-restore-fragmented PROPERTIES FIXTURES_REQUIRED llama_build_and_test(test-save-load-state.cpp LABEL "model" ARGS -m "${MODEL_DEST}") set_tests_properties(test-save-load-state PROPERTIES FIXTURES_REQUIRED test-download-model) +if (APPLE) + llama_build(test-rset-release.cpp get-model.cpp) +endif() if (NOT GGML_BACKEND_DL) # these tests use the backends directly and cannot be built with dynamic loading llama_build_and_test(test-barrier.cpp) diff --git a/tests/test-rset-release.cpp b/tests/test-rset-release.cpp new file mode 100644 index 0000000000..bf03c5e8bd --- /dev/null +++ b/tests/test-rset-release.cpp @@ -0,0 +1,53 @@ +// ref: https://github.com/ggml-org/llama.cpp/issues/25937 +// only works reliably when run with a large model that occupies 3GB+ of wired memory +// thus, this test is not run by default +// example model to run with: google/gemma-4-E4B-it-qat-q4_0-gguf + +#include +#include +#include +#include + +#include "llama.h" +#include "get-model.h" + +static uint64_t wired_memory() { + vm_statistics64_data_t vmstat; + mach_msg_type_number_t count = HOST_VM_INFO64_COUNT; + if (host_statistics64(mach_host_self(), HOST_VM_INFO64, (host_info64_t)&vmstat, &count) != KERN_SUCCESS) { + return UINT64_MAX; + } + return static_cast(vmstat.wire_count) * vm_kernel_page_size; +} + +int main(int argc, char ** argv) { + auto * model_path = get_model_or_exit(argc, argv); + + llama_backend_init(); + + const uint64_t wired_initial = wired_memory(); + + llama_model_params params = llama_model_default_params(); + params.load_mode = LLAMA_LOAD_MODE_NONE; + struct llama_model* model = llama_model_load_from_file(model_path, params); + + const uint64_t wired_loaded = wired_memory(); + const uint64_t wired_delta = wired_loaded - wired_initial; + // system memory fluctuates, so we need to allocate enough to reliably detect the release + GGML_ASSERT(wired_delta > 2'000'000'000); // 2GB + + llama_model_free(model); + + const uint64_t t_start_ms = ggml_time_ms(); + + // expect most of the allocated memory to be released within 10 seconds + // we allow for some tolerance due to system-wide memory fluctuations + while (wired_memory() > wired_loaded - 0.75 * wired_delta) { + GGML_ASSERT(ggml_time_ms() - t_start_ms < 10'000); + usleep(100'000); // 100ms + } + + llama_backend_free(); + + return 0; +}