From 425bf0f09bebbbfd9a0d4a39a4792775fe02b7a1 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 25 Feb 2026 20:29:05 +0100 Subject: [PATCH 1/3] [ggml-virtgpu-backend] Add 'backend' object management --- .../backend/backend-dispatched-backend.cpp | 94 ++++++++-- .../backend/backend-dispatched.cpp | 173 +++++++++++++++--- .../backend/backend-dispatched.gen.h | 4 + .../ggml-virtgpu/backend/backend-dispatched.h | 34 +++- .../ggml-virtgpu/backend/backend-virgl-apir.h | 1 - ggml/src/ggml-virtgpu/backend/backend.cpp | 21 ++- 6 files changed, 282 insertions(+), 45 deletions(-) diff --git a/ggml/src/ggml-virtgpu/backend/backend-dispatched-backend.cpp b/ggml/src/ggml-virtgpu/backend/backend-dispatched-backend.cpp index 03a037f1c..759224486 100644 --- a/ggml/src/ggml-virtgpu/backend/backend-dispatched-backend.cpp +++ b/ggml/src/ggml-virtgpu/backend/backend-dispatched-backend.cpp @@ -6,6 +6,34 @@ #include "shared/apir_backend.h" #include +#include +#include + +uint32_t backend_backend_initialize(apir_encoder * enc, apir_decoder * dec, virgl_apir_context * ctx) { + GGML_UNUSED(ctx); + + // Decode backend initialization request + uintptr_t function_ptr; + apir_decode_uintptr_t(dec, &function_ptr); + void * ggml_backend_reg_fct_p = (void *) function_ptr; + + // Call the actual initialization + uintptr_t device_handle = 0; + uint32_t backend_id = 0; + uint32_t result = backend_dispatch_initialize(ggml_backend_reg_fct_p, &device_handle, &backend_id); + + // Check if initialization failed + if (result != APIR_BACKEND_INITIALIZE_SUCCESS) { + // Return error without encoding anything + return 1; + } + + // Encode the device handle and backend ID separately + apir_encode_uintptr_t(enc, &device_handle); + apir_encode_uint32_t(enc, &backend_id); + + return 0; +} static uint32_t validate_graph_operation(size_t cgraph_size, uint32_t shmem_res_id, const char * operation) { if (cgraph_size == 0) { @@ -23,17 +51,6 @@ static uint32_t validate_graph_operation(size_t cgraph_size, uint32_t shmem_res_ uint32_t backend_backend_graph_compute(apir_encoder * enc, apir_decoder * dec, virgl_apir_context * ctx) { GGML_UNUSED(ctx); - static bool async_backend_initialized = false; - static bool async_backend; - - if (!async_backend_initialized) { - ggml_backend_dev_props props; - - dev->iface.get_props(dev, &props); - async_backend = props.caps.async; - async_backend_initialized = true; - } - uint32_t shmem_res_id; apir_decode_virtgpu_shmem_res_id(dec, &shmem_res_id); @@ -46,6 +63,29 @@ uint32_t backend_backend_graph_compute(apir_encoder * enc, apir_decoder * dec, v size_t cgraph_size; apir_decode_size_t(dec, &cgraph_size); + // Decode device handle first + uintptr_t device_handle; + apir_decode_uintptr_t(dec, &device_handle); + ggml_backend_dev_t device = (ggml_backend_dev_t) device_handle; + + // Decode backend ID second + uint32_t backend_id; + apir_decode_uint32_t(dec, &backend_id); + + // Get backend instance + apir_backend_instance * instance = get_backend_instance(device, backend_id); + if (instance == nullptr || instance->bck == nullptr) { + apir_decoder_set_fatal(dec); + return 1; + } + + // Get device context for async property + apir_device_context * ext = get_device_context(device); + if (ext == nullptr) { + apir_decoder_set_fatal(dec); + return 1; + } + if (validate_graph_operation(cgraph_size, shmem_res_id, __func__) != 0) { apir_decoder_set_fatal(dec); return 1; @@ -83,20 +123,34 @@ uint32_t backend_backend_graph_compute(apir_encoder * enc, apir_decoder * dec, v } #endif - // Check if backend is properly initialized - if (!bck) { - GGML_LOG_ERROR(GGML_VIRTGPU_BCK "%s: Backend not initialized (bck is null)\n", __func__); - - return 1; - } + // Backend instance is already validated above - status = bck->iface.graph_compute(bck, cgraph); + status = instance->bck->iface.graph_compute(instance->bck, cgraph); - if (async_backend && bck->iface.synchronize) { - bck->iface.synchronize(bck); + if (ext->async_backend && instance->bck->iface.synchronize) { + instance->bck->iface.synchronize(instance->bck); } apir_encode_ggml_status(enc, &status); return 0; } + +uint32_t backend_backend_cleanup(apir_encoder * enc, apir_decoder * dec, virgl_apir_context * ctx) { + GGML_UNUSED(ctx); + GGML_UNUSED(enc); + + // Decode device handle first + uintptr_t device_handle; + apir_decode_uintptr_t(dec, &device_handle); + ggml_backend_dev_t device = (ggml_backend_dev_t) device_handle; + + // Decode backend ID second + uint32_t backend_id; + apir_decode_uint32_t(dec, &backend_id); + + // Cleanup specific backend instance + cleanup_backend_instance(device, backend_id); + + return 0; +} diff --git a/ggml/src/ggml-virtgpu/backend/backend-dispatched.cpp b/ggml/src/ggml-virtgpu/backend/backend-dispatched.cpp index c80e4aabe..c9d629b61 100644 --- a/ggml/src/ggml-virtgpu/backend/backend-dispatched.cpp +++ b/ggml/src/ggml-virtgpu/backend/backend-dispatched.cpp @@ -6,46 +6,177 @@ #include "ggml-impl.h" #include +#include +#include +// Global variables for device functions ggml_backend_reg_t reg = NULL; ggml_backend_dev_t dev = NULL; -ggml_backend_t bck = NULL; + +// Device context management +static std::unordered_map device_contexts; +static std::mutex device_contexts_mutex; uint64_t timer_start = 0; uint64_t timer_total = 0; uint64_t timer_count = 0; -uint32_t backend_dispatch_initialize(void * ggml_backend_reg_fct_p) { - if (reg != NULL) { - GGML_LOG_WARN(GGML_VIRTGPU_BCK "%s: already initialized\n", __func__); - return APIR_BACKEND_INITIALIZE_ALREADY_INITED; +// Get device context (device-owned backend instances) +apir_device_context * get_device_context(ggml_backend_dev_t device) { + std::lock_guard lock(device_contexts_mutex); + auto it = device_contexts.find(device); + if (it == device_contexts.end()) { + return nullptr; + } + apir_device_context * ext = it->second; + if (ext->magic != APIR_DEVICE_EXTENSION_MAGIC) { + return nullptr; } - ggml_backend_reg_t (*ggml_backend_reg_fct)(void) = (ggml_backend_reg_t (*)()) ggml_backend_reg_fct_p; + return ext; +} + +// Ensure device context exists +void ensure_device_context(ggml_backend_dev_t device) { + std::lock_guard lock(device_contexts_mutex); + + auto it = device_contexts.find(device); + if (it == device_contexts.end()) { + apir_device_context * ext = new apir_device_context(); + ext->next_backend_id = 1; + + // Get async backend properties from the device + ggml_backend_dev_props props; + device->iface.get_props(device, &props); + ext->async_backend = props.caps.async; - reg = ggml_backend_reg_fct(); - if (reg == NULL) { - GGML_LOG_ERROR(GGML_VIRTGPU_BCK "%s: backend registration failed\n", __func__); - return APIR_BACKEND_INITIALIZE_BACKEND_REG_FAILED; + ext->magic = APIR_DEVICE_EXTENSION_MAGIC; + device_contexts[device] = ext; } +} + +// Create new backend instance for device +uintptr_t create_backend_instance(ggml_backend_dev_t device) { + ensure_device_context(device); + apir_device_context * ext = get_device_context(device); + if (ext == nullptr) { + return 0; // Failed + } + + std::lock_guard lock(ext->backends_mutex); + + apir_backend_instance * instance = new apir_backend_instance(); + instance->bck = device->iface.init_backend(device, NULL); + instance->magic = APIR_BACKEND_INSTANCE_MAGIC; - size_t device_count = reg->iface.get_device_count(reg); - if (!device_count) { - GGML_LOG_ERROR(GGML_VIRTGPU_BCK "%s: no device found\n", __func__); - return APIR_BACKEND_INITIALIZE_NO_DEVICE; + if (instance->bck == nullptr) { + GGML_LOG_ERROR(GGML_VIRTGPU_BCK "%s: device->iface.init_backend failed for device %p\n", __func__, + (void *) device); + delete instance; + return 0; // Failed } - dev = reg->iface.get_device(reg, 0); + uintptr_t backend_id = ext->next_backend_id++; + ext->backend_instances[backend_id] = instance; + + return backend_id; +} + +// Get backend instance +apir_backend_instance * get_backend_instance(ggml_backend_dev_t device, uintptr_t backend_id) { + apir_device_context * ext = get_device_context(device); + if (ext == nullptr) { + return nullptr; + } + + std::lock_guard lock(ext->backends_mutex); + auto it = ext->backend_instances.find(backend_id); + if (it == ext->backend_instances.end()) { + return nullptr; + } + + apir_backend_instance * instance = it->second; + if (instance->magic != APIR_BACKEND_INSTANCE_MAGIC) { + return nullptr; + } + + return instance; +} - if (!dev) { - GGML_LOG_ERROR(GGML_VIRTGPU_BCK "%s: failed to get device\n", __func__); - return APIR_BACKEND_INITIALIZE_NO_DEVICE; +// Cleanup specific backend instance +void cleanup_backend_instance(ggml_backend_dev_t device, uintptr_t backend_id) { + apir_device_context * ext = get_device_context(device); + if (ext == nullptr) { + return; } - bck = dev->iface.init_backend(dev, NULL); - if (!bck) { - GGML_LOG_ERROR(GGML_VIRTGPU_BCK "%s: backend initialization failed\n", __func__); + std::lock_guard lock(ext->backends_mutex); + auto it = ext->backend_instances.find(backend_id); + if (it != ext->backend_instances.end()) { + apir_backend_instance * instance = it->second; + + // Free backend + if (instance->bck) { + ggml_backend_free(instance->bck); + instance->bck = nullptr; + } + + instance->magic = 0; // Invalidate + delete instance; + ext->backend_instances.erase(it); + } +} + +// Cleanup device context and all its backend instances +void cleanup_device_context(ggml_backend_dev_t device) { + std::lock_guard lock(device_contexts_mutex); + + auto it = device_contexts.find(device); + if (it != device_contexts.end()) { + apir_device_context * ext = it->second; + + // Clean up all backend instances + { + std::lock_guard backends_lock(ext->backends_mutex); + for (auto & [backend_id, instance] : ext->backend_instances) { + if (instance->bck) { + ggml_backend_free(instance->bck); + } + + instance->magic = 0; + delete instance; + } + ext->backend_instances.clear(); + } + + ext->magic = 0; // Invalidate + delete ext; + device_contexts.erase(it); + } +} + +uint32_t backend_dispatch_initialize(void * ggml_backend_reg_fct_p, uintptr_t * out_handle, uint32_t * out_backend_id) { + GGML_UNUSED(ggml_backend_reg_fct_p); // reg/dev are already set during library loading + + if (out_handle == nullptr || out_backend_id == nullptr) { + return APIR_BACKEND_INITIALIZE_BACKEND_INIT_FAILED; + } + + // Ensure global variables are set (should be done during library loading) + if (reg == NULL || dev == NULL) { + GGML_LOG_ERROR(GGML_VIRTGPU_BCK "%s: Global reg/dev not initialized (reg=%p, dev=%p)\n", __func__, (void *) reg, + (void *) dev); return APIR_BACKEND_INITIALIZE_BACKEND_INIT_FAILED; } + // Create new backend instance + uintptr_t backend_id = create_backend_instance(dev); + if (backend_id == 0) { + return APIR_BACKEND_INITIALIZE_BACKEND_INIT_FAILED; + } + + // Set output parameters + *out_handle = (uintptr_t) dev; + *out_backend_id = (uint32_t) backend_id; + return APIR_BACKEND_INITIALIZE_SUCCESS; } diff --git a/ggml/src/ggml-virtgpu/backend/backend-dispatched.gen.h b/ggml/src/ggml-virtgpu/backend/backend-dispatched.gen.h index 3dc334e4c..9efc29cbc 100644 --- a/ggml/src/ggml-virtgpu/backend/backend-dispatched.gen.h +++ b/ggml/src/ggml-virtgpu/backend/backend-dispatched.gen.h @@ -30,7 +30,9 @@ uint32_t backend_buffer_clear(apir_encoder * enc, apir_decoder * dec, virgl_apir uint32_t backend_buffer_free_buffer(apir_encoder * enc, apir_decoder * dec, virgl_apir_context * ctx); /* backend */ +uint32_t backend_backend_initialize(apir_encoder * enc, apir_decoder * dec, virgl_apir_context * ctx); uint32_t backend_backend_graph_compute(apir_encoder * enc, apir_decoder * dec, virgl_apir_context * ctx); +uint32_t backend_backend_cleanup(apir_encoder * enc, apir_decoder * dec, virgl_apir_context * ctx); extern "C" { static const backend_dispatch_t apir_backend_dispatch_table[APIR_BACKEND_DISPATCH_TABLE_COUNT] = { @@ -68,6 +70,8 @@ static const backend_dispatch_t apir_backend_dispatch_table[APIR_BACKEND_DISPATC /* backend */ + /* APIR_COMMAND_TYPE_BACKEND_INITIALIZE = */ backend_backend_initialize, /* APIR_COMMAND_TYPE_BACKEND_GRAPH_COMPUTE = */ backend_backend_graph_compute, + /* APIR_COMMAND_TYPE_BACKEND_CLEANUP = */ backend_backend_cleanup, }; } diff --git a/ggml/src/ggml-virtgpu/backend/backend-dispatched.h b/ggml/src/ggml-virtgpu/backend/backend-dispatched.h index 740ee9e3f..7a1ff62c3 100644 --- a/ggml/src/ggml-virtgpu/backend/backend-dispatched.h +++ b/ggml/src/ggml-virtgpu/backend/backend-dispatched.h @@ -3,6 +3,8 @@ // clang-format off #include #include +#include +#include #include @@ -24,4 +26,34 @@ typedef uint32_t (*backend_dispatch_t)(apir_encoder * enc, apir_decoder * dec, v #include "backend-dispatched.gen.h" -uint32_t backend_dispatch_initialize(void * ggml_backend_reg_fct_p); +// Backend instance structure - one backend per instance +struct apir_backend_instance { + ggml_backend_t bck; // The actual backend + uint32_t magic; // For validation: 0xAB1234CD +}; + +// Device context structure - can have multiple backend instances +struct apir_device_context { + std::mutex backends_mutex; + std::unordered_map backend_instances; + uintptr_t next_backend_id; + + bool async_backend; // Whether the backend supports async operations + + uint32_t magic; // For validation: 0xAB1234CD +}; + +#define APIR_DEVICE_EXTENSION_MAGIC 0xAB1234CD +#define APIR_BACKEND_INSTANCE_MAGIC 0xCD4321BA + +// Device context management +apir_device_context * get_device_context(ggml_backend_dev_t dev); +void ensure_device_context(ggml_backend_dev_t dev); +void cleanup_device_context(ggml_backend_dev_t dev); + +// Backend instance management +uintptr_t create_backend_instance(ggml_backend_dev_t dev); +apir_backend_instance * get_backend_instance(ggml_backend_dev_t dev, uintptr_t backend_id); +void cleanup_backend_instance(ggml_backend_dev_t dev, uintptr_t backend_id); + +uint32_t backend_dispatch_initialize(void * ggml_backend_reg_fct_p, uintptr_t * out_handle, uint32_t * out_backend_id); diff --git a/ggml/src/ggml-virtgpu/backend/backend-virgl-apir.h b/ggml/src/ggml-virtgpu/backend/backend-virgl-apir.h index c65a01cdf..366379aef 100644 --- a/ggml/src/ggml-virtgpu/backend/backend-virgl-apir.h +++ b/ggml/src/ggml-virtgpu/backend/backend-virgl-apir.h @@ -11,7 +11,6 @@ extern ggml_backend_reg_t reg; extern ggml_backend_dev_t dev; -extern ggml_backend_t bck; struct virgl_apir_callbacks { const char * (*get_config)(uint32_t virgl_ctx_id, const char * key); diff --git a/ggml/src/ggml-virtgpu/backend/backend.cpp b/ggml/src/ggml-virtgpu/backend/backend.cpp index 535a05f3e..9969f8d02 100644 --- a/ggml/src/ggml-virtgpu/backend/backend.cpp +++ b/ggml/src/ggml-virtgpu/backend/backend.cpp @@ -97,9 +97,26 @@ ApirLoadLibraryReturnCode apir_backend_initialize(uint32_t virgl_ctx_id, struct return APIR_LOAD_LIBRARY_SYMBOL_MISSING; } - uint32_t ret = backend_dispatch_initialize(ggml_backend_reg_fct); + typedef ggml_backend_reg_t (*ggml_backend_reg_fct_t)(void); + ggml_backend_reg_fct_t backend_reg_fct = (ggml_backend_reg_fct_t) ggml_backend_reg_fct; - return (ApirLoadLibraryReturnCode) (APIR_LOAD_LIBRARY_INIT_BASE_INDEX + ret); + if (backend_reg_fct == NULL) { + return APIR_LOAD_LIBRARY_SYMBOL_MISSING; + } + + reg = backend_reg_fct(); + if (reg == NULL) { + return (ApirLoadLibraryReturnCode) (APIR_LOAD_LIBRARY_INIT_BASE_INDEX + + APIR_BACKEND_INITIALIZE_BACKEND_INIT_FAILED); + } + + dev = ggml_backend_reg_dev_get(reg, 0); + if (dev == NULL) { + return (ApirLoadLibraryReturnCode) (APIR_LOAD_LIBRARY_INIT_BASE_INDEX + + APIR_BACKEND_INITIALIZE_BACKEND_INIT_FAILED); + } + + return APIR_LOAD_LIBRARY_SUCCESS; } uint32_t apir_backend_dispatcher(uint32_t virgl_ctx_id, From c5090e462ab93833d3693cec57bc6dc03d017d2b Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 25 Feb 2026 20:29:48 +0100 Subject: [PATCH 2/3] [ggml-virtgpu/apir] Add 'backend' object management --- .../backend/shared/apir_backend.gen.h | 10 ++++++++-- ggml/src/ggml-virtgpu/ggmlremoting_functions.yaml | 15 +++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-virtgpu/backend/shared/apir_backend.gen.h b/ggml/src/ggml-virtgpu/backend/shared/apir_backend.gen.h index 520ac9c72..e89b6a5f2 100644 --- a/ggml/src/ggml-virtgpu/backend/shared/apir_backend.gen.h +++ b/ggml/src/ggml-virtgpu/backend/shared/apir_backend.gen.h @@ -29,10 +29,12 @@ typedef enum ApirBackendCommandType { APIR_COMMAND_TYPE_BUFFER_FREE_BUFFER = 21, /* backend */ - APIR_COMMAND_TYPE_BACKEND_GRAPH_COMPUTE = 22, + APIR_COMMAND_TYPE_BACKEND_INITIALIZE = 22, + APIR_COMMAND_TYPE_BACKEND_GRAPH_COMPUTE = 23, + APIR_COMMAND_TYPE_BACKEND_CLEANUP = 24, // last command_type index + 1 - APIR_BACKEND_DISPATCH_TABLE_COUNT = 23, + APIR_BACKEND_DISPATCH_TABLE_COUNT = 25, } ApirBackendCommandType; static inline const char * apir_dispatch_command_name(ApirBackendCommandType type) { @@ -85,8 +87,12 @@ static inline const char * apir_dispatch_command_name(ApirBackendCommandType typ case APIR_COMMAND_TYPE_BUFFER_FREE_BUFFER: return "buffer_free_buffer"; /* backend */ + case APIR_COMMAND_TYPE_BACKEND_INITIALIZE: + return "backend_initialize"; case APIR_COMMAND_TYPE_BACKEND_GRAPH_COMPUTE: return "backend_graph_compute"; + case APIR_COMMAND_TYPE_BACKEND_CLEANUP: + return "backend_cleanup"; default: return "unknown"; diff --git a/ggml/src/ggml-virtgpu/ggmlremoting_functions.yaml b/ggml/src/ggml-virtgpu/ggmlremoting_functions.yaml index 14ef2433e..a634944e4 100644 --- a/ggml/src/ggml-virtgpu/ggmlremoting_functions.yaml +++ b/ggml/src/ggml-virtgpu/ggmlremoting_functions.yaml @@ -140,9 +140,18 @@ functions: backend: group_description: "backend" functions: + initialize: + frontend_return: "int" + frontend_extra_params: + - "void *ggml_backend_reg_fct_p" + - "uintptr_t* out_device_handle" + - "uint32_t* out_backend_id" + graph_compute: frontend_return: "ggml_status" frontend_extra_params: + - "uintptr_t device_handle" + - "uint32_t backend_id" - "ggml_cgraph *cgraph" graph_optimize: @@ -151,6 +160,12 @@ functions: - "ggml_cgraph *cgraph" enabled: false + cleanup: + frontend_return: "void" + frontend_extra_params: + - "uintptr_t device_handle" + - "uint32_t backend_id" + # Naming patterns used for code generation naming_patterns: # How to generate enum names From f793662f3c8193cb9042a598c8e7c72c1823adff Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 25 Feb 2026 20:30:40 +0100 Subject: [PATCH 3/3] [ggml-virtgpu] Add 'backend' object management --- ggml/src/ggml-virtgpu/ggml-backend-reg.cpp | 10 +-- ggml/src/ggml-virtgpu/ggml-backend.cpp | 21 +++++- ggml/src/ggml-virtgpu/ggml-remoting.h | 2 + .../ggml-virtgpu/virtgpu-forward-backend.cpp | 72 ++++++++++++++++++- ggml/src/ggml-virtgpu/virtgpu-forward.gen.h | 10 ++- 5 files changed, 107 insertions(+), 8 deletions(-) diff --git a/ggml/src/ggml-virtgpu/ggml-backend-reg.cpp b/ggml/src/ggml-virtgpu/ggml-backend-reg.cpp index a4df5956a..68f25cdee 100644 --- a/ggml/src/ggml-virtgpu/ggml-backend-reg.cpp +++ b/ggml/src/ggml-virtgpu/ggml-backend-reg.cpp @@ -133,10 +133,12 @@ static void ggml_backend_remoting_reg_init_devices(ggml_backend_reg_t reg) { ggml_backend_remoting_device_context * ctx = new ggml_backend_remoting_device_context; char desc[256] = "ggml-virtgpu API Remoting device"; - ctx->device = i; - ctx->name = GGML_VIRTGPU_NAME + std::to_string(i); - ctx->description = desc; - ctx->gpu = gpu; + ctx->device = i; + ctx->name = GGML_VIRTGPU_NAME + std::to_string(i); + ctx->description = desc; + ctx->gpu = gpu; + ctx->device_handle = 0; // Will be set during backend initialization + ctx->backend_id = 0; // Will be set during backend initialization ggml_backend_dev_t dev = new ggml_backend_device{ /* .iface = */ ggml_backend_remoting_device_interface, diff --git a/ggml/src/ggml-virtgpu/ggml-backend.cpp b/ggml/src/ggml-virtgpu/ggml-backend.cpp index a63ee2b9d..fd96190c7 100644 --- a/ggml/src/ggml-virtgpu/ggml-backend.cpp +++ b/ggml/src/ggml-virtgpu/ggml-backend.cpp @@ -1,4 +1,5 @@ #include "../../include/ggml-virtgpu.h" +#include "backend/shared/apir_backend.h" #include "ggml-remoting.h" static const char * ggml_backend_remoting_get_name(ggml_backend_t backend) { @@ -8,13 +9,19 @@ static const char * ggml_backend_remoting_get_name(ggml_backend_t backend) { } static void ggml_backend_remoting_free(ggml_backend_t backend) { + ggml_backend_remoting_device_context * ctx = (ggml_backend_remoting_device_context *) backend->context; + virtgpu * gpu = ctx->gpu; + + apir_backend_cleanup(gpu, ctx->device_handle, ctx->backend_id); + delete backend; } static ggml_status ggml_backend_remoting_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) { - virtgpu * gpu = DEV_TO_GPU(backend->device); + ggml_backend_remoting_device_context * ctx = (ggml_backend_remoting_device_context *) backend->context; + virtgpu * gpu = ctx->gpu; - return apir_backend_graph_compute(gpu, cgraph); + return apir_backend_graph_compute(gpu, ctx->device_handle, ctx->backend_id, cgraph); } static void ggml_backend_remoting_graph_optimize(ggml_backend_t backend, ggml_cgraph * cgraph) { @@ -58,6 +65,16 @@ ggml_backend_t ggml_backend_remoting_device_init(ggml_backend_dev_t dev, const c ggml_backend_remoting_device_context * ctx = (ggml_backend_remoting_device_context *) dev->context; + // Initialize the backend by calling the backend + int init_result = + apir_backend_initialize(ctx->gpu, (void *) ggml_backend_virtgpu_reg, &ctx->device_handle, &ctx->backend_id); + + if (init_result != APIR_BACKEND_INITIALIZE_SUCCESS) { + // Backend initialization failed + GGML_LOG_ERROR(GGML_VIRTGPU "%s: Backend initialization failed with result=%d\n", __func__, init_result); + return nullptr; + } + ggml_backend_t remoting_backend = new ggml_backend{ /* .guid = */ ggml_backend_remoting_guid(), /* .interface = */ ggml_backend_remoting_interface, diff --git a/ggml/src/ggml-virtgpu/ggml-remoting.h b/ggml/src/ggml-virtgpu/ggml-remoting.h index 4f70326be..a65974bcb 100644 --- a/ggml/src/ggml-virtgpu/ggml-remoting.h +++ b/ggml/src/ggml-virtgpu/ggml-remoting.h @@ -36,6 +36,8 @@ struct ggml_backend_remoting_device_context { std::vector> shared_memory; virtgpu * gpu; + uintptr_t device_handle; // Device handle returned from backend initialization + uint32_t backend_id; // Backend ID within the device }; struct ggml_backend_remoting_buffer_context { diff --git a/ggml/src/ggml-virtgpu/virtgpu-forward-backend.cpp b/ggml/src/ggml-virtgpu/virtgpu-forward-backend.cpp index 4593690c6..d65ac5da5 100644 --- a/ggml/src/ggml-virtgpu/virtgpu-forward-backend.cpp +++ b/ggml/src/ggml-virtgpu/virtgpu-forward-backend.cpp @@ -6,7 +6,57 @@ static long long current_time_ms() { return (long long) ts.tv_sec * 1000000000LL + ts.tv_nsec; } -ggml_status apir_backend_graph_compute(virtgpu * gpu, ggml_cgraph * cgraph) { +int apir_backend_initialize(virtgpu * gpu, + void * ggml_backend_reg_fct_p, + uintptr_t * out_device_handle, + uint32_t * out_backend_id) { + apir_encoder * encoder; + apir_decoder * decoder; + ApirForwardReturnCode ret; + + REMOTE_CALL_PREPARE(gpu, encoder, APIR_COMMAND_TYPE_BACKEND_INITIALIZE); + + // Send the backend registration function pointer + uintptr_t function_ptr = (uintptr_t) ggml_backend_reg_fct_p; + apir_encode_uintptr_t(encoder, &function_ptr); + + REMOTE_CALL(gpu, encoder, decoder, ret); + + uintptr_t device_handle = 0; + uint32_t backend_id = 0; + + GGML_LOG_INFO(GGML_VIRTGPU "%s: Backend initialization returned: %d\n", __func__, ret); + + // Check if backend function succeeded (ret == 0) or failed (ret != 0) + if (ret == 0) { + // Success - decode the response + apir_decode_uintptr_t(decoder, &device_handle); + apir_decode_uint32_t(decoder, &backend_id); + GGML_LOG_INFO(GGML_VIRTGPU "%s: Success - device_handle=%p, backend_id=%u\n", __func__, (void *) device_handle, + backend_id); + } else { + GGML_LOG_ERROR(GGML_VIRTGPU "%s: Backend initialization failed with ret=%d\n", __func__, ret); + } + + remote_call_finish(gpu, encoder, decoder); + + // Set output parameters + if (out_device_handle) { + *out_device_handle = device_handle; + } + + if (out_backend_id) { + *out_backend_id = backend_id; + } + + // Return the result code + return (ret == 0) ? APIR_BACKEND_INITIALIZE_SUCCESS : APIR_BACKEND_INITIALIZE_BACKEND_INIT_FAILED; +} + +ggml_status apir_backend_graph_compute(virtgpu * gpu, + uintptr_t device_handle, + uint32_t backend_id, + ggml_cgraph * cgraph) { apir_encoder * encoder; apir_decoder * decoder; ApirForwardReturnCode ret; @@ -35,6 +85,10 @@ ggml_status apir_backend_graph_compute(virtgpu * gpu, ggml_cgraph * cgraph) { apir_encode_size_t(encoder, &cgraph_size); + // Send device handle and backend ID separately + apir_encode_uintptr_t(encoder, &device_handle); + apir_encode_uint32_t(encoder, &backend_id); + char * shmem_data = (char *) shmem->mmap_ptr; apir_encoder secondary_enc = apir_new_encoder(shmem_data, cgraph_size); @@ -56,3 +110,19 @@ ggml_status apir_backend_graph_compute(virtgpu * gpu, ggml_cgraph * cgraph) { return status; } + +void apir_backend_cleanup(virtgpu * gpu, uintptr_t device_handle, uint32_t backend_id) { + apir_encoder * encoder; + apir_decoder * decoder; + ApirForwardReturnCode ret; + + REMOTE_CALL_PREPARE(gpu, encoder, APIR_COMMAND_TYPE_BACKEND_CLEANUP); + + // Send device handle and backend ID separately + apir_encode_uintptr_t(encoder, &device_handle); + apir_encode_uint32_t(encoder, &backend_id); + + REMOTE_CALL(gpu, encoder, decoder, ret); + + remote_call_finish(gpu, encoder, decoder); +} diff --git a/ggml/src/ggml-virtgpu/virtgpu-forward.gen.h b/ggml/src/ggml-virtgpu/virtgpu-forward.gen.h index 44b0ad1ff..5567fdd18 100644 --- a/ggml/src/ggml-virtgpu/virtgpu-forward.gen.h +++ b/ggml/src/ggml-virtgpu/virtgpu-forward.gen.h @@ -50,4 +50,12 @@ void apir_buffer_clear(struct virtgpu * gpu, apir_buffer_context_t * buffer_co void apir_buffer_free_buffer(struct virtgpu * gpu, apir_buffer_context_t * buffer_context); /* backend */ -ggml_status apir_backend_graph_compute(struct virtgpu * gpu, ggml_cgraph * cgraph); +int apir_backend_initialize(struct virtgpu * gpu, + void * ggml_backend_reg_fct_p, + uintptr_t * out_device_handle, + uint32_t * out_backend_id); +ggml_status apir_backend_graph_compute(struct virtgpu * gpu, + uintptr_t device_handle, + uint32_t backend_id, + ggml_cgraph * cgraph); +void apir_backend_cleanup(struct virtgpu * gpu, uintptr_t device_handle, uint32_t backend_id);