Skip to content

Add backend object management #30

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
kpouget wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Add backend object management #30

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
425bf0f
[ggml-virtgpu-backend] Add 'backend' object management
kpouget Feb 25, 2026
c5090e4
[ggml-virtgpu/apir] Add 'backend' object management
kpouget Feb 25, 2026
f793662
[ggml-virtgpu] Add 'backend' object management
kpouget Feb 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
94 changes: 74 additions & 20 deletions ggml/src/ggml-virtgpu/backend/backend-dispatched-backend.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,34 @@
#include "shared/apir_backend.h"

#include <cstdint>
#include <cstdlib>
#include <cstring>

uint32_t backend_backend_initialize(apir_encoder * enc, apir_decoder * dec, virgl_apir_context * ctx) {
GGML_UNUSED(ctx);

// Decode backend initialization request
uintptr_t function_ptr;
apir_decode_uintptr_t(dec, &function_ptr);
void * ggml_backend_reg_fct_p = (void *) function_ptr;

// Call the actual initialization
uintptr_t device_handle = 0;
uint32_t backend_id = 0;
uint32_t result = backend_dispatch_initialize(ggml_backend_reg_fct_p, &device_handle, &backend_id);

// Check if initialization failed
if (result != APIR_BACKEND_INITIALIZE_SUCCESS) {
// Return error without encoding anything
return 1;
}

// Encode the device handle and backend ID separately
apir_encode_uintptr_t(enc, &device_handle);
apir_encode_uint32_t(enc, &backend_id);

return 0;
}

static uint32_t validate_graph_operation(size_t cgraph_size, uint32_t shmem_res_id, const char * operation) {
if (cgraph_size == 0) {
Expand All @@ -23,17 +51,6 @@ static uint32_t validate_graph_operation(size_t cgraph_size, uint32_t shmem_res_
uint32_t backend_backend_graph_compute(apir_encoder * enc, apir_decoder * dec, virgl_apir_context * ctx) {
GGML_UNUSED(ctx);

static bool async_backend_initialized = false;
static bool async_backend;

if (!async_backend_initialized) {
ggml_backend_dev_props props;

dev->iface.get_props(dev, &props);
async_backend = props.caps.async;
async_backend_initialized = true;
}

uint32_t shmem_res_id;
apir_decode_virtgpu_shmem_res_id(dec, &shmem_res_id);

Expand All @@ -46,6 +63,29 @@ uint32_t backend_backend_graph_compute(apir_encoder * enc, apir_decoder * dec, v
size_t cgraph_size;
apir_decode_size_t(dec, &cgraph_size);

// Decode device handle first
uintptr_t device_handle;
apir_decode_uintptr_t(dec, &device_handle);
ggml_backend_dev_t device = (ggml_backend_dev_t) device_handle;

// Decode backend ID second
uint32_t backend_id;
apir_decode_uint32_t(dec, &backend_id);

// Get backend instance
apir_backend_instance * instance = get_backend_instance(device, backend_id);
if (instance == nullptr || instance->bck == nullptr) {
apir_decoder_set_fatal(dec);
return 1;
}

// Get device context for async property
apir_device_context * ext = get_device_context(device);
if (ext == nullptr) {
apir_decoder_set_fatal(dec);
return 1;
}

if (validate_graph_operation(cgraph_size, shmem_res_id, __func__) != 0) {
apir_decoder_set_fatal(dec);
return 1;
Expand Down Expand Up @@ -83,20 +123,34 @@ uint32_t backend_backend_graph_compute(apir_encoder * enc, apir_decoder * dec, v
}
#endif

// Check if backend is properly initialized
if (!bck) {
GGML_LOG_ERROR(GGML_VIRTGPU_BCK "%s: Backend not initialized (bck is null)\n", __func__);

return 1;
}
// Backend instance is already validated above

status = bck->iface.graph_compute(bck, cgraph);
status = instance->bck->iface.graph_compute(instance->bck, cgraph);

if (async_backend && bck->iface.synchronize) {
bck->iface.synchronize(bck);
if (ext->async_backend && instance->bck->iface.synchronize) {
instance->bck->iface.synchronize(instance->bck);
}

apir_encode_ggml_status(enc, &status);

return 0;
}

uint32_t backend_backend_cleanup(apir_encoder * enc, apir_decoder * dec, virgl_apir_context * ctx) {
GGML_UNUSED(ctx);
GGML_UNUSED(enc);

// Decode device handle first
uintptr_t device_handle;
apir_decode_uintptr_t(dec, &device_handle);
ggml_backend_dev_t device = (ggml_backend_dev_t) device_handle;

// Decode backend ID second
uint32_t backend_id;
apir_decode_uint32_t(dec, &backend_id);

// Cleanup specific backend instance
cleanup_backend_instance(device, backend_id);

return 0;
}
173 changes: 152 additions & 21 deletions ggml/src/ggml-virtgpu/backend/backend-dispatched.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,46 +6,177 @@
#include "ggml-impl.h"

#include <cstdint>
#include <mutex>
#include <unordered_map>

// Global variables for device functions
ggml_backend_reg_t reg = NULL;
ggml_backend_dev_t dev = NULL;
ggml_backend_t bck = NULL;

// Device context management
static std::unordered_map<ggml_backend_dev_t, apir_device_context *> device_contexts;
static std::mutex device_contexts_mutex;

uint64_t timer_start = 0;
uint64_t timer_total = 0;
uint64_t timer_count = 0;

uint32_t backend_dispatch_initialize(void * ggml_backend_reg_fct_p) {
if (reg != NULL) {
GGML_LOG_WARN(GGML_VIRTGPU_BCK "%s: already initialized\n", __func__);
return APIR_BACKEND_INITIALIZE_ALREADY_INITED;
// Get device context (device-owned backend instances)
apir_device_context * get_device_context(ggml_backend_dev_t device) {
std::lock_guard<std::mutex> lock(device_contexts_mutex);
auto it = device_contexts.find(device);
if (it == device_contexts.end()) {
return nullptr;
}
apir_device_context * ext = it->second;
if (ext->magic != APIR_DEVICE_EXTENSION_MAGIC) {
return nullptr;
}
ggml_backend_reg_t (*ggml_backend_reg_fct)(void) = (ggml_backend_reg_t (*)()) ggml_backend_reg_fct_p;
return ext;
}

// Ensure device context exists
void ensure_device_context(ggml_backend_dev_t device) {
std::lock_guard<std::mutex> lock(device_contexts_mutex);

auto it = device_contexts.find(device);
if (it == device_contexts.end()) {
apir_device_context * ext = new apir_device_context();
ext->next_backend_id = 1;

// Get async backend properties from the device
ggml_backend_dev_props props;
device->iface.get_props(device, &props);
ext->async_backend = props.caps.async;

reg = ggml_backend_reg_fct();
if (reg == NULL) {
GGML_LOG_ERROR(GGML_VIRTGPU_BCK "%s: backend registration failed\n", __func__);
return APIR_BACKEND_INITIALIZE_BACKEND_REG_FAILED;
ext->magic = APIR_DEVICE_EXTENSION_MAGIC;
device_contexts[device] = ext;
}
}

// Create new backend instance for device
uintptr_t create_backend_instance(ggml_backend_dev_t device) {
ensure_device_context(device);
apir_device_context * ext = get_device_context(device);
if (ext == nullptr) {
return 0; // Failed
}

std::lock_guard<std::mutex> lock(ext->backends_mutex);

apir_backend_instance * instance = new apir_backend_instance();
instance->bck = device->iface.init_backend(device, NULL);
instance->magic = APIR_BACKEND_INSTANCE_MAGIC;

size_t device_count = reg->iface.get_device_count(reg);
if (!device_count) {
GGML_LOG_ERROR(GGML_VIRTGPU_BCK "%s: no device found\n", __func__);
return APIR_BACKEND_INITIALIZE_NO_DEVICE;
if (instance->bck == nullptr) {
GGML_LOG_ERROR(GGML_VIRTGPU_BCK "%s: device->iface.init_backend failed for device %p\n", __func__,
(void *) device);
delete instance;
return 0; // Failed
}

dev = reg->iface.get_device(reg, 0);
uintptr_t backend_id = ext->next_backend_id++;
ext->backend_instances[backend_id] = instance;

return backend_id;
}

// Get backend instance
apir_backend_instance * get_backend_instance(ggml_backend_dev_t device, uintptr_t backend_id) {
apir_device_context * ext = get_device_context(device);
if (ext == nullptr) {
return nullptr;
}

std::lock_guard<std::mutex> lock(ext->backends_mutex);
auto it = ext->backend_instances.find(backend_id);
if (it == ext->backend_instances.end()) {
return nullptr;
}

apir_backend_instance * instance = it->second;
if (instance->magic != APIR_BACKEND_INSTANCE_MAGIC) {
return nullptr;
}

return instance;
}

if (!dev) {
GGML_LOG_ERROR(GGML_VIRTGPU_BCK "%s: failed to get device\n", __func__);
return APIR_BACKEND_INITIALIZE_NO_DEVICE;
// Cleanup specific backend instance
void cleanup_backend_instance(ggml_backend_dev_t device, uintptr_t backend_id) {
apir_device_context * ext = get_device_context(device);
if (ext == nullptr) {
return;
}

bck = dev->iface.init_backend(dev, NULL);
if (!bck) {
GGML_LOG_ERROR(GGML_VIRTGPU_BCK "%s: backend initialization failed\n", __func__);
std::lock_guard<std::mutex> lock(ext->backends_mutex);
auto it = ext->backend_instances.find(backend_id);
if (it != ext->backend_instances.end()) {
apir_backend_instance * instance = it->second;

// Free backend
if (instance->bck) {
ggml_backend_free(instance->bck);
instance->bck = nullptr;
}

instance->magic = 0; // Invalidate
delete instance;
ext->backend_instances.erase(it);
}
}

// Cleanup device context and all its backend instances
void cleanup_device_context(ggml_backend_dev_t device) {
std::lock_guard<std::mutex> lock(device_contexts_mutex);

auto it = device_contexts.find(device);
if (it != device_contexts.end()) {
apir_device_context * ext = it->second;

// Clean up all backend instances
{
std::lock_guard<std::mutex> backends_lock(ext->backends_mutex);
for (auto & [backend_id, instance] : ext->backend_instances) {
if (instance->bck) {
ggml_backend_free(instance->bck);
}

instance->magic = 0;
delete instance;
}
ext->backend_instances.clear();
}

ext->magic = 0; // Invalidate
delete ext;
device_contexts.erase(it);
}
}

uint32_t backend_dispatch_initialize(void * ggml_backend_reg_fct_p, uintptr_t * out_handle, uint32_t * out_backend_id) {
GGML_UNUSED(ggml_backend_reg_fct_p); // reg/dev are already set during library loading

if (out_handle == nullptr || out_backend_id == nullptr) {
return APIR_BACKEND_INITIALIZE_BACKEND_INIT_FAILED;
}

// Ensure global variables are set (should be done during library loading)
if (reg == NULL || dev == NULL) {
GGML_LOG_ERROR(GGML_VIRTGPU_BCK "%s: Global reg/dev not initialized (reg=%p, dev=%p)\n", __func__, (void *) reg,
(void *) dev);
return APIR_BACKEND_INITIALIZE_BACKEND_INIT_FAILED;
}

// Create new backend instance
uintptr_t backend_id = create_backend_instance(dev);
if (backend_id == 0) {
return APIR_BACKEND_INITIALIZE_BACKEND_INIT_FAILED;
}

// Set output parameters
*out_handle = (uintptr_t) dev;
*out_backend_id = (uint32_t) backend_id;

return APIR_BACKEND_INITIALIZE_SUCCESS;
}
4 changes: 4 additions & 0 deletions ggml/src/ggml-virtgpu/backend/backend-dispatched.gen.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,9 @@ uint32_t backend_buffer_clear(apir_encoder * enc, apir_decoder * dec, virgl_apir
uint32_t backend_buffer_free_buffer(apir_encoder * enc, apir_decoder * dec, virgl_apir_context * ctx);

/* backend */
uint32_t backend_backend_initialize(apir_encoder * enc, apir_decoder * dec, virgl_apir_context * ctx);
uint32_t backend_backend_graph_compute(apir_encoder * enc, apir_decoder * dec, virgl_apir_context * ctx);
uint32_t backend_backend_cleanup(apir_encoder * enc, apir_decoder * dec, virgl_apir_context * ctx);

extern "C" {
static const backend_dispatch_t apir_backend_dispatch_table[APIR_BACKEND_DISPATCH_TABLE_COUNT] = {
Expand Down Expand Up @@ -68,6 +70,8 @@ static const backend_dispatch_t apir_backend_dispatch_table[APIR_BACKEND_DISPATC

/* backend */

/* APIR_COMMAND_TYPE_BACKEND_INITIALIZE = */ backend_backend_initialize,
/* APIR_COMMAND_TYPE_BACKEND_GRAPH_COMPUTE = */ backend_backend_graph_compute,
/* APIR_COMMAND_TYPE_BACKEND_CLEANUP = */ backend_backend_cleanup,
};
}
34 changes: 33 additions & 1 deletion ggml/src/ggml-virtgpu/backend/backend-dispatched.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
// clang-format off
#include <cstdint>
#include <cstddef>
#include <unordered_map>
#include <mutex>

#include <ggml-backend.h>

Expand All @@ -24,4 +26,34 @@ typedef uint32_t (*backend_dispatch_t)(apir_encoder * enc, apir_decoder * dec, v

#include "backend-dispatched.gen.h"

uint32_t backend_dispatch_initialize(void * ggml_backend_reg_fct_p);
// Backend instance structure - one backend per instance
struct apir_backend_instance {
ggml_backend_t bck; // The actual backend
uint32_t magic; // For validation: 0xAB1234CD
};

// Device context structure - can have multiple backend instances
struct apir_device_context {
std::mutex backends_mutex;
std::unordered_map<uintptr_t, apir_backend_instance *> backend_instances;
uintptr_t next_backend_id;

bool async_backend; // Whether the backend supports async operations

uint32_t magic; // For validation: 0xAB1234CD
};

#define APIR_DEVICE_EXTENSION_MAGIC 0xAB1234CD
#define APIR_BACKEND_INSTANCE_MAGIC 0xCD4321BA
Comment on lines +29 to +47

@coderabbitai coderabbitai Bot Apr 11, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟡 Minor

🧩 Analysis chain

🏁 Script executed:

#!/bin/bash
# Verify async_backend naming consistency
rg -n 'async_backend' --type=cpp ggml/src/ggml-virtgpu/backend/

Repository: crc-org/llama.cpp

Length of output: 422

Magic number comment is incorrect.

Line 32 comment says 0xAB1234CD but APIR_BACKEND_INSTANCE_MAGIC is defined as 0xCD4321BA on line 47. The comment should reference the correct magic value:

🔧 Fix comment to match actual magic value 
 // Backend instance structure - one backend per instance
 struct apir_backend_instance {
     ggml_backend_t bck;    // The actual backend
-    uint32_t       magic;  // For validation: 0xAB1234CD
+    uint32_t       magic;  // For validation: 0xCD4321BA
 };
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
// Backend instance structure - one backend per instance
struct apir_backend_instance {
ggml_backend_t bck; // The actual backend
uint32_t magic; // For validation: 0xAB1234CD
};
// Device context structure - can have multiple backend instances
struct apir_device_context {
std::mutex backends_mutex;
std::unordered_map<uintptr_t, apir_backend_instance *> backend_instances;
uintptr_t next_backend_id;
bool async_backend; // Whether the backend supports async operations
uint32_t magic; // For validation: 0xAB1234CD
};
#define APIR_DEVICE_EXTENSION_MAGIC 0xAB1234CD
#define APIR_BACKEND_INSTANCE_MAGIC 0xCD4321BA
// Backend instance structure - one backend per instance
struct apir_backend_instance {
ggml_backend_t bck; // The actual backend
uint32_t magic; // For validation: 0xCD4321BA
};
// Device context structure - can have multiple backend instances
struct apir_device_context {
std::mutex backends_mutex;
std::unordered_map<uintptr_t, apir_backend_instance *> backend_instances;
uintptr_t next_backend_id;
bool async_backend; // Whether the backend supports async operations
uint32_t magic; // For validation: 0xAB1234CD
};
`#define` APIR_DEVICE_EXTENSION_MAGIC 0xAB1234CD
`#define` APIR_BACKEND_INSTANCE_MAGIC 0xCD4321BA
🤖 Prompt for AI Agents 
Verify each finding against the current code and only fix it if needed.

In `@ggml/src/ggml-virtgpu/backend/backend-dispatched.h` around lines 29 - 47, The
comment for the magic field in apir_backend_instance is wrong—update the inline
comment next to the backend instance's uint32_t magic to match
APIR_BACKEND_INSTANCE_MAGIC (0xCD4321BA); also verify the comment for
apir_device_context::magic matches APIR_DEVICE_EXTENSION_MAGIC (0xAB1234CD) so
the two struct comments correctly reference APIR_BACKEND_INSTANCE_MAGIC and
APIR_DEVICE_EXTENSION_MAGIC respectively (use the symbols apir_backend_instance,
apir_device_context, APIR_BACKEND_INSTANCE_MAGIC, APIR_DEVICE_EXTENSION_MAGIC to
locate and fix the comments).


// Device context management
apir_device_context * get_device_context(ggml_backend_dev_t dev);
void ensure_device_context(ggml_backend_dev_t dev);
void cleanup_device_context(ggml_backend_dev_t dev);

// Backend instance management
uintptr_t create_backend_instance(ggml_backend_dev_t dev);
apir_backend_instance * get_backend_instance(ggml_backend_dev_t dev, uintptr_t backend_id);
void cleanup_backend_instance(ggml_backend_dev_t dev, uintptr_t backend_id);

uint32_t backend_dispatch_initialize(void * ggml_backend_reg_fct_p, uintptr_t * out_handle, uint32_t * out_backend_id);
1 change: 0 additions & 1 deletion ggml/src/ggml-virtgpu/backend/backend-virgl-apir.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@

extern ggml_backend_reg_t reg;
extern ggml_backend_dev_t dev;
extern ggml_backend_t bck;

struct virgl_apir_callbacks {
const char * (*get_config)(uint32_t virgl_ctx_id, const char * key);
Expand Down
Loading