LNXRNT/Sources/viewport_server.cpp

/**
 * Viewport Server Implementation - Shared Memory Framebuffer Export
 * 
 */

#include "viewport_server.h"
#include <kinc/log.h>
#include <kinc/window.h>
#include <kinc/graphics4/graphics.h>
#include <kinc/graphics4/rendertarget.h>
#include <string.h>
#include <stdlib.h>
#include <math.h>

#ifdef KORE_WINDOWS
#include <d3d11.h>
#include <dxgi.h>

// Forward declare structures from Kinc's Direct3D11.h
#define MAXIMUM_WINDOWS 16

struct dx_window {
    HWND hwnd;
    IDXGISwapChain *swapChain;
    ID3D11Texture2D *backBuffer;
    ID3D11RenderTargetView *renderTargetView;
    ID3D11Texture2D *depthStencil;
    ID3D11DepthStencilView *depthStencilView;
    int width;
    int height;
    int new_width;
    int new_height;
    bool vsync;
    int depth_bits;
    int stencil_bits;
};

struct dx_context {
    ID3D11Device *device;
    ID3D11DeviceContext *context;
    IDXGIDevice *dxgiDevice;
    IDXGIAdapter *dxgiAdapter;
    IDXGIFactory *dxgiFactory;
    int current_window;
    struct dx_window windows[MAXIMUM_WINDOWS];
};

// declare with C linkage to match Kinc C definition
extern "C" struct dx_context dx_ctx;
#endif

#ifdef KORE_WINDOWS
#include <Windows.h>
#else
#include <sys/mman.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <errno.h>
#endif

// global viewport server state
static ViewportServerState g_viewport_state = {0};

// ============================================================================
// Platform-specific Shared Memory Implementation
// ============================================================================

#ifdef KORE_WINDOWS

static bool shmem_create_windows(const char* name, size_t size) {
    // Create file mapping
    HANDLE hMapFile = CreateFileMappingA(
        INVALID_HANDLE_VALUE,
        NULL,
        PAGE_READWRITE,
        (DWORD)(size >> 32),
        (DWORD)(size & 0xFFFFFFFF),
        name
    );
    
    if (hMapFile == NULL) {
        kinc_log(KINC_LOG_LEVEL_ERROR, "Failed to create shared memory: %lu", GetLastError());
        return false;
    }
    
    void* pMem = MapViewOfFile(hMapFile, FILE_MAP_ALL_ACCESS, 0, 0, size);
    if (pMem == NULL) {
        kinc_log(KINC_LOG_LEVEL_ERROR, "Failed to map shared memory: %lu", GetLastError());
        CloseHandle(hMapFile);
        return false;
    }
    
    g_viewport_state.shmem_handle = hMapFile;
    g_viewport_state.shmem_ptr = pMem;
    g_viewport_state.shmem_size = size;
    
    SharedFramebufferHeader* header = (SharedFramebufferHeader*)pMem;
    memset(header, 0, sizeof(SharedFramebufferHeader));
    header->magic = VIEWPORT_MAGIC;
    header->version = VIEWPORT_VERSION;
    header->width = g_viewport_state.width;
    header->height = g_viewport_state.height;
    header->frame_id = 0;
    header->ready_flag = 0;
    header->format = 0; // RGBA8
    
    kinc_log(KINC_LOG_LEVEL_INFO, "Windows shared memory created: %s (%zu bytes)", name, size);
    return true;
}

static void shmem_destroy_windows(void) {
    if (g_viewport_state.shmem_ptr) {
        UnmapViewOfFile(g_viewport_state.shmem_ptr);
        g_viewport_state.shmem_ptr = NULL;
    }
    if (g_viewport_state.shmem_handle) {
        CloseHandle((HANDLE)g_viewport_state.shmem_handle);
        g_viewport_state.shmem_handle = NULL;
    }
}

#else

static bool shmem_create_posix(const char* name, size_t size) {
    char shmem_name[256];
    if (name[0] != '/') {
        snprintf(shmem_name, sizeof(shmem_name), "/%s", name);
    } else {
        strncpy(shmem_name, name, sizeof(shmem_name) - 1);
    }
    
    // remove already existing if any
    shm_unlink(shmem_name);
    
    // shared memory object
    int fd = shm_open(shmem_name, O_CREAT | O_RDWR, 0666);
    if (fd == -1) {
        kinc_log(KINC_LOG_LEVEL_ERROR, "Failed to create shared memory: %s", strerror(errno));
        return false;
    }
    
    if (ftruncate(fd, size) == -1) {
        kinc_log(KINC_LOG_LEVEL_ERROR, "Failed to set shared memory size: %s", strerror(errno));
        close(fd);
        shm_unlink(shmem_name);
        return false;
    }
    
    // map the memory
    void* pMem = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
    if (pMem == MAP_FAILED) {
        kinc_log(KINC_LOG_LEVEL_ERROR, "Failed to map shared memory: %s", strerror(errno));
        close(fd);
        shm_unlink(shmem_name);
        return false;
    }
    
    g_viewport_state.shmem_handle = (void*)(intptr_t)fd;
    g_viewport_state.shmem_ptr = pMem;
    g_viewport_state.shmem_size = size;
    
    SharedFramebufferHeader* header = (SharedFramebufferHeader*)pMem;
    memset(header, 0, sizeof(SharedFramebufferHeader));
    header->magic = VIEWPORT_MAGIC;
    header->version = VIEWPORT_VERSION;
    header->width = g_viewport_state.width;
    header->height = g_viewport_state.height;
    header->frame_id = 0;
    header->ready_flag = 0;
    header->format = 0; // RGBA8
    
    kinc_log(KINC_LOG_LEVEL_INFO, "POSIX shared memory created: %s (%zu bytes)", shmem_name, size);
    return true;
}

static void shmem_destroy_posix(void) {
    if (g_viewport_state.shmem_ptr) {
        munmap(g_viewport_state.shmem_ptr, g_viewport_state.shmem_size);
        g_viewport_state.shmem_ptr = NULL;
    }
    if (g_viewport_state.shmem_handle) {
        close((int)(intptr_t)g_viewport_state.shmem_handle);
        g_viewport_state.shmem_handle = NULL;
        
        // unlink shared memory
        char shmem_name[256];
        if (g_viewport_state.shmem_name[0] != '/') {
            snprintf(shmem_name, sizeof(shmem_name), "/%s", g_viewport_state.shmem_name);
        } else {
            strncpy(shmem_name, g_viewport_state.shmem_name, sizeof(shmem_name) - 1);
        }
        shm_unlink(shmem_name);
    }
}

#endif

static bool shmem_create(const char* name, size_t size) {
#ifdef KORE_WINDOWS
    return shmem_create_windows(name, size);
#else
    return shmem_create_posix(name, size);
#endif
}

static void shmem_destroy(void) {
#ifdef KORE_WINDOWS
    shmem_destroy_windows();
#else
    shmem_destroy_posix();
#endif
}

bool viewport_server_init(const char* shmem_name, int width, int height) {
    if (g_viewport_state.initialized) {
        kinc_log(KINC_LOG_LEVEL_WARNING, "Viewport server already initialized");
        return true;
    }
    
    if (width <= 0 || width > VIEWPORT_MAX_WIDTH || 
        height <= 0 || height > VIEWPORT_MAX_HEIGHT) {
        kinc_log(KINC_LOG_LEVEL_ERROR, "Invalid viewport dimensions: %dx%d", width, height);
        return false;
    }
    
    g_viewport_state.width = width;
    g_viewport_state.height = height;
    strncpy(g_viewport_state.shmem_name, shmem_name ? shmem_name : VIEWPORT_SHMEM_NAME, 
            sizeof(g_viewport_state.shmem_name) - 1);
    
    // pre-allocate shared memory for MAXIMUM size to avoid recreation on resize
    // *** this prevents access denied errors when Blender still has the memory mapped
    size_t max_pixel_size = (size_t)VIEWPORT_MAX_WIDTH * VIEWPORT_MAX_HEIGHT * 4; // RGBA8
    size_t total_size = VIEWPORT_HEADER_SIZE + max_pixel_size;
    
    // create shared memory
    if (!shmem_create(g_viewport_state.shmem_name, total_size)) {
        return false;
    }
    
    // now allocate CPU side pixel buffer for readback with a max size for the resize support
    g_viewport_state.pixel_buffer = (uint8_t*)malloc(max_pixel_size);
    if (!g_viewport_state.pixel_buffer) {
        kinc_log(KINC_LOG_LEVEL_ERROR, "Failed to allocate pixel buffer");
        shmem_destroy();
        return false;
    }

    // TODO: needs investigation
    // clear pixel buffer to prevent stale data from previous scenes
    memset(g_viewport_state.pixel_buffer, 0, max_pixel_size);
    // also clear shared memory pixel area to prevent flickering with old data
    uint8_t* pixel_dest = (uint8_t*)g_viewport_state.shmem_ptr + VIEWPORT_HEADER_SIZE;
    memset(pixel_dest, 0, max_pixel_size);
    
    kinc_g4_render_target_t* rt = (kinc_g4_render_target_t*)malloc(sizeof(kinc_g4_render_target_t));
    if (!rt) {
        kinc_log(KINC_LOG_LEVEL_ERROR, "Failed to allocate render target");
        free(g_viewport_state.pixel_buffer);
        shmem_destroy();
        return false;
    }
    
    kinc_g4_render_target_init(rt, width, height, KINC_G4_RENDER_TARGET_FORMAT_32BIT, 24, 0);
    g_viewport_state.render_target = rt;
    
    g_viewport_state.enabled = true;
    g_viewport_state.initialized = true;
    g_viewport_state.frame_count = 0;
    
    kinc_log(KINC_LOG_LEVEL_INFO, "Viewport server initialized: %dx%d, shmem=%s", 
             width, height, g_viewport_state.shmem_name);
    
    return true;
}

#ifdef KORE_WINDOWS
static ID3D11Texture2D* g_stagingTexture = NULL;
static int g_stagingWidth = 0, g_stagingHeight = 0;
#endif

void viewport_server_shutdown(void) {
    if (!g_viewport_state.initialized) {
        return;
    }
    
#ifdef KORE_WINDOWS
    if (g_stagingTexture) {
        g_stagingTexture->Release();
        g_stagingTexture = NULL;
        g_stagingWidth = 0;
        g_stagingHeight = 0;
    }
#endif
    
    if (g_viewport_state.render_target) {
        kinc_g4_render_target_destroy((kinc_g4_render_target_t*)g_viewport_state.render_target);
        free(g_viewport_state.render_target);
        g_viewport_state.render_target = NULL;
    }
    
    if (g_viewport_state.pixel_buffer) {
        free(g_viewport_state.pixel_buffer);
        g_viewport_state.pixel_buffer = NULL;
    }
    
    shmem_destroy();
    
    g_viewport_state.enabled = false;
    g_viewport_state.initialized = false;
    g_viewport_state.frame_count = 0;  
    
    kinc_log(KINC_LOG_LEVEL_INFO, "Viewport server shutdown complete");
}

bool viewport_server_is_enabled(void) {
    return g_viewport_state.enabled && g_viewport_state.initialized;
}

void viewport_server_begin_frame(void) {
    // no-op we let iron render normally to framebuffer and capture the backbuffer in end_frame BEFORE swap_buffers
}

void viewport_server_end_frame(void) {
    if (!viewport_server_is_enabled()) return;
    
    if (g_viewport_state.pixel_buffer == NULL ||
        g_viewport_state.shmem_ptr == NULL) {
        kinc_log(KINC_LOG_LEVEL_WARNING, "Viewport server: invalid state in end_frame");
        g_viewport_state.frame_count++;
        return;
    }
    
    if (g_viewport_state.frame_count < 3) {
        g_viewport_state.frame_count++;
        return;
    }
    
    SharedFramebufferHeader* header = (SharedFramebufferHeader*)g_viewport_state.shmem_ptr;
    
    header->width = g_viewport_state.width;
    header->height = g_viewport_state.height;
    
    size_t pixel_size = (size_t)g_viewport_state.width * g_viewport_state.height * 4;
    uint8_t* pixels = g_viewport_state.pixel_buffer;
    uint8_t* pixel_dest = (uint8_t*)g_viewport_state.shmem_ptr + VIEWPORT_HEADER_SIZE;
    
#ifdef KORE_WINDOWS
    // Direct3D11 - read from backbuffer BEFORE swap_buffers
    ID3D11DeviceContext* context = dx_ctx.context;
    ID3D11Device* device = dx_ctx.device;
    struct dx_window* window = &dx_ctx.windows[0];
    ID3D11Texture2D* backBuffer = window->backBuffer;
    
    if (backBuffer && context && device) {
        D3D11_TEXTURE2D_DESC bbDesc;
        backBuffer->GetDesc(&bbDesc);
        
        int bbWidth = (int)bbDesc.Width;
        int bbHeight = (int)bbDesc.Height;
        
        // capture dimensions are the actual size of whats renderedso we clamp to max shared memory size
        int captureWidth = bbWidth;
        int captureHeight = bbHeight;
        if (captureWidth > VIEWPORT_MAX_WIDTH) captureWidth = VIEWPORT_MAX_WIDTH;
        if (captureHeight > VIEWPORT_MAX_HEIGHT) captureHeight = VIEWPORT_MAX_HEIGHT;
        
        header->width = captureWidth;
        header->height = captureHeight;
        pixel_size = (size_t)captureWidth * captureHeight * 4;
        
        // same dimensions between source and destination
        if (!g_stagingTexture || g_stagingWidth != bbWidth || g_stagingHeight != bbHeight) {
            if (g_stagingTexture) g_stagingTexture->Release();
            
            D3D11_TEXTURE2D_DESC desc = {};
            desc.Width = bbWidth;
            desc.Height = bbHeight;
            desc.MipLevels = 1;
            desc.ArraySize = 1;
            desc.Format = bbDesc.Format;
            desc.SampleDesc.Count = 1;
            desc.Usage = D3D11_USAGE_STAGING;
            desc.CPUAccessFlags = D3D11_CPU_ACCESS_READ;
            
            HRESULT hr = device->CreateTexture2D(&desc, NULL, &g_stagingTexture);
            if (FAILED(hr)) {
                kinc_log(KINC_LOG_LEVEL_ERROR, "Failed to create staging texture: 0x%08X", hr);
                g_stagingTexture = NULL;
                g_stagingWidth = 0;
                g_stagingHeight = 0;
                g_viewport_state.frame_count++;
                return;
            }
            g_stagingWidth = bbWidth;
            g_stagingHeight = bbHeight;
        }
        
        context->CopyResource(g_stagingTexture, backBuffer);
        
        // only read the portion we need
        D3D11_MAPPED_SUBRESOURCE mapped;
        if (SUCCEEDED(context->Map(g_stagingTexture, 0, D3D11_MAP_READ, 0, &mapped))) {
            for (int y = 0; y < captureHeight; y++) {
                memcpy(pixels + y * captureWidth * 4, 
                       (uint8_t*)mapped.pData + y * mapped.RowPitch, 
                       captureWidth * 4);
            }
            context->Unmap(g_stagingTexture, 0);
            
            // BGRA to RGBA conversion
            for (size_t i = 0; i < pixel_size; i += 4) {
                uint8_t t = pixels[i]; pixels[i] = pixels[i+2]; pixels[i+2] = t;
            }
            
            memcpy(pixel_dest, pixels, pixel_size);
        }
    }
#else
    // other platforms use render target
    if (g_viewport_state.render_target) {
        kinc_g4_render_target_t* rt = (kinc_g4_render_target_t*)g_viewport_state.render_target;
        kinc_g4_render_target_get_pixels(rt, pixels);
        memcpy(pixel_dest, pixels, pixel_size);
    }
#endif
    
    g_viewport_state.frame_count++;
    header->frame_id = g_viewport_state.frame_count;
    
    // NOTE: Camera sync from RunT to Blender is handled via viewport_server_set_camera()
    // which is called explicitly when RunT's internal camera changes.
    // We do NOT extract camera from view_matrix here as that would create a feedback loop
    // (Blender sends view_matrix -> we extract camera -> Blender applies it -> repeat)
    
    // Memory barrier to ensure writes complete before setting ready flag
#ifdef KORE_WINDOWS
    MemoryBarrier();
#else
    __sync_synchronize();
#endif
    
    header->ready_flag = 1;
}

bool viewport_server_check_resize(int* new_width, int* new_height) {
    if (!viewport_server_is_enabled()) return false;
    
    SharedFramebufferHeader* header = (SharedFramebufferHeader*)g_viewport_state.shmem_ptr;
    
    if (header->resize_request) {
        *new_width = header->viewport_width;
        *new_height = header->viewport_height;
        header->resize_request = 0; 
        return true;
    }
    
    return false;
}

bool viewport_server_resize(int new_width, int new_height) {
    if (!g_viewport_state.initialized) return false;
    
    if (new_width <= 0 || new_width > VIEWPORT_MAX_WIDTH || 
        new_height <= 0 || new_height > VIEWPORT_MAX_HEIGHT) {
        kinc_log(KINC_LOG_LEVEL_ERROR, "Invalid resize dimensions: %dx%d", new_width, new_height);
        return false;
    }
    
    if (new_width == g_viewport_state.width && new_height == g_viewport_state.height) {
        return true;
    }
    
    kinc_log(KINC_LOG_LEVEL_INFO, "Viewport server resizing: %dx%d -> %dx%d", 
             g_viewport_state.width, g_viewport_state.height, new_width, new_height);
    
    // resize the actual Kinc window so the D3D11 backbuffer is the correct size critical because viewport_server_end_frame captures from the backbuffer
    kinc_window_resize(0, new_width, new_height);
    
    if (g_viewport_state.render_target) {
        kinc_g4_render_target_destroy((kinc_g4_render_target_t*)g_viewport_state.render_target);
        // dont free here we reuse the allocation
    }
    
    // update dimensions pre-allocated and reinitialize render target with new size using same memory
    g_viewport_state.width = new_width;
    g_viewport_state.height = new_height;
    
    kinc_g4_render_target_t* rt = (kinc_g4_render_target_t*)g_viewport_state.render_target;
    if (rt) {
        kinc_g4_render_target_init(rt, new_width, new_height, KINC_G4_RENDER_TARGET_FORMAT_32BIT, 24, 0);
    }
    
    SharedFramebufferHeader* header = (SharedFramebufferHeader*)g_viewport_state.shmem_ptr;
    if (header) {
        header->width = new_width;
        header->height = new_height;
        header->ready_flag = 0;
    }
    
    kinc_log(KINC_LOG_LEVEL_INFO, "Viewport server resize complete");
    return true;
}

bool viewport_server_check_shutdown(void) {
    if (!viewport_server_is_enabled()) return false;
    
    SharedFramebufferHeader* header = (SharedFramebufferHeader*)g_viewport_state.shmem_ptr;
    return header->shutdown_flag != 0;
}

bool viewport_server_get_view_matrix(float* matrix) {
    if (!viewport_server_is_enabled() || !matrix) return false;
    
    SharedFramebufferHeader* header = (SharedFramebufferHeader*)g_viewport_state.shmem_ptr;
    memcpy(matrix, header->view_matrix, 16 * sizeof(float));
    return true;
}

bool viewport_server_get_proj_matrix(float* matrix) {
    if (!viewport_server_is_enabled() || !matrix) return false;
    
    SharedFramebufferHeader* header = (SharedFramebufferHeader*)g_viewport_state.shmem_ptr;
    memcpy(matrix, header->proj_matrix, 16 * sizeof(float));
    return true;
}

ViewportServerState* viewport_server_get_state(void) {
    return &g_viewport_state;
}

void viewport_server_set_camera(float pos_x, float pos_y, float pos_z,
                                 float rot_x, float rot_y, float rot_z, float rot_w) {
    if (!viewport_server_is_enabled()) return;
    
    SharedFramebufferHeader* header = (SharedFramebufferHeader*)g_viewport_state.shmem_ptr;
    if (!header) return;
    
    header->runt_camera_pos[0] = pos_x;
    header->runt_camera_pos[1] = pos_y;
    header->runt_camera_pos[2] = pos_z;
    
    header->runt_camera_rot[0] = rot_x;
    header->runt_camera_rot[1] = rot_y;
    header->runt_camera_rot[2] = rot_z;
    header->runt_camera_rot[3] = rot_w;
    
    header->runt_camera_dirty = 1;
}

bool viewport_server_read_input(InputEvent* event) {
    if (!viewport_server_is_enabled()) return false;
    if (!event) return false;
    
    SharedFramebufferHeader* header = (SharedFramebufferHeader*)g_viewport_state.shmem_ptr;
    if (!header) return false;
    
    uint32_t read_idx = header->input_read_idx;
    uint32_t write_idx = header->input_write_idx;
    
    if (read_idx == write_idx) {
        return false;  
    }
    
    *event = header->input_events[read_idx % MAX_INPUT_EVENTS];
    
    header->input_read_idx = (read_idx + 1) % MAX_INPUT_EVENTS;
    
    return true;
}

bool viewport_server_input_enabled(void) {
    if (!viewport_server_is_enabled()) return false;
    
    SharedFramebufferHeader* header = (SharedFramebufferHeader*)g_viewport_state.shmem_ptr;
    if (!header) return false;
    
    return header->input_enabled != 0;
}