File offload_rendering_server.hpp

File List > offload_rendering_server > offload_rendering_server.hpp

Go to the documentation of this file

#pragma once
#define MONADO_IS_SOURCE
#define DOUBLE_INCLUDE

#ifdef _WIN32
#    ifndef XRT_OS_WINDOWS
#        define XRT_OS_WINDOWS
#    endif
#endif

#include "drivers/illixr/illixr_framebuffer.h"
#ifdef USING_OPENXR
#    include "illixr/data_format/poses/combined_pose.hpp"
#endif
#include "illixr/data_format/frame.hpp"
#include "illixr/data_format/hmd_config.hpp"
#include "illixr/data_format/pose_id.hpp"
#include "illixr/data_format/pose_prediction.hpp"
#include "illixr/data_format/serialization/frame.hpp"
#include "illixr/data_format/serialization/head_pose.hpp"
#include "illixr/switchboard.hpp"
#include "illixr/threadloop.hpp"
#include "illixr/vk/display_provider.hpp"
#include "illixr/vk/render_pass.hpp"
#include "illixr/vk/vulkan_utils.hpp"
#include "pose_relay.hpp"

#ifdef NVENC_ENCODER
#    include "nvenc/nvenc_encoder.hpp"
#    define OFFLOAD_RENDERING_BITRATE 100000000
#else
#    include "illixr/vk/ffmpeg_utils.hpp"
#endif

#undef DOUBLE_INCLUDE

#include 
#include 
#include 
#include 

namespace ILLIXR {

class MY_EXPORT_API offload_rendering_server
    : public threadloop
    , public vulkan::timewarp
    , public data_format::pose_prediction {
public:
    offload_rendering_server(const std::string& name, phonebook* pb);
    void start() override;
    void stop() override;
    void _p_thread_setup() override;

    void setup(VkRenderPass render_pass, uint32_t subpass, std::shared_ptr<vulkan::buffer_pool<BUFFER_TYPE>> buffer_pool,
               bool input_texture_vulkan_coordinates, struct illixr_framebuffer* framebuffer_array, VkExtent2D extent) override;

    bool is_external() override {
        return true;
    }

    void destroy() override;

    [[nodiscard]] POSE_TYPE get_fast_pose() const override {
        return pose_relay_->get_pose();
    }

#ifndef USING_OPENXR
    ILLIXR::data_format::pose::head_pose_type get_true_pose() const override {
        return get_fast_pose().pose;
    }
#endif

    [[nodiscard]] POSE_TYPE get_fast_pose(POSE_TIME_TYPE future_time) const override {
        return pose_relay_->get_pose(future_time);
    }

    [[nodiscard]] bool fast_pose_reliable() const override {
        return pose_relay_->fast_pose_reliable();
    }

    [[nodiscard]] bool true_pose_reliable() const override {
        return false;
    }

    void set_offset(const Eigen::Quaternionf& orientation) override {
        (void) orientation;
    }

    Eigen::Quaternionf get_offset() override {
        return {};
    }

    [[nodiscard]] data_format::pose::head_pose_type correct_pose(const data_format::pose::head_pose_type& pose) const override {
        (void) pose;
        return {};
    }

    void record_command_buffer(VkCommandBuffer commandBuffer, VkFramebuffer framebuffer, int buffer_ind, bool left) override {
        (void) commandBuffer;
        (void) framebuffer;
        (void) buffer_ind;
        (void) left;
    }

protected:
    threadloop::skip_option _p_should_skip() override {
        return threadloop::_p_should_skip();
    }

    void _p_one_iteration() override;

private:
    void enqueue_for_network_send(BUFFER_TYPE& pose
#ifdef USING_OPENXR
                                  ,
                                  uint64_t pose_id
#endif
    );

#ifdef NVENC_ENCODER
    // ========================================================================
    // NVENC-specific methods (no FFmpeg dependency)
    // ========================================================================

    void nvenc_init_vulkan_context();

    void nvenc_init_encoders();

    void nvenc_import_buffer_pool_images();

    void nvenc_encode_frames(int ind);

#else
    // ========================================================================
    // FFmpeg-specific methods
    // ========================================================================

    void ffmpeg_init_device();

    void ffmpeg_init_cuda_device();

    void ffmpeg_init_frame_ctx();

    void ffmpeg_init_cuda_frame_ctx();

    // Linux depth is a separate RGBA grayscale stream, not the color context.
    void ffmpeg_init_depth_frame_ctx();
    void ffmpeg_transfer_depth(size_t buffer_index, size_t eye);

    void ffmpeg_init_buffer_pool();

    void ffmpeg_init_encoder();

    void ffmpeg_populate_buffer_pool_from_framebuffers();
#endif
    void sender_loop();

    std::shared_ptr<spdlog::logger>                            log_;
    std::shared_ptr<vulkan::display_provider>                  display_provider_;
    std::shared_ptr<switchboard>                               switchboard_;
    switchboard::network_writer<data_format::compressed_frame> frames_topic_;

    mutable data_format::pose::fast_head_pose_type cached_head_pose_;

    mutable time_point last_processed_time_{};

    std::shared_ptr<vulkan::buffer_pool<BUFFER_TYPE>> buffer_pool_;

#ifdef OPENXR_CLIENT
    int framerate_ = 90;
#else
    int framerate_ = 144;
#endif
    long bitrate_ = OFFLOAD_RENDERING_BITRATE;

    bool use_pass_depth_ = false;
#ifdef _WIN32
    // These are currently only supported with Unity, which only supports OpenXR on Windows
    bool use_pass_motion_vectors_ = false;
#endif
    bool nalu_only_ = false;

    std::atomic<bool> framebuffers_imported_{false};

    // Set after each color encode call and copied into compressed_frame::is_keyframe.
    bool color_frame_is_keyframe_ = false;

#ifdef NVENC_ENCODER
    // ========================================================================
    // NVENC-specific members
    // ========================================================================

    // Vulkan context for CUDA interop
    vulkan_context vk_ctx_;

    // NVENC encoders (one per eye for color, optionally for depth and motion vectors)
    std::array<std::unique_ptr<nvenc_encoder>, 2> color_encoder_;
    std::array<std::unique_ptr<nvenc_encoder>, 2> depth_encoder_;
    std::array<std::unique_ptr<nvenc_encoder>, 2> motion_vec_encoder_;

    // Imported image indices: [buffer_index][eye] for color, depth, and motion vectors
    std::vector<std::array<int, 2>> color_imported_indices_;
    std::vector<std::array<int, 2>> depth_imported_indices_;
    std::vector<std::array<int, 2>> motion_vec_imported_indices_;

    // Set by nvenc_encode_frames() immediately after each color encode call.
    // Read by enqueue_for_network_send() to populate compressed_frame::is_keyframe.
    // Using last_frame_was_keyframe() from the encoder is authoritative for both
    // HEVC (IDR) and AV1 (KEY_FRAME / auto-GOP I-frame), avoiding any need for
    // bitstream parsing on the client side.
    // bool color_frame_is_keyframe_ = false;

#    ifdef COMBINED_ENCODING
    // Under COMBINED_ENCODING a single encoder handles both eyes at double width.
    // color_encoder_[0] is used; color_encoder_[1] is unused.
    // encode_out_combined_color_packet_ carries the single combined bitstream;
    // encode_out_color_packets_ is not used for color in this mode.
    PACKET_TYPE encode_out_combined_color_packet_{};
#    endif // COMBINED_ENCODING

#else
    // ========================================================================
    // FFmpeg-specific members
    // ========================================================================

    std::vector<std::array<vulkan::ffmpeg_utils::ffmpeg_vk_frame, 2>> avvk_color_frames_;
    std::vector<std::array<vulkan::ffmpeg_utils::ffmpeg_vk_frame, 2>> avvk_depth_frames_;

    AVBufferRef*         device_ctx_            = nullptr;
    AVBufferRef*         cuda_device_ctx_       = nullptr;
    AVBufferRef*         frame_ctx_             = nullptr;
    AVBufferRef*         cuda_frame_ctx_        = nullptr;
    AVBufferRef*         depth_frame_ctx_       = nullptr;
    AVBufferRef*         cuda_depth_frame_ctx_  = nullptr;
    VkCommandPool        depth_transfer_pool_   = VK_NULL_HANDLE;
    VkCommandBuffer      depth_transfer_cmd_    = VK_NULL_HANDLE;
    VkFence              depth_transfer_fence_  = VK_NULL_HANDLE;
    PFN_vkWaitSemaphores depth_wait_semaphores_ = nullptr;

    AVCodecContext*         codec_color_ctx_ = nullptr;
    std::array<AVFrame*, 2> encode_src_color_frames_{};
#endif
    std::array<PACKET_TYPE, 2> encode_out_color_packets_{};
#ifndef NVENC_ENCODER
    AVCodecContext*         codec_depth_ctx_ = nullptr;
    std::array<AVFrame*, 2> encode_src_depth_frames_{};
#endif
    std::array<PACKET_TYPE, 2> encode_out_depth_packets_{};
    std::array<PACKET_TYPE, 2> encode_out_motion_vec_packets_{};

    uint64_t frame_count_ = 0;

    float near_z_{0.};
    float far_z_{0.};
    // Boxcar FPS: timestamps of frames encoded within the last 1 second.
    // Updated every frame in _p_one_iteration(); get_fps() returns the count.
    std::deque<std::chrono::high_resolution_clock::time_point> fps_window_;
    std::map<std::string, long long>                           metrics_;

    int32_t last_frame_ind_ = -1;
#ifdef OPENXR_CLIENT
    xrt_pose last_sent_pose_{};
#else
    data_format::pose::fast_head_pose_type last_sent_pose_{};
#endif

    struct illixr_framebuffer* framebuffer_array_ = nullptr;
    VkExtent2D                 extent_            = {0, 0};

    std::shared_ptr<pose_relay> pose_relay_;
#ifdef OPENXR_CLIENT
    hmd_config hmd_setup_;
#endif

    std::atomic<bool> ready_{false};
    uint64_t          frame_number_{0};
    double            current_encode_time_{0.};

    std::deque<std::shared_ptr<data_format::compressed_frame>> send_queue_;

    std::mutex                  send_queue_mutex_;
    std::condition_variable     send_queue_cv_;
    std::thread                 sender_thread_;
    static constexpr size_t     MAX_QUEUE_DEPTH = 6;
    std::atomic<bool>           sender_running_{false};
    std::map<uint64_t, uint8_t> pose_usage_{};
#ifdef _WIN32
    std::map<uint64_t, std::chrono::steady_clock::time_point> frame_timing_{};
#else
    std::map<uint64_t, std::chrono::time_point<std::chrono::high_resolution_clock>> frame_timing_{};
#endif
#ifdef OPENXR_CLIENT
    float overscan_ = 1.f;
#endif
};
} // namespace ILLIXR