File offload_rendering_client.hpp

File List > offload_rendering_client > offload_rendering_client.hpp

Go to the documentation of this file

#pragma once

#define DOUBLE_INCLUDE
// ILLIXR core headers
#ifdef USING_OPENXR
#    include "illixr/data_format/poses/combined_pose.hpp"
#endif

#include "illixr/data_format/frame.hpp"
#include "illixr/data_format/latency_data.hpp"
#include "illixr/data_format/pose_prediction.hpp"
#include "illixr/data_format/serialization/frame.hpp"
#include "illixr/data_format/serialization/head_pose.hpp"
#include "illixr/switchboard.hpp"
#include "illixr/threadloop.hpp"
#undef DOUBLE_INCLUDE
#ifdef USING_OPENXR
#    include 
#    include 
#    include 
#    include 
#    include 
#    ifdef __ANDROID__
#        include "android/stereo_surface_decoder.hpp"
#        include "illixr/quest3_params.hpp"

#        include 
#    endif
#else
// ILLIXR Vulkan headers
#    include "illixr/vk/display_provider.hpp"
#    include "illixr/vk/ffmpeg_utils.hpp"
#    include "illixr/vk/render_pass.hpp"
#    include "illixr/vk/vk_extension_request.hpp"
#    include "illixr/vk/vulkan_utils.hpp"

// FFmpeg headers (C interface)
extern "C" {
#    include "libavfilter_illixr/buffersink.h"
#    include "libavfilter_illixr/buffersrc.h"
#    include "libswscale_illixr/swscale.h"
}

// NVIDIA nppi headers
#    include "nppi.h"
#endif

namespace ILLIXR {

class offload_rendering_client
    : public threadloop
#ifndef __ANDROID__
    , public vulkan::app
#endif
{
public:
    offload_rendering_client(const std::string& name, phonebook* pb);

#ifdef __ANDROID__
    ~offload_rendering_client() override;
#else
    void start() override;

    void setup(VkRenderPass render_pass, uint32_t subpass,
               std::shared_ptr<vulkan::buffer_pool<data_format::pose::fast_head_pose_type>> buffer_pool) override;

    void record_command_buffer(VkCommandBuffer commandBuffer, VkFramebuffer framebuffer, int buffer_ind, bool left) override {
        (void) commandBuffer;
        (void) framebuffer;
        (void) buffer_ind;
        (void) left;
    }

    void update_uniforms(const BUFFER_TYPE& render_pose) override {
        (void) render_pose;
    }

    bool is_external() override {
        return true;
    }

    void destroy() override;
#endif // __ANDROID__

protected:
    void _p_thread_setup() override;

    skip_option _p_should_skip() override {
        return threadloop::_p_should_skip();
    }

#ifndef __ANDROID__
    [[maybe_unused]] void copy_image_to_cpu_and_save_file(AVFrame* frame);
    [[maybe_unused]] void save_nv12_img_to_png(AVFrame* cuda_frame) const;
    void transition_layout(VkCommandBuffer cmd_buf, AVFrame* frame, VkImageLayout old_layout, VkImageLayout new_layout);
#endif
    void _p_one_iteration() override;

private:
#ifdef __ANDROID__
    struct frame_meta {
        BUFFER_TYPE pose;
        uint64_t    frame_number{0};
        uint64_t    frame_time{0};
        uint64_t    pose_id{0};
        float       near_z{0.f};
        float       far_z{0.f};
        double      encode_time{0.};
        bool        consumed{false};
#    ifdef ILLIXR_ENABLE_BOBA
        data_format::stereo_presentation_mode presentation_mode{data_format::stereo_presentation_mode::stereo_fullscreen};
        float                                 content_aspect_ratio{0.0F};
        data_format::boba_frame_overlay       boba_overlay{};
        data_format::boba_modal_overlay       boba_modal{};
        std::array<float, 2>                  fov_left{0.0F, 0.0F};
        std::array<float, 2>                  fov_right{0.0F, 0.0F};
        std::array<float, 2>                  fov_up{0.0F, 0.0F};
        std::array<float, 2>                  fov_down{0.0F, 0.0F};
#    endif
    };

#    ifdef ILLIXR_ENABLE_BOBA
    struct modal_texture_cache_entry {
        std::uint32_t                                    width{0};
        std::uint32_t                                    height{0};
        std::shared_ptr<const std::vector<std::uint8_t>> rgba;
    };
#    endif

    void receiver_loop();

#    ifdef ILLIXR_ENABLE_BOBA
    void drain_modal_texture_updates();
#    endif

    void log_android_decode_timing();

    data_format::dual_frames construct_dual_frames(time_point render_time);

    void release_stale_frames();

#else
    void push_pose();

    bool network_receive();

    [[maybe_unused]] void submit_command_buffer(VkCommandBuffer vk_command_buffer);

    void ffmpeg_init_device();

    void ffmpeg_init_cuda_device();

    void ffmpeg_init_frame_ctx();

    AVBufferRef* create_cuda_frame_ctx(AVPixelFormat fmt);

    void ffmpeg_init_cuda_frame_ctx();

    void ffmpeg_init_buffer_pool();

    void ffmpeg_init_decoder();
#endif // __ANDROID__

    std::shared_ptr<switchboard>    switchboard_;
    std::shared_ptr<spdlog::logger> log_;
#ifdef __ANDROID__
    switchboard::writer<data_format::dual_frames> frame_writer_;
#else
    std::shared_ptr<vulkan::display_provider> display_provider_;
#endif
    switchboard::buffered_reader<data_format::compressed_frame> frames_reader_;
    switchboard::reader<data_format::network_latency_result>    network_latency_reader_;
#ifdef ILLIXR_ENABLE_BOBA
    switchboard::buffered_reader<data_format::boba_modal_texture> modal_texture_reader_;
#endif

#ifndef USING_OPENXR
    // Pose transmission to server
    switchboard::network_writer<data_format::pose::fast_head_pose_type> pose_writer_;
    std::shared_ptr<data_format::pose_prediction>                       pose_prediction_;
#endif
    std::atomic<bool>               ready_ = false;
    std::shared_ptr<relative_clock> clock_;

#ifndef __ANDROID__
    std::shared_ptr<vulkan::buffer_pool<data_format::pose::fast_head_pose_type>> buffer_pool_;
#endif
    bool use_depth_ = false;

#ifdef __ANDROID__

    bool use_motion_vectors_ = false;

    // Receiver thread: feeds encoded data to decoders independently of the
    // consumer (_p_one_iteration) so decoder latency cannot block reception.
    std::thread       receiver_thread_;
    std::atomic<bool> receiver_running_{false};
    // Maximum number of encoded frames allowed in the color decoder input
    // queue before incoming frames are dropped to prevent growing latency.
    // At 90fps a frame arrives every ~11.1ms; the feeder thread's
    // dequeueInputBuffer has a 2ms timeout and may loop several times per
    // frame under normal scheduling jitter.  A threshold of 3 fires on any
    // brief stall; 8 gives ~89ms of headroom while still bounding latency
    // to under 200ms if the decoder genuinely falls behind.
    static constexpr size_t MAX_DECODER_QUEUE_DEPTH = 8;

    // Maps server frame_number -> complete frame metadata for that frame.
    // Written by receiver_thread_ as each compressed_frame arrives.
    // Read and pruned by _p_one_iteration once the decoded frame_number is known.
    // Entries for frame numbers older than the one just consumed are erased at
    // the same time, so the map stays bounded even if the decoder skips frames.
    std::map<uint64_t, frame_meta> frame_meta_map_;
    std::mutex                     frame_meta_map_mutex_;
#    ifdef ILLIXR_ENABLE_BOBA
    std::unordered_map<std::uint64_t, modal_texture_cache_entry> modal_texture_cache_;
#    endif
    // Bounds how many submitted dual_frames are kept alive awaiting release.
    // Rather than requiring an explicit GPU-completion signal from the
    // render-side consumer (oxr_interface), a submitted frame's
    // AHardwareBuffers are released once this many *newer* frames have been
    // submitted after it - i.e. once enough real time has passed that the
    // render pass reading it is essentially certain to be done.  At the
    // slowest supported refresh rate (72Hz, ~13.9ms/frame) this guarantees
    // at least ~139ms between a frame's submission and its release; at 90Hz
    // it's ~111ms.  The render-pass read itself takes on the order of a few
    // ms, so this leaves a wide margin under normal operation.  A sustained
    // render stall longer than this window could still release a buffer
    // while the GPU is reading it; if corruption is ever suspected, that's
    // the first thing to check (e.g. via frame-pacing logs around the same
    // time) before raising this value further.
    static constexpr size_t MAX_IN_FLIGHT_FRAMES = 10;

    // Frames already submitted on "unity_rendered_frame", retained here
    // purely so their AHardwareBuffers can be released later - not read for
    // any other purpose.  Pushed to in _p_one_iteration() right after each
    // submission; drained from the front by release_stale_frames().
    std::deque<data_format::dual_frames> in_flight_frames_;

    android_app* app_;

    std::unique_ptr<stereo_surface_decoder> color_decoder_;
    std::unique_ptr<stereo_surface_decoder> depth_decoder_;
    // Motion-vector decoder (432x432 HEVC 10-bit)
    std::unique_ptr<stereo_surface_decoder> motion_vec_decoder_;

    // Android timing metrics (accumulated between reports)
    // uint64_t android_queue_time_us_{0};         // Time spent queueing data to decoders
    // uint64_t android_texture_update_time_us_{0}; // Time spent updating GPU textures
    // uint64_t android_total_frame_time_us_{0};    // Total end-to-end frame processing time
    // uint64_t android_timing_frame_count_{0};     // Number of frames in current timing window

#else
    std::vector<std::array<vulkan::ffmpeg_utils::ffmpeg_vk_frame, 2>> avvk_color_frames_;
    std::vector<std::array<vulkan::ffmpeg_utils::ffmpeg_vk_frame, 2>> avvk_depth_frames_;
    std::vector<std::array<VkCommandBuffer, 2>>                       layout_transition_start_cmd_bufs_;
    std::vector<std::array<VkCommandBuffer, 2>>                       layout_transition_end_cmd_bufs_;
    AVBufferRef*                                                      device_ctx_          = nullptr;
    AVBufferRef*                                                      cuda_device_ctx_     = nullptr;
    AVBufferRef*                                                      frame_ctx_           = nullptr;
    AVBufferRef*                                                      cuda_nv12_frame_ctx_ = nullptr;
    AVBufferRef*                                                      cuda_bgra_frame_ctx_ = nullptr;

    AVCodecContext*          codec_color_ctx_               = nullptr;
    std::array<AVPacket*, 2> decode_src_color_packets_      = {nullptr, nullptr};
    std::array<AVFrame*, 2>  decode_out_color_frames_       = {nullptr, nullptr};
    std::array<AVFrame*, 2>  decode_converted_color_frames_ = {nullptr, nullptr};

    AVCodecContext*          codec_depth_ctx_               = nullptr;
    std::array<AVPacket*, 2> decode_src_depth_packets_      = {nullptr, nullptr};
    std::array<AVFrame*, 2>  decode_out_depth_frames_       = {nullptr, nullptr};
    std::array<AVFrame*, 2>  decode_converted_depth_frames_ = {nullptr, nullptr};

    data_format::pose::fast_head_pose_type decoded_frame_pose_;

    VkCommandPool    command_pool{};
    Npp8u*           yuv420_y_plane_ = nullptr;
    Npp8u*           yuv420_u_plane_ = nullptr;
    Npp8u*           yuv420_v_plane_ = nullptr;
    int              y_step_         = 0;
    int              u_step_         = 0;
    int              v_step_         = 0;
    NppStreamContext npp_ctx_        = {};
#endif // __ANDROID__

    uint64_t frame_count_ = 0;

#ifndef __ANDROID__
    VkFence fence_{};
#endif

    uint16_t                                       fps_counter_    = 0;
    std::chrono::high_resolution_clock::time_point fps_start_time_ = std::chrono::high_resolution_clock::now();
    std::map<std::string, uint32_t>                metrics_{};
#if defined(USING_OPENXR) && !defined(ILLIXR_ENABLE_BOBA)
    std::array<float, 2> cached_fov_left_  = {0.0f, 0.0f};
    std::array<float, 2> cached_fov_right_ = {0.0f, 0.0f};
    std::array<float, 2> cached_fov_up_    = {0.0f, 0.0f};
    std::array<float, 2> cached_fov_down_  = {0.0f, 0.0f};
    bool                 fov_cached_       = false;
#endif
    uint64_t last_submitted_frame_{0};
#ifdef __ANDROID__
#    ifdef ILLIXR_ENABLE_BOBA
    int headset_width_  = NATIVE_STREAM_EYE_WIDTH;
    int headset_height_ = NATIVE_STREAM_EYE_HEIGHT;
#    else
    int    headset_width_  = HEADSET_WIDTH;
    int    headset_height_ = HEADSET_HEIGHT;
    double overscan_;
#    endif
#endif
};

} // namespace ILLIXR