File plugin.cpp
File List > openxr_interface > plugin.cpp
Go to the documentation of this file
#ifdef USING_OPENXR
# include "plugin.hpp"
# include (current_frames_->pose_id);
spdlog::get("illixr")->debug(" Got new frame {}", current_frames_->frame_number);
// Resolve the correlation
}
// current_frames_ retains last valid frame if nothing new arrived
if (current_frames_ && current_frames_->is_valid()) {
// Import AHardwareBuffers into Vulkan (cached — no-op if buffer unchanged).
renderer_->receive_frame(*current_frames_);
// Look up the original pose measurement that was used to render this
// frame so we can log the end-to-end pose tracking latency.
{
// static uint64_t pose_log_counter = 0;
// const bool should_log_pose = (++pose_log_counter % 300) == 1;
if (current_frames_->pose_id != 0) {
ILLIXR::pose_history_entry history_entry{};
if (oxr_relay_->get_pose_history(current_frames_->pose_id, history_entry)) {
// current time and XrTime for comparison
auto now_tp = time_point{std::chrono::duration<long, std::nano>{
std::chrono::high_resolution_clock::now().time_since_epoch()}};
auto now_xr = static_cast<int64_t>(frame_state.predictedDisplayTime);
// Age of the pose at display time
auto age_ns =
std::chrono::duration_cast<std::chrono::nanoseconds>(now_tp - history_entry.generated_time).count();
XrSpaceLocation loc = {XR_TYPE_SPACE_LOCATION};
OXR(xrLocateSpace(view_space_, local_space_, frame_state.predictedDisplayTime, &loc))
spdlog::get("illixr")->info(
"[pose_tracking] frame pose_id={} frame_id={} "
"initial_pose=({:.3f},{:.3f},{:.3f};{:.3f},{:.3f},{:.3f},{:.3f};{:.3f},{:.3f},{:.3f};{:.3f},{:.3f},"
"{:.3f}) "
"render_pose=({:.3f},{:.3f},{:.3f};{:.3f},{:.3f},{:.3f},{:.3f}) "
"currentPose=({:.3f},{:.3f},{:.3f};{:.3f},{:.3f},{:.3f},{:.3f}) "
"generated_xr_time={} "
"current_xr_time={} "
"xr_time_delta_ms={:.3f} "
"pose_age_ms={:.3f} "
"encode_time={:.3f}",
current_frames_->pose_id, current_frames_->frame_number, history_entry.pose.pose.position.x,
history_entry.pose.pose.position.y, history_entry.pose.pose.position.z,
history_entry.pose.pose.orientation.w, history_entry.pose.pose.orientation.x,
history_entry.pose.pose.orientation.y, history_entry.pose.pose.orientation.z,
history_entry.pose.linear_velocity.x, history_entry.pose.linear_velocity.y,
history_entry.pose.linear_velocity.z, history_entry.pose.angular_velocity.x,
history_entry.pose.angular_velocity.y, history_entry.pose.angular_velocity.z,
(current_frames_->pose[0].position.x + current_frames_->pose[1].position.x) / 2.,
(current_frames_->pose[0].position.y + current_frames_->pose[1].position.y) / 2.,
(current_frames_->pose[0].position.z + current_frames_->pose[1].position.z) / 2.,
current_frames_->pose[0].orientation.w, current_frames_->pose[0].orientation.x,
current_frames_->pose[0].orientation.y, current_frames_->pose[0].orientation.z, loc.pose.position.x,
loc.pose.position.y, loc.pose.position.z, loc.pose.orientation.w, loc.pose.orientation.x,
loc.pose.orientation.y, loc.pose.orientation.z, history_entry.xr_time, now_xr,
static_cast<double>(now_xr - static_cast<int64_t>(history_entry.xr_time)) / 1'000'000.0,
static_cast<double>(age_ns) / 1'000'000.0, current_frames_->encode_time);
} else {
spdlog::get("illixr")->debug("[pose_tracking] frame pose_id={} not found in history "
"(may have been pruned)",
current_frames_->pose_id);
}
} else {
spdlog::get("illixr")->debug("[pose_tracker] No current pose");
}
}
# ifdef ILLIXR_ENABLE_BOBA
const bool render_as_panel =
current_frames_->presentation_mode != data_format::stereo_presentation_mode::stereo_fullscreen;
if (current_frames_->presentation_mode == data_format::stereo_presentation_mode::mono_panel &&
(previous_presentation_mode_ != data_format::stereo_presentation_mode::mono_panel ||
!world_panel_anchor_initialized_)) {
world_panel_pose_ = panel_pose_from_view(views_[0].pose);
world_panel_anchor_initialized_ = true;
}
previous_presentation_mode_ = current_frames_->presentation_mode;
// Panel modes are monoscopic and use one compositor quad. Fullscreen mode renders both projection eyes.
const int render_eye_count = render_as_panel ? 1 : 2;
# else
constexpr int render_eye_count = 2;
# endif
for (int eye = 0; eye < render_eye_count; eye++) {
swapchain_info& sc = swapchains_[eye];
uint32_t img_idx = 0;
XrSwapchainImageAcquireInfo acq{XR_TYPE_SWAPCHAIN_IMAGE_ACQUIRE_INFO};
OXR(xrAcquireSwapchainImage(sc.swapchain, &acq, &img_idx))
XrSwapchainImageWaitInfo wait{XR_TYPE_SWAPCHAIN_IMAGE_WAIT_INFO};
wait.timeout = XR_INFINITE_DURATION;
OXR(xrWaitSwapchainImage(sc.swapchain, &wait))
VkImage swapchain_vk_image = sc.images[img_idx].image;
// Render color into swapchain VkImage.
renderer_->render_eye(eye, swapchain_vk_image, sc.width, sc.height);
XrSwapchainImageReleaseInfo rel{XR_TYPE_SWAPCHAIN_IMAGE_RELEASE_INFO};
OXR(xrReleaseSwapchainImage(sc.swapchain, &rel))
# ifdef ILLIXR_ENABLE_BOBA
if (render_as_panel) {
panelLayer.layerFlags =
XR_COMPOSITION_LAYER_BLEND_TEXTURE_SOURCE_ALPHA_BIT | XR_COMPOSITION_LAYER_UNPREMULTIPLIED_ALPHA_BIT;
panelLayer.space =
current_frames_->presentation_mode == data_format::stereo_presentation_mode::head_locked_panel
? view_space_
: local_space_;
panelLayer.eyeVisibility = XR_EYE_VISIBILITY_BOTH;
panelLayer.pose =
current_frames_->presentation_mode == data_format::stereo_presentation_mode::head_locked_panel
? identity_pose()
: world_panel_pose_;
if (current_frames_->presentation_mode == data_format::stereo_presentation_mode::head_locked_panel) {
panelLayer.pose.position.z = -BOBA_PANEL_DISTANCE_METERS;
}
const float aspect =
current_frames_->content_aspect_ratio > 0.0F ? current_frames_->content_aspect_ratio : 1.0F;
panelLayer.size = {BOBA_PANEL_WIDTH_METERS, BOBA_PANEL_WIDTH_METERS / aspect};
panelLayer.subImage.swapchain = sc.swapchain;
panelLayer.subImage.imageRect.offset = {0, 0};
panelLayer.subImage.imageRect.extent = {static_cast<int32_t>(sc.width), static_cast<int32_t>(sc.height)};
panelLayer.subImage.imageArrayIndex = 0;
continue;
}
# endif
// Set up projection layer
projectionViews[eye].pose = current_frames_->pose[eye];
// Use the render FOV from the server if available (supports overdraw margins).
// Fall back to the headset's native FOV if not yet received.
if (current_frames_->fov_left[eye] != 0.0f) {
projectionViews[eye].fov.angleLeft = current_frames_->fov_left[eye];
projectionViews[eye].fov.angleRight = current_frames_->fov_right[eye];
projectionViews[eye].fov.angleUp = current_frames_->fov_up[eye];
projectionViews[eye].fov.angleDown = current_frames_->fov_down[eye];
} else {
projectionViews[eye].fov = views_[eye].fov;
}
projectionViews[eye].subImage.imageArrayIndex = 0;
projectionViews[eye].subImage.swapchain = sc.swapchain;
projectionViews[eye].subImage.imageRect.offset = {0, 0};
projectionViews[eye].subImage.imageRect.extent.width = static_cast<int32_t>(sc.width);
projectionViews[eye].subImage.imageRect.extent.height = static_cast<int32_t>(sc.height);
// Depth swapchain (if enabled and frame has depth)
if (use_depth_ && depth_extension_supported_ && current_frames_->has_valid_depth() &&
depth_swapchains_[eye].swapchain != XR_NULL_HANDLE) {
swapchain_info& dsc = depth_swapchains_[eye];
uint32_t depth_img_idx = 0;
OXR(xrAcquireSwapchainImage(dsc.swapchain, &acq, &depth_img_idx))
OXR(xrWaitSwapchainImage(dsc.swapchain, &wait))
VkImage depth_vk_image = dsc.images[depth_img_idx].image;
renderer_->render_eye_depth(eye, depth_vk_image, depth_format_, dsc.width, dsc.height);
OXR(xrReleaseSwapchainImage(dsc.swapchain, &rel))
// App Spacewarp layer
// Submitted when the frame carries both motion vectors and depth.
// The motion-vector and spacewarp-depth swapchains are at
// 432×432 (MV_SWAPCHAIN_WIDTH × MV_SWAPCHAIN_HEIGHT).
if (spacewarp_supported_ && current_frames_->has_valid_motion_vectors() &&
mv_swapchains_[eye].swapchain != XR_NULL_HANDLE) {
// Render motion vectors into the MV swapchain
swapchain_info& mv_sc = mv_swapchains_[eye];
uint32_t mv_img_idx = 0;
OXR(xrAcquireSwapchainImage(mv_sc.swapchain, &acq, &mv_img_idx))
OXR(xrWaitSwapchainImage(mv_sc.swapchain, &wait))
VkImage mv_vk_image = mv_sc.images[mv_img_idx].image;
renderer_->render_eye_motion_vec(eye, mv_vk_image, mv_sc.width, mv_sc.height);
OXR(xrReleaseSwapchainImage(mv_sc.swapchain, &rel))
// Build XrCompositionLayerSpaceWarpInfoFB
// appSpaceDeltaPose: relative transform of the app's tracking
// origin between the rendered frame and now. For offload
// rendering with a fixed world origin this is identity.
XrPosef identity_delta{};
identity_delta.orientation.w = 1.0f;
XrCompositionLayerSpaceWarpInfoFB& sw = spacewarp_infos[eye];
sw.layerFlags = 0;
sw.motionVectorSubImage.swapchain = mv_sc.swapchain;
sw.motionVectorSubImage.imageArrayIndex = 0;
sw.motionVectorSubImage.imageRect.offset = {0, 0};
sw.motionVectorSubImage.imageRect.extent = {static_cast<int32_t>(mv_sc.width),
static_cast<int32_t>(mv_sc.height)};
sw.appSpaceDeltaPose = identity_delta;
sw.depthSubImage.swapchain = dsc.swapchain;
sw.depthSubImage.imageArrayIndex = 0;
sw.depthSubImage.imageRect.offset = {0, 0};
sw.depthSubImage.imageRect.extent = {static_cast<int32_t>(dsc.width), static_cast<int32_t>(dsc.height)};
sw.minDepth = 0.0f;
sw.maxDepth = 1.0f;
sw.nearZ = current_frames_->near_z;
sw.farZ = current_frames_->far_z;
// Chain spacewarp info onto the projection view.
// If a depth_info was already chained, spacewarp takes
// precedence (it incorporates the same depth information).
projectionViews[eye].next = &spacewarp_infos[eye];
} else {
// Chain XrCompositionLayerDepthInfoKHR onto this projection view.
depth_infos[eye].subImage.swapchain = dsc.swapchain;
depth_infos[eye].subImage.imageArrayIndex = 0;
depth_infos[eye].subImage.imageRect.offset = {0, 0};
depth_infos[eye].subImage.imageRect.extent = {static_cast<int32_t>(dsc.width),
static_cast<int32_t>(dsc.height)};
depth_infos[eye].minDepth = 0.0f;
depth_infos[eye].maxDepth = 1.0f;
depth_infos[eye].nearZ = current_frames_->near_z;
depth_infos[eye].farZ = current_frames_->far_z;
projectionViews[eye].next = &depth_infos[eye];
}
}
}
# ifdef ILLIXR_ENABLE_BOBA
if (render_as_panel) {
layers[0] = reinterpret_cast<XrCompositionLayerBaseHeader*>(&panelLayer);
} else
# endif
{
projectionLayer.space = local_space_;
projectionLayer.viewCount = 2;
projectionLayer.views = projectionViews;
projectionLayer.layerFlags = XR_COMPOSITION_LAYER_CORRECT_CHROMATIC_ABERRATION_BIT |
XR_COMPOSITION_LAYER_BLEND_TEXTURE_SOURCE_ALPHA_BIT | XR_COMPOSITION_LAYER_UNPREMULTIPLIED_ALPHA_BIT;
layers[0] = reinterpret_cast<XrCompositionLayerBaseHeader*>(&projectionLayer);
}
layer_count = 1;
}
}
// End frame (normally you'd submit layers here)
XrFrameEndInfo endInfo = {XR_TYPE_FRAME_END_INFO};
endInfo.displayTime = frame_state.predictedDisplayTime;
endInfo.environmentBlendMode = XR_ENVIRONMENT_BLEND_MODE_OPAQUE;
endInfo.layerCount = layer_count;
endInfo.layers = layers;
auto now_ns = std::chrono::steady_clock::now().time_since_epoch().count();
int64_t margin_ms = (frame_state.predictedDisplayTime - now_ns) / 1'000'000;
spdlog::get("illixr")->debug("xrEndFrame margin: {}ms", margin_ms);
OXR(xrEndFrame(session_, &endInfo))
frame_counter_++;
}
void oxr_interface::create_swapchains() {
// Enumerate supported Vulkan swapchain formats and pick a suitable one.
uint32_t fmt_count = 0;
xrEnumerateSwapchainFormats(session_, 0, &fmt_count, nullptr);
std::vector<int64_t> formats(fmt_count);
xrEnumerateSwapchainFormats(session_, fmt_count, &fmt_count, formats.data());
# ifdef ILLIXR_ENABLE_BOBA
// Prefer an sRGB swapchain. The decoded video is converted back to linear
// RGB in boba_color.frag before this attachment applies its output transfer.
VkFormat chosen_fmt = formats.empty() ? VK_FORMAT_R8G8B8A8_UNORM : static_cast<VkFormat>(formats.front());
const std::array<VkFormat, 4> preferred_formats{
VK_FORMAT_R8G8B8A8_SRGB,
VK_FORMAT_R8G8B8A8_UNORM,
VK_FORMAT_B8G8R8A8_SRGB,
VK_FORMAT_B8G8R8A8_UNORM,
};
for (const VkFormat preferred : preferred_formats) {
if (std::find(formats.begin(), formats.end(), static_cast<int64_t>(preferred)) != formats.end()) {
chosen_fmt = preferred;
break;
}
}
# else
// Prefer R8G8B8A8_SRGB → R8G8B8A8_UNORM → B8G8R8A8_UNORM
VkFormat chosen_fmt = VK_FORMAT_R8G8B8A8_UNORM;
for (int64_t f : formats) {
if (f == VK_FORMAT_R8G8B8A8_SRGB) {
chosen_fmt = VK_FORMAT_R8G8B8A8_SRGB;
break;
}
if (f == VK_FORMAT_R8G8B8A8_UNORM) {
chosen_fmt = VK_FORMAT_R8G8B8A8_UNORM;
break;
}
if (f == VK_FORMAT_B8G8R8A8_SRGB) {
chosen_fmt = VK_FORMAT_B8G8R8A8_SRGB;
break;
}
}
# endif
spdlog::get("illixr")->info("Swapchain format selected: 0x{:X}", static_cast<uint32_t>(chosen_fmt));
// Create one swapchain per eye
for (int eye = 0; eye < 2; eye++) {
swapchain_info& sc = swapchains_[eye];
sc.format = chosen_fmt;
sc.width = headset_width_; // Your input resolution
sc.height = headset_height_; // Your input resolution
XrSwapchainCreateInfo swapchainInfo = {XR_TYPE_SWAPCHAIN_CREATE_INFO};
swapchainInfo.arraySize = 1;
swapchainInfo.format = static_cast<int64_t>(chosen_fmt);
swapchainInfo.width = sc.width;
swapchainInfo.height = sc.height;
swapchainInfo.mipCount = 1;
swapchainInfo.faceCount = 1;
swapchainInfo.sampleCount = 1;
swapchainInfo.usageFlags = XR_SWAPCHAIN_USAGE_SAMPLED_BIT | XR_SWAPCHAIN_USAGE_COLOR_ATTACHMENT_BIT;
OXR(xrCreateSwapchain(session_, &swapchainInfo, &sc.swapchain))
// Enumerate swapchain images as Vulkan image handles.
uint32_t img_count = 0;
OXR(xrEnumerateSwapchainImages(sc.swapchain, 0, &img_count, nullptr))
sc.images.resize(img_count, {XR_TYPE_SWAPCHAIN_IMAGE_VULKAN2_KHR});
OXR(xrEnumerateSwapchainImages(sc.swapchain, img_count, &img_count,
reinterpret_cast<XrSwapchainImageBaseHeader*>(sc.images.data())))
spdlog::get("illixr")->info("Eye {} swapchain: {}x{} format=0x{:X} images={}", eye, sc.width, sc.height,
static_cast<uint32_t>(chosen_fmt), img_count);
}
if (use_depth_ && depth_extension_supported_) {
// App Spacewarp swapchains
// Both the motion-vector swapchain and the spacewarp depth swapchain are
// created at 432×432, independent of the full-resolution color swapchains.
if (spacewarp_supported_) {
// Motion-vector swapchain — R16G16B16A16_SFLOAT
// XR_SWAPCHAIN_USAGE_MUTABLE_FORMAT_BIT lets the runtime use an
// aliased format internally if needed.
for (int eye = 0; eye < 2; eye++) {
swapchain_info& mv_sc = mv_swapchains_[eye];
mv_sc.format = mv_swapchain_format_;
mv_sc.width = MOTION_VEC_WIDTH;
mv_sc.height = MOTION_VEC_HEIGHT;
XrSwapchainCreateInfo mv_info{XR_TYPE_SWAPCHAIN_CREATE_INFO};
mv_info.arraySize = 1;
mv_info.format = static_cast<int64_t>(mv_swapchain_format_);
mv_info.width = MOTION_VEC_WIDTH;
mv_info.height = MOTION_VEC_HEIGHT;
mv_info.mipCount = 1;
mv_info.faceCount = 1;
mv_info.sampleCount = 1;
mv_info.usageFlags = XR_SWAPCHAIN_USAGE_SAMPLED_BIT | XR_SWAPCHAIN_USAGE_COLOR_ATTACHMENT_BIT |
XR_SWAPCHAIN_USAGE_MUTABLE_FORMAT_BIT;
OXR(xrCreateSwapchain(session_, &mv_info, &mv_sc.swapchain))
uint32_t img_count = 0;
OXR(xrEnumerateSwapchainImages(mv_sc.swapchain, 0, &img_count, nullptr))
mv_sc.images.resize(img_count, {XR_TYPE_SWAPCHAIN_IMAGE_VULKAN2_KHR});
OXR(xrEnumerateSwapchainImages(mv_sc.swapchain, img_count, &img_count,
reinterpret_cast<XrSwapchainImageBaseHeader*>(mv_sc.images.data())))
spdlog::get("illixr")->info("Eye {} MV swapchain: {}x{} format=0x{:X} images={}", eye, MOTION_VEC_WIDTH,
MOTION_VEC_HEIGHT, static_cast<uint32_t>(mv_swapchain_format_), img_count);
}
// Spacewarp depth swapchain — D32_SFLOAT (fallback D16_UNORM)
depth_format_ = VK_FORMAT_D32_SFLOAT;
for (int64_t f : formats) {
if (f == VK_FORMAT_D32_SFLOAT) {
depth_format_ = VK_FORMAT_D32_SFLOAT;
break;
}
if (f == VK_FORMAT_D16_UNORM) {
depth_format_ = VK_FORMAT_D16_UNORM;
break;
}
}
for (int eye = 0; eye < 2; eye++) {
swapchain_info& sw_dsc = depth_swapchains_[eye];
sw_dsc.format = depth_format_;
sw_dsc.width = MOTION_VEC_WIDTH;
sw_dsc.height = MOTION_VEC_HEIGHT;
XrSwapchainCreateInfo sw_depth_info{XR_TYPE_SWAPCHAIN_CREATE_INFO};
sw_depth_info.arraySize = 1;
sw_depth_info.format = static_cast<int64_t>(depth_format_);
sw_depth_info.width = MOTION_VEC_WIDTH;
sw_depth_info.height = MOTION_VEC_HEIGHT;
sw_depth_info.mipCount = 1;
sw_depth_info.faceCount = 1;
sw_depth_info.sampleCount = 1;
sw_depth_info.usageFlags = XR_SWAPCHAIN_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT;
OXR(xrCreateSwapchain(session_, &sw_depth_info, &sw_dsc.swapchain))
uint32_t img_count = 0;
OXR(xrEnumerateSwapchainImages(sw_dsc.swapchain, 0, &img_count, nullptr))
sw_dsc.images.resize(img_count, {XR_TYPE_SWAPCHAIN_IMAGE_VULKAN2_KHR});
OXR(xrEnumerateSwapchainImages(sw_dsc.swapchain, img_count, &img_count,
reinterpret_cast<XrSwapchainImageBaseHeader*>(sw_dsc.images.data())))
spdlog::get("illixr")->info("Eye {} spacewarp depth swapchain: {}x{} format=0x{:X} images={}", eye,
MOTION_VEC_WIDTH, MOTION_VEC_HEIGHT, static_cast<uint32_t>(depth_format_),
img_count);
}
} else {
// Prefer D32_SFLOAT for maximum precision; fall back to D16_UNORM.
depth_format_ = VK_FORMAT_D32_SFLOAT;
for (int64_t f : formats) {
if (f == VK_FORMAT_D16_UNORM) {
depth_format_ = VK_FORMAT_D16_UNORM;
break;
}
if (f == VK_FORMAT_D32_SFLOAT) {
depth_format_ = VK_FORMAT_D32_SFLOAT;
break;
}
}
spdlog::get("illixr")->info("Depth swapchain format: 0x{:X}", static_cast<uint32_t>(depth_format_));
for (int eye = 0; eye < 2; eye++) {
swapchain_info& sc = depth_swapchains_[eye];
sc.format = depth_format_;
sc.width = headset_width_;
sc.height = headset_height_;
XrSwapchainCreateInfo depth_info = {XR_TYPE_SWAPCHAIN_CREATE_INFO};
depth_info.arraySize = 1;
depth_info.format = static_cast<int64_t>(depth_format_);
depth_info.width = sc.width;
depth_info.height = sc.height;
depth_info.mipCount = 1;
depth_info.faceCount = 1;
depth_info.sampleCount = 1;
depth_info.usageFlags = XR_SWAPCHAIN_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT;
OXR(xrCreateSwapchain(session_, &depth_info, &sc.swapchain))
uint32_t img_count = 0;
OXR(xrEnumerateSwapchainImages(sc.swapchain, 0, &img_count, nullptr))
sc.images.resize(img_count, {XR_TYPE_SWAPCHAIN_IMAGE_VULKAN2_KHR});
OXR(xrEnumerateSwapchainImages(sc.swapchain, img_count, &img_count,
reinterpret_cast<XrSwapchainImageBaseHeader*>(sc.images.data())))
spdlog::get("illixr")->info("Eye {} depth swapchain: {}x{} format=0x{:X} images={}", eye, sc.width, sc.height,
static_cast<uint32_t>(depth_format_), img_count);
}
}
}
}
extern "C" plugin* this_plugin_factory(phonebook* pb) {
auto* obj = new oxr_interface("openxr_interface", pb);
// The runtime owns the plugin returned by this factory. Register a non-owning
// service alias so the phonebook does not try to delete the same object again.
pb->register_impl<vk::vulkan_context_provider>(std::shared_ptr<vk::vulkan_context_provider>(
static_cast<vk::vulkan_context_provider*>(obj), [](vk::vulkan_context_provider*) { }));
return obj;
}
#endif