I added a sample code if someone want's to try fixing it: https://www.dropbox.com/s/6t6neo40qjganra/To%20be%20fixed.zip?dl=1
Im making an AR app using Vuforia SDK. Their sample code contains video background rendering using metal. It works fine for vertical orientation, but I need to change it to portrait. The problem is, that after changing it, the video is rendered upside down. Detected targets are in correct orientation, so I think this should be fixed in Metal rendering class. Could someone help me do that? Below is the code Im using to draw that background. How can I rotate it 180 degrees?
private var mMetalDevice: MTLDevice
private var mVideoBackgroundPipelineState: MTLRenderPipelineState!
private var mUniformColorShaderPipelineState: MTLRenderPipelineState!
private var mTexturedVertexShaderPipelineState: MTLRenderPipelineState!
private var mDefaultSamplerState: MTLSamplerState?
private var mVideoBackgroundVertices: MTLBuffer!
private var mVideoBackgroundIndices: MTLBuffer!
private var mVideoBackgroundTextureCoordinates: MTLBuffer!
/// Initialize the renderer ready for use
init(metalDevice: MTLDevice, layer: CAMetalLayer, library: MTLLibrary?, textureDepth: MTLTexture) {
mMetalDevice = metalDevice
let stateDescriptor = MTLRenderPipelineDescriptor()
//
// Video background
//
stateDescriptor.vertexFunction = library?.makeFunction(name: "texturedVertex")
stateDescriptor.fragmentFunction = library?.makeFunction(name: "texturedFragment")
stateDescriptor.colorAttachments[0].pixelFormat = layer.pixelFormat
stateDescriptor.depthAttachmentPixelFormat = textureDepth.pixelFormat
// And create the pipeline state with the descriptor
do {
try mVideoBackgroundPipelineState = metalDevice.makeRenderPipelineState(descriptor: stateDescriptor)
} catch {
print("Failed to create video background render pipeline state:",error)
}
//
// Augmentations
//
// Create pipeline for transparent object overlays
stateDescriptor.vertexFunction = library?.makeFunction(name: "uniformColorVertex")
stateDescriptor.fragmentFunction = library?.makeFunction(name: "uniformColorFragment")
stateDescriptor.colorAttachments[0].pixelFormat = layer.pixelFormat
stateDescriptor.colorAttachments[0].isBlendingEnabled = true
stateDescriptor.colorAttachments[0].rgbBlendOperation = .add
stateDescriptor.colorAttachments[0].alphaBlendOperation = .add
stateDescriptor.colorAttachments[0].sourceRGBBlendFactor = .sourceAlpha
stateDescriptor.colorAttachments[0].sourceAlphaBlendFactor = .sourceAlpha
stateDescriptor.colorAttachments[0].destinationRGBBlendFactor = .oneMinusSourceAlpha
stateDescriptor.colorAttachments[0].destinationAlphaBlendFactor = .oneMinusSourceAlpha
stateDescriptor.depthAttachmentPixelFormat = textureDepth.pixelFormat
do {
try mUniformColorShaderPipelineState = metalDevice.makeRenderPipelineState(descriptor: stateDescriptor)
} catch {
print("Failed to create augmentation render pipeline state:",error)
return
}
stateDescriptor.vertexFunction = library?.makeFunction(name: "texturedVertex")
stateDescriptor.fragmentFunction = library?.makeFunction(name: "texturedFragment")
// Create pipeline for rendering textures
do {
try mTexturedVertexShaderPipelineState = metalDevice.makeRenderPipelineState(descriptor: stateDescriptor)
} catch {
print("Failed to create guide view render pipeline state:", error)
return
}
mDefaultSamplerState = MetalRenderer.defaultSampler(device: metalDevice)
// Allocate space for rendering data for Video background
mVideoBackgroundVertices = mMetalDevice.makeBuffer(length: MemoryLayout<Float>.size * 3 * 4, options: [.optionCPUCacheModeWriteCombined])
mVideoBackgroundTextureCoordinates = mMetalDevice.makeBuffer(length: MemoryLayout<Float>.size * 2 * 4, options: [.optionCPUCacheModeWriteCombined])
mVideoBackgroundIndices = mMetalDevice.makeBuffer(length: MemoryLayout<UInt16>.size * 6, options: [.optionCPUCacheModeWriteCombined])
}
/// Render the video background
func renderVideoBackground(encoder: MTLRenderCommandEncoder?, projectionMatrix: MTLBuffer, mesh: VuforiaMesh) {
// Copy mesh data into metal buffers
mVideoBackgroundVertices.contents().copyMemory(from: mesh.vertices, byteCount: MemoryLayout<Float>.size * Int(mesh.numVertices) * 3)
mVideoBackgroundTextureCoordinates.contents().copyMemory(from: mesh.textureCoordinates, byteCount: MemoryLayout<Float>.size * Int(mesh.numVertices) * 2)
mVideoBackgroundIndices.contents().copyMemory(from: mesh.indices, byteCount: MemoryLayout<CShort>.size * Int(mesh.numIndices))
// Set the render pipeline state
encoder?.setRenderPipelineState(mVideoBackgroundPipelineState)
// Set the vertex buffer
encoder?.setVertexBuffer(mVideoBackgroundVertices, offset: 0, index: 0)
// Set the projection matrix
encoder?.setVertexBuffer(projectionMatrix, offset: 0, index: 1)
// Set the texture coordinate buffer
encoder?.setVertexBuffer(mVideoBackgroundTextureCoordinates, offset: 0, index: 2)
encoder?.setFragmentSamplerState(mDefaultSamplerState, index: 0)
// Draw the geometry
encoder?.drawIndexedPrimitives(
type: .triangle,
indexCount: 6,
indexType: .uint16,
indexBuffer: mVideoBackgroundIndices,
indexBufferOffset: 0
)
}
}
extension MetalRenderer {
class func defaultSampler(device: MTLDevice) -> MTLSamplerState? {
let sampler = MTLSamplerDescriptor()
sampler.minFilter = .linear
sampler.magFilter = .linear
sampler.mipFilter = .linear
sampler.maxAnisotropy = 1
sampler.sAddressMode = .clampToEdge
sampler.tAddressMode = .clampToEdge
sampler.rAddressMode = .clampToEdge
sampler.normalizedCoordinates = true
sampler.lodMinClamp = 0
sampler.lodMaxClamp = .greatestFiniteMagnitude
return device.makeSamplerState(descriptor: sampler)
}
}
Adding code from the view that creates renderer:
import UIKit
import MetalKit
protocol VuforiaViewDelegate: AnyObject {
func renderFrame(vuforiaView: VuforiaView)
}
class VuforiaView: UIView {
weak var delegate: VuforiaViewDelegate?
var mVuforiaStarted = false
private var mConfigurationChanged = true
private var mRenderer: MetalRenderer!
private var mMetalDevice: MTLDevice!
private var mMetalCommandQueue: MTLCommandQueue!
private var mCommandExecutingSemaphore: DispatchSemaphore!
private var mDepthStencilState: MTLDepthStencilState!
private var mDepthTexture: MTLTexture!
private var mVideoBackgroundProjectionBuffer: MTLBuffer!
private lazy var metalLayer = layer as! CAMetalLayer
override class var layerClass: AnyClass { CAMetalLayer.self }
// Transformations and variables - constantly updated by vuforia frame updates
private var viewport = MTLViewport()
private var trackableProjection = matrix_float4x4()
private var trackableModelView = matrix_float4x4()
private var trackableScaledModelView = matrix_float4x4()
private(set) var worldOriginProjectionMatrix = matrix_float4x4()
private(set) var worldOriginModelViewMatrix = matrix_float4x4()
private(set) var targetPose = matrix_float4x4()
private(set) var targetSize = simd_float3()
override init(frame: CGRect) {
super.init(frame: frame)
setup()
}
required init?(coder: NSCoder) {
super.init(coder: coder)
setup()
}
private func setup() {
contentScaleFactor = UIScreen.main.nativeScale
// Get the system default metal device
mMetalDevice = MTLCreateSystemDefaultDevice()
// Metal command queue
mMetalCommandQueue = mMetalDevice.makeCommandQueue()
// Create a dispatch semaphore, used to synchronise command execution
mCommandExecutingSemaphore = DispatchSemaphore(value: 1)
// Create a CAMetalLayer and set its frame to match that of the view
let layer = self.layer as! CAMetalLayer
layer.device = mMetalDevice
layer.pixelFormat = .bgra8Unorm
layer.framebufferOnly = true
layer.contentsScale = contentScaleFactor
// Get the default library from the bundle (Metal shaders)
let library = mMetalDevice.makeDefaultLibrary()
// Create a depth texture that is needed when rendering the augmentation.
let screenSize = UIScreen.main.bounds.size
let depthTextureDescriptor = MTLTextureDescriptor.texture2DDescriptor(
pixelFormat: .depth32Float,
width: Int(screenSize.width * contentScaleFactor),
height: Int(screenSize.height * contentScaleFactor),
mipmapped: false
)
depthTextureDescriptor.usage = .renderTarget
mDepthTexture = mMetalDevice.makeTexture(descriptor: depthTextureDescriptor)
// Video background projection matrix buffer
mVideoBackgroundProjectionBuffer = mMetalDevice.makeBuffer(length: MemoryLayout<Float>.size * 16, options: [])
// Fragment depth stencil
let depthStencilDescriptor = MTLDepthStencilDescriptor()
depthStencilDescriptor.depthCompareFunction = .less
depthStencilDescriptor.isDepthWriteEnabled = true
mDepthStencilState = mMetalDevice.makeDepthStencilState(descriptor: depthStencilDescriptor)
mRenderer = MetalRenderer(
metalDevice: mMetalDevice,
layer: layer,
library: library,
textureDepth: mDepthTexture
)
}
private func configureVuforia() {
let orientationValue: Int32 = {
let orientation = UIApplication.shared.windows.first(where: { $0.isKeyWindow })?.windowScene?.interfaceOrientation ?? .portrait
switch orientation {
case .portrait: return 0
case .portraitUpsideDown: return 1
case .landscapeLeft: return 2
case .landscapeRight: return 3
case .unknown: return 4
@unknown default: return 4
}
}()
let screenSize = UIScreen.main.bounds.size
configureRendering(
Int32(screenSize.width * contentScaleFactor),
Int32(screenSize.height * contentScaleFactor),
orientationValue
)
}
@objc private func renderFrameVuforia() {
objc_sync_enter(self)
if mVuforiaStarted {
if mConfigurationChanged {
mConfigurationChanged = false
configureVuforia()
}
renderFrameVuforiaInternal()
delegate?.renderFrame(vuforiaView: self)
}
objc_sync_exit(self)
}
private func renderFrameVuforiaInternal() {
//Check if Camera is Started
guard isCameraStarted() else { return }
// ========== Set up ==========
var viewportsValue = Array(arrayLiteral: 0.0, 0.0, Double(metalLayer.drawableSize.width), Double(metalLayer.drawableSize.height), 0.0, 1.0)
// --- Command buffer ---
// Get the command buffer from the command queue
let commandBuffer = mMetalCommandQueue.makeCommandBuffer()
// Get the next drawable from the CAMetalLayer
let drawable = metalLayer.nextDrawable()
// It's possible for nextDrawable to return nil, which means a call to
// renderCommandEncoderWithDescriptor will fail
guard drawable != nil else { return }
// Wait for exclusive access to the GPU
let _ = mCommandExecutingSemaphore.wait(timeout: .distantFuture)
// -- Render pass descriptor ---
// Set up a render pass decriptor
let renderPassDescriptor = MTLRenderPassDescriptor()
// Draw to the drawable's texture
renderPassDescriptor.colorAttachments[0].texture = drawable?.texture
// Clear the colour attachment in case there is no video frame
renderPassDescriptor.colorAttachments[0].loadAction = .clear
// Store the data in the texture when rendering is complete
renderPassDescriptor.colorAttachments[0].storeAction = .store
// Use textureDepth for depth operations.
renderPassDescriptor.depthAttachment.texture = mDepthTexture
// Get a command encoder to encode into the command buffer
let encoder = commandBuffer?.makeRenderCommandEncoder(descriptor: renderPassDescriptor)
if prepareToRender(
&viewportsValue,
UnsafeMutableRawPointer(Unmanaged.passRetained(mMetalDevice!).toOpaque()),
UnsafeMutableRawPointer(Unmanaged.passRetained(drawable!.texture).toOpaque()),
UnsafeMutableRawPointer(Unmanaged.passRetained(encoder!).toOpaque())
) {
viewport.originX = viewportsValue[0]
viewport.originY = viewportsValue[1]
viewport.width = viewportsValue[2]
viewport.height = viewportsValue[3]
viewport.znear = viewportsValue[4]
viewport.zfar = viewportsValue[5]
encoder?.setViewport(viewport)
// Once the camera is initialized we can get the video background rendering values
getVideoBackgroundProjection(mVideoBackgroundProjectionBuffer.contents())
// Call the renderer to draw the video background
mRenderer.renderVideoBackground(encoder: encoder, projectionMatrix: mVideoBackgroundProjectionBuffer, mesh: getVideoBackgroundMesh())
encoder?.setDepthStencilState(mDepthStencilState)
getOrigin(
&worldOriginProjectionMatrix.columns,
&worldOriginModelViewMatrix.columns
)
getImageTargetResult(
&trackableProjection.columns,
&trackableModelView.columns,
&trackableScaledModelView.columns,
&targetPose.columns,
&targetSize
)
}
// Pass Metal context data to Vuforia Engine (we may have changed the encoder since
// calling Vuforia::Renderer::begin)
finishRender(
UnsafeMutableRawPointer(Unmanaged.passRetained(drawable!.texture).toOpaque()),
UnsafeMutableRawPointer(Unmanaged.passRetained(encoder!).toOpaque())
)
// ========== Finish Metal rendering ==========
encoder?.endEncoding()
// Commit the rendering commands
// Command completed handler
commandBuffer?.addCompletedHandler { _ in self.mCommandExecutingSemaphore.signal() }
// Present the drawable when the command buffer has been executed (Metal
// calls to CoreAnimation to tell it to put the texture on the display when
// the rendering is complete)
commandBuffer?.present(drawable!)
// Commit the command buffer for execution as soon as possible
commandBuffer?.commit()
}
}
Another problem is that in portrait mode something is wrong with aspect ratio, camera background is drawn distorted. But this is for another subject.
Shaders.metal:
/*===============================================================================
Copyright (c) 2020, PTC Inc. All rights reserved.
Vuforia is a trademark of PTC Inc., registered in the United States and other
countries.
===============================================================================*/
#include <metal_stdlib>
using namespace metal;
// === Texture sampling shader ===
struct VertexTextureOut
{
float4 m_Position [[ position ]];
float2 m_TexCoord;
};
vertex VertexTextureOut texturedVertex(constant packed_float3* pPosition [[ buffer(0) ]],
constant float4x4* pMVP [[ buffer(1) ]],
constant float2* pTexCoords [[ buffer(2) ]],
uint vid [[ vertex_id ]])
{
VertexTextureOut out;
float4 in(pPosition[vid], 1.0f);
out.m_Position = *pMVP * in;
out.m_TexCoord = pTexCoords[vid];
return out;
}
fragment half4 texturedFragment(VertexTextureOut inFrag [[ stage_in ]],
texture2d<half> tex2D [[ texture(0) ]],
sampler sampler2D [[ sampler(0) ]])
{
return tex2D.sample(sampler2D, inFrag.m_TexCoord);
}
// === Uniform color shader ===
struct VertexOut
{
float4 m_Position [[ position ]];
};
vertex VertexOut uniformColorVertex(constant packed_float3* pPosition [[ buffer(0) ]],
constant float4x4* pMVP [[ buffer(1) ]],
uint vid [[ vertex_id ]])
{
VertexOut out;
float4 in(pPosition[vid], 1.0f);
out.m_Position = *pMVP * in;
return out;
}
fragment float4 uniformColorFragment(constant float4 &color [[ buffer(0) ]])
{
return color;
}
// === Vertex color shader ===
struct VertexColorOut
{
float4 m_Position [[ position ]];
float4 m_Color;
};
vertex VertexColorOut vertexColorVertex(constant packed_float3* pPosition [[ buffer(0) ]],
constant float4* pColor [[ buffer(1) ]],
constant float4x4* pMVP [[ buffer(2) ]],
uint vid [[ vertex_id ]])
{
VertexColorOut out;
float4 in(pPosition[vid], 1.0f);
out.m_Position = *pMVP * in;
out.m_Color = pColor[vid];
return out;
}
fragment float4 vertexColorFragment(VertexColorOut inFrag [[ stage_in ]])
{
return inFrag.m_Color;
}