func sampleDepthRaw(_ pointer: UnsafeMutablePointer, size: SIMD2, at: SIMD2) -> Float { let baseAddressIndex = at.y * size.x + at.x return Float(pointer[baseAddressIndex]) } // This also works. Adapted from: // https://developer.apple.com/forums/thread/676368 func worldPoint(depthMapPixelPoint: SIMD2, depth: Float, cameraIntrinsicsInverted: simd_float3x3, viewMatrixInverted: simd_float4x4) -> SIMD3 { let localPoint = cameraIntrinsicsInverted * simd_float3(depthMapPixelPoint, 1) * -depth let localPointSwappedX = simd_float3(-localPoint.x, localPoint.y, localPoint.z) let worldPoint = viewMatrixInverted * simd_float4(localPointSwappedX, 1) return (worldPoint / worldPoint.w)[SIMD3(0,1,2)] } // This one is adapted from: // http://nicolas.burrus.name/index.php/Research/KinectCalibration func worldPoint(depthMapPixelPoint: SIMD2, depth: Float, cameraIntrinsics: simd_float3x3, viewMatrixInverted: simd_float4x4) -> SIMD3 { let xrw = ((depthMapPixelPoint.x - cameraIntrinsics[2][0]) * depth / cameraIntrinsics[0][0]) let yrw = (depthMapPixelPoint.y - cameraIntrinsics[2][1]) * depth / cameraIntrinsics[1][1] // Y is UP in camera space, vs it being DOWN in image space. let localPoint = simd_float3(xrw, -yrw, -depth) let worldPoint = viewMatrixInverted * simd_float4(localPoint, 1) return simd_float3(worldPoint.x, worldPoint.y, worldPoint.z) } extension CVPixelBuffer { var size: SIMD2 { let width = CVPixelBufferGetWidthOfPlane(self, 0) let height = CVPixelBufferGetHeightOfPlane(self, 0) return .init(x: width, y: height) } }