Upgrade to Pro — share decks privately, control downloads, hide ads and more …

100日間AR表現を実装して見つけた面白い実装を全力解説

satoshi0212
September 17, 2021

 100日間AR表現を実装して見つけた面白い実装を全力解説

satoshi0212

September 17, 2021
Tweet

More Decks by satoshi0212

Other Decks in Programming

Transcript

  1. class DrawnImageView: UIImageView { private var swiped: Bool = false

    private var lastPoint: CGPoint = .zero ... override func touchesMoved(_ touches: Set<UITouch>, with event: UIEvent?) { super.touchesMoved(touches, with: event) guard let touch = touches.first else { return } swiped = true let currentPoint = touch.location(in: self) if lastPoint == .zero { lastPoint = currentPoint } drawLine(from: lastPoint, to: currentPoint) lastPoint = currentPoint } override func touchesEnded(_ touches: Set<UITouch>, with event: UIEvent?) { super.touchesEnded(touches, with: event) lastPoint = .zero } private func drawLine(from fromPoint: CGPoint, to toPoint: CGPoint) { ... } }
  2. class DrawnImageView: UIImageView { private var swiped: Bool = false

    private var lastPoint: CGPoint = .zero ... override func touchesMoved(_ touches: Set<UITouch>, with event: UIEvent?) { super.touchesMoved(touches, with: event) guard let touch = touches.first else { return } swiped = true let currentPoint = touch.location(in: self) if lastPoint == .zero { lastPoint = currentPoint } drawLine(from: lastPoint, to: currentPoint) lastPoint = currentPoint } override func touchesEnded(_ touches: Set<UITouch>, with event: UIEvent?) { super.touchesEnded(touches, with: event) lastPoint = .zero } private func drawLine(from fromPoint: CGPoint, to toPoint: CGPoint) { ... } }
  3. private func drawLine(from fromPoint: CGPoint, to toPoint: CGPoint) { UIGraphicsBeginImageContext(frame.size)

    guard let context = UIGraphicsGetCurrentContext() else { return } image?.draw(in: bounds) context.move(to: fromPoint) context.addLine(to: toPoint) context.setLineCap(.round) context.setBlendMode(.normal) context.setLineWidth(8) context.setStrokeColor(UIColor.black.cgColor) context.strokePath() image = UIGraphicsGetImageFromCurrentImageContext() UIGraphicsEndImageContext() }
  4. func renderer(_ renderer: SCNSceneRenderer, didUpdate node: SCNNode, for anchor: ARAnchor)

    { guard let faceGeometry = node.geometry as? ARSCNFaceGeometry, let faceAnchor = anchor as? ARFaceAnchor else { return } faceGeometry.update(from: faceAnchor.geometry) if let imageView = imageView { DispatchQueue.main.async { let material = faceGeometry.firstMaterial! material.diffuse.contents = imageView.image } } } "34$/7JFX%FMFHBUF
  5. 8*1

  6. override func viewWillAppear(_ animated: Bool) { super.viewWillAppear(animated) let configuration =

    ARWorldTrackingConfiguration() configuration.sceneReconstruction = .mesh sceneView.session.run(configuration) }
  7. func session(_ session: ARSession, didAdd anchors: [ARAnchor]) { for anchor

    in anchors { if let meshAnchor = anchor as? ARMeshAnchor { guard let camera = session.currentFrame?.camera else { return } let meshGeometry = buildGeometry(meshAnchor: meshAnchor, camera: camera) let node = SCNNode(geometry: meshGeometry) node.simdTransform = meshAnchor.transform let shape = SCNPhysicsShape(geometry: node.geometry!, options: [.type: SCNPhysicsShape.ShapeType.concavePolyhedron]) node.physicsBody = SCNPhysicsBody(type: .static, shape: shape) knownAnchors[anchor.identifier] = node sceneView.scene.rootNode.addChildNode(node) } } } "34FTTJPO%FMFHBUF
  8. func session(_ session: ARSession, didAdd anchors: [ARAnchor]) { for anchor

    in anchors { if let meshAnchor = anchor as? ARMeshAnchor { guard let camera = session.currentFrame?.camera else { return } let meshGeometry = buildGeometry(meshAnchor: meshAnchor, camera: camera) let node = SCNNode(geometry: meshGeometry) node.simdTransform = meshAnchor.transform let shape = SCNPhysicsShape(geometry: node.geometry!, options: [.type: SCNPhysicsShape.ShapeType.concavePolyhedron]) node.physicsBody = SCNPhysicsBody(type: .static, shape: shape) knownAnchors[anchor.identifier] = node sceneView.scene.rootNode.addChildNode(node) } } } "34FTTJPO%FMFHBUF
  9. func session(_ session: ARSession, didUpdate anchors: [ARAnchor]) { for anchor

    in anchors { if let node = knownAnchors[anchor.identifier] { if let meshAnchor = anchor as? ARMeshAnchor { guard let camera = session.currentFrame?.camera else { return } let meshGeometry = buildGeometry(meshAnchor: meshAnchor, camera: camera) node.geometry = meshGeometry node.simdTransform = meshAnchor.transform let shape = SCNPhysicsShape(geometry: node.geometry!, options: [.type: SCNPhysicsShape.ShapeType.concavePolyhedron]) node.physicsBody = SCNPhysicsBody(type: .static, shape: shape) knownAnchors[anchor.identifier] = node } } } } "34FTTJPO%FMFHBUF
  10. func buildGeometry(meshAnchor: ARMeshAnchor, camera: ARCamera) -> SCNGeometry { let vertices

    = meshAnchor.geometry.vertices let faces = meshAnchor.geometry.faces let size = camera.imageResolution let vertexSource = SCNGeometrySource(buffer: vertices.buffer, vertexFormat: vertices.format, semantic: .vertex, vertexCount: vertices.count, dataOffset: vertices.offset, dataStride: vertices.stride) let modelMatrix = meshAnchor.transform var textCords = [CGPoint]() for index in 0..<vertices.count { let vertexPointer = vertices.buffer.contents().advanced(by: vertices.offset + vertices.stride * index) let vertex = vertexPointer.assumingMemoryBound(to: (Float, Float, Float).self).pointee let vertex4 = SIMD4<Float>(vertex.0, vertex.1, vertex.2, 1) let world_vertex4 = simd_mul(modelMatrix, vertex4) let world_vector3 = simd_float3(x: world_vertex4.x, y: world_vertex4.y, z: world_vertex4.z) let pt = camera.projectPoint(world_vector3, orientation: .portrait, viewportSize: CGSize(width: CGFloat(size.height), height: CGFloat(size.width))) let v = 1.0 - Float(pt.x) / Float(size.height) let u = Float(pt.y) / Float(size.width) let c = CGPoint(x: CGFloat(v), y: CGFloat(u)) textCords.append(c) } let textureSource = SCNGeometrySource(textureCoordinates: textCords) let normalsSource = SCNGeometrySource(meshAnchor.geometry.normals, semantic: .normal) let faceData = Data(bytes: faces.buffer.contents(), count: faces.buffer.length) let geometryElement = SCNGeometryElement(data: faceData, primitiveType: .triangles, primitiveCount: faces.count, bytesPerIndex: faces.bytesPerIndex) let nodeGeometry = SCNGeometry(sources: [vertexSource, textureSource, normalsSource], elements: [geometryElement]) let material = SCNMaterial() material.isDoubleSided = true let skScene = SKScene(size: CGSize(width: SPRITE_SIZE, height: SPRITE_SIZE)) skScene.backgroundColor = .clear material.diffuse.contents = skScene nodeGeometry.materials = [material] return nodeGeometry }
  11. let vertices = meshAnchor.geometry.vertices let vertexSource = SCNGeometrySource(buffer: vertices.buffer, vertexFormat:

    vertices.format, semantic: .vertex, vertexCount: vertices.count, dataOffset: vertices.offset, dataStride: vertices.stride)
  12. var textCords = [CGPoint]() for index in 0..<vertices.count { let

    vertexPointer = vertices.buffer.contents().advanced(by: vertices.offset + vertices.stride * index) let vertex = vertexPointer.assumingMemoryBound(to: (Float, Float, Float).self).pointee let vertex4 = SIMD4<Float>(vertex.0, vertex.1, vertex.2, 1) let world_vertex4 = simd_mul(modelMatrix, vertex4) let world_vector3 = simd_float3(x: world_vertex4.x, y: world_vertex4.y, z: world_vertex4.z) let pt = camera.projectPoint(world_vector3, orientation: .portrait, viewportSize: CGSize(width: CGFloat(size.height), height: CGFloat(size.width))) let v = 1.0 - Float(pt.x) / Float(size.height) let u = Float(pt.y) / Float(size.width) let c = CGPoint(x: CGFloat(v), y: CGFloat(u)) textCords.append(c) } let textureSource = SCNGeometrySource(textureCoordinates: textCords)
  13. let faceData = Data(bytes: faces.buffer.contents(), count: faces.buffer.length) let geometryElement =

    SCNGeometryElement(data: faceData, primitiveType: .triangles, primitiveCount: faces.count, bytesPerIndex: faces.bytesPerIndex)
  14. let nodeGeometry = SCNGeometry(sources: [vertexSource, textureSource, normalsSource], elements: [geometryElement]) let

    material = SCNMaterial() material.isDoubleSided = true let skScene = SKScene(size: CGSize(width: SPRITE_SIZE, height: SPRITE_SIZE)) skScene.backgroundColor = .clear material.diffuse.contents = skScene nodeGeometry.materials = [material] return nodeGeometry
  15. private func launchColorBall() { let ball = SCNNode() let sphere

    = SCNSphere(radius: 0.08) ball.geometry = sphere let hue = CGFloat(arc4random()) / CGFloat(UInt32.max) ball.geometry!.firstMaterial?.diffuse.contents = SKColor(hue: hue, saturation: 1, brightness: 1, alpha: 1) ball.geometry!.firstMaterial?.fresnelExponent = 1.0 ball.physicsBody = .dynamic() ball.physicsBody!.restitution = 0.9 ball.physicsBody!.categoryBitMask = 0x4 ball.physicsBody!.contactTestBitMask = ~0 ball.physicsBody!.collisionBitMask = ~(0x4) guard let camera = sceneView.pointOfView else { return } let cameraPos = SCNVector3Make(0, 0, -0.5) let position = camera.convertPosition(cameraPos, to: nil) ball.position = position let (direction, _) = getUserVector() ball.physicsBody!.velocity = SCNVector3Make( direction.x * 8.0, direction.y * 8.0, direction.z * 8.0) sceneView.scene.rootNode.addChildNode(ball) }
  16. private func getUserVector() -> (SCNVector3, SCNVector3) { if let frame

    = sceneView.session.currentFrame { let mat = SCNMatrix4(frame.camera.transform) // 4x4 transform matrix describing camera in world space let dir = SCNVector3(-1 * mat.m31, -1 * mat.m32, -1 * mat.m33) // orientation of camera in world space let pos = SCNVector3(mat.m41, mat.m42, mat.m43) // location of camera in world space return (dir, pos) } return (SCNVector3(0, 0, -1), SCNVector3(0, 0, -0.2)) }
  17. func physicsWorld(_ world: SCNPhysicsWorld, didBegin contact: SCNPhysicsContact) { var ball:

    SCNNode? = nil var target: SCNNode? = nil if contact.nodeA.physicsBody!.type == .dynamic { ball = contact.nodeA target = contact.nodeB } else if contact.nodeB.physicsBody!.type == .dynamic { ball = contact.nodeB target = contact.nodeA } if let ball = ball { let pointA = SCNVector3Make(contact.contactPoint.x, contact.contactPoint.y, contact.contactPoint.z + 20) let pointB = SCNVector3Make(contact.contactPoint.x, contact.contactPoint.y, contact.contactPoint.z - 20) let results = sceneView.scene.rootNode.hitTestWithSegment(from: pointA, to: pointB, options: [:]) if !results.isEmpty, let hit = results.first { let color = ball.geometry!.firstMaterial!.diffuse.contents as! SKColor addPaintAtLocation(hit.textureCoordinates(withMappingChannel: 0), color: color, targetNode: target!) } ball.removeFromParentNode() } } 4$/1IZTJDT$POUBDU%FMFHBUF
  18. private func addPaintAtLocation(_ _p: CGPoint, color: SKColor, targetNode: SCNNode) {

    guard let skScene = targetNode.geometry!.firstMaterial!.diffuse.contents as? SKScene else { return } var p = _p p.x *= SPRITE_SIZE p.y *= SPRITE_SIZE let node: SKNode = SKSpriteNode() node.position = p let subNode = SKSpriteNode(imageNamed: "splash.png") subNode.color = color subNode.colorBlendFactor = 1 node.addChild(subNode) skScene.addChild(node) }
  19. 8*1

  20. private func addNode(image: UIImage, position: SCNVector3, rotation: SCNVector4? = nil)

    { let node = SCNNode() let geometry = SCNPlane(width: 0.4, height: 0.4) geometry.firstMaterial?.diffuse.contents = image geometry.firstMaterial?.isDoubleSided = true node.geometry = geometry if let rotation = rotation { node.rotation = rotation } node.position = position sceneView.scene.rootNode.addChildNode(node) } private func addNodes() { addNode(image: UIImage(named: "nazo001")!, position: SCNVector3(0, 0, -1.1)) addNode(image: UIImage(named: "nazo002")!, position: SCNVector3(-0.5, 0.15, -1.5), rotation: SCNVector4(0, 1, 0, 0.5 * Double.pi)) addNode(image: UIImage(named: "nazo003")!, position: SCNVector3(0, 0.1, -1.5), rotation: SCNVector4(0, 1, 0, 0.5 * Double.pi)) }
  21. 8*1

  22. func update() { let commandBuffer = commandQueue.makeCommandBuffer()! ... guard let

    currentFrame = session.currentFrame else { return } let pixelBuffer = currentFrame.capturedImage if CVPixelBufferGetPlaneCount(pixelBuffer) < 2 { return } capturedImageTextureY = createTexture(fromPixelBuffer: pixelBuffer, pixelFormat: .r8Unorm, planeIndex: 0) capturedImageTextureCbCr = createTexture(fromPixelBuffer: pixelBuffer, pixelFormat: .rg8Unorm, planeIndex:1) ... alphaTexture = matteGenerator.generateMatte(from: currentFrame, commandBuffer: commandBuffer) ... }
  23. func update() { ... if let width = alphaTexture?.width, let

    height = alphaTexture?.height { let colorDesc = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: .bgra8Unorm, width: width, height: height, mipmapped: false) colorDesc.usage = [.shaderRead, .shaderWrite] whiteBlurTexture = device.makeTexture(descriptor: colorDesc) outerBlurTexture = device.makeTexture(descriptor: colorDesc) let threadCountW = (width + threadgroupSize.width - 1) / threadgroupSize.width let threadCountH = (height + threadgroupSize.height - 1) / threadgroupSize.height let threadgroupCount = MTLSizeMake(threadCountW, threadCountH, 1) let computeEncoder = commandBuffer.makeComputeCommandEncoder()! computeEncoder.setComputePipelineState(computeState) computeEncoder.setTexture(alphaTexture, index: 0) computeEncoder.setTexture(whiteBlurTexture, index: 1) computeEncoder.setTexture(outerBlurTexture, index: 2) computeEncoder.dispatchThreadgroups(threadgroupCount, threadsPerThreadgroup: threadgroupSize) computeEncoder.endEncoding() } time += 1 let whiteIntensity = Int(6) | 0x01 let kernel1 = MPSImageTent(device: device, kernelWidth: whiteIntensity, kernelHeight: whiteIntensity) kernel1.encode(commandBuffer: commandBuffer, inPlaceTexture: &whiteBlurTexture!, fallbackCopyAllocator: nil) let outerIntensity = Int((sin(Float(time)/10) + 2) * 40) | 0x01 let kernel2 = MPSImageTent(device: device, kernelWidth: outerIntensity, kernelHeight: outerIntensity) kernel2.encode(commandBuffer: commandBuffer, inPlaceTexture: &outerBlurTexture!, fallbackCopyAllocator: nil) ... }
  24. kernel void matteConvert(texture2d<half, access::read> inTexture [[ texture(0) ]], texture2d<half, access::write>

    outWhiteTexture [[ texture(1) ]], texture2d<half, access::write> outOuterTexture [[ texture(2) ]], uint2 gid [[thread_position_in_grid]]) { uint2 textureIndex(gid); if (inTexture.read(textureIndex).r > 0.1) { outWhiteTexture.write(half4(0.0), gid); outOuterTexture.write(half4(0.0), gid); return; } constexpr int scale = 15; constexpr int radius = scale / 2; half color = 0.0; for (int i = 0; i < scale; i++) { for (int j = 0; j < scale; j++) { uint2 textureIndex(gid.x + (i - radius), gid.y + (j - radius)); half alpha = inTexture.read(textureIndex).r; if (alpha > 0.1) { color = 1.0; break; } } if (color > 0.0) { break; } } outWhiteTexture.write(half4(color, color, color, 1.0), gid); outOuterTexture.write(half4(color, 0.0, color, 1.0), gid); } 4IBEFSNFUBM
  25. func update() { ... guard let renderPassDescriptor = mtkView.currentRenderPassDescriptor, let

    currentDrawable = mtkView.currentDrawable else { return } let compositeRenderEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: renderPassDescriptor)! compositeImagesWithEncoder(renderEncoder: compositeRenderEncoder) compositeRenderEncoder.endEncoding() commandBuffer.present(currentDrawable) commandBuffer.commit() }
  26. fragment half4 compositeImageFragmentShader(CompositeColorInOut in [[ stage_in ]], texture2d<float, access::sample> capturedImageTextureY

    [[ texture(0) ]], texture2d<float, access::sample> capturedImageTextureCbCr [[ texture(1) ]], texture2d<float, access::sample> whiteColorTexture [[ texture(2) ]], texture2d<float, access::sample> outerColorTexture [[ texture(3) ]], texture2d<float, access::sample> alphaTexture [[ texture(4) ]]) { constexpr sampler s(address::clamp_to_edge, filter::linear); float2 cameraTexCoord = in.texCoordCamera; float4 rgb = ycbcrToRGBTransform(capturedImageTextureY.sample(s, cameraTexCoord), capturedImageTextureCbCr.sample(s, cameraTexCoord)); half4 cameraColor = half4(rgb); half4 whiteColor = half4(whiteColorTexture.sample(s, cameraTexCoord)); half4 outerColor = half4(outerColorTexture.sample(s, cameraTexCoord)) * 2.0; return cameraColor + whiteColor + outerColor; } 4IBEFSNFUBM
  27. var m_data: [UInt8] = [UInt8](repeating: 0, count: 375*3 * 812*3)

    var heatMapNode: SCNNode = { let node = SCNNode(geometry: SCNPlane(width: 2, height: 2)) let program = SCNProgram() program.vertexFunctionName = "heatMapVert" program.fragmentFunctionName = "heatMapFrag" node.geometry?.firstMaterial?.program = program node.geometry?.firstMaterial?.blendMode = SCNBlendMode.add return node }()
  28. func renderer(_ renderer: SCNSceneRenderer, updateAtTime time: TimeInterval) { virtualPhoneNode.transform =

    (sceneView.pointOfView?.transform)! let options: [String: Any] = [SCNHitTestOption.backFaceCulling.rawValue: false, SCNHitTestOption.searchMode.rawValue: 1, SCNHitTestOption.ignoreChildNodes.rawValue : false, SCNHitTestOption.ignoreHiddenNodes.rawValue : false] let hitTestLeftEye = virtualPhoneNode.hitTestWithSegment( from: virtualPhoneNode.convertPosition(eyeRaycastData!.leftEye.worldPosition, from: nil), to: virtualPhoneNode.convertPosition(eyeRaycastData!.leftEyeEnd.worldPosition, from: nil), options: options) let hitTestRightEye = virtualPhoneNode.hitTestWithSegment( from: virtualPhoneNode.convertPosition(eyeRaycastData!.rightEye.worldPosition, from: nil), to: virtualPhoneNode.convertPosition(eyeRaycastData!.rightEyeEnd.worldPosition, from: nil), options: options) if (hitTestLeftEye.count > 0 && hitTestRightEye.count > 0) { let coords = screenPositionFromHittest(hitTestLeftEye[0], secondResult: hitTestRightEye[0]) incrementHeatMapAtPosition(x:Int(coords.x * 3), y:Int(coords.y * 3)) let data = NSData(bytes: &m_data, length: phoneWidth * phoneHeight) heatMapNode.geometry?.firstMaterial?.setValue(data, forKey: "heatmapTexture") DispatchQueue.main.async(execute: {() -> Void in self.target.center = CGPoint(x: CGFloat(coords.x), y:CGFloat(coords.y)) }) } } "34$/7JFX%FMFHBUF
  29. private func incrementHeatMapAtPosition(x: Int, y: Int) { let radius: Int

    = 46 // in pixels let maxIncrement: Float = 25 for curX in x - radius ... x + radius { for curY in y - radius ... y + radius { let idx = posToIndex(x: curX, y: curY) if (idx == -1) { continue } let offset = simd_float2(Float(curX - x), Float(curY - y)) let len = simd_length(offset) if (len >= Float(radius)) { continue } let incrementValue = Int((1 - (len / Float(radius))) * maxIncrement) if (255 - m_data[idx] > incrementValue) { m_data[idx] = UInt8(Int(m_data[idx]) + incrementValue) } else { m_data[idx] = 255 } } } } private func posToIndex(x: Int, y: Int) -> Int { if (x < 0 || x >= phoneWidth || y < 0 || y >= phoneHeight) { return -1 } return x + y * phoneWidth }
  30. fragment half4 heatMapFrag(SimpleVertex in [[stage_in]], device r8unorm<float> *heatmapTexture [[buffer(0)]]) {

    int x = round(in.uv.x * 375 * 3); int y = round(in.uv.y * 812 * 3); int index = x + y * 375 * 3; float value = heatmapTexture[index]; return ColorForHeat(value); } 4IBEFSNFUBM
  31. half4 ColorForHeat(float heat) { float pval = 4.0 * (1.0

    - heat) + 0.99; float lb = pval - floor (pval); int pvalCategory = abs(int(floor(pval))); switch (pvalCategory) { case 0: return half4(1.0, 1.0 - lb, 1.0 - lb, heat); case 1: return half4(1.0, lb, 0.0, heat); case 2: return half4(1.0 - lb, 1.0, 0.0, heat); case 3: return half4(0.0, 1.0, lb, heat); case 4: return half4(0.0, 1.0 - lb, 1.0, heat); case 5: return half4(0.0, 0.0, 1.0 - lb, heat); } return half4(1.0, 1.0, 1.0, heat); } 4IBEFSNFUBM
  32. 8*1