Upgrade to Pro — share decks privately, control downloads, hide ads and more …

100日間AR表現を実装して見つけた面白い実装を全力解説

Sponsored · Your Podcast. Everywhere. Effortlessly. Share. Educate. Inspire. Entertain. You do you. We'll handle the rest.
Avatar for satoshi0212 satoshi0212
September 17, 2021

 100日間AR表現を実装して見つけた面白い実装を全力解説

Avatar for satoshi0212

satoshi0212

September 17, 2021
Tweet

More Decks by satoshi0212

Other Decks in Programming

Transcript

  1. class DrawnImageView: UIImageView { private var swiped: Bool = false

    private var lastPoint: CGPoint = .zero ... override func touchesMoved(_ touches: Set<UITouch>, with event: UIEvent?) { super.touchesMoved(touches, with: event) guard let touch = touches.first else { return } swiped = true let currentPoint = touch.location(in: self) if lastPoint == .zero { lastPoint = currentPoint } drawLine(from: lastPoint, to: currentPoint) lastPoint = currentPoint } override func touchesEnded(_ touches: Set<UITouch>, with event: UIEvent?) { super.touchesEnded(touches, with: event) lastPoint = .zero } private func drawLine(from fromPoint: CGPoint, to toPoint: CGPoint) { ... } }
  2. class DrawnImageView: UIImageView { private var swiped: Bool = false

    private var lastPoint: CGPoint = .zero ... override func touchesMoved(_ touches: Set<UITouch>, with event: UIEvent?) { super.touchesMoved(touches, with: event) guard let touch = touches.first else { return } swiped = true let currentPoint = touch.location(in: self) if lastPoint == .zero { lastPoint = currentPoint } drawLine(from: lastPoint, to: currentPoint) lastPoint = currentPoint } override func touchesEnded(_ touches: Set<UITouch>, with event: UIEvent?) { super.touchesEnded(touches, with: event) lastPoint = .zero } private func drawLine(from fromPoint: CGPoint, to toPoint: CGPoint) { ... } }
  3. private func drawLine(from fromPoint: CGPoint, to toPoint: CGPoint) { UIGraphicsBeginImageContext(frame.size)

    guard let context = UIGraphicsGetCurrentContext() else { return } image?.draw(in: bounds) context.move(to: fromPoint) context.addLine(to: toPoint) context.setLineCap(.round) context.setBlendMode(.normal) context.setLineWidth(8) context.setStrokeColor(UIColor.black.cgColor) context.strokePath() image = UIGraphicsGetImageFromCurrentImageContext() UIGraphicsEndImageContext() }
  4. func renderer(_ renderer: SCNSceneRenderer, didUpdate node: SCNNode, for anchor: ARAnchor)

    { guard let faceGeometry = node.geometry as? ARSCNFaceGeometry, let faceAnchor = anchor as? ARFaceAnchor else { return } faceGeometry.update(from: faceAnchor.geometry) if let imageView = imageView { DispatchQueue.main.async { let material = faceGeometry.firstMaterial! material.diffuse.contents = imageView.image } } } "34$/7JFX%FMFHBUF
  5. 8*1

  6. override func viewWillAppear(_ animated: Bool) { super.viewWillAppear(animated) let configuration =

    ARWorldTrackingConfiguration() configuration.sceneReconstruction = .mesh sceneView.session.run(configuration) }
  7. func session(_ session: ARSession, didAdd anchors: [ARAnchor]) { for anchor

    in anchors { if let meshAnchor = anchor as? ARMeshAnchor { guard let camera = session.currentFrame?.camera else { return } let meshGeometry = buildGeometry(meshAnchor: meshAnchor, camera: camera) let node = SCNNode(geometry: meshGeometry) node.simdTransform = meshAnchor.transform let shape = SCNPhysicsShape(geometry: node.geometry!, options: [.type: SCNPhysicsShape.ShapeType.concavePolyhedron]) node.physicsBody = SCNPhysicsBody(type: .static, shape: shape) knownAnchors[anchor.identifier] = node sceneView.scene.rootNode.addChildNode(node) } } } "34FTTJPO%FMFHBUF
  8. func session(_ session: ARSession, didAdd anchors: [ARAnchor]) { for anchor

    in anchors { if let meshAnchor = anchor as? ARMeshAnchor { guard let camera = session.currentFrame?.camera else { return } let meshGeometry = buildGeometry(meshAnchor: meshAnchor, camera: camera) let node = SCNNode(geometry: meshGeometry) node.simdTransform = meshAnchor.transform let shape = SCNPhysicsShape(geometry: node.geometry!, options: [.type: SCNPhysicsShape.ShapeType.concavePolyhedron]) node.physicsBody = SCNPhysicsBody(type: .static, shape: shape) knownAnchors[anchor.identifier] = node sceneView.scene.rootNode.addChildNode(node) } } } "34FTTJPO%FMFHBUF
  9. func session(_ session: ARSession, didUpdate anchors: [ARAnchor]) { for anchor

    in anchors { if let node = knownAnchors[anchor.identifier] { if let meshAnchor = anchor as? ARMeshAnchor { guard let camera = session.currentFrame?.camera else { return } let meshGeometry = buildGeometry(meshAnchor: meshAnchor, camera: camera) node.geometry = meshGeometry node.simdTransform = meshAnchor.transform let shape = SCNPhysicsShape(geometry: node.geometry!, options: [.type: SCNPhysicsShape.ShapeType.concavePolyhedron]) node.physicsBody = SCNPhysicsBody(type: .static, shape: shape) knownAnchors[anchor.identifier] = node } } } } "34FTTJPO%FMFHBUF
  10. func buildGeometry(meshAnchor: ARMeshAnchor, camera: ARCamera) -> SCNGeometry { let vertices

    = meshAnchor.geometry.vertices let faces = meshAnchor.geometry.faces let size = camera.imageResolution let vertexSource = SCNGeometrySource(buffer: vertices.buffer, vertexFormat: vertices.format, semantic: .vertex, vertexCount: vertices.count, dataOffset: vertices.offset, dataStride: vertices.stride) let modelMatrix = meshAnchor.transform var textCords = [CGPoint]() for index in 0..<vertices.count { let vertexPointer = vertices.buffer.contents().advanced(by: vertices.offset + vertices.stride * index) let vertex = vertexPointer.assumingMemoryBound(to: (Float, Float, Float).self).pointee let vertex4 = SIMD4<Float>(vertex.0, vertex.1, vertex.2, 1) let world_vertex4 = simd_mul(modelMatrix, vertex4) let world_vector3 = simd_float3(x: world_vertex4.x, y: world_vertex4.y, z: world_vertex4.z) let pt = camera.projectPoint(world_vector3, orientation: .portrait, viewportSize: CGSize(width: CGFloat(size.height), height: CGFloat(size.width))) let v = 1.0 - Float(pt.x) / Float(size.height) let u = Float(pt.y) / Float(size.width) let c = CGPoint(x: CGFloat(v), y: CGFloat(u)) textCords.append(c) } let textureSource = SCNGeometrySource(textureCoordinates: textCords) let normalsSource = SCNGeometrySource(meshAnchor.geometry.normals, semantic: .normal) let faceData = Data(bytes: faces.buffer.contents(), count: faces.buffer.length) let geometryElement = SCNGeometryElement(data: faceData, primitiveType: .triangles, primitiveCount: faces.count, bytesPerIndex: faces.bytesPerIndex) let nodeGeometry = SCNGeometry(sources: [vertexSource, textureSource, normalsSource], elements: [geometryElement]) let material = SCNMaterial() material.isDoubleSided = true let skScene = SKScene(size: CGSize(width: SPRITE_SIZE, height: SPRITE_SIZE)) skScene.backgroundColor = .clear material.diffuse.contents = skScene nodeGeometry.materials = [material] return nodeGeometry }
  11. let vertices = meshAnchor.geometry.vertices let vertexSource = SCNGeometrySource(buffer: vertices.buffer, vertexFormat:

    vertices.format, semantic: .vertex, vertexCount: vertices.count, dataOffset: vertices.offset, dataStride: vertices.stride)
  12. var textCords = [CGPoint]() for index in 0..<vertices.count { let

    vertexPointer = vertices.buffer.contents().advanced(by: vertices.offset + vertices.stride * index) let vertex = vertexPointer.assumingMemoryBound(to: (Float, Float, Float).self).pointee let vertex4 = SIMD4<Float>(vertex.0, vertex.1, vertex.2, 1) let world_vertex4 = simd_mul(modelMatrix, vertex4) let world_vector3 = simd_float3(x: world_vertex4.x, y: world_vertex4.y, z: world_vertex4.z) let pt = camera.projectPoint(world_vector3, orientation: .portrait, viewportSize: CGSize(width: CGFloat(size.height), height: CGFloat(size.width))) let v = 1.0 - Float(pt.x) / Float(size.height) let u = Float(pt.y) / Float(size.width) let c = CGPoint(x: CGFloat(v), y: CGFloat(u)) textCords.append(c) } let textureSource = SCNGeometrySource(textureCoordinates: textCords)
  13. let faceData = Data(bytes: faces.buffer.contents(), count: faces.buffer.length) let geometryElement =

    SCNGeometryElement(data: faceData, primitiveType: .triangles, primitiveCount: faces.count, bytesPerIndex: faces.bytesPerIndex)
  14. let nodeGeometry = SCNGeometry(sources: [vertexSource, textureSource, normalsSource], elements: [geometryElement]) let

    material = SCNMaterial() material.isDoubleSided = true let skScene = SKScene(size: CGSize(width: SPRITE_SIZE, height: SPRITE_SIZE)) skScene.backgroundColor = .clear material.diffuse.contents = skScene nodeGeometry.materials = [material] return nodeGeometry
  15. private func launchColorBall() { let ball = SCNNode() let sphere

    = SCNSphere(radius: 0.08) ball.geometry = sphere let hue = CGFloat(arc4random()) / CGFloat(UInt32.max) ball.geometry!.firstMaterial?.diffuse.contents = SKColor(hue: hue, saturation: 1, brightness: 1, alpha: 1) ball.geometry!.firstMaterial?.fresnelExponent = 1.0 ball.physicsBody = .dynamic() ball.physicsBody!.restitution = 0.9 ball.physicsBody!.categoryBitMask = 0x4 ball.physicsBody!.contactTestBitMask = ~0 ball.physicsBody!.collisionBitMask = ~(0x4) guard let camera = sceneView.pointOfView else { return } let cameraPos = SCNVector3Make(0, 0, -0.5) let position = camera.convertPosition(cameraPos, to: nil) ball.position = position let (direction, _) = getUserVector() ball.physicsBody!.velocity = SCNVector3Make( direction.x * 8.0, direction.y * 8.0, direction.z * 8.0) sceneView.scene.rootNode.addChildNode(ball) }
  16. private func getUserVector() -> (SCNVector3, SCNVector3) { if let frame

    = sceneView.session.currentFrame { let mat = SCNMatrix4(frame.camera.transform) // 4x4 transform matrix describing camera in world space let dir = SCNVector3(-1 * mat.m31, -1 * mat.m32, -1 * mat.m33) // orientation of camera in world space let pos = SCNVector3(mat.m41, mat.m42, mat.m43) // location of camera in world space return (dir, pos) } return (SCNVector3(0, 0, -1), SCNVector3(0, 0, -0.2)) }
  17. func physicsWorld(_ world: SCNPhysicsWorld, didBegin contact: SCNPhysicsContact) { var ball:

    SCNNode? = nil var target: SCNNode? = nil if contact.nodeA.physicsBody!.type == .dynamic { ball = contact.nodeA target = contact.nodeB } else if contact.nodeB.physicsBody!.type == .dynamic { ball = contact.nodeB target = contact.nodeA } if let ball = ball { let pointA = SCNVector3Make(contact.contactPoint.x, contact.contactPoint.y, contact.contactPoint.z + 20) let pointB = SCNVector3Make(contact.contactPoint.x, contact.contactPoint.y, contact.contactPoint.z - 20) let results = sceneView.scene.rootNode.hitTestWithSegment(from: pointA, to: pointB, options: [:]) if !results.isEmpty, let hit = results.first { let color = ball.geometry!.firstMaterial!.diffuse.contents as! SKColor addPaintAtLocation(hit.textureCoordinates(withMappingChannel: 0), color: color, targetNode: target!) } ball.removeFromParentNode() } } 4$/1IZTJDT$POUBDU%FMFHBUF
  18. private func addPaintAtLocation(_ _p: CGPoint, color: SKColor, targetNode: SCNNode) {

    guard let skScene = targetNode.geometry!.firstMaterial!.diffuse.contents as? SKScene else { return } var p = _p p.x *= SPRITE_SIZE p.y *= SPRITE_SIZE let node: SKNode = SKSpriteNode() node.position = p let subNode = SKSpriteNode(imageNamed: "splash.png") subNode.color = color subNode.colorBlendFactor = 1 node.addChild(subNode) skScene.addChild(node) }
  19. 8*1

  20. private func addNode(image: UIImage, position: SCNVector3, rotation: SCNVector4? = nil)

    { let node = SCNNode() let geometry = SCNPlane(width: 0.4, height: 0.4) geometry.firstMaterial?.diffuse.contents = image geometry.firstMaterial?.isDoubleSided = true node.geometry = geometry if let rotation = rotation { node.rotation = rotation } node.position = position sceneView.scene.rootNode.addChildNode(node) } private func addNodes() { addNode(image: UIImage(named: "nazo001")!, position: SCNVector3(0, 0, -1.1)) addNode(image: UIImage(named: "nazo002")!, position: SCNVector3(-0.5, 0.15, -1.5), rotation: SCNVector4(0, 1, 0, 0.5 * Double.pi)) addNode(image: UIImage(named: "nazo003")!, position: SCNVector3(0, 0.1, -1.5), rotation: SCNVector4(0, 1, 0, 0.5 * Double.pi)) }
  21. 8*1

  22. func update() { let commandBuffer = commandQueue.makeCommandBuffer()! ... guard let

    currentFrame = session.currentFrame else { return } let pixelBuffer = currentFrame.capturedImage if CVPixelBufferGetPlaneCount(pixelBuffer) < 2 { return } capturedImageTextureY = createTexture(fromPixelBuffer: pixelBuffer, pixelFormat: .r8Unorm, planeIndex: 0) capturedImageTextureCbCr = createTexture(fromPixelBuffer: pixelBuffer, pixelFormat: .rg8Unorm, planeIndex:1) ... alphaTexture = matteGenerator.generateMatte(from: currentFrame, commandBuffer: commandBuffer) ... }
  23. func update() { ... if let width = alphaTexture?.width, let

    height = alphaTexture?.height { let colorDesc = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: .bgra8Unorm, width: width, height: height, mipmapped: false) colorDesc.usage = [.shaderRead, .shaderWrite] whiteBlurTexture = device.makeTexture(descriptor: colorDesc) outerBlurTexture = device.makeTexture(descriptor: colorDesc) let threadCountW = (width + threadgroupSize.width - 1) / threadgroupSize.width let threadCountH = (height + threadgroupSize.height - 1) / threadgroupSize.height let threadgroupCount = MTLSizeMake(threadCountW, threadCountH, 1) let computeEncoder = commandBuffer.makeComputeCommandEncoder()! computeEncoder.setComputePipelineState(computeState) computeEncoder.setTexture(alphaTexture, index: 0) computeEncoder.setTexture(whiteBlurTexture, index: 1) computeEncoder.setTexture(outerBlurTexture, index: 2) computeEncoder.dispatchThreadgroups(threadgroupCount, threadsPerThreadgroup: threadgroupSize) computeEncoder.endEncoding() } time += 1 let whiteIntensity = Int(6) | 0x01 let kernel1 = MPSImageTent(device: device, kernelWidth: whiteIntensity, kernelHeight: whiteIntensity) kernel1.encode(commandBuffer: commandBuffer, inPlaceTexture: &whiteBlurTexture!, fallbackCopyAllocator: nil) let outerIntensity = Int((sin(Float(time)/10) + 2) * 40) | 0x01 let kernel2 = MPSImageTent(device: device, kernelWidth: outerIntensity, kernelHeight: outerIntensity) kernel2.encode(commandBuffer: commandBuffer, inPlaceTexture: &outerBlurTexture!, fallbackCopyAllocator: nil) ... }
  24. kernel void matteConvert(texture2d<half, access::read> inTexture [[ texture(0) ]], texture2d<half, access::write>

    outWhiteTexture [[ texture(1) ]], texture2d<half, access::write> outOuterTexture [[ texture(2) ]], uint2 gid [[thread_position_in_grid]]) { uint2 textureIndex(gid); if (inTexture.read(textureIndex).r > 0.1) { outWhiteTexture.write(half4(0.0), gid); outOuterTexture.write(half4(0.0), gid); return; } constexpr int scale = 15; constexpr int radius = scale / 2; half color = 0.0; for (int i = 0; i < scale; i++) { for (int j = 0; j < scale; j++) { uint2 textureIndex(gid.x + (i - radius), gid.y + (j - radius)); half alpha = inTexture.read(textureIndex).r; if (alpha > 0.1) { color = 1.0; break; } } if (color > 0.0) { break; } } outWhiteTexture.write(half4(color, color, color, 1.0), gid); outOuterTexture.write(half4(color, 0.0, color, 1.0), gid); } 4IBEFSNFUBM
  25. func update() { ... guard let renderPassDescriptor = mtkView.currentRenderPassDescriptor, let

    currentDrawable = mtkView.currentDrawable else { return } let compositeRenderEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: renderPassDescriptor)! compositeImagesWithEncoder(renderEncoder: compositeRenderEncoder) compositeRenderEncoder.endEncoding() commandBuffer.present(currentDrawable) commandBuffer.commit() }
  26. fragment half4 compositeImageFragmentShader(CompositeColorInOut in [[ stage_in ]], texture2d<float, access::sample> capturedImageTextureY

    [[ texture(0) ]], texture2d<float, access::sample> capturedImageTextureCbCr [[ texture(1) ]], texture2d<float, access::sample> whiteColorTexture [[ texture(2) ]], texture2d<float, access::sample> outerColorTexture [[ texture(3) ]], texture2d<float, access::sample> alphaTexture [[ texture(4) ]]) { constexpr sampler s(address::clamp_to_edge, filter::linear); float2 cameraTexCoord = in.texCoordCamera; float4 rgb = ycbcrToRGBTransform(capturedImageTextureY.sample(s, cameraTexCoord), capturedImageTextureCbCr.sample(s, cameraTexCoord)); half4 cameraColor = half4(rgb); half4 whiteColor = half4(whiteColorTexture.sample(s, cameraTexCoord)); half4 outerColor = half4(outerColorTexture.sample(s, cameraTexCoord)) * 2.0; return cameraColor + whiteColor + outerColor; } 4IBEFSNFUBM
  27. var m_data: [UInt8] = [UInt8](repeating: 0, count: 375*3 * 812*3)

    var heatMapNode: SCNNode = { let node = SCNNode(geometry: SCNPlane(width: 2, height: 2)) let program = SCNProgram() program.vertexFunctionName = "heatMapVert" program.fragmentFunctionName = "heatMapFrag" node.geometry?.firstMaterial?.program = program node.geometry?.firstMaterial?.blendMode = SCNBlendMode.add return node }()
  28. func renderer(_ renderer: SCNSceneRenderer, updateAtTime time: TimeInterval) { virtualPhoneNode.transform =

    (sceneView.pointOfView?.transform)! let options: [String: Any] = [SCNHitTestOption.backFaceCulling.rawValue: false, SCNHitTestOption.searchMode.rawValue: 1, SCNHitTestOption.ignoreChildNodes.rawValue : false, SCNHitTestOption.ignoreHiddenNodes.rawValue : false] let hitTestLeftEye = virtualPhoneNode.hitTestWithSegment( from: virtualPhoneNode.convertPosition(eyeRaycastData!.leftEye.worldPosition, from: nil), to: virtualPhoneNode.convertPosition(eyeRaycastData!.leftEyeEnd.worldPosition, from: nil), options: options) let hitTestRightEye = virtualPhoneNode.hitTestWithSegment( from: virtualPhoneNode.convertPosition(eyeRaycastData!.rightEye.worldPosition, from: nil), to: virtualPhoneNode.convertPosition(eyeRaycastData!.rightEyeEnd.worldPosition, from: nil), options: options) if (hitTestLeftEye.count > 0 && hitTestRightEye.count > 0) { let coords = screenPositionFromHittest(hitTestLeftEye[0], secondResult: hitTestRightEye[0]) incrementHeatMapAtPosition(x:Int(coords.x * 3), y:Int(coords.y * 3)) let data = NSData(bytes: &m_data, length: phoneWidth * phoneHeight) heatMapNode.geometry?.firstMaterial?.setValue(data, forKey: "heatmapTexture") DispatchQueue.main.async(execute: {() -> Void in self.target.center = CGPoint(x: CGFloat(coords.x), y:CGFloat(coords.y)) }) } } "34$/7JFX%FMFHBUF
  29. private func incrementHeatMapAtPosition(x: Int, y: Int) { let radius: Int

    = 46 // in pixels let maxIncrement: Float = 25 for curX in x - radius ... x + radius { for curY in y - radius ... y + radius { let idx = posToIndex(x: curX, y: curY) if (idx == -1) { continue } let offset = simd_float2(Float(curX - x), Float(curY - y)) let len = simd_length(offset) if (len >= Float(radius)) { continue } let incrementValue = Int((1 - (len / Float(radius))) * maxIncrement) if (255 - m_data[idx] > incrementValue) { m_data[idx] = UInt8(Int(m_data[idx]) + incrementValue) } else { m_data[idx] = 255 } } } } private func posToIndex(x: Int, y: Int) -> Int { if (x < 0 || x >= phoneWidth || y < 0 || y >= phoneHeight) { return -1 } return x + y * phoneWidth }
  30. fragment half4 heatMapFrag(SimpleVertex in [[stage_in]], device r8unorm<float> *heatmapTexture [[buffer(0)]]) {

    int x = round(in.uv.x * 375 * 3); int y = round(in.uv.y * 812 * 3); int index = x + y * 375 * 3; float value = heatmapTexture[index]; return ColorForHeat(value); } 4IBEFSNFUBM
  31. half4 ColorForHeat(float heat) { float pval = 4.0 * (1.0

    - heat) + 0.99; float lb = pval - floor (pval); int pvalCategory = abs(int(floor(pval))); switch (pvalCategory) { case 0: return half4(1.0, 1.0 - lb, 1.0 - lb, heat); case 1: return half4(1.0, lb, 0.0, heat); case 2: return half4(1.0 - lb, 1.0, 0.0, heat); case 3: return half4(0.0, 1.0, lb, heat); case 4: return half4(0.0, 1.0 - lb, 1.0, heat); case 5: return half4(0.0, 0.0, 1.0 - lb, heat); } return half4(1.0, 1.0, 1.0, heat); } 4IBEFSNFUBM
  32. 8*1