仮想カメラで切り開く拡張現実の世界

Transcript

1. Ծ૝ΧϝϥͰ੾Γ։֦͘ுݱ࣮ͷੈք 93,BJHJ
     ෰෦
   ஐ
   5XJUUFS
   !TINEFWFMPQ
2. Ծ૝Χϝϥʹڵຯ͕͋ΔΤϯδχΞʹಧ͚
   େܕελδΦͰ͸ͳ͘σεΫτοϓ΍εϚʔτϑΥϯ ૝ఆλʔήοτ
3. όʔνϟϧΧϝϥ࡞ΕΔʂָͦ͠͏ʂ
   ͱ͍͏஍ฏΛ։͍ͯ΄͍͠ʂ
4. ࣮૷ํ๏
   ݱ࣮֦ுͱͯ͠ͷར༻
5. Ծ૝Χϝϥ 4OBQ
   $BNFSB NNINN
13. ࣮૷ํ๏
    ݱ࣮֦ுͱͯ͠ͷར༻
14. 5XJUUFSͰ
    (8όʔνϟϧΧϝϥ࡞੒νϟϨϯδ
    Ͱ࡞੒աఔπΠʔτ͍ͯ͠·͢
15. ࡞ͬͨࡍͷۤ࿑ΛৼΓฦΓ
16. ֎෦ϥΠϒϥϦͳ͠ɺ4XJGUͷΈͰ࣮૷Մೳ
17. Ծ૝Χϝϥ࣮૷αϯϓϧ
    IUUQTHJUIVCDPNTBUPTIJ7JSUVBM$BNFSB4BNQMF
18. ߏ੒ཁૉ
    
19. ߏ੒ཁૉ
    ɹ$PSF.FEJB*0
    %"-
    1MVHJO
    ɹίϯτϩʔϥΞϓϦ
    
20. $PSF.FEJB*0
    %"-
    1MVHJO ίϯτϩʔϥΞϓϦ /41BTUFCPBSE -JCSBSZ$PSF.FEJB*01MVH*OT%"- Ծ૝Χϝϥͷ࣮ମ 6*Λ࣋ͨͳ͍1MVHJOʹ
    ஋Λ౉͢ΞϓϦ
21. ϓϩδΣΫτ࡞੒
    QMJTU৘ใ௥Ճ
    ΤϯτϦʔϙΠϯτ࣮૷
    ΠϯλʔϑΣΠε࣮૷
    Χϝϥө૾ͷऔಘͱग़ྗ $PSF.FEJB*0
    %"-
    1MVHJO
22. ϓϩδΣΫτ࡞੒
    QMJTU৘ใ௥Ճ
    ΤϯτϦʔϙΠϯτ࣮૷
    ΠϯλʔϑΣΠε࣮૷
    Χϝϥө૾ͷऔಘͱग़ྗ $PSF.FEJB*0
    %"-
    1MVHJO
25. ϓϩδΣΫτ࡞੒
    QMJTU৘ใ௥Ճ
    ΤϯτϦʔϙΠϯτ࣮૷
    ΠϯλʔϑΣΠε࣮૷
    Χϝϥө૾Λग़ྗઃఆ $PSF.FEJB*0
    %"-
    1MVHJO
26. 1MVHJO
    GBDUPSZ
    JOUFSGBDFT
    1MVHJO
    UZQFT
    Λ௥Ճ
28. ࣗ෼Ͱࢦఆ͢Δ*%஋ɻ66*%ͳͲઃఆ
29. ΤϯτϦʔϙΠϯτ
    GVODUJPO໊
30. Ծ૝Χϝϥ1MVH*O
    ݻఆ஋ ઌड़ͷ66*%஋
31. ϓϩδΣΫτ࡞੒
    QMJTU৘ใ௥Ճ
    ΤϯτϦʔϙΠϯτ࣮૷
    ΠϯλʔϑΣΠε࣮૷
    Χϝϥө૾Λग़ྗઃఆ $PSF.FEJB*0
    %"-
    1MVHJO
32. import Foundation import CoreMediaIO @_cdecl("VirtualCameraSampleMain") func VirtualCameraSampleMain(allocator: CFAllocator, requestedTypeUUID: CFUUID) -> CMIOHardwarePlugInRef { return pluginRef } Main.swift
33. import Foundation import CoreMediaIO @_cdecl("VirtualCameraSampleMain") func VirtualCameraSampleMain(allocator: CFAllocator, requestedTypeUUID: CFUUID) -> CMIOHardwarePlugInRef { return pluginRef } Main.swift ΤϯτϦʔϙΠϯτ
    ΠϯλʔϑΣΠεͷࢀরΛฦ٫
34. ϓϩδΣΫτ࡞੒
    QMJTU৘ใ௥Ճ
    ΤϯτϦʔϙΠϯτ࣮૷
    ΠϯλʔϑΣΠε࣮૷
    Χϝϥө૾Λग़ྗઃఆ $PSF.FEJB*0
    %"-
    1MVHJO
35. private func createPluginInterface() -> CMIOHardwarePlugInInterface { return CMIOHardwarePlugInInterface( _reserved: nil, QueryInterface: QueryInterface, AddRef: AddRef, Release: Release, Initialize: Initialize, InitializeWithObjectID: InitializeWithObjectID, Teardown: Teardown, ObjectShow: ObjectShow, ObjectHasProperty: ObjectHasProperty, ObjectIsPropertySettable: ObjectIsPropertySettable, ObjectGetPropertyDataSize: ObjectGetPropertyDataSize, ObjectGetPropertyData: ObjectGetPropertyData, ObjectSetPropertyData: ObjectSetPropertyData, DeviceSuspend: DeviceSuspend, DeviceResume: DeviceResume, DeviceStartStream: DeviceStartStream, DeviceStopStream: DeviceStopStream, DeviceProcessAVCCommand: DeviceProcessAVCCommand, DeviceProcessRS422Command: DeviceProcessRS422Command, StreamCopyBufferQueue: StreamCopyBufferQueue, StreamDeckPlay: StreamDeckPlay, StreamDeckStop: StreamDeckStop, StreamDeckJog: StreamDeckJog, StreamDeckCueTo: StreamDeckCueTo) } PluginInterface.swift (ൈਮ)
36. private func createPluginInterface() -> CMIOHardwarePlugInInterface { return CMIOHardwarePlugInInterface( _reserved: nil, QueryInterface: QueryInterface, AddRef: AddRef, Release: Release, Initialize: Initialize, InitializeWithObjectID: InitializeWithObjectID, Teardown: Teardown, ObjectShow: ObjectShow, ObjectHasProperty: ObjectHasProperty, ObjectIsPropertySettable: ObjectIsPropertySettable, ObjectGetPropertyDataSize: ObjectGetPropertyDataSize, ObjectGetPropertyData: ObjectGetPropertyData, ObjectSetPropertyData: ObjectSetPropertyData, DeviceSuspend: DeviceSuspend, DeviceResume: DeviceResume, DeviceStartStream: DeviceStartStream, DeviceStopStream: DeviceStopStream, DeviceProcessAVCCommand: DeviceProcessAVCCommand, DeviceProcessRS422Command: DeviceProcessRS422Command, StreamCopyBufferQueue: StreamCopyBufferQueue, StreamDeckPlay: StreamDeckPlay, StreamDeckStop: StreamDeckStop, StreamDeckJog: StreamDeckJog, StreamDeckCueTo: StreamDeckCueTo) } PluginInterface.swift (ൈਮ) ΠϯλʔϑΣΠε͕ظ଴͢Δ΋ͷΛฦ٫
37. PluginInterface.swift (ൈਮ) ৄࡉ͸ιʔεࢀর
38. ϓϩδΣΫτ࡞੒
    QMJTU৘ใ௥Ճ
    ΤϯτϦʔϙΠϯτ࣮૷
    ΠϯλʔϑΣΠε࣮૷
    Χϝϥө૾औಘͱग़ྗ $PSF.FEJB*0
    %"-
    1MVHJO
39. func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, ɹɹɹɹɹɹɹɹɹɹɹ ɹɹfrom connection: AVCaptureConnection) { if output == cameraCapture.output { guard let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return } let cameraImage = CIImage(cvImageBuffer: imageBuffer) let compositedImage = compose(bgImage: cameraImage, overlayImage: self.textImage) var pixelBuffer: CVPixelBuffer? _ = CVPixelBufferCreate( kCFAllocatorDefault, Int(compositedImage.extent.size.width), Int(compositedImage.extent.height), kCVPixelFormatType_32BGRA, self.CVPixelBufferCreateOptions as CFDictionary, &pixelBuffer ) if let pixelBuffer = pixelBuffer { context.render(compositedImage, to: pixelBuffer) delegate?.videoComposer(self, didComposeImageBuffer: pixelBuffer) } } } VideoComposer.swift (ൈਮ)
40. func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, ɹɹɹɹɹɹɹɹɹɹɹ ɹɹfrom connection: AVCaptureConnection) { if output == cameraCapture.output { guard let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return } let cameraImage = CIImage(cvImageBuffer: imageBuffer) let compositedImage = compose(bgImage: cameraImage, overlayImage: self.textImage) var pixelBuffer: CVPixelBuffer? _ = CVPixelBufferCreate( kCFAllocatorDefault, Int(compositedImage.extent.size.width), Int(compositedImage.extent.height), kCVPixelFormatType_32BGRA, self.CVPixelBufferCreateOptions as CFDictionary, &pixelBuffer ) if let pixelBuffer = pixelBuffer { context.render(compositedImage, to: pixelBuffer) delegate?.videoComposer(self, didComposeImageBuffer: pixelBuffer) } } } VideoComposer.swift (ൈਮ) ผ్ੜ੒ͨ͠ΦʔόʔϨΠςΩετը૾ͱ߹੒ EFMFHBUFܦ༝ͰQJYFM#V⒎FSΛ౉͢
41. private lazy var timer: DispatchSourceTimer = { let interval = 1.0 / Double(frameRate) let timer = DispatchSource.makeTimerSource() timer.schedule(deadline: .now() + interval, repeating: interval) timer.setEventHandler(handler: { [weak self] in self?.enqueueBuffer() }) return timer }() Stream.swift (ൈਮ)
42. private lazy var timer: DispatchSourceTimer = { let interval = 1.0 / Double(frameRate) let timer = DispatchSource.makeTimerSource() timer.schedule(deadline: .now() + interval, repeating: interval) timer.setEventHandler(handler: { [weak self] in self?.enqueueBuffer() }) return timer }() Stream.swift (ൈਮ) GSBNF3BUFࢦఆ͠ߋ৽ॲཧݺͼग़͠
43. private func enqueueBuffer() { (தུ) var sampleBufferUnmanaged: Unmanaged<CMSampleBuffer>? = nil error = CMIOSampleBufferCreateForImageBuffer( kCFAllocatorDefault, pixelBuffer, formatDescription, &timing, sequenceNumber, UInt32(kCMIOSampleBufferNoDiscontinuities), &sampleBufferUnmanaged ) guard error == noErr else { log("CMIOSampleBufferCreateForImageBuffer Error: \(error)") return } CMSimpleQueueEnqueue(queue, element: sampleBufferUnmanaged!.toOpaque()) queueAlteredProc?(objectID, sampleBufferUnmanaged!.toOpaque(), queueAlteredRefCon) sequenceNumber += 1 } Stream.swift (ൈਮ)
44. private func enqueueBuffer() { (தུ) var sampleBufferUnmanaged: Unmanaged<CMSampleBuffer>? = nil error = CMIOSampleBufferCreateForImageBuffer( kCFAllocatorDefault, pixelBuffer, formatDescription, &timing, sequenceNumber, UInt32(kCMIOSampleBufferNoDiscontinuities), &sampleBufferUnmanaged ) guard error == noErr else { log("CMIOSampleBufferCreateForImageBuffer Error: \(error)") return } CMSimpleQueueEnqueue(queue, element: sampleBufferUnmanaged!.toOpaque()) queueAlteredProc?(objectID, sampleBufferUnmanaged!.toOpaque(), queueAlteredRefCon) sequenceNumber += 1 } Stream.swift (ൈਮ) QJYFM#V⒎FS͔Βੜ੒ͨ͠
    $.4BNQMF#V⒎FSΛRVFVFʹ௥Ճ
45. @IBAction func sendButton_action(_ sender: Any) { SettingsPasteboard.shared.settings["text1"] = mainTextField.stringValue SettingsPasteboard.shared.update() } ViewController.swift (ൈਮ)
47. QMVHJOϑΝΠϧΛ഑ஔ
48. -JCSBSZ$PSF.FEJB*01MVH*OT%"-
49. දࣔ͞Εͨʂ
51. ߏ੒ཁૉ
    ɹ$PSF.FEJB*0
    %"-
    1MVHJO
    ɹίϯτϩʔϥΞϓϦ
    
52. ίϯτϩʔϥΞϓϦ
53. $PSF.FEJB*0
    %"-
    1MVHJO ίϯτϩʔϥΞϓϦ /41BTUFCPBSE -JCSBSZ$PSF.FEJB*01MVH*OT%"- Ծ૝Χϝϥͷ࣮ମ 6*Λ࣋ͨͳ͍1MVHJOʹ
    ஋Λ౉͢ΞϓϦ
54. 1BTUFCPBSEʹ஋Λஔ͘໾ׂ
55. ϓϩδΣΫτ࡞੒
    ίϯτϩʔϧ഑ஔ
    /41BTUFCPBSEͰσʔλڞ༗ ίϯτϩʔϥΞϓϦ
58. ϓϩδΣΫτ࡞੒
    ίϯτϩʔϧ഑ஔ
    /41BTUFCPBSEͰσʔλڞ༗ ίϯτϩʔϥΞϓϦ
61. ϓϩδΣΫτ࡞੒
    ίϯτϩʔϧ഑ஔ
    /41BTUFCPBSEͰσʔλڞ༗ ίϯτϩʔϥΞϓϦ
62. extension NSPasteboard.Name { static let main = NSPasteboard.Name(Config.mainAppBundleIdentifier) } extension NSPasteboard.PasteboardType { static let plain = NSPasteboard.PasteboardType(rawValue: "public.utf8-plain-text") } class SettingsPasteboard { static let shared = SettingsPasteboard() open var settings = [String: Any]() open func current() -> [String: Any] { let pasteboard = NSPasteboard(name: .main) if let element = pasteboard.pasteboardItems?.last, let str = element.string(forType: .plain), let data = str.data(using: .utf8) { do { let json = try JSONSerialization.jsonObject(with: data, options : .allowFragments) as! [String: Any] settings = json } catch { print("can't convert json") } } return settings } open func update() { let jsonStr = stringify(json: settings) let pasteboard = NSPasteboard(name: .main) pasteboard.declareTypes([.string], owner: nil) pasteboard.setString(jsonStr, forType: .string) } ... } SettingsPasteboard.swift (ൈਮ)
64. ׬੒
65. $PSF.FEJB*0
    %"-
    1MVHJO ίϯτϩʔϥΞϓϦ /41BTUFCPBSE -JCSBSZ$PSF.FEJB*01MVH*OT%"- Ծ૝Χϝϥͷ࣮ମ 6*Λ࣋ͨͳ͍1MVHJOʹ
    ஋Λ౉͢ΞϓϦ
66. ίϯτϩʔϥΞϓϦ͔Β஋Λૹ৴
70. ࣮૷ํ๏
    ݱ࣮֦ுͱͯ͠ͷར༻
71. 'BDFUJNFΧϝϥө૾ Ի੠ೖྗ ը૾Ճ޻ "1*Ϩεϙϯε ΩʔϘʔυೖྗ ,FZOPUFը໘ FUD ֎෦ Ծ૝Χϝϥ
    
    ग़ྗՃ޻
    ػߏ ݱ࣮֦ுͱͯ͠ͷԾ૝Χϝϥ ֤छೖྗ
72. NNINN
    ϏοάϋϯυϞʔυ
73. Ή͜͏ͷ͘ʹ
    ϑΟϧλɺϚΠϖʔδɺϝλόʔε
74. ϑΟϧλ
    ҟٞ͋Γɺ଴ͬͨ
75. ϑΟϧλ
    ΠϩϞωΞత
76. UPOBSJ
    ԕִϦΞϧλΠϜө૾
77. ϚΠϯυϚοϓ
79. δΣωϨΠςΟϒදݱ
80. ελϯϓϦΞΫγϣϯ
81. Ի੠ೝࣝͱ຋༁
84. ݱ࣮֦ுͱͯ͠ͷԾ૝Χϝϥ
85. ࢀߟ IUUQTEFWFMPQFSBQQMFDPNMJCSBSZBSDIJWFTBNQMFDPEF$PSF.FEJB*0*OUSPEVDUJPO*OUSPIUNM
    IUUQTHJUIVCDPNKPIOCPJMFTDPSFNFEJBJPEBMNJOJNBMFYBNQMF
    IUUQTHJUIVCDPNTFBODIBT4JNQMF%"-1MVHJO
    IUUQTTQFBLFSEFDLDPNLJTIJLBXBLBUTVNJWJSUVBMXFCDBNFSBXP[VPSPV
86. 5XJUUFSͰ࠷৽৘ใൃ৴த
    !TINEFWFMPQ
    Ծ૝Χϝϥ࣮૷αϯϓϧ
    IUUQTHJUIVCDPNTBUPTIJ7JSUVBM$BNFSB4BNQMF