Upgrade to Pro — share decks privately, control downloads, hide ads and more …

Enhancing Applications with Accessibility API

Enhancing Applications with Accessibility API

Kishikawa Katsumi

March 20, 2024
Tweet

More Decks by Kishikawa Katsumi

Other Decks in Programming

Transcript

  1. 3FUSJFWF4FMFDUFE5FYU L"94FMFDUFE5FYU"UUSJCVUF var selectedTextValue: AnyObject? let selectedTextValueError = AXUIElementCopyAttributeValue( focusedElement

    as! AXUIElement, kAXSelectedTextAttribute as CFString, &selectedTextValue ) guard let selectedTextValue, selectedTextValueError == .success else { return }
  2. 3FUSJFWF4FMFDUFE5FYU L"94FMFDUFE5FYU"UUSJCVUF var selectedTextValue: AnyObject? let selectedTextValueError = AXUIElementCopyAttributeValue( focusedElement

    as! AXUIElement, kAXSelectedTextAttribute as CFString, &selectedTextValue ) guard let selectedTextValue, selectedTextValueError == .success else { return }
  3. 3FUSJFWF4FMFDUFE5FYU L"94FMFDUFE5FYU"UUSJCVUF var selectedTextValue: AnyObject? let selectedTextValueError = AXUIElementCopyAttributeValue( focusedElement

    as! AXUIElement, kAXSelectedTextAttribute as CFString, &selectedTextValue ) guard let selectedTextValue, selectedTextValueError == .success else { return }
  4. 4ZTUFN8JEF"96*&MFNFOU r "DDFTTTZTUFNXJEFLFZCPBSEGPDVT NPVTFDVSTPSQPTJUJPO BOEPUIFS JOGPSNBUJPO r "DDFTTTZTUFNMFWFM6*FMFNFOUTUIBUEPOPUCFMPOHUPBTQFDJ fi D

    BQQMJDBUJPO FH NFOVCBS OPUJ fi DBUJPODFOUFS  r 3FUSJFWFJOGPSNBUJPOBCPVUUIFDPOUFYUPGUIFDVSSFOUVTFSJOUFSBDUJPO  TVDIBTUIFBDUJWFXJOEPXPSUIF6*FMFNFOUUIBUDVSSFOUMZIBTGPDVT
  5. 3FUSJFWF'PDVTFE6*&MFNFOU L"9'PDVTFE6*&MFNFOU"UUSJCVUF let systemWideElement = AXUIElementCreateSystemWide() var focusedElement: AnyObject? let

    focusedElementError = AXUIElementCopyAttributeValue( systemWideElement, kAXFocusedUIElementAttribute as CFString, &focusedElement )
  6. 3FUSJFWF4FMFDUFE5FYU L"94FMFDUFE5FYU"UUSJCVUF var selectedTextValue: AnyObject? let selectedTextValueError = AXUIElementCopyAttributeValue( focusedElement

    as! AXUIElement, kAXSelectedTextAttribute as CFString, &selectedTextValue ) guard let selectedTextValue, selectedTextValueError == .success else { return }
  7. .POJUPS(MPCBM,FZ&WFOU /4&WFOUBEE(MPCBM.POJUPS'PS&WFOUT NBUDIJOHIBOEMFS open class NSEvent : NSObject, NSCopying, NSCoding

    { open class func addGlobalMonitorForEvents( matching mask: NSEvent.EventTypeMask, handler block: @escaping (NSEvent) -> Void ) -> Any? }
  8. .POJUPS(MPCBM,FZ&WFOU NSEvent.addGlobalMonitorForEvents( matching: [.keyDown] ) { (event) in switch event.type

    { case .keyDown: guard event.modifierFlags.contains([.command, .control]) && event.keyCode == kVK_ANSI_A else { return } ... default: break } }
  9. .POJUPS(MPCBM,FZ&WFOU NSEvent.addGlobalMonitorForEvents( matching: [.keyDown] ) { (event) in switch event.type

    { case .keyDown: guard event.modifierFlags.contains([.command, .control]) && event.keyCode == kVK_ANSI_A else { return } ... default: break } }
  10. .POJUPS(MPCBM,FZ&WFOU &WFOU5BQ ➡IUUQTHJUIVCDPNLJTIJLBXBLBUTVNJUSZTXJGU let mask: CGEventMask = (1 << CGEventType.keyDown.rawValue)

    let tap = CGEvent.tapCreate( tap: .cgSessionEventTap, place: .headInsertEventTap, options: .defaultTap, eventsOfInterest: mask, callback: { (proxy, type, event, refcon) in ... }, userInfo: UnsafeMutableRawPointer(Unmanaged.passUnretained(self).toOpaque()) ) let runLoopSource = CFMachPortCreateRunLoopSource(kCFAllocatorDefault, tap, 0) CFRunLoopAddSource(CFRunLoopGetCurrent(), runLoopSource, .commonModes) CGEvent.tapEnable(tap: tap!, enable: true)
  11. 3FUSJFWF4FMFDUFE5FYU 'SPN8FC7JFX var selectedTextMarkerRangeValue: AnyObject? let selectedTextMarkerRangeValueError = AXUIElementCopyAttributeValue( focusedElement

    as! AXUIElement, "AXSelectedTextMarkerRange" as CFString, &selectedTextMarkerRangeValue ) ... var stringForTextMarkerRangeValue: AnyObject? let stringForTextMarkerRangeValueError = AXUIElementCopyParameterizedAttributeValue( focusedElement as! AXUIElement, "AXStringForTextMarkerRange" as CFString, selectedTextMarkerRangeValue, &stringForTextMarkerRangeValue ) ...
  12. 3FUSJFWF4FMFDUFE5FYU 'SPN8FC7JFX var selectedTextMarkerRangeValue: AnyObject? let selectedTextMarkerRangeValueError = AXUIElementCopyAttributeValue( focusedElement

    as! AXUIElement, "AXSelectedTextMarkerRange" as CFString, &selectedTextMarkerRangeValue ) ... var stringForTextMarkerRangeValue: AnyObject? let stringForTextMarkerRangeValueError = AXUIElementCopyParameterizedAttributeValue( focusedElement as! AXUIElement, "AXStringForTextMarkerRange" as CFString, selectedTextMarkerRangeValue, &stringForTextMarkerRangeValue ) ...
  13. "96*&MFNFOU$PQZ1BSBNFUFSJ[FE"UUSJCVUF7BMVF var selectedTextMarkerRangeValue: AnyObject? let selectedTextMarkerRangeValueError = AXUIElementCopyAttributeValue( focusedElement as!

    AXUIElement, "AXSelectedTextMarkerRange" as CFString, &selectedTextMarkerRangeValue ) ... var stringForTextMarkerRangeValue: AnyObject? let stringForTextMarkerRangeValueError = AXUIElementCopyParameterizedAttributeValue( focusedElement as! AXUIElement, "AXStringForTextMarkerRange" as CFString, selectedTextMarkerRangeValue, &stringForTextMarkerRangeValue ) ...
  14. var selectedTextMarkerRangeValue: AnyObject? let selectedTextMarkerRangeValueError = AXUIElementCopyAttributeValue( focusedElement as! AXUIElement,

    "AXSelectedTextMarkerRange" as CFString, &selectedTextMarkerRangeValue ) ... var stringForTextMarkerRangeValue: AnyObject? let stringForTextMarkerRangeValueError = AXUIElementCopyParameterizedAttributeValue( focusedElement as! AXUIElement, "AXStringForTextMarkerRange" as CFString, selectedTextMarkerRangeValue, &stringForTextMarkerRangeValue ) ... "96*&MFNFOU$PQZ1BSBNFUFSJ[FE"UUSJCVUF7BMVF
  15. var selectedTextMarkerRangeValue: AnyObject? let selectedTextMarkerRangeValueError = AXUIElementCopyAttributeValue( focusedElement as! AXUIElement,

    "AXSelectedTextMarkerRange" as CFString, &selectedTextMarkerRangeValue ) ... var stringForTextMarkerRangeValue: AnyObject? let stringForTextMarkerRangeValueError = AXUIElementCopyParameterizedAttributeValue( focusedElement as! AXUIElement, "AXStringForTextMarkerRange" as CFString, selectedTextMarkerRangeValue, &stringForTextMarkerRangeValue ) ... "96*&MFNFOU$PQZ1BSBNFUFSJ[FE"UUSJCVUF7BMVF
  16. )PXUP,OPX6OEPDVNFOUFE,FZT var selectedTextMarkerRangeValue: AnyObject? let selectedTextMarkerRangeValueError = AXUIElementCopyAttributeValue( focusedElement as!

    AXUIElement, "AXSelectedTextMarkerRange" as CFString, &selectedTextMarkerRangeValue ) ... var stringForTextMarkerRangeValue: AnyObject? let stringForTextMarkerRangeValueError = AXUIElementCopyParameterizedAttributeValue( focusedElement as! AXUIElement, "AXStringForTextMarkerRange" as CFString, selectedTextMarkerRangeValue, &stringForTextMarkerRangeValue ) ...
  17. /PUJGJDBUJPOT var observer: AXObserver? let observerCreateError = AXObserverCreate( app.processIdentifier, {

    (observer, element, notification, userData) in ... }, &observer ) AXObserverAddNotification( observer, appElement, kAXSelectedTextChangedNotification as CFString, nil ) CFRunLoopAddSource( CFRunLoopGetCurrent(), AXObserverGetRunLoopSource(observer), .commonModes )
  18. /PUJGJDBUJPOT var observer: AXObserver? let observerCreateError = AXObserverCreate( app.processIdentifier, {

    (observer, element, notification, userData) in ... }, &observer ) AXObserverAddNotification( observer, appElement, kAXSelectedTextChangedNotification as CFString, nil ) CFRunLoopAddSource( CFRunLoopGetCurrent(), AXObserverGetRunLoopSource(observer), .commonModes )
  19. /PUJGJDBUJPOT var observer: AXObserver? let observerCreateError = AXObserverCreate( app.processIdentifier, {

    (observer, element, notification, userData) in ... }, &observer ) AXObserverAddNotification( observer, appElement, kAXSelectedTextChangedNotification as CFString, nil ) CFRunLoopAddSource( CFRunLoopGetCurrent(), AXObserverGetRunLoopSource(observer), .commonModes ) let app = NSRunningApplication .runningApplications(withBundleIdentifier: "com.apple.TextEdit") .first
  20. /PUJGJDBUJPOT var observer: AXObserver? let observerCreateError = AXObserverCreate( app.processIdentifier, {

    (observer, element, notification, userData) in ... }, &observer ) AXObserverAddNotification( observer, appElement, kAXSelectedTextChangedNotification as CFString, nil ) CFRunLoopAddSource( CFRunLoopGetCurrent(), AXObserverGetRunLoopSource(observer), .commonModes )
  21. /PUJGJDBUJPOT var observer: AXObserver? let observerCreateError = AXObserverCreate( app.processIdentifier, {

    (observer, element, notification, userData) in ... }, &observer ) AXObserverAddNotification( observer, appElement, kAXSelectedTextChangedNotification as CFString, nil ) CFRunLoopAddSource( CFRunLoopGetCurrent(), AXObserverGetRunLoopSource(observer), .commonModes ) kAXSelectedTextChangedNotification kAXValueChangedNotification ...
  22. .PEJGZ5FYU var isAttributeSettableValue = DarwinBoolean(false) let isAttributeSettableValueError = AXUIElementIsAttributeSettable( element,

    kAXValueAttribute as CFString, &isAttributeSettableValue ) guard isAttributeSettableValueError == .success else { return } if isAttributeSettableValue.boolValue { AXUIElementSetAttributeValue( element, kAXValueAttribute as CFString, "\(precedingTextValue)\n\(codeBlock)" as CFString ) return }
  23. .PEJGZ5FYU var isAttributeSettableValue = DarwinBoolean(false) let isAttributeSettableValueError = AXUIElementIsAttributeSettable( element,

    kAXValueAttribute as CFString, &isAttributeSettableValue ) guard isAttributeSettableValueError == .success else { return } if isAttributeSettableValue.boolValue { AXUIElementSetAttributeValue( element, kAXValueAttribute as CFString, "\(precedingTextValue)\n\(codeBlock)" as CFString ) return }
  24. .PEJGZ5FYU var isAttributeSettableValue = DarwinBoolean(false) let isAttributeSettableValueError = AXUIElementIsAttributeSettable( element,

    kAXValueAttribute as CFString, &isAttributeSettableValue ) guard isAttributeSettableValueError == .success else { return } if isAttributeSettableValue.boolValue { AXUIElementSetAttributeValue( element, kAXValueAttribute as CFString, "\(precedingTextValue)\n\(codeBlock)" as CFString ) return }
  25. 4$4DSFFOTIPU.BOBHFSDBQUVSF*NBHF  func captureScreen(rect: CGRect) async throws -> CGImage? {

    let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: false) guard let mainDisplayID = NSScreen.main?.deviceDescription[NSDeviceDescriptionKey("NSScreenNumber")] as? CGDirectDisplayID, let display = content.displays.first(where: { $0.displayID == mainDisplayID }) else { return nil } let streamConfig = SCStreamConfiguration() streamConfig.captureResolution = .automatic streamConfig.sourceRect = rect streamConfig.preservesAspectRatio = true streamConfig.scalesToFit = true streamConfig.capturesAudio = false streamConfig.excludesCurrentProcessAudio = true let filter = SCContentFilter(display: display, excludingWindows: []) let capturedImage = try await SCScreenshotManager.captureImage( contentFilter: filter, configuration: streamConfig ) return capturedImage }
  26. 3FUSJFWFBO*NBHF $(8JOEPX-JTU$SFBUF*NBHF @@@@ func captureScreen(rect: CGRect) -> CGImage? { let

    screenShot = CGWindowListCreateImage( rect, [.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID, [.boundsIgnoreFraming, .bestResolution] ) return screenShot }
  27. (SBOU4DSFFO3FDPSEJOH1FSNJTTJPO $(1SF fl JHIU4DSFFO$BQUVSF"DDFTT r $(1SF fl JHIU4DSFFO$BQUVSF"DDFTT  r

    *OEJDBUFTXIFUIFSTDSFFOSFDPSEJOHJTBMMPXFE r $(3FRVFTU4DSFFO$BQUVSF"DDFTT  r 1SFTFOUUIFTZTUFNQSPNQU
  28. &YUSBDUUIF5FYUGSPNBO*NBHF *NBHF"OBMZ[FS let analyzer = ImageAnalyzer() let analysis = try

    await analyzer.analyze( image, orientation: .up, configuration: configuration ) let transcript = analysis.transcript
  29. &YUSBDUUIF5FYUGSPNBO*NBHF *NBHF"OBMZ[FS let analyzer = ImageAnalyzer() let analysis = try

    await analyzer.analyze( image, orientation: .up, configuration: configuration ) let transcript = analysis.transcript
  30. 8SBQ6Q r 8JOEPX$IBU"TTJTUBOU r #BTJD6TBHFPG"DDFTTJCJMJUZ"1* r *OMJOF$IBU"TTJTUBOU r /PUJ fi

    DBUJPOT r .PEJGZ$POUFOU r $PNJD5SBOTMBUPS r 4DSFFO$BQUVSF"1*BOE0$3 r "96*&MFNFOU*OTQFDUPS r *OTQFDU"UUSJCVUF/BNFPG"96*&MFNFOU
  31. 3FGFSFODFT r 4BNQMF$PEF IUUQTHJUIVCDPNLJTIJLBXBLBUTVNJUSZTXJGU r "DDFTTJOHUFYUWBMVFGSPNBOZ4ZTUFNXJEF"QQMJDBUJPOWJB"DDFTTJCJMJUZ "1* IUUQTNBDEFWFMPQFSTXPSEQSFTTDPNBDDFTTJOHUFYUWBMVFGSPNBOZTZTUFNXJEFBQQMJDBUJPOWJB BDDFTTJCJMJUZBQJ r

    )PXUPHFUTFMFDUFEUFYUBOEJUTDPPSEJOBUFTGSPNBOZTZTUFNXJEFBQQMJDBUJPOVTJOH "DDFTTJCJMJUZ "1*  IUUQTNBDEFWFMPQFSTXPSEQSFTTDPNIPXUPHFUTFMFDUFEUFYUBOEJUTDPPSEJOBUFTGSPNBOZTZTUFNXJEF BQQMJDBUJPOVTJOHBDDFTTJCJMJUZBQJ r 8IZBSFOUUIFNPTUVTFGVM.BDBQQTPOUIF"QQ4UPSF  IUUQTBMJOQBOBJUJVDPNCMPHBQQTPVUTJEFBQQTUPSF r "MMBCPVUNBD04FWFOUPCTFSWBUJPO IUUQTEPDTHPPHMFDPNQSFTFOUBUJPOEO&BJ16EVIWKLTS%735D+B&6-C488IU7E()'@946IUNMQSFTFOU