Upgrade to Pro — share decks privately, control downloads, hide ads and more …

Machine Learning On Mobile With MLKit

Machine Learning On Mobile With MLKit

Machine Learning is becoming more approachable for developers with no or very little background in Data Science. With the release of MLKit from Google, developers can now easily add features based on Machine Learning to their mobile apps. In this session, we will go through a number of use-cases and how to implement it for a mobile app.

Erik Hellman

November 20, 2018
Tweet

More Decks by Erik Hellman

Other Decks in Programming

Transcript

  1. class TensorFlowLiteDemo(modelFile: File, labels: Array<String>) { private val labelProbArray: Array<FloatArray>

    = arrayOf(FloatArray(labels.size)) private val tflite: Interpreter = Interpreter(modelFile) fun classifyImage(bitmap: Bitmap) { val imageData = convertBitmapToByteBuffer(bitmap) tflite.run(imageData, labelProbArray) } } Machine Learning on Mobile with MLKit - @ErikHellman, Øredev 2018
  2. Offline/On-device Machine Learning → Reduced accuracy → Larger downloads →

    Increased resource usage Machine Learning on Mobile with MLKit - @ErikHellman, Øredev 2018
  3. MLKit on iOS - Setup pod 'Firebase/Core' pod 'Firebase/MLVision' #

    If using an on-device API: pod 'Firebase/MLVisionTextModel' Machine Learning on Mobile with MLKit - @ErikHellman, Øredev 2018
  4. Text recognition (iOS) let vision = Vision.vision() let textRecognizer =

    vision.onDeviceTextRecognizer() let image = VisionImage(image: uiImage) textRecognizer.process(visionImage) { result, error in guard error == nil, let result = result else { // ... return } // Recognized text } Machine Learning on Mobile with MLKit - @ErikHellman, Øredev 2018
  5. let resultText = result.text for block in result.blocks { let

    blockText = block.text let blockConfidence = block.confidence let blockLanguages = block.recognizedLanguages let blockCornerPoints = block.cornerPoints let blockFrame = block.frame for line in block.lines { let lineText = line.text let lineConfidence = line.confidence let lineLanguages = line.recognizedLanguages let lineCornerPoints = line.cornerPoints let lineFrame = line.frame for element in line.elements { let elementText = element.text let elementConfidence = element.confidence let elementLanguages = element.recognizedLanguages let elementCornerPoints = element.cornerPoints let elementFrame = element.frame } } } Machine Learning on Mobile with MLKit - @ErikHellman, Øredev 2018
  6. MLKit on Android - Setup dependencies { // ... implementation

    'com.google.firebase:firebase-ml-vision:18.0.1' implementation 'com.google.firebase:firebase-ml-vision-image-label-model:17.0.2' implementation 'com.google.firebase:firebase-ml-vision-face-model:17.0.2' } <application ...> ... <meta-data android:name="com.google.firebase.ml.vision.DEPENDENCIES" android:value="ocr" /> <!-- To use multiple models: android:value="ocr,face,barcode,label" --> </application> Machine Learning on Mobile with MLKit - @ErikHellman, Øredev 2018
  7. Text recognition (Android) val image = FirebaseVisionImage.fromBitmap(bitmap) val textRecognizer =

    FirebaseVision.instance.getOnDeviceTextRecognizer(); textRecognizer.processImage(image) .addOnSuccessListener({ result: FirebaseVisionText -> /* Task completed successfully */ }) .addOnFailureListener({ error: Exception -> /* Task failed with an exception */ }) Machine Learning on Mobile with MLKit - @ErikHellman, Øredev 2018
  8. val resultText = result.getText result.getTextBlocks.forEach {block -> val blockText =

    block.text val blockConfidence = block.confidence val blockLanguages = block.recognizedLanguages val blockCornerPoints = block.cornerPoints val blockFrame = block.boundingBox block.lines.forEach { line -> val lineText = line.text val lineConfidence = line.confidence val lineLanguages = line.recognizedLanguages val lineCornerPoints = line.cornerPoints val lineFrame = line.boundingBox line.elements.forEach {element -> val elementText = element.text val elementConfidence = element.confidence val elementLanguages = element.recognizedLanguages val elementCornerPoints = element.cornerPoints() val elementFrame = element.boundingBox } } } Machine Learning on Mobile with MLKit - @ErikHellman, Øredev 2018
  9. Dealing with camera rotation object ORIENTATIONS: SparseIntArray() { init {

    append(Surface.ROTATION_0, 90) append(Surface.ROTATION_90, 0) append(Surface.ROTATION_180, 270) append(Surface.ROTATION_270, 180) } } Machine Learning on Mobile with MLKit - @ErikHellman, Øredev 2018
  10. private fun calculateRotationCompensation(cameraId: String, activity: Activity): Int { val deviceRotation

    = activity.windowManager.defaultDisplay.rotation var rotationCompensation = ORIENTATIONS.get(deviceRotation) val cameraManager = activity.getSystemService(CAMERA_SERVICE) as CameraManager val sensorOrientation = cameraManager .getCameraCharacteristics(cameraId) .get(CameraCharacteristics.SENSOR_ORIENTATION)!! rotationCompensation = (rotationCompensation + sensorOrientation + 270) % 360 // Return the corresponding FirebaseVisionImageMetadata rotation value. return when (rotationCompensation) { 0 -> FirebaseVisionImageMetadata.ROTATION_0 90 -> FirebaseVisionImageMetadata.ROTATION_90 180 -> FirebaseVisionImageMetadata.ROTATION_180 270 -> FirebaseVisionImageMetadata.ROTATION_270 else -> { FirebaseVisionImageMetadata.ROTATION_0 } } } Machine Learning on Mobile with MLKit - @ErikHellman, Øredev 2018
  11. Face Detection (Android) // High-accuracy landmark detection and face classification

    fun setupFacedetection(): FirebaseVisionFaceDetector { val options = FirebaseVisionFaceDetectorOptions.Builder() .setPerformanceMode(FirebaseVisionFaceDetectorOptions.ACCURATE) .setLandmarkMode(FirebaseVisionFaceDetectorOptions.ALL_LANDMARKS) .setClassificationMode(FirebaseVisionFaceDetectorOptions.ALL_CLASSIFICATIONS) .build() return FirebaseVision.getInstance().getVisionFaceDetector(options) } Machine Learning on Mobile with MLKit - @ErikHellman, Øredev 2018
  12. fun detectFaces(detector: FirebaseVisionFaceDetector, image: Bitmap) { val visionImage = FirebaseVisionImage.fromBitmap(image)

    detector.detectInImage(visionImage) .addOnSuccessListener { faces -> for (face in faces) { // Faces detected } } .addOnFailureListener { // TODO Error handling } } Machine Learning on Mobile with MLKit - @ErikHellman, Øredev 2018
  13. val bounds = face.boundingBox val rotY = face.headEulerAngleY // Head

    is rotated to the right rotY degrees val rotZ = face.headEulerAngleZ // Head is tilted sideways rotZ degrees // If landmark detection was enabled (mouth, ears, eyes, cheeks, and // nose available): val leftEar = face.getLandmark(FirebaseVisionFaceLandmark.LEFT_EAR) if (leftEar != null) { val leftEarPos = leftEar.position } Machine Learning on Mobile with MLKit - @ErikHellman, Øredev 2018
  14. // If contour detection was enabled: val leftEyeContour = face.getContour(FirebaseVisionFaceContour.LEFT_EYE).points

    val upperLipBottomContour = face.getContour(FirebaseVisionFaceContour.UPPER_LIP_BOTTOM).points Machine Learning on Mobile with MLKit - @ErikHellman, Øredev 2018
  15. // If classification was enabled: if (face.smilingProbability != FirebaseVisionFace.UNCOMPUTED_PROBABILITY) {

    val smileProb = face.smilingProbability } if (face.rightEyeOpenProbability != FirebaseVisionFace.UNCOMPUTED_PROBABILITY) { val rightEyeOpenProb = face.rightEyeOpenProbability } // If face tracking was enabled: if (face.trackingId != FirebaseVisionFace.INVALID_ID) { val id = face.trackingId } Machine Learning on Mobile with MLKit - @ErikHellman, Øredev 2018
  16. Cloud Image Labeling (Android) fun createCloudLabelDetector(): FirebaseVisionCloudLabelDetector? { val options

    = FirebaseVisionCloudDetectorOptions.Builder() .setModelType(FirebaseVisionCloudDetectorOptions.LATEST_MODEL) .setMaxResults(15) .build() return FirebaseVision.getInstance().getVisionCloudLabelDetector(options) } Machine Learning on Mobile with MLKit - @ErikHellman, Øredev 2018
  17. fun labelImage(detector: FirebaseVisionCloudLabelDetector, bitmap: Bitmap) { val image = FirebaseVisionImage.fromBitmap(bitmap)

    detector.detectInImage(image) .addOnSuccessListener { result -> result.forEach { label -> val text = label.label val entityId = label.entityId val confidence = label.confidence } } .addOnFailureListener { // Error handling } } Machine Learning on Mobile with MLKit - @ErikHellman, Øredev 2018
  18. Determine input and output sizes import tensorflow as tf interpreter

    = tf.contrib.lite.Interpreter(model_path="my_custom_model.tflite") interpreter.allocate_tensors() # Print input shape and type print(interpreter.get_input_details()[0]['shape']) # Example: [1 224 224 3] print(interpreter.get_input_details()[0]['dtype']) # Example: <class 'numpy.float32'> # Print output shape and type print(interpreter.get_output_details()[0]['shape']) # Example: [1 1000] print(interpreter.get_output_details()[0]['dtype']) # Example: <class 'numpy.float32'> Machine Learning on Mobile with MLKit - @ErikHellman, Øredev 2018
  19. fun configureCustomModel(modelName: String): FirebaseModelInterpreter? { val conditions = FirebaseModelDownloadConditions.Builder() .requireWifi()

    .build() val cloudSource = FirebaseCloudModelSource.Builder(modelName) .enableModelUpdates(true) .setInitialDownloadConditions(conditions) .setUpdatesDownloadConditions(conditions) .build() FirebaseModelManager.getInstance().registerCloudModelSource(cloudSource) val options = FirebaseModelOptions.Builder() .setCloudModelName(modelName) .setLocalModelName(modelName) .build() return FirebaseModelInterpreter.getInstance(options) } Machine Learning on Mobile with MLKit - @ErikHellman, Øredev 2018
  20. fun runInterpreter(input: Any, firebaseInterpreter: FirebaseModelInterpreter) { val inputs = FirebaseModelInputs.Builder()

    .add(input) // add() as many input arrays as your model requires .build() val inputOutputOptions = FirebaseModelInputOutputOptions.Builder() .setInputFormat(/* input format defined here */) .setOutputFormat(/* output format defined here */) .build() firebaseInterpreter.run(inputs, inputOutputOptions) .addOnSuccessListener { // Interpret results } .addOnFailureListener { // Error handling } } Machine Learning on Mobile with MLKit - @ErikHellman, Øredev 2018
  21. Resources → Firebase MLKit - https://firebase.google.com/ docs/ml-kit/ → TensorFlow Lite

    - https://www.tensorflow.org/lite → TensorFlow models - https://github.com/ tensorflow/models Machine Learning on Mobile with MLKit - @ErikHellman, Øredev 2018