Put some 3D sparkles in your 2D app

Trigonometry, rototranslations, vectors, matrices... when thinking about 3D modeling and visualization most mobile apps developers start to get worried, but it doesn’t need to be all that scary!
In this talk we’ll see how any developer, even without any prior experience with the topic, can integrate some 3D model and environment into an Android app, blending it seamlessly with the existing UI. We’ll do this thanks to the great Filament library from Google.

At the end of the talk the audience will learn:
- how a 3D model is made
- which formats are supported by Filament
- the main components needed to properly display a 3D model into an app
- how to harness the powerful Filament lightning system to add more realism to the scene
- how to blend 3D components into a Compose-based UI
- advanced 3D animations

Transcript

1. Daniele Bonaldo Put some 3D sparkles in your 2D app
2. Daniele Bonaldo @danybony_ @daniele@androiddev.social danielebonaldo.com
4. Filament https://google.github.io/filament
5. Physically based renderer
6. Physically based renderer
7. Documentation
8. Materials
9. GLTF GLB Graphics Library Transmission Format Graphics Library Binary
10. GL files preview
11. Samples
12. On Android
13. App base class MainActivity : ComponentActivity() { private val viewModel: FilamentViewModel by viewModels() override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) setContent { FilamentDemo3dPrinterTheme { Scaffold( ... ) { innerPadding -> ItemScreen( ... ) } } } } }
14. Setup dependencies { ... val filamentVersion = "1.53.1" implementation("com.google.android.filament:filament-android:$filamentVersion") implementation("com.google.android.filament:filament-utils-android:$filamentVersion") implementation("com.google.android.filament:gltfio-android:$filamentVersion") }
15. Init class FilamentViewModel(private val application: Application) : ViewModel() { init {_ Utils.init() }_ }=
16. Init private val engine by lazy { Engine.create() } private val assetLoader: AssetLoader by lazy { AssetLoader(engine, UbershaderProvider(engine), EntityManager.get()) } private val resourceLoader: ResourceLoader by lazy { val normalizeSkinningWeights = true ResourceLoader(engine, normalizeSkinningWeights) } class FilamentViewModel(private val application: Application) : ViewModel() { init {_ Utils.init() }_ }= init {_ Utils.init() initFilament() }_
17. ... private fun initFilament() { viewModelScope.launch { val newScene = engine.createScene() val asset = readCompressedAsset(application, "models/vesa_support.glb").let { val asset = loadModelGlb(assetLoader, resourceLoader, it) transformToUnitCube(engine, asset) asset } newScene.addEntities(asset.entities) } } class FilamentViewModel(private val application: Application) : ViewModel() { }= init {_ Utils.init() initFilament() }_ private fun initFilament() { viewModelScope.launch { val newScene = engine.createScene() } }
18. Programmatic Lights ... class FilamentViewModel(private val application: Application) : ViewModel() { }= } } private fun initFilament() { viewModelScope.launch { val newScene = engine.createScene() ... light = EntityManager.get().create() val (r, g, b) = Colors.cct(6_000.0f) LightManager.Builder(LightManager.Type.SUN) .color(r, g, b) .intensity(70_000.0f) .direction(0.28f, -0.6f, -0.76f) .build(engine, light) newScene.addEntity(light) Directional Point Spot
19. Ambient Lights ... class FilamentViewModel(private val application: Application) : ViewModel() { } } private fun initFilament() { viewModelScope.launch { val newScene = engine.createScene() ... val ibl = "default_env" readCompressedAsset(application, "envs/${ibl}/${ibl}_ibl.ktx").let { indirectLight = KTX1Loader.createIndirectLight(engine, it) indirectLight.intensity = 30_000.0f } newScene.indirectLight = indirectLight readCompressedAsset(application, "envs/${ibl}/${ibl}_skybox.ktx").let { skybox = KTX1Loader.createSkybox(engine, it) } newScene.skybox = skybox
20. Models and state var itemsUiState by mutableStateOf(ItemsUiState()) private set data class Item( val name: String, val printTime: String, val itemScene: ItemScene ) data class ItemScene( val engine: Engine, val scene: Scene, val asset: FilamentAsset, val resourceLoader: ResourceLoader )
21. Filament and Compose @Composable fun FilamentViewer(item: Item) { var modelViewer by remember { mutableStateOf<ModelViewer?>(null) } LaunchedEffect(true) { while (true) { withFrameNanos { nano -> modelViewer?.render(nano) } } } SideEffect { modelViewer?.scene = item.itemScene.scene } AndroidView({ context -> SurfaceView(context).also { surfaceView -> val (engine) = item.itemScene modelViewer = ModelViewer(engine, surfaceView).also { modelViewer -> setupModelViewer(modelViewer) } } }) }
22. Filament and Compose @Composable fun FilamentViewer(item: Item) { var modelViewer by remember { mutableStateOf<ModelViewer?>(null) } LaunchedEffect(true) { while (true) { withFrameNanos { nano -> modelViewer?.render(nano) } } } SideEffect { modelViewer?.scene = item.itemScene.scene } AndroidView({ context -> SurfaceView(context).also { surfaceView -> val (engine) = item.itemScene modelViewer = ModelViewer(engine, surfaceView).also { modelViewer -> setupModelViewer(modelViewer) } } }) }
23. Filament and Compose @Composable fun FilamentViewer(item: Item) { var modelViewer by remember { mutableStateOf<ModelViewer?>(null) } LaunchedEffect(true) { while (true) { withFrameNanos { nano -> modelViewer?.render(nano) } } } SideEffect { modelViewer?.scene = item.itemScene.scene } AndroidView({ context -> SurfaceView(context).also { surfaceView -> val (engine) = item.itemScene modelViewer = ModelViewer(engine, surfaceView).also { modelViewer -> setupModelViewer(modelViewer) } } }) }
24. ModelViewer setup fun setupModelViewer(viewer: ModelViewer) { val options = viewer.view.dynamicResolutionOptions options.enabled = true viewer.view.dynamicResolutionOptions = options viewer.view.antiAliasing = View.AntiAliasing.FXAA viewer.view.sampleCount = 4 val bloom = viewer.view.bloomOptions bloom.enabled = true viewer.view.bloomOptions = bloom }
25. ModelViewer class ModelViewer( val engine: Engine, val surfaceView: SurfaceView ) { val view: View = engine.createView() val camera: Camera = ... var scene: Scene? = ... private val uiHelper: UiHelper = ... private var displayHelper: DisplayHelper = ... private var swapChain: SwapChain? = ... private val renderer: Renderer = ... ... fun render(frameTimeNanos: Long) { ... } }
26. Camera private const val kNearPlane = 0.5 private const val kFarPlane = 10000.0 private const val kAperture = 3.5f private const val kShutterSpeed = 1f / 125f private const val kSensitivity = 100f private const val kFocalLength = 50.0 //lens mm private const val kAspectRatio = 3.0 / 2.0 val camera: Camera = engine.createCamera(engine.entityManager.create()) .apply { setLensProjection(kFocalLength, kAspectRatio, kNearPlane, kFarPlane) setExposure(kAperture, kShutterSpeed, kSensitivity) }
28. Camera Rotation val animator = ValueAnimator.ofFloat(0.0f, (2.0 * PI).toFloat()) animator.interpolator = LinearInterpolator() animator.duration = 36_000 animator.repeatMode = ValueAnimator.RESTART animator.repeatCount = ValueAnimator.INFINITE animator.addUpdateListener { a -> val v = (a.animatedValue as Float) camera.lookAt( cos(v) * 5.0, 0.5, sin(v) * 5.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0 ) } animator.start()
29. Camera Rotation val animator = ValueAnimator.ofFloat(0.0f, (2.0 * PI).toFloat()) animator.interpolator = LinearInterpolator() animator.duration = 36_000 animator.repeatMode = ValueAnimator.RESTART animator.repeatCount = ValueAnimator.INFINITE animator.addUpdateListener { a -> val v = (a.animatedValue as Float) camera.lookAt( cos(v) * 5.0, 0.5, sin(v) * 5.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0 ) } animator.start() override fun onViewDetachedFromWindow(v: android.view.View) { ... animator.cancel() }
30. Touch Movements @Composable fun FilamentViewer(item: Item) { ... AndroidView({ context -> SurfaceView(context).also { surfaceView -> ... modelViewer = ModelViewer(engine, surfaceView).also { modelViewer -> setupModelViewer(modelViewer) surfaceView.setOnTouchListener { _, event -> modelViewer.onTouchEvent(event) true } } } }) }
31. Touch Movements class ModelViewer(...) { init { ... cameraManipulator = Manipulator.Builder() .orbitHomePosition(4.0f, 0.5f,4.0f) .viewport(surfaceView.width, surfaceView.height) .orbitSpeed(0.005f, 0.005f) .build(Manipulator.Mode.ORBIT) gestureDetector = GestureDetector(surfaceView, cameraManipulator) }
32. class ModelViewer(...) { init { ... cameraManipulator = Manipulator.Builder() .orbitHomePosition(4.0f, 0.5f,4.0f) .viewport(surfaceView.width, surfaceView.height) .orbitSpeed(0.005f, 0.005f) .build(Manipulator.Mode.ORBIT) gestureDetector = GestureDetector(surfaceView, cameraManipulator) } Touch Movements fun onTouchEvent(event: MotionEvent) { gestureDetector.onTouchEvent(event) }
33. .build(Manipulator.Mode.ORBIT) gestureDetector = GestureDetector(surfaceView, cameraManipulator) } Touch Movements fun onTouchEvent(event: MotionEvent) { gestureDetector.onTouchEvent(event) } fun render(frameTimeNanos: Long) { ... cameraManipulator.getLookAt(eyePos, target, upward) camera.lookAt( eyePos[0], eyePos[1], eyePos[2], target[0], target[1], target[2], upward[0], upward[1], upward[2] ) } }
34. Skyboxes
35. No background class FilamentViewModel(...): ViewModel() { init { ... newScene.skybox = null } }
36. No background class FilamentViewModel(...): ViewModel() { init { ... newScene.skybox = null } } class ModelViewer() { val view: View = engine.createView().also { it.blendMode = View.BlendMode.TRANSLUCENT } init { textureView.isOpaque = false uiHelper.isOpaque = false renderer.clearOptions = renderer.clearOptions.apply { clear = true } } }
37. @Composable fun FilamentViewer(item: Item) { ... SideEffect { val (engine, _, asset) = item.itemScene modelViewer?.scene = item.itemScene.scene Material manipulation: color } ... }
38. @Composable fun FilamentViewer(item: Item) { ... SideEffect { val (engine, _, asset) = item.itemScene modelViewer?.scene = item.itemScene.scene Material manipulation: color asset.entities.find { asset.getName(it)?.startsWith("imagetostl") ?: false }?.also { entity -> val manager = engine.renderableManager val instance = manager.getInstance(entity) val material = manager.getMaterialInstanceAt(instance, 0) val r = item.currentColor.red val g = item.currentColor.green val b = item.currentColor.blue material.setParameter( "baseColorFactor", Colors.RgbaType.SRGB, r, g, b, 1.0f ) } } ... val manager = engine.renderableManager val instance = manager.getInstance(entity) val material = manager.getMaterialInstanceAt(instance, 0) material.material.parameters
39. deEffect { val (engine, _, asset) = item.itemScene modelViewer?.scene = item.itemScene.scene asset.entities.find { asset.getName(it)?.startsWith("imagetostl") ?: false }?.also { entity -> val manager = engine.renderableManager val instance = manager.getInstance(entity) val material = manager.getMaterialInstanceAt(instance, 0) val r = item.currentColor.red val g = item.currentColor.green val b = item.currentColor.blue material.setParameter( "baseColorFactor", Colors.RgbaType.SRGB, r, g, b, 1.0f ) } } ... val manager = engine.renderableManager val instance = manager.getInstance(entity) val material = manager.getMaterialInstanceAt(instance, 0) material.material.parameters Name: specularFactor, type: FLOAT3, prec: DEFAULT Name: glossinessFactor, type: FLOAT, prec: DEFAULT Name: baseColorIndex, type: INT, prec: DEFAULT Name: baseColorFactor, type: FLOAT4, prec: DEFAULT Name: baseColorUvMatrix, type: MAT3, prec: HIGH Name: metallicRoughnessIndex, type: INT, prec: DEFAULT Name: metallicFactor, type: FLOAT, prec: DEFAULT Name: roughnessFactor, type: FLOAT, prec: DEFAULT Name: metallicRoughnessUvMatrix, type: MAT3, prec: HIGH Name: normalIndex, type: INT, prec: DEFAULT Name: normalScale, type: FLOAT, prec: DEFAULT Name: normalUvMatrix, type: MAT3, prec: HIGH Name: aoIndex, type: INT, prec: DEFAULT Name: specularFactor, type: FLOAT3, prec: DEFAULT Name: glossinessFactor, type: FLOAT, prec: DEFAULT Name: baseColorIndex, type: INT, prec: DEFAULT Name: baseColorFactor, type: FLOAT4, prec: DEFAULT Name: baseColorUvMatrix, type: MAT3, prec: HIGH Name: metallicRoughnessIndex, type: INT, prec: DEFAULT Name: metallicFactor, type: FLOAT, prec: DEFAULT Name: roughnessFactor, type: FLOAT, prec: DEFAULT Name: metallicRoughnessUvMatrix, type: MAT3, prec: HIGH Name: normalIndex, type: INT, prec: DEFAULT Name: normalScale, type: FLOAT, prec: DEFAULT Name: normalUvMatrix, type: MAT3, prec: HIGH Name: aoIndex, type: INT, prec: DEFAULT Material Parameters
40. @Composable fun FilamentViewer(item: Item) { ... SideEffect { val (engine, _, asset) = item.itemScene modelViewer?.scene = item.itemScene.scene asset.entities.find { asset.getName(it)?.startsWith("imagetostl") ?: false }?.also { entity -> val manager = engine.renderableManager val instance = manager.getInstance(entity) val material = manager.getMaterialInstanceAt(instance, 0) val r = item.currentColor.red val g = item.currentColor.green val b = item.currentColor.blue material.setParameter( "baseColorFactor", Colors.RgbaType.SRGB, r, g, b, 1.0f ) } } ... Material Parameters val manager = engine.renderableManager val instance = manager.getInstance(entity) val material = manager.getMaterialInstanceAt(instance, 0) material.material.parameters Name: specularFactor, type: FLOAT3, prec: DEFAULT Name: glossinessFactor, type: FLOAT, prec: DEFAULT Name: baseColorIndex, type: INT, prec: DEFAULT Name: baseColorFactor, type: FLOAT4, prec: DEFAULT Name: baseColorUvMatrix, type: MAT3, prec: HIGH Name: metallicRoughnessIndex, type: INT, prec: DEFAULT Name: metallicFactor, type: FLOAT, prec: DEFAULT Name: roughnessFactor, type: FLOAT, prec: DEFAULT Name: metallicRoughnessUvMatrix, type: MAT3, prec: HIGH Name: specularFactor, type: FLOAT3, prec: DEFAULT Name: glossinessFactor, type: FLOAT, prec: DEFAULT Name: baseColorIndex, type: INT, prec: DEFAULT Name: baseColorFactor, type: FLOAT4, prec: DEFAULT Name: baseColorUvMatrix, type: MAT3, prec: HIGH Name: metallicRoughnessIndex, type: INT, prec: DEFAULT Name: metallicFactor, type: FLOAT, prec: DEFAULT Name: roughnessFactor, type: FLOAT, prec: DEFAULT Name: metallicRoughnessUvMatrix, type: MAT3, prec: HIGH material.setParameter("metallicFactor", item.material.metallicFactor) material.setParameter("roughnessFactor", item.material.roughness)
41. Model Animations
42. data class Animation( val name: String, val durationSeconds: Float, val targetState: State, val startNano: Long ) { enum class State { On, Off } } Model Animations
43. fun extractAnimations(asset: FilamentAsset): ImmutableList<Animation> { val animator = asset.instance.animator return buildList { repeat(animator.animationCount) { animationId -> add( Animation( name = animator.getAnimationName(animationId), durationSeconds = animator.getAnimationDuration(animationId), targetState = Animation.State.Off, startNano = 0L ) ) } }.toImmutableList() } Model Animations fun extractAnimations(asset: FilamentAsset): ImmutableList<Animation> { val animator = asset.instance.animator return buildList { repeat(animator.animationCount) { animationId -> add( Animation( name = animator.getAnimationName(animationId), durationSeconds = animator.getAnimationDuration(animationId), targetState = Animation.State.Off, startNano = 0L ) ) } }.toImmutableList() }
44. Model Animations class ModelViewer(...) { inner class FrameCallback : Choreographer.FrameCallback { override fun doFrame(frameTimeNanos: Long) { choreographer.postFrameCallback(this) animations.forEachIndexed { id, animation -> filamentAnimator?.apply { val elapsedTimeSeconds = (frameTimeNanos - animation.startNano).toDouble() / 1_000_000_000 if (elapsedTimeSeconds <= animation.durationSeconds) { applyAnimation( id, when (animation.targetState) { Animation.State.On -> elapsedTimeSeconds Animation.State.Off -> animation.durationSeconds - elapsedTimeSeconds } ) } updateBoneMatrices() } } render(frameTimeNanos) } } }
45. Model Animations er(...) { FrameCallback : Choreographer.FrameCallback { e fun doFrame(frameTimeNanos: Long) { reographer.postFrameCallback(this) mations.forEachIndexed { id, animation -> filamentAnimator?.apply { val elapsedTimeSeconds = (frameTimeNanos - animation.startNano).toDouble() / 1_000_000_000 if (elapsedTimeSeconds <= animation.durationSeconds) { applyAnimation( id, when (animation.targetState) { Animation.State.On -> elapsedTimeSeconds Animation.State.Off -> animation.durationSeconds - elapsedTimeSeconds } ) } updateBoneMatrices() } der(frameTimeNanos)
46. @Composable fun ItemsListScreen( items: ImmutableList<Item>, modifier: Modifier ) { LazyColumn(modifier = modifier) { items( items = items, itemContent = { item -> ItemCard(item) } ) } } List of items
47. List of items @Composable fun ItemCard(item: Item) { Surface( modifier = Modifier.padding(horizontal = 16.dp, vertical = 8.dp), shape = RoundedCornerShape(8.dp), shadowElevation = 4.dp ) { Column { Box(Modifier.fillMaxWidth().height(200.dp)) { FilamentViewer(item = item, autoRotate = true) } Row(Modifier.padding(8.dp).fillMaxWidth()) { Text(text = item.name, Modifier.padding(start = 8.dp)) } } } }
48. viewModelScope.launch { ... allItems.forEachIndexed { index, item -> val newScene = engine.createScene() val asset = readCompressedAsset(application, "models/${item.assetModel}.glb").let { ... } newScene.addEntities(asset.entities) newScene.indirectLight = indirectLight newScene.skybox = null newScene.addEntity(light) val itemScene = ItemScene( engine = engine, scene = newScene, resourceLoader = resourceLoader, asset = asset ) items.add( Item( id = index.toString(), name = item.name, printTime = formatPrintTime(item.printTimeMin), itemScene = itemScene, material = ItemMaterial() ) ) } itemsUiState = itemsUiState.copy( items = items.toImmutableList() ) } Shared Filament components
49. viewModelScope.launch { ... allItems.forEachIndexed { index, item -> val newScene = engine.createScene() val asset = readCompressedAsset(application, "models/${item.assetModel}.glb").let { ... } newScene.addEntities(asset.entities) newScene.indirectLight = indirectLight newScene.skybox = null newScene.addEntity(light) val itemScene = ItemScene( engine = engine, scene = newScene, resourceLoader = resourceLoader, asset = asset ) items.add( Item( id = index.toString(), name = item.name, printTime = formatPrintTime(item.printTimeMin), itemScene = itemScene, material = ItemMaterial() ) ) } viewModelScope.launch { ... allItems.forEachIndexed { index, item -> val newScene = engine.createScene() val asset = readCompressedAsset(application, "models/${item.assetModel}.glb").let { ... } newScene.addEntities(asset.entities) newScene.indirectLight = indirectLight newScene.skybox = null newScene.addEntity(light) val itemScene = ItemScene( engine = engine, scene = newScene, resourceLoader = resourceLoader, asset = asset ) items.add( Item( id = index.toString(), name = item.name, printTime = formatPrintTime(item.printTimeMin), itemScene = itemScene, material = ItemMaterial() ) ) } Shared Filament components
50. Navigation val navController = rememberNavController() NavHost(navController = navController, startDestination = "itemsList") { composable("itemsList") { ItemsListScreen( items = itemsUiState.items, onItemSelected = { navController.navigate("itemScreen/$it") }, ... ) } composable("itemScreen/{itemId}") { backStackEntry -> val itemId = backStackEntry.arguments?.getString("itemId") val item = itemsUiState.items.first { it.id == itemId } ItemScreen( item = item, ... ) } }
51. Resources Filament 3D skyboxes, textures and models Demo app repo https://github.com/danybony/filament-demo-3d-printer https://polyhaven.com https://google.github.io/filament
52. Daniele Bonaldo Thank You! @danybony_ @daniele@androiddev.social danielebonaldo.com