Share
How to write and automate performance tests in Android
Performance testing has always been neglected by developers. We write unit tests and integration tests but for some reason we never test the performance of our apps and when we do is already too late because our users are complaining about it. It has not been historically easy to write these tests and we usually rely on manual testing, which is not really accurate, or complicated tools that we don’t even know how to use or we don’t understand. But, is there an easy way to write and automate performance tests? Yes there is, and I’m going to show you how you can do it.
In this talk you will learn how you can write and automate performance tests for your Android apps. We will explore different ways of writing performance tests and we will see how we can take advantage of a few not well-known APIs that the Android platform offer us to help us in our journey towards the perfect automation testing suite.
Marcos
Transcript
195 slides
-
@orbycius Slow & clunky Some user “ ” “ ”
Laggs massively Some other user -
@orbycius Slow & clunky Some user “ ” “ ”
Laggs massively Some other user “ ” So laggy it makes me sad Another user -
It doesn’t matter how much value your app gives to
users if they can’t use it @orbycius -
@RunWith(AndroidJUnit4::class) @LargeTest class FirstTest { @get:Rule val activityRule = ActivityTestRule(MainActivity::class.java)
@Before fun setUp() { // do something } @After fun after() { // do something } @Test fun testFirst() { // do something } } @orbycius @RunWith(AndroidJUnit4::class) @LargeTest class FirstTest { @get:Rule val activityRule = ActivityTestRule(MainActivity::class.java) @Before fun setUp() { // do something } @After fun after() { // do something } @Test fun testFirst() { // do something } } -
@RunWith(AndroidJUnit4::class) @LargeTest class FirstTest { @get:Rule val activityRule = ActivityTestRule(MainActivity::class.java)
@Before fun setUp() { // do something } @After fun after() { // do something } @Test fun testFirst() { // do something } } @RunWith(AndroidJUnit4::class) @LargeTest class FirstTest { @get:Rule val activityRule = ActivityTestRule(MainActivity::class.java) @Before fun setUp() { // do something } @After fun after() { // do something } @Test fun testFirst() { // do something } } @orbycius -
@RunWith(AndroidJUnit4::class) @LargeTest class FirstTest { @get:Rule val activityRule = ActivityTestRule(MainActivity::class.java)
@Before fun setUp() { // do something } @After fun after() { // do something } @Test fun testFirst() { // do something } } @RunWith(AndroidJUnit4::class) @LargeTest class FirstTest { @get:Rule val activityRule = ActivityTestRule(MainActivity::class.java) @Before fun setUp() { // do something } @After fun after() { // do something } @Test fun testFirst() { // do something } } @orbycius -
@RunWith(AndroidJUnit4::class) @LargeTest class FirstTest { @get:Rule val activityRule = ActivityTestRule(MainActivity::class.java)
@Before fun setUp() { // do something } @After fun after() { // do something } @Test fun testFirst() { // do something } } @orbycius -
class MyActivityTestRule<T: Activity>(activityClass: Class<T>): ActivityTestRule<T>(activityClass) { var timestamp: Long =
0 @orbycius } class MyActivityTestRule<T: Activity>(activityClass: Class<T>): ActivityTestRule<T>(activityClass) { var timestamp: Long = 0 override fun beforeActivityLaunched() { timestamp = System.currentTimeMillis() super.beforeActivityLaunched() } -
class MyActivityTestRule<T: Activity>(activityClass: Class<T>): ActivityTestRule<T>(activityClass) { var timestamp: Long =
0 @orbycius } class MyActivityTestRule<T: Activity>(activityClass: Class<T>): ActivityTestRule<T>(activityClass) { var timestamp: Long = 0 override fun beforeActivityLaunched() { timestamp = System.currentTimeMillis() super.beforeActivityLaunched() } class MyActivityTestRule<T: Activity>(activityClass: Class<T>): ActivityTestRule<T>(activityClass) { var timestamp: Long = 0 override fun beforeActivityLaunched() { timestamp = System.currentTimeMillis() super.beforeActivityLaunched() } override fun afterActivityLaunched() { super.afterActivityLaunched() timestamp = System.currentTimeMillis() - timestamp } -
@orbycius @RunWith(AndroidJUnit4::class) class FirstTest { @get:Rule val activityRule = MyActivityTestRule(MainActivity::class.java)
@Test fun startUpTest() { assertWithMessage("Start up time: ${activityRule.timestamp}") .that(activityRule.timestamp) .isLessThan(300) } } -
@orbycius FrameMetrics Animation Duration Command Issue Duration Draw Duration First
Draw Frame Input Handling Duration Intended Vsync Timestamp Layout Measure Duration Swap Buffers Duration Sync Duration Total Duration Unknown Delay Duration Vsync Timestamp -
@orbycius class MainAdapter : RecyclerView.Adapter<MainViewHolder>() { override fun onCreateViewHolder(parent: ViewGroup,
viewType: Int): MainViewHolder { val view = LayoutInflater .from(parent.context) .inflate(R.layout.item, parent, false) return MainViewHolder(view) } override fun onBindViewHolder(holder: MainViewHolder, pos: Int) { (holder.itemView as TextView).text = "This is item $pos" // Add sleep to recreate work on the main thread Thread.sleep(150) } override fun getItemCount(): Int = 200 } -
class MainAdapter : RecyclerView.Adapter<MainViewHolder>() { override fun onCreateViewHolder(parent: ViewGroup, viewType:
Int): MainViewHolder { val view = LayoutInflater .from(parent.context) .inflate(R.layout.item, parent, false) return MainViewHolder(view) } override fun onBindViewHolder(holder: MainViewHolder, pos: Int) { (holder.itemView as TextView).text = "This is item $pos" // Add sleep to recreate work on the main thread Thread.sleep(150) } override fun getItemCount(): Int = 200 } @orbycius -
@orbycius @Test fun testFirst() { for (i in 0..30) {
Espresso.onView(ViewMatchers.withId(R.id.recyclerView)) .perform(RecyclerViewActions.scrollToPosition<RecyclerView.ViewHolder>(i)) } for (i in 30.downTo(1)) { Espresso.onView(ViewMatchers.withId(R.id.recyclerView)) .perform(RecyclerViewActions.scrollToPosition<RecyclerView.ViewHolder>(i)) } assertWithMessage("Janky frames over $PERCENTAGE% value was $percJankyFrames%") .that(percJankyFrames) .isLessThan(PERCENTAGE) } var percJankyFrames = 0f -
@Test fun testFirst() { @orbycius for (i in 0..30) {
Espresso.onView(ViewMatchers.withId(R.id.recyclerView)) .perform(RecyclerViewActions.scrollToPosition<RecyclerView.ViewHolder>(i)) val handler = Handler(Looper.getMainLooper()) activityRule.activity.window .addOnFrameMetricsAvailableListener(object : Window.OnFrameMetricsAvailableListener { private var totalFrames = 0 private var jankyFrames = 0 override fun onFrameMetricsAvailable( window: Window, frameMetrics: FrameMetrics, drop: Int ) { totalFrames++ val duration = (0.000001 * frameMetrics.getMetric(FrameMetrics.TOTAL_DURATION)).toFloat() if (duration > 16f) { jankyFrames++ percJankyFrames = jankyFrames.toFloat() / totalFrames * 100 } } }, handler) var percJankyFrames = 0f -
val handler = Handler(Looper.getMainLooper()) activityRule.activity.window .addOnFrameMetricsAvailableListener(object : Window.OnFrameMetricsAvailableListener { private
var totalFrames = 0 private var jankyFrames = 0 override fun onFrameMetricsAvailable( window: Window, frameMetrics: FrameMetrics, drop: Int ) { totalFrames++ val duration = (0.000001 * frameMetrics.getMetric(FrameMetrics.TOTAL_DURATION)).toFloat() if (duration > 16f) { jankyFrames++ percJankyFrames = jankyFrames.toFloat() / totalFrames * 100 } } }, handler) val handler = Handler(Looper.getMainLooper()) activityRule.activity.window .addOnFrameMetricsAvailableListener(object : Window.OnFrameMetricsAvailableListener { private var totalFrames = 0 private var jankyFrames = 0 override fun onFrameMetricsAvailable( window: Window, frameMetrics: FrameMetrics, drop: Int ) { totalFrames++ val duration = (0.000001 * frameMetrics.getMetric(FrameMetrics.TOTAL_DURATION)).toFloat() if (duration > 16f) { jankyFrames++ percJankyFrames = jankyFrames.toFloat() / totalFrames * 100 } } }, handler) @orbycius var percJankyFrames = 0f @Test fun testFirst() { for (i in 0..30) { Espresso.onView(ViewMatchers.withId(R.id.recyclerView)) .perform(RecyclerViewActions.scrollToPosition<RecyclerView.ViewHolder>(i)) -
val handler = Handler(Looper.getMainLooper()) activityRule.activity.window .addOnFrameMetricsAvailableListener(object : Window.OnFrameMetricsAvailableListener { private
var totalFrames = 0 private var jankyFrames = 0 override fun onFrameMetricsAvailable( window: Window, frameMetrics: FrameMetrics, drop: Int ) { totalFrames++ val duration = (0.000001 * frameMetrics.getMetric(FrameMetrics.TOTAL_DURATION)).toFloat() if (duration > 16f) { jankyFrames++ percJankyFrames = jankyFrames.toFloat() / totalFrames * 100 } } }, handler) val handler = Handler(Looper.getMainLooper()) activityRule.activity.window .addOnFrameMetricsAvailableListener(object : Window.OnFrameMetricsAvailableListener { private var totalFrames = 0 private var jankyFrames = 0 override fun onFrameMetricsAvailable( window: Window, frameMetrics: FrameMetrics, drop: Int ) { totalFrames++ val duration = (0.000001 * frameMetrics.getMetric(FrameMetrics.TOTAL_DURATION)).toFloat() if (duration > 16f) { jankyFrames++ percJankyFrames = jankyFrames.toFloat() / totalFrames * 100 } } }, handler) @orbycius var percJankyFrames = 0f @Test fun testFirst() { for (i in 0..30) { Espresso.onView(ViewMatchers.withId(R.id.recyclerView)) .perform(RecyclerViewActions.scrollToPosition<RecyclerView.ViewHolder>(i)) -
val handler = Handler(Looper.getMainLooper()) activityRule.activity.window .addOnFrameMetricsAvailableListener(object : Window.OnFrameMetricsAvailableListener { private
var totalFrames = 0 private var jankyFrames = 0 override fun onFrameMetricsAvailable( window: Window, frameMetrics: FrameMetrics, drop: Int ) { totalFrames++ val duration = (0.000001 * frameMetrics.getMetric(FrameMetrics.TOTAL_DURATION)).toFloat() if (duration > 16f) { jankyFrames++ percJankyFrames = jankyFrames.toFloat() / totalFrames * 100 } } }, handler) val handler = Handler(Looper.getMainLooper()) activityRule.activity.window .addOnFrameMetricsAvailableListener(object : Window.OnFrameMetricsAvailableListener { private var totalFrames = 0 private var jankyFrames = 0 override fun onFrameMetricsAvailable( window: Window, frameMetrics: FrameMetrics, drop: Int ) { totalFrames++ val duration = (0.000001 * frameMetrics.getMetric(FrameMetrics.TOTAL_DURATION)).toFloat() if (duration > 16f) { jankyFrames++ percJankyFrames = jankyFrames.toFloat() / totalFrames * 100 } } }, handler) @orbycius var percJankyFrames = 0f @Test fun testFirst() { for (i in 0..30) { Espresso.onView(ViewMatchers.withId(R.id.recyclerView)) .perform(RecyclerViewActions.scrollToPosition<RecyclerView.ViewHolder>(i)) -
val handler = Handler(Looper.getMainLooper()) activityRule.activity.window .addOnFrameMetricsAvailableListener(object : Window.OnFrameMetricsAvailableListener { private
var totalFrames = 0 private var jankyFrames = 0 override fun onFrameMetricsAvailable( window: Window, frameMetrics: FrameMetrics, drop: Int ) { totalFrames++ val duration = (0.000001 * frameMetrics.getMetric(FrameMetrics.TOTAL_DURATION)).toFloat() if (duration > 16f) { jankyFrames++ percJankyFrames = jankyFrames.toFloat() / totalFrames * 100 } } }, handler) val handler = Handler(Looper.getMainLooper()) activityRule.activity.window .addOnFrameMetricsAvailableListener(object : Window.OnFrameMetricsAvailableListener { private var totalFrames = 0 private var jankyFrames = 0 override fun onFrameMetricsAvailable( window: Window, frameMetrics: FrameMetrics, drop: Int ) { totalFrames++ val duration = (0.000001 * frameMetrics.getMetric(FrameMetrics.TOTAL_DURATION)).toFloat() if (duration > 16f) { jankyFrames++ percJankyFrames = jankyFrames.toFloat() / totalFrames * 100 } } }, handler) @orbycius var percJankyFrames = 0f @Test fun testFirst() { for (i in 0..30) { Espresso.onView(ViewMatchers.withId(R.id.recyclerView)) .perform(RecyclerViewActions.scrollToPosition<RecyclerView.ViewHolder>(i)) -
val handler = Handler(Looper.getMainLooper()) activityRule.activity.window .addOnFrameMetricsAvailableListener(object : Window.OnFrameMetricsAvailableListener { private
var totalFrames = 0 private var jankyFrames = 0 override fun onFrameMetricsAvailable( window: Window, frameMetrics: FrameMetrics, drop: Int ) { totalFrames++ val duration = (0.000001 * frameMetrics.getMetric(FrameMetrics.TOTAL_DURATION)).toFloat() if (duration > 16f) { jankyFrames++ percJankyFrames = jankyFrames.toFloat() / totalFrames * 100 } } }, handler) val handler = Handler(Looper.getMainLooper()) activityRule.activity.window .addOnFrameMetricsAvailableListener(object : Window.OnFrameMetricsAvailableListener { private var totalFrames = 0 private var jankyFrames = 0 override fun onFrameMetricsAvailable( window: Window, frameMetrics: FrameMetrics, drop: Int ) { totalFrames++ val duration = (0.000001 * frameMetrics.getMetric(FrameMetrics.TOTAL_DURATION)).toFloat() if (duration > 16f) { jankyFrames++ percJankyFrames = jankyFrames.toFloat() / totalFrames * 100 } } }, handler) @orbycius var percJankyFrames = 0f @Test fun testFirst() { for (i in 0..30) { Espresso.onView(ViewMatchers.withId(R.id.recyclerView)) .perform(RecyclerViewActions.scrollToPosition<RecyclerView.ViewHolder>(i)) -
val handler = Handler(Looper.getMainLooper()) activityRule.activity.window .addOnFrameMetricsAvailableListener(object : Window.OnFrameMetricsAvailableListener { private
var totalFrames = 0 private var jankyFrames = 0 override fun onFrameMetricsAvailable( window: Window, frameMetrics: FrameMetrics, drop: Int ) { totalFrames++ val duration = (0.000001 * frameMetrics.getMetric(FrameMetrics.TOTAL_DURATION)).toFloat() if (duration > 16f) { jankyFrames++ percJankyFrames = jankyFrames.toFloat() / totalFrames * 100 } } }, handler) @orbycius var percJankyFrames = 0f @Test fun testFirst() { for (i in 0..30) { Espresso.onView(ViewMatchers.withId(R.id.recyclerView)) .perform(RecyclerViewActions.scrollToPosition<RecyclerView.ViewHolder>(i)) -
@orbycius Results companion object { private const val PERCENTAGE =
20f } // with Thread.sleep() Percentage of janky frames was 81.08% // without Thread.sleep() Percentage of janky frames was 0% -
@orbycius for (i in 0..30) { Espresso.onView(ViewMatchers.withId(R.id.recyclerView)) .perform(RecyclerViewActions.scrollToPosition<RecyclerView.ViewHolder>(i)) } for
(i in 30.downTo(1)) { Espresso.onView(ViewMatchers.withId(R.id.recyclerView)) .perform(RecyclerViewActions.scrollToPosition<RecyclerView.ViewHolder>(i)) } Espresso UIAutomator -
@orbycius for (i in 0..30) { Espresso.onView(ViewMatchers.withId(R.id.recyclerView)) .perform(RecyclerViewActions.scrollToPosition<RecyclerView.ViewHolder>(i)) } for
(i in 30.downTo(1)) { Espresso.onView(ViewMatchers.withId(R.id.recyclerView)) .perform(RecyclerViewActions.scrollToPosition<RecyclerView.ViewHolder>(i)) } Espresso @Before fun setUp() { device = UiDevice.getInstance(InstrumentationRegistry.getInstrumentation()) } val appViews = UiScrollable(UiSelector().scrollable(true)) appViews.setAsVerticalList() appViews.scrollTextIntoView("This is item 24") appViews.scrollTextIntoView("This is item 1") UIAutomator -
@orbycius // with Thread.sleep() Percentage of janky frames was 81.08%
// without Thread.sleep() Percentage of janky frames was 0% Espresso UIAutomator // with Thread.sleep() Percentage of janky frames was 9.89% // without Thread.sleep() Percentage of janky frames was 0% -
@UseMonitorFactory interface IMonitor interface IMonitorFactory @GfxFrameStatsMonitor @GfxMonitor @JankTest androidx.test.jank androidx.test.jank.annotations
@orbycius androidx.test.jank androidx.test.jank.annotations @GfxFrameStatsMonitor @GfxMonitor @JankTest @UseMonitorFactory interface IMonitor interface IMonitorFactory -
@UseMonitorFactory interface IMonitor interface IMonitorFactory @GfxFrameStatsMonitor @GfxMonitor @JankTest androidx.test.jank androidx.test.jank.annotations
@orbycius androidx.test.jank androidx.test.jank.annotations @GfxFrameStatsMonitor @GfxMonitor @JankTest @UseMonitorFactory interface IMonitor interface IMonitorFactory class JankTestBase class JankTestBase -
@UseMonitorFactory interface IMonitor interface IMonitorFactory @GfxFrameStatsMonitor @GfxMonitor @JankTest androidx.test.jank androidx.test.jank.annotations
@orbycius class JankTestBase class JankTestBase -
class SecondTest : JankTestBase() { private lateinit var device: UiDevice
@Throws(Exception::class) public override fun setUp() { super.setUp() device = UiDevice.getInstance(instrumentation) } private fun launchApp(packageName: String) { val pm = instrumentation.context.packageManager val appIntent = pm.getLaunchIntentForPackage(packageName) appIntent!!.flags = Intent.FLAG_ACTIVITY_NEW_TASK instrumentation.context.startActivity(appIntent) } @Throws(OperationApplicationException::class, RemoteException::class) fun launchApp() { launchApp(PACKAGE_NAME) device.waitForIdle() } } @orbycius class SecondTest : JankTestBase() { private lateinit var device: UiDevice @Throws(Exception::class) public override fun setUp() { super.setUp() device = UiDevice.getInstance(instrumentation) } private fun launchApp(packageName: String) { val pm = instrumentation.context.packageManager val appIntent = pm.getLaunchIntentForPackage(packageName) appIntent!!.flags = Intent.FLAG_ACTIVITY_NEW_TASK instrumentation.context.startActivity(appIntent) } @Throws(OperationApplicationException::class, RemoteException::class) fun launchApp() { launchApp(PACKAGE_NAME) device.waitForIdle() } } -
class SecondTest : JankTestBase() { private lateinit var device: UiDevice
@Throws(Exception::class) public override fun setUp() { super.setUp() device = UiDevice.getInstance(instrumentation) } private fun launchApp(packageName: String) { val pm = instrumentation.context.packageManager val appIntent = pm.getLaunchIntentForPackage(packageName) appIntent!!.flags = Intent.FLAG_ACTIVITY_NEW_TASK instrumentation.context.startActivity(appIntent) } @Throws(OperationApplicationException::class, RemoteException::class) fun launchApp() { launchApp(PACKAGE_NAME) device.waitForIdle() } } class SecondTest : JankTestBase() { private lateinit var device: UiDevice @Throws(Exception::class) public override fun setUp() { super.setUp() device = UiDevice.getInstance(instrumentation) } private fun launchApp(packageName: String) { val pm = instrumentation.context.packageManager val appIntent = pm.getLaunchIntentForPackage(packageName) appIntent!!.flags = Intent.FLAG_ACTIVITY_NEW_TASK instrumentation.context.startActivity(appIntent) } @Throws(OperationApplicationException::class, RemoteException::class) fun launchApp() { launchApp(PACKAGE_NAME) device.waitForIdle() } } @orbycius -
class SecondTest : JankTestBase() { private lateinit var device: UiDevice
@Throws(Exception::class) public override fun setUp() { super.setUp() device = UiDevice.getInstance(instrumentation) } private fun launchApp(packageName: String) { val pm = instrumentation.context.packageManager val appIntent = pm.getLaunchIntentForPackage(packageName) appIntent!!.flags = Intent.FLAG_ACTIVITY_NEW_TASK instrumentation.context.startActivity(appIntent) } @Throws(OperationApplicationException::class, RemoteException::class) fun launchApp() { launchApp(PACKAGE_NAME) device.waitForIdle() } } @orbycius -
@orbycius class SecondTest : JankTestBase() { @JankTest( beforeTest = "launchApp",
expectedFrames = EXPECTED_FRAMES, defaultIterationCount = 1 ) @GfxMonitor(processName = PACKAGE_NAME) fun testSecond() { for (i in 0 until INNER_LOOP) { val appViews = UiScrollable(UiSelector().scrollable(true)) appViews.setAsVerticalList() appViews.scrollTextIntoView("This is item 24") appViews.scrollTextIntoView("This is item 1") } SystemClock.sleep(200) } companion object { private const val INNER_LOOP = 2 private const val EXPECTED_FRAMES = 450 private const val PACKAGE_NAME = "com.marcosholgado.performancetest" } -
@JankTest( beforeTest = "launchApp", expectedFrames = EXPECTED_FRAMES, defaultIterationCount = 1
) @GfxMonitor(processName = PACKAGE_NAME) fun testSecond() { for (i in 0 until INNER_LOOP) { val appViews = UiScrollable(UiSelector().scrollable(true)) appViews.setAsVerticalList() appViews.scrollTextIntoView("This is item 24") appViews.scrollTextIntoView("This is item 1") } SystemClock.sleep(200) } companion object { private const val INNER_LOOP = 2 private const val EXPECTED_FRAMES = 450 private const val PACKAGE_NAME = "com.marcosholgado.performancetest" } @JankTest( beforeTest = "launchApp", expectedFrames = EXPECTED_FRAMES, defaultIterationCount = 1 ) @GfxMonitor(processName = PACKAGE_NAME) fun testSecond() { for (i in 0 until INNER_LOOP) { val appViews = UiScrollable(UiSelector().scrollable(true)) appViews.setAsVerticalList() appViews.scrollTextIntoView("This is item 24") appViews.scrollTextIntoView("This is item 1") } SystemClock.sleep(200) } companion object { private const val INNER_LOOP = 2 private const val EXPECTED_FRAMES = 450 private const val PACKAGE_NAME = "com.marcosholgado.performancetest" } @orbycius class SecondTest : JankTestBase() { -
@JankTest( beforeTest = "launchApp", expectedFrames = EXPECTED_FRAMES, defaultIterationCount = 1
) @GfxMonitor(processName = PACKAGE_NAME) fun testSecond() { for (i in 0 until INNER_LOOP) { val appViews = UiScrollable(UiSelector().scrollable(true)) appViews.setAsVerticalList() appViews.scrollTextIntoView("This is item 24") appViews.scrollTextIntoView("This is item 1") } SystemClock.sleep(200) } companion object { private const val INNER_LOOP = 2 private const val EXPECTED_FRAMES = 450 private const val PACKAGE_NAME = "com.marcosholgado.performancetest" } @JankTest( beforeTest = "launchApp", expectedFrames = EXPECTED_FRAMES, defaultIterationCount = 1 ) @GfxMonitor(processName = PACKAGE_NAME) fun testSecond() { for (i in 0 until INNER_LOOP) { val appViews = UiScrollable(UiSelector().scrollable(true)) appViews.setAsVerticalList() appViews.scrollTextIntoView("This is item 24") appViews.scrollTextIntoView("This is item 1") } SystemClock.sleep(200) } companion object { private const val INNER_LOOP = 2 private const val EXPECTED_FRAMES = 450 private const val PACKAGE_NAME = "com.marcosholgado.performancetest" } @orbycius class SecondTest : JankTestBase() { -
@JankTest( beforeTest = "launchApp", expectedFrames = EXPECTED_FRAMES, defaultIterationCount = 1
) @GfxMonitor(processName = PACKAGE_NAME) fun testSecond() { for (i in 0 until INNER_LOOP) { val appViews = UiScrollable(UiSelector().scrollable(true)) appViews.setAsVerticalList() appViews.scrollTextIntoView("This is item 24") appViews.scrollTextIntoView("This is item 1") } SystemClock.sleep(200) } companion object { private const val INNER_LOOP = 2 private const val EXPECTED_FRAMES = 450 private const val PACKAGE_NAME = "com.marcosholgado.performancetest" } @orbycius class SecondTest : JankTestBase() { -
@orbycius // with Thread.sleep() Expected frames was 450. Received frames
is 420 // without Thread.sleep() Expected frames was 450. Received frames is 476 -
@orbycius How to calculate totalFrames (threshold) Problems with GfxMonitor Frame
deadline errors junit.framework.AssertionFailedError: Failed to parse NUM_FRAME_DEADLINE_MISSED -
androidx.test.jank androidx.test.jank.annotations @GfxFrameStatsMonitor @GfxMonitor @JankTest @UseMonitorFactory interface IMonitor interface IMonitorFactory
class JankTestBase @UseMonitorFactory @orbycius androidx.test.jank androidx.test.jank.annotations @GfxFrameStatsMonitor @GfxMonitor @JankTest @UseMonitorFactory interface IMonitor interface IMonitorFactory class JankTestBase @UseMonitorFactory -
androidx.test.jank androidx.test.jank.annotations @GfxFrameStatsMonitor @GfxMonitor @JankTest @UseMonitorFactory interface IMonitor interface IMonitorFactory
class JankTestBase @UseMonitorFactory @orbycius -
@orbycius class ThirdTest : JankTestBase() { ... } class ThirdTest
: JankTestBase() { ... } -
} class MyFactory(private val instrumentation: Instrumentation): IMonitorFactory { @orbycius class
MyFactory(private val instrumentation: Instrumentation): IMonitorFactory { } -
override fun getMonitors(testMethod: Method?, instance: Any?): MutableList<IMonitor> { } }
class MyFactory(private val instrumentation: Instrumentation): IMonitorFactory { @orbycius override fun getMonitors(testMethod: Method?, instance: Any?): MutableList<IMonitor> { } -
val monitors = mutableListOf<IMonitor>() override fun getMonitors(testMethod: Method?, instance: Any?):
MutableList<IMonitor> { } } class MyFactory(private val instrumentation: Instrumentation): IMonitorFactory { @orbycius val monitors = mutableListOf<IMonitor>() -
val gfxMonitorArgs = testMethod?.getAnnotation(GfxMonitor::class.java) val monitors = mutableListOf<IMonitor>() override fun
getMonitors(testMethod: Method?, instance: Any?): MutableList<IMonitor> { } } class MyFactory(private val instrumentation: Instrumentation): IMonitorFactory { @orbycius val gfxMonitorArgs = testMethod?.getAnnotation(GfxMonitor::class.java) -
val gfxMonitorArgs = testMethod?.getAnnotation(GfxMonitor::class.java) val monitors = mutableListOf<IMonitor>() override fun
getMonitors(testMethod: Method?, instance: Any?): MutableList<IMonitor> { } } class MyFactory(private val instrumentation: Instrumentation): IMonitorFactory { @orbycius if (gfxMonitorArgs != null) { // GfxMonitor only works on M+ if (API_LEVEL_ACTUAL <= 22) { Log.w("PerfTest", "Skipping GfxMonitor. Not supported”) } else { monitors.add(MyMonitor( instrumentation, gfxMonitorArgs.processName)) } } return monitors -
class GfxMonitorImpl( private val instrumentation: Instrumentation, private val process: String
) : IMonitor { } @orbycius class GfxMonitorImpl( private val instrumentation: Instrumentation, private val process: String ) : IMonitor { } -
class GfxMonitorImpl( private val instrumentation: Instrumentation, private val process: String
) : IMonitor { } @orbycius override fun startIteration() { } override fun stopIteration(): Bundle { } override fun getMetrics(): Bundle { } -
@orbycius override fun startIteration() { // Clear out any previous
data val stdout = executeShellCommand( String.format("dumpsys gfxinfo %s reset", process) ) val reader = BufferedReader(InputStreamReader(stdout)) reader.use { reader -> // Read the output, but don't do anything with it while (reader.readLine() != null) { } } } override fun startIteration() { } override fun stopIteration(): Bundle { } override fun getMetrics(): Bundle { } -
override fun stopIteration(): Bundle { } @orbycius override fun stopIteration():
Bundle { } override fun startIteration() { } override fun stopIteration(): Bundle { } override fun getMetrics(): Bundle { } -
// Dump the latest stats val stdout = executeShellCommand( String.format("dumpsys
gfxinfo %s", process) ) override fun stopIteration(): Bundle { } @orbycius override fun startIteration() { } override fun stopIteration(): Bundle { } override fun getMetrics(): Bundle { } // Dump the latest stats val stdout = executeShellCommand( String.format("dumpsys gfxinfo %s", process) ) -
val reader = BufferedReader(InputStreamReader(stdout)) reader.use { reader -> var line:
String? = reader.readLine() do { // Parse line as frame stat value } } // Dump the latest stats val stdout = executeShellCommand( String.format("dumpsys gfxinfo %s", process) ) override fun stopIteration(): Bundle { } @orbycius override fun startIteration() { } override fun stopIteration(): Bundle { } override fun getMetrics(): Bundle { } val reader = BufferedReader(InputStreamReader(stdout)) reader.use { reader -> var line: String? = reader.readLine() do { // Parse line as frame stat value } } -
val reader = BufferedReader(InputStreamReader(stdout)) reader.use { reader -> var line:
String? = reader.readLine() do { // Parse line as frame stat value } } // Dump the latest stats val stdout = executeShellCommand( String.format("dumpsys gfxinfo %s", process) ) override fun stopIteration(): Bundle { } @orbycius override fun startIteration() { } override fun stopIteration(): Bundle { } override fun getMetrics(): Bundle { } // Make sure we found all the stats ... val ret = Bundle() ret.putInt("num-frames", jankyFreeFrames[jankyFreeFrames.size - 1]) return ret -
@orbycius // Parse line as frame stat value // Make
sure we found all the stats -
@orbycius // Parse line as frame stat value enum class
JankStat private constructor( private val parsePattern: Pattern, private val groupIndex: Int, internal val type: Class<*>, optional: Boolean = false ) { TOTAL_FRAMES( Pattern.compile("\\s*Total frames rendered: (\\d+)"), 1, Int::class.java ) } -
@orbycius // Parse line as frame stat value enum class
JankStat private constructor( private val parsePattern: Pattern, private val groupIndex: Int, internal val type: Class<*>, optional: Boolean = false ) { TOTAL_FRAMES( Pattern.compile("\\s*Total frames rendered: (\\d+)"), 1, Int::class.java ) } Total frames rendered: ### Janky frames: ### (##.##%) 50th percentile: ##ms 90th percentile: ##ms 95th percentile: ##ms 99th percentile: ##ms Number Missed Vsync: # Number High input latency: # Number Slow UI thread: # Number Slow bitmap uploads: # Number Slow draw: # Number Frame deadline missed: # -
@orbycius // Parse line as frame stat value for (stat
in JankStat.values()) { val part: String? = stat.parse(line!!) if (part != null) { when { stat.type == Int::class.java -> { val stats = accumulatedStats[stat] as MutableList<Int> stats.add(Integer.valueOf(part)) } stat.type == Double::class.java -> { val stats = accumulatedStats[stat] as MutableList<Double> stats.add(java.lang.Double.valueOf(part)) } else -> // Shouldn't get here throw IllegalStateException("Unsupported JankStat type") } break } } -
for (stat in JankStat.values()) { val part: String? = stat.parse(line!!)
if (part != null) { when { stat.type == Int::class.java -> { val stats = accumulatedStats[stat] as MutableList<Int> stats.add(Integer.valueOf(part)) } stat.type == Double::class.java -> { val stats = accumulatedStats[stat] as MutableList<Double> stats.add(java.lang.Double.valueOf(part)) } else -> // Shouldn't get here throw IllegalStateException("Unsupported JankStat type") } break } } @orbycius private val accumulatedStats = EnumMap<JankStat, List<Number>>(JankStat::class.java) // Parse line as frame stat value -
for (stat in JankStat.values()) { if (!stat.wasParsedSuccessfully() && !stat.isOptional) {
Assert.fail(String.format("Failed to parse %s", stat.name)) } stat.reset() } val jankyFreeFrames = accumulatedStats[JankStat.TOTAL_FRAMES] as List<Int> @orbycius // Make sure we found all the stats for (stat in JankStat.values()) { if (!stat.wasParsedSuccessfully() && !stat.isOptional) { Assert.fail(String.format("Failed to parse %s", stat.name)) } stat.reset() } val jankyFreeFrames = accumulatedStats[JankStat.TOTAL_FRAMES] as List<Int> -
for (stat in JankStat.values()) { if (!stat.wasParsedSuccessfully() && !stat.isOptional) {
Assert.fail(String.format("Failed to parse %s", stat.name)) } stat.reset() } val jankyFreeFrames = accumulatedStats[JankStat.TOTAL_FRAMES] as List<Int> @orbycius // Make sure we found all the stats val ret = Bundle() ret.putInt("num-frames", jankyFreeFrames[jankyFreeFrames.size - 1]) return ret -
override fun getMetrics(): Bundle { } @orbycius override fun startIteration()
{ } override fun stopIteration(): Bundle { } override fun getMetrics(): Bundle { } override fun getMetrics(): Bundle { } -
override fun getMetrics(): Bundle { } @orbycius override fun startIteration()
{ } override fun stopIteration(): Bundle { } override fun getMetrics(): Bundle { } val metrics = Bundle() // Retrieve the total number of frames val totals = accumulatedStats[JankStat.TOTAL_FRAMES] as List<Int> // Store avg, min and max of total frames metrics.putInt(GfxMonitor.KEY_AVG_TOTAL_FRAMES, computeAverage(totals)) metrics.putInt(GfxMonitor.KEY_MAX_TOTAL_FRAMES, Collections.max(totals)) metrics.putInt(GfxMonitor.KEY_MIN_TOTAL_FRAMES, Collections.min(totals)) ... return metrics -
@orbycius class GfxMonitorImpl( private val instrumentation: Instrumentation, private val process:
String ) : IMonitor { override fun startIteration() { } override fun stopIteration(): Bundle { } override fun getMetrics(): Bundle { } } -
@orbycius class GfxMonitorImpl( private val instrumentation: Instrumentation, private val process:
String ) : IMonitor { override fun startIteration() { } override fun stopIteration(): Bundle { } override fun getMetrics(): Bundle { } } junit.framework.AssertionFailedError: Failed to parse NUM_FRAME_DEADLINE_MISSED -
junit.framework.AssertionFailedError: Failed to parse NUM_FRAME_DEADLINE_MISSED for (stat in JankStat.values()) {
if (!stat.wasParsedSuccessfully() && !stat.isOptional) { Assert.fail(String.format("Failed to parse %s", stat.name)) } stat.reset() } @orbycius for (stat in JankStat.values()) { if (!stat.wasParsedSuccessfully() && !stat.isOptional) { Assert.fail(String.format("Failed to parse %s", stat.name)) } stat.reset() } junit.framework.AssertionFailedError: Failed to parse NUM_FRAME_DEADLINE_MISSED -
for (stat in JankStat.values()) { if (!stat.wasParsedSuccessfully() && !stat.isOptional) {
Assert.fail(String.format("Failed to parse %s", stat.name)) } stat.reset() } junit.framework.AssertionFailedError: Failed to parse NUM_FRAME_DEADLINE_MISSED for (stat in JankStat.values()) { if (!stat.wasParsedSuccessfully() && !stat.isOptional) { Assert.fail(String.format("Failed to parse %s", stat.name)) } stat.reset() } @orbycius -
enum class JankStat private constructor( private val parsePattern: Pattern, private
val groupIndex: Int, internal val type: Class<*>, optional: Boolean = false ) { TOTAL_FRAMES( Pattern.compile("\\s*Total frames rendered: (\\d+)"), 1, Int::class.java ), NUM_FRAME_DEADLINE_MISSED( Pattern.compile("\\s*Number Frame deadline missed: (\\d+)"), 1, Int::class.java junit.framework.AssertionFailedError: Failed to parse NUM_FRAME_DEADLINE_MISSED @orbycius junit.framework.AssertionFailedError: Failed to parse NUM_FRAME_DEADLINE_MISSED enum class JankStat private constructor( private val parsePattern: Pattern, private val groupIndex: Int, internal val type: Class<*>, optional: Boolean = false ) { TOTAL_FRAMES( Pattern.compile("\\s*Total frames rendered: (\\d+)"), 1, Int::class.java ), NUM_FRAME_DEADLINE_MISSED( Pattern.compile("\\s*Number Frame deadline missed: (\\d+)"), 1, Int::class.java ); ); -
enum class JankStat private constructor( private val parsePattern: Pattern, private
val groupIndex: Int, internal val type: Class<*>, optional: Boolean = false ) { TOTAL_FRAMES( Pattern.compile("\\s*Total frames rendered: (\\d+)"), 1, Int::class.java ), NUM_FRAME_DEADLINE_MISSED( Pattern.compile("\\s*Number Frame deadline missed: (\\d+)"), 1, Int::class.java junit.framework.AssertionFailedError: Failed to parse NUM_FRAME_DEADLINE_MISSED @orbycius junit.framework.AssertionFailedError: Failed to parse NUM_FRAME_DEADLINE_MISSED ); -
enum class JankStat private constructor( private val parsePattern: Pattern, private
val groupIndex: Int, internal val type: Class<*>, optional: Boolean = false ) { TOTAL_FRAMES( Pattern.compile("\\s*Total frames rendered: (\\d+)"), 1, Int::class.java ), NUM_FRAME_DEADLINE_MISSED( Pattern.compile("\\s*Number Frame deadline missed: (\\d+)"), 1, Int::class.java junit.framework.AssertionFailedError: Failed to parse NUM_FRAME_DEADLINE_MISSED @orbycius junit.framework.AssertionFailedError: Failed to parse NUM_FRAME_DEADLINE_MISSED ); , true -
@orbycius class MyMonitor( private val instrumentation: Instrumentation, private val process:
String ) : } GfxMonitorImpl { -
@orbycius class MyMonitor( private val instrumentation: Instrumentation, private val process:
String ) : } GfxMonitorImpl { -
@orbycius class MyMonitor( private val instrumentation: Instrumentation, private val process:
String ) : } IMonitor { -
val jankyFreeFrames = accumulatedStats[JankStat.TOTAL_FRAMES] as List<Int> val ret = Bundle()
ret.putInt("num-frames", jankyFreeFrames[jankyFreeFrames.size - 1]) return ret @orbycius val jankyFreeFrames = accumulatedStats[JankStat.TOTAL_FRAMES] as List<Int> val ret = Bundle() ret.putInt("num-frames", jankyFreeFrames[jankyFreeFrames.size - 1]) return ret -
val jankyFreeFrames = accumulatedStats[JankStat.TOTAL_FRAMES] as List<Int> val ret = Bundle()
ret.putInt("num-frames", jankyFreeFrames[jankyFreeFrames.size - 1]) return ret @orbycius -
val jankyFreeFrames = accumulatedStats[JankStat.TOTAL_FRAMES] as List<Int> val ret = Bundle()
ret.putInt("num-frames", jankyFreeFrames[jankyFreeFrames.size - 1]) return ret @orbycius Total frames rendered: ### Janky frames: ### (##.##%) 50th percentile: ##ms 90th percentile: ##ms 95th percentile: ##ms 99th percentile: ##ms Number Missed Vsync: # Number High input latency: # Number Slow UI thread: # Number Slow bitmap uploads: # Number Slow draw: # Number Frame deadline missed: # -
val jankyFreeFrames = accumulatedStats[JankStat.TOTAL_FRAMES] as List<Int> val ret = Bundle()
ret.putInt("num-frames", jankyFreeFrames[jankyFreeFrames.size - 1]) return ret @orbycius Total frames rendered: ### Janky frames: ### (##.##%) 50th percentile: ##ms 90th percentile: ##ms 95th percentile: ##ms 99th percentile: ##ms Number Missed Vsync: # Number High input latency: # Number Slow UI thread: # Number Slow bitmap uploads: # Number Slow draw: # Number Frame deadline missed: # JankStat.TOTAL_FRAMES JankStat.NUM_JANKY JankStat.FRAME_TIME_50TH JankStat.FRAME_TIME_90TH JankStat.FRAME_TIME_95TH JankStat.FRAME_TIME_99TH JankStat.NUM_MISSED_VSYNC JankStat.NUM_HIGH_INPUT_LATENCY JankStat.NUM_SLOW_UI_THREAD JankStat.NUM_SLOW_BITMAP_UPLOADS JankStat.NUM_SLOW_DRAW JankStat.NUM_FRAME_DEADLINE_MISSED -
@orbycius val jankyFreeFrames = accumulatedStats[JankStat.NUM_JANKY] as List<Int> val ret =
Bundle() ret.putInt("num-frames", jankyFreeFrames[jankyFreeFrames.size - 1]) return ret -
public class JankTestBase extends InstrumentationTestCase { protected void runTest() throws
Throwable { ... Bundle results = monitor.stopIteration(); int numFrames = results.getInt("num-frames", 0); // Fail the test if we didn't get enough frames assertTrue(String.format( "Too few frames received. Monitor: %s, Expected: %d, Received: %d.", monitor.getClass().getSimpleName(), annotation.expectedFrames(), numFrames), numFrames >= annotation.expectedFrames()); ... } } @orbycius val jankyFreeFrames = accumulatedStats[JankStat.NUM_JANKY] as List<Int> val ret = Bundle() ret.putInt("num-frames", jankyFreeFrames[jankyFreeFrames.size - 1]) return ret public class JankTestBase extends InstrumentationTestCase { protected void runTest() throws Throwable { ... Bundle results = monitor.stopIteration(); int numFrames = results.getInt("num-frames", 0); // Fail the test if we didn't get enough frames assertTrue(String.format( "Too few frames received. Monitor: %s, Expected: %d, Received: %d.", monitor.getClass().getSimpleName(), annotation.expectedFrames(), numFrames), numFrames >= annotation.expectedFrames()); ... } } -
public class JankTestBase extends InstrumentationTestCase { protected void runTest() throws
Throwable { ... Bundle results = monitor.stopIteration(); int numFrames = results.getInt("num-frames", 0); // Fail the test if we didn't get enough frames assertTrue(String.format( "Too few frames received. Monitor: %s, Expected: %d, Received: %d.", monitor.getClass().getSimpleName(), annotation.expectedFrames(), numFrames), numFrames >= annotation.expectedFrames()); ... } } public class JankTestBase extends InstrumentationTestCase { protected void runTest() throws Throwable { ... Bundle results = monitor.stopIteration(); int numFrames = results.getInt("num-frames", 0); // Fail the test if we didn't get enough frames assertTrue(String.format( "Too few frames received. Monitor: %s, Expected: %d, Received: %d.", monitor.getClass().getSimpleName(), annotation.expectedFrames(), numFrames), numFrames >= annotation.expectedFrames()); ... } } @orbycius val jankyFreeFrames = accumulatedStats[JankStat.NUM_JANKY] as List<Int> val ret = Bundle() ret.putInt("num-frames", jankyFreeFrames[jankyFreeFrames.size - 1]) return ret -
public class JankTestBase extends InstrumentationTestCase { protected void runTest() throws
Throwable { ... Bundle results = monitor.stopIteration(); int numFrames = results.getInt("num-frames", 0); // Fail the test if we didn't get enough frames assertTrue(String.format( "Too few frames received. Monitor: %s, Expected: %d, Received: %d.", monitor.getClass().getSimpleName(), annotation.expectedFrames(), numFrames), numFrames >= annotation.expectedFrames()); ... } } @orbycius val jankyFreeFrames = accumulatedStats[JankStat.NUM_JANKY] as List<Int> val ret = Bundle() ret.putInt("num-frames", jankyFreeFrames[jankyFreeFrames.size - 1]) return ret -
@orbycius val jankyFreeFrames = accumulatedStats[JankStat.NUM_JANKY] as List<Int> val ret =
Bundle() ret.putInt("num-frames", jankyFreeFrames[jankyFreeFrames.size - 1]) return ret monitor -
@orbycius val jankyFreeFrames = accumulatedStats[JankStat.NUM_JANKY] as List<Int> val ret =
Bundle() ret.putInt("num-frames", jankyFreeFrames[jankyFreeFrames.size - 1]) return ret Janky frames: ### (##.##%) numFrames >= annotation.expectedFrames() monitor JankTestBase -
@orbycius val jankyFreeFrames = accumulatedStats[JankStat.NUM_JANKY] as List<Int> val ret =
Bundle() ret.putInt("num-frames", jankyFreeFrames[jankyFreeFrames.size - 1]) return ret Janky frames: ### (##.##%) 10.50% numFrames >= annotation.expectedFrames() numFrames monitor JankTestBase -
@orbycius val jankyFreeFrames = accumulatedStats[JankStat.NUM_JANKY] as List<Int> val ret =
Bundle() ret.putInt("num-frames", jankyFreeFrames[jankyFreeFrames.size - 1]) return ret Janky frames: ### (##.##%) 10.50% numFrames >= annotation.expectedFrames() 5.00% numFrames expectedFrames monitor JankTestBase -
@orbycius val jankyFreeFrames = accumulatedStats[JankStat.NUM_JANKY] as List<Int> val ret =
Bundle() ret.putInt("num-frames", jankyFreeFrames[jankyFreeFrames.size - 1]) return ret Janky frames: ### (##.##%) 10.50% numFrames >= annotation.expectedFrames() >= 5.00% numFrames expectedFrames monitor JankTestBase -
@orbycius val jankyFreeFrames = accumulatedStats[JankStat.NUM_JANKY] as List<Int> val ret =
Bundle() ret.putInt("num-frames", jankyFreeFrames[jankyFreeFrames.size - 1]) return ret Janky frames: ### (##.##%) 10.50% numFrames >= annotation.expectedFrames() >= 5.00% PASS numFrames expectedFrames monitor JankTestBase -
@orbycius val jankyFreeFrames = accumulatedStats[JankStat.NUM_JANKY] as List<Int> val ret =
Bundle() ret.putInt("num-frames", return ret Janky frames: ### (##.##%) 10.50% numFrames >= annotation.expectedFrames() >= 5.00% numFrames expectedFrames monitor JankTestBase jankyFreeFrames[jankyFreeFrames.size - 1]) 100 - -
@orbycius val jankyFreeFrames = accumulatedStats[JankStat.NUM_JANKY] as List<Int> val ret =
Bundle() ret.putInt("num-frames", return ret Janky frames: ### (##.##%) numFrames >= annotation.expectedFrames() >= 5.00% numFrames expectedFrames monitor JankTestBase jankyFreeFrames[jankyFreeFrames.size - 1]) 100 - 89.50% -
@orbycius val jankyFreeFrames = accumulatedStats[JankStat.NUM_JANKY] as List<Int> val ret =
Bundle() ret.putInt("num-frames", return ret Janky frames: ### (##.##%) numFrames >= annotation.expectedFrames() >= numFrames expectedFrames monitor JankTestBase jankyFreeFrames[jankyFreeFrames.size - 1]) 100 - 89.50% 95.00% -
@orbycius val jankyFreeFrames = accumulatedStats[JankStat.NUM_JANKY] as List<Int> val ret =
Bundle() ret.putInt("num-frames", return ret Janky frames: ### (##.##%) numFrames >= annotation.expectedFrames() >= numFrames expectedFrames monitor JankTestBase jankyFreeFrames[jankyFreeFrames.size - 1]) 100 - 89.50% 95.00% FAIL -
@orbycius 0 40 80 120 160 0 40 80 120
160 16.299 Average -
@orbycius avg 99th perc 95th per 90th per 50th per
No sleep 4ms 7ms 5ms 5ms 5ms -
@orbycius avg 99th perc 95th per 90th per 50th per
No sleep 4ms 7ms 5ms 5ms 5ms Sleep 150ms 16ms 150ms 150ms 12ms 5ms -
@orbycius avg 99th perc 95th per 90th per 50th per
No sleep 4ms 7ms 5ms 5ms 5ms Sleep 150ms 16ms 150ms 150ms 12ms 5ms Sleep 150ms %20 4.5ms 150ms 5ms 5ms 5ms -
@orbycius JankStat.TOTAL_FRAMES JankStat.NUM_JANKY JankStat.FRAME_TIME_50TH JankStat.FRAME_TIME_90TH JankStat.FRAME_TIME_95TH JankStat.FRAME_TIME_99TH JankStat.NUM_MISSED_VSYNC JankStat.NUM_HIGH_INPUT_LATENCY JankStat.NUM_SLOW_UI_THREAD
JankStat.NUM_SLOW_BITMAP_UPLOADS JankStat.NUM_SLOW_DRAW JankStat.NUM_FRAME_DEADLINE_MISSED -
private var monitor: IMonitor init { if (API_LEVEL_ACTUAL <= 22)
{ error("Not supported by current platform.") } else { monitor = PercentileMonitor( InstrumentationRegistry.getInstrumentation(), PACKAGE_NAME ) } } @RunWith(AndroidJUnit4::class) class FifthTest { @orbycius private var monitor: IMonitor init { if (API_LEVEL_ACTUAL <= 22) { error("Not supported by current platform.") } else { monitor = PercentileMonitor( InstrumentationRegistry.getInstrumentation(), PACKAGE_NAME ) } } -
private var monitor: IMonitor init { if (API_LEVEL_ACTUAL <= 22)
{ error("Not supported by current platform.") } else { monitor = PercentileMonitor( InstrumentationRegistry.getInstrumentation(), PACKAGE_NAME ) } } @RunWith(AndroidJUnit4::class) class FifthTest { @orbycius @get:Rule val activityRule = ActivityTestRule(MainActivity::class.java) -
@get:Rule var activityRule: ActivityTestRule<MainActivity> = object : ActivityTestRule<MainActivity>(MainActivity::class.java) { override
fun beforeActivityLaunched() { monitor.startIteration() super.beforeActivityLaunched() } override fun afterActivityFinished() { val results = monitor.stopIteration() val percentile = results.getInt("percentilesValue", Integer.MAX_VALUE) TestCase.assertTrue( String.format( "Monitor: %s, Expected: %d, Received: %d.", monitor::class.java.simpleName, EXPECTED_MSECS, percentile ), percentile < EXPECTED_MSECS ) super.afterActivityFinished() } } @orbycius @get:Rule var activityRule: ActivityTestRule<MainActivity> = object : ActivityTestRule<MainActivity>(MainActivity::class.java) { override fun beforeActivityLaunched() { monitor.startIteration() super.beforeActivityLaunched() } override fun afterActivityFinished() { val results = monitor.stopIteration() val percentile = results.getInt("percentilesValue", Integer.MAX_VALUE) TestCase.assertTrue( String.format( "Monitor: %s, Expected: %d, Received: %d.", monitor::class.java.simpleName, EXPECTED_MSECS, percentile ), percentile < EXPECTED_MSECS ) super.afterActivityFinished() } } -
@get:Rule var activityRule: ActivityTestRule<MainActivity> = object : ActivityTestRule<MainActivity>(MainActivity::class.java) { override
fun beforeActivityLaunched() { monitor.startIteration() super.beforeActivityLaunched() } override fun afterActivityFinished() { val results = monitor.stopIteration() val percentile = results.getInt("percentilesValue", Integer.MAX_VALUE) TestCase.assertTrue( String.format( "Monitor: %s, Expected: %d, Received: %d.", monitor::class.java.simpleName, EXPECTED_MSECS, percentile ), percentile < EXPECTED_MSECS ) super.afterActivityFinished() } } @get:Rule var activityRule: ActivityTestRule<MainActivity> = object : ActivityTestRule<MainActivity>(MainActivity::class.java) { override fun beforeActivityLaunched() { monitor.startIteration() super.beforeActivityLaunched() } override fun afterActivityFinished() { val results = monitor.stopIteration() val percentile = results.getInt("percentilesValue", Integer.MAX_VALUE) TestCase.assertTrue( String.format( "Monitor: %s, Expected: %d, Received: %d.", monitor::class.java.simpleName, EXPECTED_MSECS, percentile ), percentile < EXPECTED_MSECS ) super.afterActivityFinished() } } @orbycius -
@get:Rule var activityRule: ActivityTestRule<MainActivity> = object : ActivityTestRule<MainActivity>(MainActivity::class.java) { override
fun beforeActivityLaunched() { monitor.startIteration() super.beforeActivityLaunched() } override fun afterActivityFinished() { val results = monitor.stopIteration() val percentile = results.getInt("percentilesValue", Integer.MAX_VALUE) TestCase.assertTrue( String.format( "Monitor: %s, Expected: %d, Received: %d.", monitor::class.java.simpleName, EXPECTED_MSECS, percentile ), percentile < EXPECTED_MSECS ) super.afterActivityFinished() } } @get:Rule var activityRule: ActivityTestRule<MainActivity> = object : ActivityTestRule<MainActivity>(MainActivity::class.java) { override fun beforeActivityLaunched() { monitor.startIteration() super.beforeActivityLaunched() } override fun afterActivityFinished() { val results = monitor.stopIteration() val percentile = results.getInt("percentilesValue", Integer.MAX_VALUE) TestCase.assertTrue( String.format( "Monitor: %s, Expected: %d, Received: %d.", monitor::class.java.simpleName, EXPECTED_MSECS, percentile ), percentile < EXPECTED_MSECS ) super.afterActivityFinished() } } @orbycius -
@get:Rule var activityRule: ActivityTestRule<MainActivity> = object : ActivityTestRule<MainActivity>(MainActivity::class.java) { override
fun beforeActivityLaunched() { monitor.startIteration() super.beforeActivityLaunched() } override fun afterActivityFinished() { val results = monitor.stopIteration() val percentile = results.getInt("percentilesValue", Integer.MAX_VALUE) TestCase.assertTrue( String.format( "Monitor: %s, Expected: %d, Received: %d.", monitor::class.java.simpleName, EXPECTED_MSECS, percentile ), percentile < EXPECTED_MSECS ) super.afterActivityFinished() } } @orbycius -
@orbycius @Test @PerformanceTest( processName = PACKAGE_NAME, perfType = PerformanceTest.PerfType.AVG_FRAME_TIME_95TH, threshold
= 18, assertionType = PerformanceTest.AssertionType.LESS_OR_EQUAL ) fun testSixth() { for (i in 0 until INNER_LOOP) { val appViews = UiScrollable(UiSelector().scrollable(true)) appViews.setAsVerticalList() appViews.scrollTextIntoView("This is item 24") appViews.scrollTextIntoView("This is item 1") } } -
@orbycius val jankyFreeFrames = accumulatedStats[JankStat.FRAME_TIME_99TH] as List<Int> val ret =
Bundle() ret.putInt("percentilesValue", percentileTimeInMs[percentileTimeInMs.size - 1]) return ret monitor -
@orbycius override fun startIteration() { } override fun stopIteration(): Bundle
{ } override fun getMetrics(): Bundle { } override fun getMetrics(): Bundle { } val metrics = Bundle() // Retrieve the total number of frames val totals = accumulatedStats[JankStat.TOTAL_FRAMES] as List<Int> // Store avg, min and max of total frames metrics.putInt(GfxMonitor.KEY_AVG_TOTAL_FRAMES, computeAverage(totals)) metrics.putInt(GfxMonitor.KEY_MAX_TOTAL_FRAMES, Collections.max(totals)) metrics.putInt(GfxMonitor.KEY_MIN_TOTAL_FRAMES, Collections.min(totals)) ... return metrics val jankyFreeFrames = accumulatedStats[JankStat.FRAME_TIME_99TH] as List<Int> val ret = Bundle() ret.putInt("percentilesValue", percentileTimeInMs[percentileTimeInMs.size - 1]) return ret monitor -
@orbycius override fun startIteration() { } override fun stopIteration(): Bundle
{ } override fun getMetrics(): Bundle { } override fun getMetrics(): Bundle { } val metrics = Bundle() // Retrieve the total number of frames val totals = accumulatedStats[JankStat.TOTAL_FRAMES] as List<Int> // Store avg, min and max of total frames metrics.putInt(GfxMonitor.KEY_AVG_TOTAL_FRAMES, computeAverage(totals)) metrics.putInt(GfxMonitor.KEY_MAX_TOTAL_FRAMES, Collections.max(totals)) metrics.putInt(GfxMonitor.KEY_MIN_TOTAL_FRAMES, Collections.min(totals)) ... return metrics monitor return getMetrics() -
} @Retention(AnnotationRetention.RUNTIME) @Target(AnnotationTarget.FUNCTION) annotation class PerformanceTest( val processName: String =
"", val perfType: PerfType val threshold: Int = Int.MAX_VALUE, val assertionType: AssertionType ) { @orbycius @Retention(AnnotationRetention.RUNTIME) @Target(AnnotationTarget.FUNCTION) annotation class PerformanceTest( val processName: String = "", val perfType: PerfType val threshold: Int = Int.MAX_VALUE, val assertionType: AssertionType ) { } , -
} @Retention(AnnotationRetention.RUNTIME) @Target(AnnotationTarget.FUNCTION) annotation class PerformanceTest( val processName: String =
"", val perfType: PerfType val threshold: Int = Int.MAX_VALUE, val assertionType: AssertionType ) { @orbycius , enum class PerfType(val type: String) { TOTAL_FRAMES("gfx-avg-total-frames"), MAX_TOTAL_FRAMES("gfx-max-total-frames"), MIN_TOTAL_FRAMES("gfx-min-total-frames"), … } = PerfType.TOTAL_FRAMES -
@Retention(AnnotationRetention.RUNTIME) @Target(AnnotationTarget.FUNCTION) annotation class PerformanceTest( val processName: String = "",
val perfType: PerfType = PerfType.TOTAL_FRAMES, val threshold: Int = Int.MAX_VALUE, val assertionType: AssertionType ) { } @orbycius enum class PerfType(val type: String) { TOTAL_FRAMES("gfx-avg-total-frames"), MAX_TOTAL_FRAMES("gfx-max-total-frames"), MIN_TOTAL_FRAMES("gfx-min-total-frames"), … } enum class AssertionType(val type: Int) { LESS(0), LESS_OR_EQUAL(1), GREATER(2), GREATER_OR_EQUAL(3), EQUAL(4) } = AssertionType.LESS_OR_EQUAL -
} open class ActivityPerfTestRule<T: Activity>(activityClass: Class<T>): ActivityTestRule<T>(activityClass) { @orbycius open
class ActivityPerfTestRule<T: Activity>(activityClass: Class<T>): ActivityTestRule<T>(activityClass) { } -
} open class ActivityPerfTestRule<T: Activity>(activityClass: Class<T>): ActivityTestRule<T>(activityClass) { @orbycius private
var monitor: IMonitor? = null private var annotation: PerformanceTest? = null init { if (API_LEVEL_ACTUAL <= 22) { error("Not supported by current platform.") } } -
} open class ActivityPerfTestRule<T: Activity>(activityClass: Class<T>): ActivityTestRule<T>(activityClass) { @orbycius private
var monitor: IMonitor? = null private var annotation: PerformanceTest? = null init { if (API_LEVEL_ACTUAL <= 22) { error("Not supported by current platform.") } } override fun apply(base: Statement?, description: Description?): Statement { annotation = description?.getAnnotation(PerformanceTest::class.java) annotation?.let { monitor = PerfMonitor( InstrumentationRegistry.getInstrumentation(), it.processName ) } return super.apply(base, description) } -
override fun beforeActivityLaunched() { monitor?.startIteration() super.beforeActivityLaunched() } @orbycius open class
ActivityPerfTestRule<T: Activity>(activityClass: Class<T>): ActivityTestRule<T>(activityClass) { // ... } override fun beforeActivityLaunched() { monitor?.startIteration() super.beforeActivityLaunched() } -
override fun beforeActivityLaunched() { monitor?.startIteration() super.beforeActivityLaunched() } override fun afterActivityFinished()
{ monitor?.let { val results = it.stopIteration() val res: Double = results?.get(annotation?.perfType?.type) as Double val assertion = when(annotation?.assertionType) { PerformanceTest.AssertionType.LESS -> res < annotation!!.threshold PerformanceTest.AssertionType.GREATER -> res > annotation!!.threshold … } TestCase.assertTrue("Error msg here.", assertion) } super.afterActivityFinished() } @orbycius open class ActivityPerfTestRule<T: Activity>(activityClass: Class<T>): ActivityTestRule<T>(activityClass) { // ... } override fun beforeActivityLaunched() { monitor?.startIteration() super.beforeActivityLaunched() } override fun afterActivityFinished() { monitor?.let { val results = it.stopIteration() val res: Double = results?.get(annotation?.perfType?.type) as Double val assertion = when(annotation?.assertionType) { PerformanceTest.AssertionType.LESS -> res < annotation!!.threshold PerformanceTest.AssertionType.GREATER -> res > annotation!!.threshold … } TestCase.assertTrue("Error msg here.", assertion) } super.afterActivityFinished() } -
override fun beforeActivityLaunched() { monitor?.startIteration() super.beforeActivityLaunched() } override fun afterActivityFinished()
{ monitor?.let { val results = it.stopIteration() val res: Double = results?.get(annotation?.perfType?.type) as Double val assertion = when(annotation?.assertionType) { PerformanceTest.AssertionType.LESS -> res < annotation!!.threshold PerformanceTest.AssertionType.GREATER -> res > annotation!!.threshold … } TestCase.assertTrue("Error msg here.", assertion) } super.afterActivityFinished() } override fun afterActivityFinished() { monitor?.let { val results = it.stopIteration() val res: Double = results?.get(annotation?.perfType?.type) as Double val assertion = when(annotation?.assertionType) { PerformanceTest.AssertionType.LESS -> res < annotation!!.threshold PerformanceTest.AssertionType.GREATER -> res > annotation!!.threshold … } TestCase.assertTrue("Error msg here.", assertion) } super.afterActivityFinished() } @orbycius open class ActivityPerfTestRule<T: Activity>(activityClass: Class<T>): ActivityTestRule<T>(activityClass) { // ... } -
override fun beforeActivityLaunched() { monitor?.startIteration() super.beforeActivityLaunched() } override fun afterActivityFinished()
{ monitor?.let { val results = it.stopIteration() val res: Double = results?.get(annotation?.perfType?.type) as Double val assertion = when(annotation?.assertionType) { PerformanceTest.AssertionType.LESS -> res < annotation!!.threshold PerformanceTest.AssertionType.GREATER -> res > annotation!!.threshold … } TestCase.assertTrue("Error msg here.", assertion) } super.afterActivityFinished() } @orbycius open class ActivityPerfTestRule<T: Activity>(activityClass: Class<T>): ActivityTestRule<T>(activityClass) { // ... } -
@orbycius @Test @PerformanceTest( processName = PACKAGE_NAME, perfType = PerformanceTest.PerfType.AVG_FRAME_TIME_95TH, threshold
= 18, assertionType = PerformanceTest.AssertionType.LESS_OR_EQUAL ) fun testSixth() { for (i in 0 until INNER_LOOP) { val appViews = UiScrollable(UiSelector().scrollable(true)) appViews.setAsVerticalList() appViews.scrollTextIntoView("This is item 24") appViews.scrollTextIntoView("This is item 1") } } -
public class LeakActivity extends Activity { @Override protected void onCreate(Bundle
savedInstanceState) { super.onCreate(savedInstanceState); setContentView(R.layout.activity_leak); View button = findViewById(R.id.button); button.setOnClickListener(new View.OnClickListener() { @Override public void onClick(View v) { startAsyncWork(); } }); } void startAsyncWork() { Runnable work = new Runnable() { @Override public void run() { SystemClock.sleep(20000); } }; new Thread(work).start(); } } @orbycius public class LeakActivity extends Activity { @Override protected void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); setContentView(R.layout.activity_leak); View button = findViewById(R.id.button); button.setOnClickListener(new View.OnClickListener() { @Override public void onClick(View v) { startAsyncWork(); } }); } void startAsyncWork() { Runnable work = new Runnable() { @Override public void run() { SystemClock.sleep(20000); } }; new Thread(work).start(); } } -
public class LeakActivity extends Activity { @Override protected void onCreate(Bundle
savedInstanceState) { super.onCreate(savedInstanceState); setContentView(R.layout.activity_leak); View button = findViewById(R.id.button); button.setOnClickListener(new View.OnClickListener() { @Override public void onClick(View v) { startAsyncWork(); } }); } void startAsyncWork() { Runnable work = new Runnable() { @Override public void run() { SystemClock.sleep(20000); } }; new Thread(work).start(); } } public class LeakActivity extends Activity { @Override protected void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); setContentView(R.layout.activity_leak); View button = findViewById(R.id.button); button.setOnClickListener(new View.OnClickListener() { @Override public void onClick(View v) { startAsyncWork(); } }); } void startAsyncWork() { Runnable work = new Runnable() { @Override public void run() { SystemClock.sleep(20000); } }; new Thread(work).start(); } } @orbycius -
public class LeakActivity extends Activity { @Override protected void onCreate(Bundle
savedInstanceState) { super.onCreate(savedInstanceState); setContentView(R.layout.activity_leak); View button = findViewById(R.id.button); button.setOnClickListener(new View.OnClickListener() { @Override public void onClick(View v) { startAsyncWork(); } }); } void startAsyncWork() { Runnable work = new Runnable() { @Override public void run() { SystemClock.sleep(20000); } }; new Thread(work).start(); } } @orbycius -
@orbycius class SeventhTest { @get:Rule var mainActivityActivityTestRule = ActivityTestRule(LeakActivity::class.java) @Test
fun testLeaks() { onView(withId(R.id.button)).perform(click()) } } -
@orbycius dependencies { ... debugImplementation 'com.squareup.leakcanary:leakcanary-android:1.6.3' androidTestImplementation 'com.squareup.leakcanary:leakcanary-android-instrumentation:1.6.3' } defaultConfig
{ ... testInstrumentationRunnerArgument "listener", "com.squareup.leakcanary.FailTestOnLeakRunListener" } -
@orbycius class SeventhTest { @get:Rule var mainActivityActivityTestRule = ActivityTestRule(LeakActivity::class.java) @Test
fun testLeaks() { onView(withId(R.id.button)).perform(click()) } } -
@orbycius class SeventhTest { @get:Rule var mainActivityActivityTestRule = ActivityTestRule(LeakActivity::class.java) @Test
fun testLeaks() { onView(withId(R.id.button)).perform(click()) } } Test failed because memory leaks were detected, see leak traces below. ###################################### In com.marcosholgado.performancetest.test:null:0. * com.marcosholgado.performancetest.LeakActivity has leaked: -
adb shell am instrument -w \ > -e listener com.squareup.leakcanary.FailTestOnLeakRunListener
\ > -e class com.marcosholgado.performancetest.SeventhTest#testLeaks \ > com.marcosholgado.performancetest.test/androidx.test.runner.AndroidJUnitRunner @orbycius adb shell am instrument -w \ > -e listener com.squareup.leakcanary.FailTestOnLeakRunListener \ > -e class com.marcosholgado.performancetest.SeventhTest#testLeaks \ > com.marcosholgado.performancetest.test/androidx.test.runner.AndroidJUnitRunner -
adb shell am instrument -w \ > -e listener com.squareup.leakcanary.FailTestOnLeakRunListener
\ > -e class com.marcosholgado.performancetest.SeventhTest#testLeaks \ > com.marcosholgado.performancetest.test/androidx.test.runner.AndroidJUnitRunner @orbycius -
@orbycius class SeventhTest { @get:Rule var mainActivityActivityTestRule = ActivityTestRule(LeakActivity::class.java) @Test
fun testLeaks() { onView(withId(R.id.button)).perform(click()) } } -
@orbycius class SeventhTest { @get:Rule var mainActivityActivityTestRule = ActivityTestRule(LeakActivity::class.java) @Test
fun testLeaks() { onView(withId(R.id.button)).perform(click()) } } @LeakTest -
@orbycius @Retention(AnnotationRetention.RUNTIME) @Target(AnnotationTarget.FUNCTION) annotation class LeakTest class MyOtherLeakRunListener: FailTestOnLeakRunListener() {
} override fun skipLeakDetectionReason(description: Description): String? { return if(description.getAnnotation(LeakTest::class.java) != null) null else "Skip Leak test" } -
@orbycius Janky frames adb shell dumpsys gfxinfo adb shell dumpsys
batterystats adb shell dumpsys netstats -
How to write and automate performance tests @Orbycius Marcos Holgado
https://github.com/marcosholgado/performance-test https://github.com/marcosholgado/dagger-playground