Share
Pragmatic Testing
(given at Droidcon Italy 2018 and Android Makers 2018)
You open a pull request only to find that some tests are now failing. What goes through your mind?
Did I break something? Do I need to update the tests? Just what is that test testing anyway?
Ideally, #2 would almost never be the case, as rewriting a test usually means it's lost the regression testing value of its original form. How can we get there though? There’s no one right way to write tests, and it’s surprisingly easy to fall into the trap of writing change tests. There are good patterns you can leverage to scale them and maximize their long term value. This talk will detail simple patterns you can follow for healthy, pragmatic testing with a focus on test design, correctness, and how these values can translate into better overall API design of your code.
Zac Sweers
Transcript
143 slides
-
Non-Goals • Convince you testing is worthwhile • UI Testing
(Espresso, layouts, etc) -
Non-Goals • Convince you testing is worthwhile • UI Testing
(Espresso, layouts, etc) • Knock specific libraries -
@Test fun testAdd() { val calculator = Calculator() }a InputManager
DisplayManager CPU RAM -
@Test fun testAdd() { val cpu = mock(CPU::class.java) val calculator
= Calculator() }a InputManager DisplayManager CPU RAM -
@Test fun testAdd() { val cpu = mock(CPU::class.java) val ram
= mock(RAM::class.java) val calculator = Calculator() }a InputManager DisplayManager CPU RAM -
@Test fun testAdd() { val cpu = mock(CPU::class.java) val ram
= mock(RAM::class.java) val display = mock(DisplayManager::class.java) val calculator = Calculator() }a InputManager DisplayManager CPU RAM -
@Test fun testAdd() { val cpu = mock(CPU::class.java) val ram
= mock(RAM::class.java) val display = mock(DisplayManager::class.java) val input = mock(InputManager::class.java) val calculator = Calculator() }a InputManager DisplayManager CPU RAM -
@Test fun testAdd() { val cpu = mock(CPU::class.java) val ram
= mock(RAM::class.java) val display = mock(DisplayManager::class.java) val input = mock(InputManager::class.java) val calculator = Calculator(cpu, ram, display, input) }a InputManager DisplayManager CPU RAM -
@Test fun testAdd() { // Mocks... val calculator = Calculator(cpu,
ram, display, input) }a InputManager DisplayManager CPU RAM -
@Test fun testAdd() { // Mocks... val calculator = Calculator(cpu,
ram, display, input) input.add(2, 2) }a InputManager DisplayManager CPU RAM -
@Test fun testAdd() { // Mocks... val calculator = Calculator(cpu,
ram, display, input) input.add(2, 2) verify(cpu).setRam(ram) }a InputManager DisplayManager CPU RAM -
@Test fun testAdd() { // Mocks... val calculator = Calculator(cpu,
ram, display, input) input.add(2, 2) verify(cpu).setRam(ram) verify(input).onInputReceived(2) }a InputManager DisplayManager CPU RAM -
@Test fun testAdd() { // Mocks... val calculator = Calculator(cpu,
ram, display, input) input.add(2, 2) verify(cpu).setRam(ram) verify(input).onInputReceived(2) verify(cpu).loadInputToCpu(2) }a InputManager DisplayManager CPU RAM -
@Test fun testAdd() { // Mocks... val calculator = Calculator(cpu,
ram, display, input) input.add(2, 2) verify(cpu).setRam(ram) verify(input).onInputReceived(2) verify(cpu).loadInputToCpu(2) verify(input).onInputReceived("+") }a InputManager DisplayManager CPU RAM -
@Test fun testAdd() { // Mocks... val calculator = Calculator(cpu,
ram, display, input) input.add(2, 2) verify(cpu).setRam(ram) verify(input).onInputReceived(2) verify(cpu).loadInputToCpu(2) verify(input).onInputReceived("+") verify(ram).storeInRAM(2) }a InputManager DisplayManager CPU RAM -
@Test fun testAdd() { // Mocks... val calculator = Calculator(cpu,
ram, display, input) input.add(2, 2) verify(cpu).setRam(ram) verify(input).onInputReceived(2) verify(cpu).loadInputToCpu(2) verify(input).onInputReceived("+") verify(ram).storeInRAM(2) verify(cpu).loadInputToCpu("+") }a InputManager DisplayManager CPU RAM -
@Test fun testAdd() { // Mocks... val calculator = Calculator(cpu,
ram, display, input) input.add(2, 2) verify(cpu).setRam(ram) verify(input).onInputReceived(2) verify(cpu).loadInputToCpu(2) verify(input).onInputReceived("+") verify(ram).storeInRAM(2) verify(cpu).loadInputToCpu("+") verify(input).onInputReceived(2) }a InputManager DisplayManager CPU RAM -
InputManager DisplayManager CPU RAM @Test fun testAdd() { // Mocks...
val calculator = Calculator(cpu, ram, display, input) input.add(2, 2) verify(cpu).setRam(ram) verify(input).onInputReceived(2) verify(cpu).loadInputToCpu(2) verify(input).onInputReceived("+") verify(ram).storeInRAM(2) verify(cpu).loadInputToCpu("+") verify(input).onInputReceived(2) verify(cpu).loadInputToCpu(2) }a -
InputManager DisplayManager CPU RAM @Test fun testAdd() { // Mocks...
val calculator = Calculator(cpu, ram, display, input) input.add(2, 2) verify(cpu).setRam(ram) verify(input).onInputReceived(2) verify(cpu).loadInputToCpu(2) verify(input).onInputReceived("+") verify(ram).storeInRAM(2) verify(cpu).loadInputToCpu("+") verify(input).onInputReceived(2) verify(cpu).loadInputToCpu(2) verify(ram).loadFromRam() }a -
InputManager DisplayManager CPU RAM @Test fun testAdd() { // Mocks...
val calculator = Calculator(cpu, ram, display, input) input.add(2, 2) verify(cpu).setRam(ram) verify(input).onInputReceived(2) verify(cpu).loadInputToCpu(2) verify(input).onInputReceived("+") verify(ram).storeInRAM(2) verify(cpu).loadInputToCpu("+") verify(input).onInputReceived(2) verify(cpu).loadInputToCpu(2) verify(ram).loadFromRam() verify(cpu).calculateSum(2, 2) }a -
InputManager DisplayManager CPU RAM @Test fun testAdd() { // Mocks...
val calculator = Calculator(cpu, ram, display, input) input.add(2, 2) verify(cpu).setRam(ram) verify(input).onInputReceived(2) verify(cpu).loadInputToCpu(2) verify(input).onInputReceived("+") verify(ram).storeInRAM(2) verify(cpu).loadInputToCpu("+") verify(input).onInputReceived(2) verify(cpu).loadInputToCpu(2) verify(ram).loadFromRam() verify(cpu).calculateSum(2, 2) verify(display).displayResult(4) }a -
InputManager DisplayManager CPU RAM @Test fun testAdd() { // Mocks...
val calculator = Calculator(cpu, ram, display, input) input.add(2, 2) verify(cpu).setRam(ram) verify(input).onInputReceived(2) verify(cpu).loadInputToCpu(2) verify(input).onInputReceived("+") verify(ram).storeInRAM(2) verify(cpu).loadInputToCpu("+") verify(input).onInputReceived(2) verify(cpu).loadInputToCpu(2) verify(ram).loadFromRam() verify(cpu).calculateSum(2, 2) verify(display).displayResult(4) }a -
@Test fun testAdd() { val captor = ArgumentCaptor.forClass(Listener::class.java) verify(input).registerInputListener(captor.capture()) val
listener = captor.value input.add(2, 2) }a InputManager DisplayManager CPU RAM -
@Test fun testAdd() { // Captor stuff input.add(2, 2) }a
InputManager DisplayManager CPU RAM -
InputManager DisplayManager CPU RAM @Test fun testAdd() { // Mocks...
val calculator = Calculator(cpu, ram, display, input) // Captor stuff... input.add(2, 2) verify(cpu).setRam(ram) listener.onNewInput(2) verify(input).onInputReceived(2) verify(cpu).loadInputToCpu(2) listener.onNewInput("+") verify(input).onInputReceived("+") verify(ram).storeInRAM(2) verify(cpu).loadInputToCpu("+") listener.onNewInput(2) verify(input).onInputReceived(2) verify(cpu).loadInputToCpu(2) verify(ram).loadFromRam() verify(cpu).calculateSum(2, 2) verify(display).displayResult(4) }a -
InputManager DisplayManager CPU RAM @Test fun testAdd() { // Mocks...
val calculator = Calculator(cpu, ram, display, input) // Captor stuff... input.add(2, 2) verify(cpu).setRam(ram) listener.onNewInput(2) verify(input).onInputReceived(2) verify(cpu).loadInputToCpu(2) listener.onNewInput("+") verify(input).onInputReceived("+") verify(ram).storeInRAM(2) verify(cpu).loadInputToCpu("+") listener.onNewInput(2) verify(input).onInputReceived(2) verify(cpu).loadInputToCpu(2) verify(ram).loadFromRam() verify(cpu).calculateSum(2, 2) verify(display).displayResult(4) }a -
InputManager DisplayManager CPU RAM @Test fun testAdd() { // Mocks...
val calculator = Calculator(cpu, ram, display, input) // Captor stuff... input.add(2, 2) verify(cpu).setRam(ram) listener.onNewInput(2) verify(input).onInputReceived(2) verify(cpu).loadInputToCpu(2) listener.onNewInput("+") verify(input).onInputReceived("+") verify(ram).storeInRAM(2) verify(cpu).loadInputToCpu("+") listener.onNewInput(2) verify(input).onInputReceived(2) verify(cpu).loadInputToCpu(2) verify(ram).loadFromRam() verify(cpu).calculateSum(2, 2) verify(display).displayResult(4) }a -
InputManager DisplayManager CPU RAM @Test fun testAdd() { // Mocks...
val calculator = Calculator(cpu, ram, display, input) // Captor stuff... input.add(2, 2) verify(cpu).setRam(ram) listener.onNewInput(2) verify(input).onInputReceived(2) verify(cpu).loadInputToCpu(2) listener.onNewInput("+") verify(input).onInputReceived("+") verify(ram).storeInRAM(2) verify(cpu).loadInputToCpu("+") listener.onNewInput(2) verify(input).onInputReceived(2) verify(cpu).loadInputToCpu(2) verify(ram).loadFromRam() verify(cpu).calculateSum(2, 2) verify(display).displayResult(4) }a -
InputManager DisplayManager CPU RAM @Test fun testAdd() { // Mocks...
val calculator = Calculator(cpu, ram, display, input) // Captor stuff... input.add(2, 2) verify(cpu).setRam(ram) listener.onNewInput(2) verify(input).onInputReceived(2) verify(cpu).loadInputToCpu(2) listener.onNewInput("+") verify(input).onInputReceived("+") verify(ram).storeInRAM(2) verify(cpu).loadInputToCpu("+") listener.onNewInput(2) verify(input).onInputReceived(2) verify(cpu).loadInputToCpu(2) verify(ram).loadFromRam() verify(cpu).calculateSum(2, 2) verify(display).displayResult(4) }a -
@Test fun testAdd() { verify(cpu).calculateSum(2, 2) // Actually 0 verify(display).displayResult(4)
}a InputManager DisplayManager CPU RAM -
@Test fun testAdd() { when(cpu.calculateSum(2, 2)).thenReturn(4) verify(cpu).calculateSum(2, 2) verify(display).displayResult(4) }a
InputManager DisplayManager CPU RAM -
InputManager DisplayManager CPU RAM @Test fun testAdd() { // Mocks...
val calculator = Calculator(cpu, ram, display, input) // Captor stuff... input.add(2, 2) verify(cpu).setRam(ram) listener.onNewInput(2) verify(input).onInputReceived(2) verify(cpu).loadInputToCpu(2) listener.onNewInput("+") verify(input).onInputReceived("+") verify(ram).storeInRAM(2) verify(cpu).loadInputToCpu("+") listener.onNewInput(2) verify(input).onInputReceived(2) verify(cpu).loadInputToCpu(2) verify(ram).loadFromRam() when(cpu.calculateSum(2, 2)).thenReturn(4) verify(cpu).calculateSum(2, 2) verify(display).displayResult(4) -
InputManager DisplayManager CPU RAM @Test fun testAdd() { // Mocks...
val calculator = Calculator(cpu, ram, display, input) // Captor stuff... input.add(2, 2) verify(cpu).setRam(ram) listener.onNewInput(2) verify(input).onInputReceived(2) verify(cpu).loadInputToCpu(2) listener.onNewInput("+") verify(input).onInputReceived("+") verify(ram).storeInRAM(2) verify(cpu).loadInputToCpu("+") listener.onNewInput(2) verify(input).onInputReceived(2) verify(cpu).loadInputToCpu(2) verify(ram).loadFromRam() when(cpu.calculateSum(2, 2)).thenReturn(4) verify(cpu).calculateSum(2, 2) verify(display).displayResult(4) -
InputManager DisplayManager CPU RAM @Test fun testAdd() { // Mocks...
val calculator = Calculator(cpu, ram, display, input) // Captor stuff... input.add(2, 2) verify(cpu).setRam(ram) listener.onNewInput(2) verify(input).onInputReceived(2) verify(cpu).loadInputToCpu(2) listener.onNewInput("+") verify(input).onInputReceived("+") verify(ram).storeInRAM(2) verify(cpu).loadInputToCpu("+") listener.onNewInput(2) verify(input).onInputReceived(2) verify(cpu).loadInputToCpu(2) verify(ram).loadFromRam() when(cpu.calculateSum(2, 2)).thenReturn(4) verify(cpu).calculateSum(2, 2) verify(display).displayResult(4) -
@Test fun testAdd() { // Mocks... val calculator = Calculator(cpu,
ram, display, input) // Captor stuff... input.add(2, 2) listener.onNewInput(2) listener.onNewInput("+") listener.onNewInput(2) when(cpu.calculateSum(2, 2)).thenReturn(4) verify(cpu).calculateSum(2, 2) verify(display).displayResult(4) }a InputManager DisplayManager CPU RAM -
InputManager DisplayManager CPU RAM @Test fun testAdd() { val calculator
= Calculator(cpu, ram, display, input) }a -
InputManager DisplayManager CPU RAM @Test fun testAdd() { val calculator
= Calculator(cpu, ram, display, input) assertThat(calculator.add(2, 2)).isEqualTo(4) }a -
InputManager DisplayManager CPU RAM InputManager DisplayManager CPU RAM @Test fun
testAdd() { val calculator = Calculator(cpu, ram, display, input) assertThat(calculator.add(2, 2)).isEqualTo(4) }a -
DisplayManager CPU RAM InputManager DisplayManager CPU RAM @Test fun testAdd()
{ val calculator = Calculator(cpu, ram, display, input) assertThat(calculator.add(2, 2)).isEqualTo(4) }a -
DisplayManager CPU RAM DisplayManager CPU RAM @Test fun testAdd() {
val calculator = Calculator(cpu, ram, display) assertThat(calculator.add(2, 2)).isEqualTo(4) }a -
DisplayManager RAM DisplayManager CPU RAM @Test fun testAdd() { val
calculator = Calculator(cpu, ram, display) assertThat(calculator.add(2, 2)).isEqualTo(4) }a -
DisplayManager CPU RAM @Test fun testAdd() { val calculator =
Calculator(CPU(), ram, display) assertThat(calculator.add(2, 2)).isEqualTo(4) }a -
@Test fun testAdd() { val calculator = Calculator(CPU(), ram, display)
assertThat(calculator.add(2, 2)).isEqualTo(4) }a DisplayManager CPU RAM DisplayManager CPU RAM -
DisplayManager CPU DisplayManager CPU RAM @Test fun testAdd() { val
calculator = Calculator(CPU(), ram, display) assertThat(calculator.add(2, 2)).isEqualTo(4) }a -
class InMemoryRAM : RAMa{ private val store = mutableMapOf<String, Any>()
override <T>afunastoreInRAM(key:aString,avalue:aT) { return store[key] = value } override <T>afunaloadFromRam(key:aString):aT? { return store[key] } } -
DisplayManager CPU InMemoryRAM @Test fun testAdd() { val ram =
mock(RAM::class.java) val calculator = Calculator(CPU(), ram, display) assertThat(calculator.add(2, 2)).isEqualTo(4) }a -
DisplayManager CPU InMemoryRAM @Test fun testAdd() { val ram =
InMemoryRAM() val calculator = Calculator(CPU(), ram, display) assertThat(calculator.add(2, 2)).isEqualTo(4) }a -
CPU DisplayManager CPU @Test fun testAdd() { val ram =
InMemoryRAM() val calculator = Calculator(CPU(), ram, display) assertThat(calculator.add(2, 2)).isEqualTo(4) }a -
CPU DisplayManager CPU @Test fun testAdd() { val cpu =
CPU(InMemoryRAM()) val calculator = Calculator(cpu, display) assertThat(calculator.add(2, 2)).isEqualTo(4) }a -
CPU CPU DisplayManager @Test fun testAdd() { val cpu =
CPU(InMemoryRAM()) val calculator = Calculator(cpu, display) assertThat(calculator.add(2, 2)).isEqualTo(4) }a DisplayManager -
CPU ResultHandler CPU @Test fun testAdd() { val cpu =
CPU(InMemoryRAM()) val resultHandler = RecordingResultHandler() val calculator = Calculator(cpu, resultHandler) calculator.add(2, 2) assertThat(resultHandler.results[0]).isEqualTo(4) }a -
CPU @Test fun testAdd() { val cpu = CPU(InMemoryRAM()) val
calculator = Calculator(cpu) assertThat(calculator.add(2, 2)).isEqualTo(4) }a -
val cpu = CPU(InMemoryRAM()) val calculator = Calculator(cpu) @Test fun
testAdd() { assertThat(calculator.add(2, 2)).isEqualTo(4) }a -
val cpu = CPU(InMemoryRAM()) val calculator = Calculator(cpu) @Testa fun
testAdd() { assertThat(calculator.add(2, 2)).isEqualTo(4) }a @Testb fun testSub() { assertThat(calculator.subtract(2, 2)).isEqualTo(0) }b @Testc fun testMultiply() { assertThat(calculator.multiply(2, 2)).isEqualTo(4) }c -
val cpu = CPU(InMemoryRAM()) val calculator = Calculator(cpu) @Testa fun
testAdd() { assertThat(calculator.add(2, 2)).isEqualTo(4) }a -
val cpu = CPU(InMemoryRAM()) val resultHandler = RecordingResultHandler() val calculator
= Calculator(cpu, resultHandler) @Test fun testAdd() { calculator.add(2, 2) calculator.add(2, 3) calculator.add(2, -1) assertThat(resultHandler.results) .containsExactly(4, 5, 1) }a -
@Test fun foo() { try { foo.bar() } catch (e:
Exception) { // Expected } }a -
@Test() fun foo() { try { foo.bar() } catch (e:
ExpectedException) { assertThat(e) .hasMessageThat() .contains("Oh no") }b }a -
@Test() fun foo() { try { foo.bar() throw AssertionError("No") }
catch (e: ExpectedException) { assertThat(e) .hasMessageThat() .contains("Oh no") }b }a -
// throw error Thread currentThread = Thread.currentThread(); UncaughtExceptionHandler handler =
currentThread.getUncaughtExceptionHandler(); handler.uncaughtException(currentThread, error); -
// throw error Thread currentThread = Thread.currentThread(); UncaughtExceptionHandler handler =
currentThread.getUncaughtExceptionHandler(); handler.uncaughtException(currentThread, error); https://github.com/uber/AutoDispose/blob/master/autodispose/src/test/java/com/uber/ autodispose/RxErrorsRule.java -
class CalculatorPresenter(val cpu: CPU, val resultHandler: ResultHandler, val inputManager: InputManager)
Courtesy of Nick Butcher (@crafty) -
class CalculatorPresenter(val cpu: CPU, val resultHandler: ResultHandler, val inputManager: InputManager)
0.0 7 8 9 4 5 6 1 2 3 C 0 ⏎ -
class CurrentValueView : ResultHandler 0.0 7 8 9 4 5
6 1 2 3 C 0 ⏎ -
class NumberInputManager : InputManager class CurrentValueView : ResultHandler 0.0 7
8 9 4 5 6 1 2 3 C 0 ⏎ -
class NumberInputManager : InputManager class CurrentValueView : ResultHandler 0.0 7
8 9 4 5 6 1 2 3 C 0 ⏎ class ImeInputManager : InputManager -
class NumberInputManager : InputManager class CurrentValueView : ResultHandler 7 8
9 4 5 6 1 2 3 C 0 ⏎ 0.0 -
class MainActivity : Activity { @Inject lateinit var foo: Foo
@Inject lateinit var bar: Bar } -
@Test fun testActivity() { val activity = ... activity.foo =
TestFoo() activity.bar = MockBar() } -
@Test fun testActivity() { val activity = MainActivityTestHelper.create( TestFoo(), MockBar()
)a }b RIBS: https://github.com/uber/RIBs/blob/master/android/libraries/rib- compiler-test/src/main/java/com/uber/rib/compiler/ InteractorTestGenerator.java Injection Helper: https://github.com/dinosaurwithakatana/injection- helper -
Fakes are worth writing class InMemoryRAM : RAM { private
val store = mutableMapOf<String, Any>() override <T> fun storeInRAM(key: String, value: T) { return store[key] = value } override <T> fun loadFromRam(key: String): T? { return store[key] } } -
Failures should be designed for throw AssertionError("Should have crashed before
this") catch (e: ExpectedException) { assertThat(e).isWhatWeExpected() } @Rule val errorsRule = RxErrorsRule()