EmberFest 2017: Testing against Time - Meaningful testing in Ember apps when timing matters

Ember offers a rich API and a wide set of helpers and blueprints to make testing in your apps fast and straight-forward. But as the applications we write tests against grow more complex, we might find ourselves stumbling into test timeouts, eventually succeeding tests and other hard-to-reason about test errors brought on by asynchronous and time-dependent behaviour.

This talk will give an insight into how to create meaningful test cases for async or other, time-related operations in our applications. We will see how we can leverage Ember's test helpers, newest JavaScript features and Ember community addons to reduce non-determinism in our test suites and make those tests turn green even as time passes by.

Transcript

1. TESTING AGAINST TIME MEANINGFUL TESTING IN EMBER APPS WHEN TIMING MATTERS
3. EMBER.JS TIMES
4. EMBER.JS BERLIN MEETUP
5. TESTING HOW & WHY
6. TESTING: HOW AND WHY WHY WE DO TESTING WORK WITH COOL AND FANCY TESTING FRAMEWORKS ✨ PUSHING BACK RELEASE DATES SPEND MORE TIME AT WORK
7. TESTING: HOW AND WHY WHY WE DO TESTING SHIP CONFIDENTLY PREVENT SUBSEQUENT BUGS INTERNAL DOCUMENTATION
9. FALSE NEGATIVES
10. TEST TIMEOUTS
11. OCCASSIONALLY PASSING TESTS
12. TESTING: HOW AND WHY
13. ASYNC OPERATIONS MAKING TESTS WAIT
14. EDITORIAL BOARD FOR EMBER.JS TIMES
15. WAITING FOR ASYNCHRONOUS OPERATIONS TESTING ROUTE TRANSITIONS
16. WAITING FOR ASYNCHRONOUS OPERATIONS TESTING ROUTE TRANSITIONS export default Route.extend({ model() { const store = this.get('store'); const repoFetches = this.get('reposList').map((repo) => { return store.findRecord('github-org', repo); }); return all(repoFetches); }, });
17. WAITING FOR ASYNCHRONOUS OPERATIONS EMBER ASYNC HELPERS: VISIT export default function visit(app, url) { let router = app.__container__.lookup('router:main'); app.boot().then(() => { router.location.setURL(url); // ... }); // teardown work return app.testHelpers.wait(); }
18. WAITING FOR ASYNCHRONOUS OPERATIONS import { test } from 'qunit'; import moduleForAcceptance from 'whats-new-in-emberland/tests/helpers/ module-for-acceptance'; moduleForAcceptance('Acceptance | overview'); test('visiting /overview', function(assert) { visit('overview'); andThen(function() { assert.equal(currentURL(), '/overview'); }); });
20. WAITING FOR ASYNCHRONOUS OPERATIONS visit() click() fillIn() keyEvent() triggerEvent() export default function visit(app, url){ // … return app.testHelpers.wait(); } BUILT-IN ASYNCHRONOUS HELPERS FOR ACCEPTANCE TESTING
21. WAITING FOR ASYNCHRONOUS OPERATIONS TESTING ASYNC USER INTERACTIONS
22. WAITING FOR ASYNCHRONOUS OPERATIONS TESTING ASYNC USER INTERACTIONS import Component from '@ember/component'; import { debounce } from '@ember/runloop'; export default Component.extend({ pull: null, actions: { loadComments(pull) { this.set('isLoadingComments', true); debounce(this, this.loadComments, pull, 800); }, }, });
23. WAITING FOR ASYNCHRONOUS OPERATIONS TESTING ASYNC USER INTERACTIONS import Component from '@ember/component'; import { debounce } from '@ember/runloop'; export default Component.extend({ pull: null, comments: null, loadComments(pull) { const commentsUrl = pull.get('commentsUrl'); return this.get(‘request’).fetch(commentsUrl).then((comments) => { this.set('comments', comments); this.set('isLoadingComments', false); }); }, actions: { loadComments(pull) { this.set('isLoadingComments', true); debounce(this, this.loadComments, pull, 800); }, }, });
24. WAITING FOR ASYNCHRONOUS OPERATIONS moduleForComponent('news-item', 'Integration | Component | news item'); test(‘loading comments via user interaction', function(assert) { // ember install ember-data-factory-guy this.set('pull', make('github-pull')); this.render(hbs`{{news-item pull=pull repo=repo}}`); $(‘[data-test-load-comments]').click(); assert.equal($('[data-test-num-of-comments]').text(), `2`, 'displays comments'); });
25. NOT OK
26. WAITING FOR ASYNCHRONOUS OPERATIONS YARN ADD TESTDOUBLE
27. WAITING FOR ASYNCHRONOUS OPERATIONS moduleForComponent(‘news-item', 'Integration | Component | news item', { beforeEach() { this.register('service:request', Service.extend({ fetch: td.function(), }); }, });
28. WAITING FOR ASYNCHRONOUS OPERATIONS moduleForComponent('news-item', 'Integration | Component | news item'); test('loading comments via user interaction', function(assert) { const pull = make(‘github-pull') this.set('pull', pull); this.set('comments', makeList('github-comment', 2)); const commentsUrl = 'https://api.github.com/repos/user1/repository/pulls/1/comments'; td.when(this.get(‘request’).fetch(commentsUrl)) .thenResolve(this.comments); this.render(hbs`{{news-item pull=pull repo=repo}}`); $(‘[data-test-load-comments]').click(); assert.equal($(‘[data-test-num-of-comments]’).text().trim(), `2`, 'displays comments'); });
29. WAITING FOR ASYNCHRONOUS OPERATIONS const fetch = td.function(); td.when(this.get(‘request’) .fetch(commentsUrl)) .thenResolve(this.comments); loadComments(pull) { const commentsUrl = pull.get('commentsUrl'); return this.get(‘request’) .fetch(commentsUrl) .then((comments) => { this.set('comments', comments); this.set('isLoadingComments', false); }); }, app/components/news-item.js tests/integration/components/news-item-test.js
30. WAITING FOR ASYNCHRONOUS OPERATIONS td.when().thenResolve() td.when().thenReject() td.when().thenCallback() MOCKING PROMISES SYNCHRONOUSLY IMMEDIATE RESOLUTION / REJECTION OF MOCKED PROMISES
31. WAITING FOR ASYNCHRONOUS OPERATIONS moduleForComponent('news-item', 'Integration | Component | news item'); test('loading comments via user interaction', function(assert) { const pull = make(‘github-pull') this.set('pull', pull); this.set('comments', makeList('github-comment', 2)); const commentsUrl = 'https://api.github.com/repos/user1/repository/pulls/1/comments'; td.when(this.get(‘request’) .fetch(commentsUrl)) .thenResolve(this.comments); this.render(hbs`{{news-item pull=pull repo=repo}}`); $(‘[data-test-load-comments]’).click(); assert.equal($('[data-test-num-of-comments]').text(), `2`, 'displays comments'); });
32. NOT OK
33. WAITING FOR ASYNCHRONOUS OPERATIONS wait().then(callback) LETTING PROMISES AND TIMERS SETTLE POLLING FOR ANY PENDING REQUESTS AND ACTIONS IN RUN LOOP
34. WAITING FOR ASYNCHRONOUS OPERATIONS import { wait } from 'ember-test-helpers/wait'; moduleForComponent('news-item', 'Integration | Component | news item'); test('loading comments via user interaction', function(assert) { const pull = make(‘github-pull') this.set('pull', pull); this.set('comments', makeList('github-comment', 2)); const commentsUrl = 'https://api.github.com/repos/user1/repository/pulls/1/comments'; td.when(this.get(‘request’).fetch(commentsUrl)) .thenResolve(this.comments); this.render(hbs`{{news-item pull=pull repo=repo}}`); $('[data-test-load-comments]').click(); return wait(() => { assert.equal($('[data-test-num-of-comments]').text(), `2`, 'displays comments'); }); });
35. WAITING FOR ASYNCHRONOUS OPERATIONS import { wait } from 'ember-test-helpers/wait'; moduleForComponent('news-item', 'Integration | Component | news item'); test('loading comments via user interaction', function(assert) { const pull = make(‘github-pull') this.set('pull', pull); this.set('comments', makeList('github-comment', 2)); const commentsUrl = 'https://api.github.com/repos/user1/repository/pulls/1/comments'; td.when(this.get(‘request’).fetch(commentsUrl)) .thenResolve(this.comments); this.render(hbs`{{news-item pull=pull repo=repo}}`); $('[data-test-load-comments]').click(); return wait(() => { assert.equal($('[data-test-num-of-comments]').text(), `2`, 'displays comments'); }); });
36. WAITING FOR ASYNCHRONOUS OPERATIONS await fillIn() await click() await keyEvent() await triggerEvent() await focus() await blur() await tap() EMBER-NATIVE-DOM-HELPERS & AWAIT / ASYNC SUITABLE FOR BOTH YOUR ACCEPTANCE & INTEGRATION TESTS
37. WAITING FOR ASYNCHRONOUS OPERATIONS import { click } from 'ember-native-dom-helpers'; moduleForComponent('news-item', 'Integration | Component | news item'); test('loading comments via user interaction', async function(assert) { this.set('pull', make('github-pull')); this.set('comments', makeList('github-comment', 2)); const commentsUrl = 'https://api.github.com/repos/user1/repository/pulls/1/comments'; td.when(this.get(‘request’).fetch(commentsUrl)) .thenResolve(this.comments); this.render(hbs`{{news-item pull=pull repo=repo}}`); await click(‘[data-test-load-comments]’); assert.equal($('[data-test-num-of-comments]').text(), `2`, 'displays comments'); });
39. WAITING FOR ASYNCHRONOUS OPERATIONS export default Component.extend({ //… reloadComments: task(function * () { const commentsUrl = this.get('pull.commentsUrl'); const comments = yield this.get(‘request’) .fetch(this.get(‘pull.commentsUrl’)); this.set('comments', comments); }).drop(), });
40. WAITING FOR ASYNCHRONOUS OPERATIONS export default Component.extend({ startReloading: task(function *(){ while(true) { this.get(‘reloadComments').perform(); yield timeout(50000); }, reloadComments: task(function * () { const commentsUrl = this.get('pull.commentsUrl'); const comments = yield this.get(‘request’) .fetch(this.get(‘pull.commentsUrl’)); this.set('comments', comments); }).drop(), });
41. WAITING FOR ASYNCHRONOUS OPERATIONS td.when(this.get(‘request’) .fetch(commentsUrl)) .thenResolve(this.comments); this.render(hbs`{{news-item pull=pull repo=repo}}`); return wait().then(() => { assert.equal(find('[data-test-num-of-comments]').textContent.trim(), `2`, 'displays comments'); });
42. TEST TIMEOUT
43. WAITING FOR ASYNCHRONOUS OPERATIONS startReloading: task(function *(){ while(true) { this.get(‘reloadComments').perform(); if (Ember.testing) { return; } yield timeout(50000); } }
44. WAITING FOR ASYNCHRONOUS OPERATIONS startReloading: task(function *(){ while(true) { this.get(‘reloadComments').perform(); if (Ember.testing) { return; } yield timeout(50000); } }
45. WAITING FOR ASYNCHRONOUS OPERATIONS …CREATES A PERFECT HOME FOR BUGS
46. WAITING FOR ASYNCHRONOUS OPERATIONS AND WHAT IF WE NEED TO TEST SEVERAL ITERATIONS?
47. WAITING FOR ASYNCHRONOUS OPERATIONS this.render(hbs`{{news-item pull=pull repo=repo}}`); later(() => { run.cancelTimers(); }, 500); return wait().then(() => { assert.equal(find('[data-test-num-of-comments]').textContent.trim(), `2`, 'displays comments'); });
48. NOT OK
49. WAITING FOR ASYNCHRONOUS OPERATIONS this.render(hbs`{{news-item pull=pull repo=repo}}`); later(() => { run.cancelTimers(); }, 500000); return wait().then(() => { assert.equal(find('[data-test-num-of-comments]').textContent.trim(), `2`, 'displays comments'); });
50. WAITING FOR ASYNCHRONOUS OPERATIONS
51. WAITING FOR ASYNCHRONOUS OPERATIONS const TIMEOUT_INTERVAL = Ember.testing ? 1 : 500000; //… startReloading: task(function *(){ while(true) { yield timeout(TIMEOUT_INTERVAL); this.get(‘reloadComments').perform(); } }
52. this.render(hbs`{{news-item pull=pull repo=repo}}`); later(() => { run.cancelTimers(); }, 50); return wait().then(() => { assert.equal(find('[data-test-num-of-comments]').textContent.trim(), `2`, 'displays comments'); });
54. WAITING FOR ASYNCHRONOUS OPERATIONS FURTHER READING Ember Concurrency Docs on Testing: https://ember-concurrency.com/#/docs/testing-debugging Ember Testing Unificationn RFC: https://github.com/emberjs/rfcs/pull/119
55. TIME TRAVEL MAKING TESTS PASS AT ANY TIME
56. SETTING AND TRAVELLING THROUGH TIME
57. WAITING FOR ASYNCHRONOUS OPERATIONS test('the overview page doesn\'t stress me out with release date disclaimers', function(assert) { visit('/overview'); andThen(function() { assert.notOk(find('[data-test-is-thursday-disclaimer]'), 'doesn\'t display Thu disclaimer'); assert.notOk(find('[data-test-is-friday-disclaimer]'), 'doesn\'t display Fri disclaimer'); }); });
58. WAITING FOR ASYNCHRONOUS OPERATIONS test('the overview page doesn\'t stress me out with release date disclaimers', function(assert) { visit('/overview'); andThen(function() { assert.notOk(find('[data-test-is-thursday-disclaimer]'), 'doesn\'t display Thu disclaimer'); assert.notOk(find('[data-test-is-friday-disclaimer]'), 'doesn\'t display Fri disclaimer'); }); }); WHEN THIS TEST IS RUN ON A THURSDAY OR FRIDAY NOT OK
59. WAITING FOR ASYNCHRONOUS OPERATIONS moduleForAcceptance('Acceptance | overview', { beforeEach() Timecop.install(); Timecop.travel(new Date(2017, 9, 11, 11, 45)); }, afterEach(){ Timecop.uninstall(); } }); EMBER INSTALL EMBER-CLI-TIMECOP
60. WAITING FOR ASYNCHRONOUS OPERATIONS test('the overview page doesn\'t stress me out with release date disclaimers', function(assert) { visit('/overview'); andThen(function() { assert.notOk(find('[data-test-is-thursday-disclaimer]'), 'doesn\'t display Thursday disclaimer'); assert.notOk(find('[data-test-is-friday-disclaimer]'), 'doesn\'t display Friday disclaimer'); }); }); moduleForAcceptance('Acceptance | overview', { beforeEach() Timecop.install(); Timecop.travel(new Date(2017, 9, 11, 11, 45)); // is a Wednesday: 11.10.2017 }, afterEach(){ Timecop.uninstall(); } });
61. WAITING FOR ASYNCHRONOUS OPERATIONS test('the overview page…oh no, it’s Friday’, function(assert) { Timecop.travel(new Date(2017, 9, 13, 16, 20)); // is today’s Friday! :o visit('/overview'); andThen(function() { assert.notOk(find('[data-test-is-thursday-disclaimer]'), 'doesn\'t display Thursday disclaimer'); assert.ok(find('[data-test-is-friday-disclaimer]'), ‘the newsletter has to get out - oh no!’); }); });
62. TIMING IN YOUR TESTS MATTERS
63. TIMING IN YOUR TESTS MATTERS
64. DANKESCHÖN! @JESSICA @JJORDAN_DEV