Harnessing the Power of Abstract Syntax Trees

A look at building powerful tooling with abstract syntax trees in JavaScript

Transcript

1. HARNESSING THE POWER OF Abstract Syntax Trees
2. Parsers
3. console.log("UtahJS");
5. { type: "Program", body: [ { type: "ExpressionStatement", expression: { type: "CallExpression", callee: { type: "MemberExpression", computed: false, object: { type: "Identifier", name: "console" }, property: { type: "Identifier", name: "log" } }, arguments: [ { type: "Literal", value: "UtahJS", raw: "\"UtahJS\"" } ] } } ] }
7. Acorn
8. Esprima
9. Espree
10. PARSING // generate an AST from a string of code espree.parse("console.log('UtahJS')");
11. PARSING // generate an AST from a string of code acorn.parse("console.log('UtahJS')");
12. PARSING ES6 // generate an AST from a string of code acorn.parse("console.log('UtahJS')", { ecmaVersion: 6 });
13. ASTS ARE EVERYWHERE
14. BUILD-TIME CODE TRANFORMATION ONE-OFF CODE MIGRATION CODE INSPECTION
15. CODE Tranformation
16. How many of you use babel?
18. BABEL PLUGIN module.exports = function (Babel) { return new Babel.Plugin("plugin-example", { // visitor: { // "FunctionDeclaration": swapWithExpression // } }); };
19. BABEL PLUGIN module.exports = function (Babel) { return new Babel.Plugin("plugin-example", { visitor: { "FunctionDeclaration": swapWithExpression } }); };
20. GUTS function swapWithExpression(node, parent) { var id = node.id; // change the node type // node.type = "FunctionExpression"; // node.id = null; // return a variable declaration // return Babel.types.variableDeclaration("var", [ // Babel.types.variableDeclarator(id, node) // ]); }
21. GUTS function swapWithExpression(node, parent) { var id = node.id; // change the node type node.type = "FunctionExpression"; node.id = null; // return a variable declaration // return Babel.types.variableDeclaration("var", [ // Babel.types.variableDeclarator(id, node) // ]); }
22. GUTS function swapWithExpression(node, parent) { var id = node.id; // change the node type node.type = "FunctionExpression"; node.id = null; // return a variable declaration return Babel.types.variableDeclaration("var", [ Babel.types.variableDeclarator(id, node) ]); }
23. BABEL PLUGIN RESULTS // function declaration function help() {} // transformed to a function expression var help = function() {}
24. CODE Migration
26. JSCODESHIFT EXAMPLE module.exports = function(fileInfo, api) { return api.jscodeshift(fileInfo.source) // .findVariableDeclarators('foo') // .renameTo('bar') // .toSource(); };
27. JSCODESHIFT EXAMPLE module.exports = function(fileInfo, api) { return api.jscodeshift(fileInfo.source) .findVariableDeclarators('foo') // .renameTo('bar') // .toSource(); };
28. JSCODESHIFT EXAMPLE module.exports = function(fileInfo, api) { return api.jscodeshift(fileInfo.source) .findVariableDeclarators('foo') .renameTo('bar') // .toSource(); };
29. JSCODESHIFT EXAMPLE module.exports = function(fileInfo, api) { return api.jscodeshift(fileInfo.source) .findVariableDeclarators('foo') .renameTo('bar') .toSource(); };
30. JSCODESHIFT RESULTS // this gets var foo = "UtahJS"; // transformed to var bar = "UtahJS"
31. CODE Inspection
33. CUSTOM LINT RULES // lib/rules/no-class.js module.exports = function(context) { return { "ClassDeclaration": function(node) { context.report(node, "Your code has no class"); } } }
34. KNOW YOUR NODES
36. AN AST GIVES YOU SUPER POWERS
38. "Making easy things easy & hard things possible" - Learning Perl
39. JS AWARE git diff
40. console.log("UtahJS");
41. console.log("SomeOtherConf");
42. console.error("SomeOtherConf");
46. git diff --raw | node compare.js
47. A BETTER DIFF // turn stdin into an array of lines var lines = fs.readFileSync('/dev/stdin').toString().split('\n');
48. A BETTER DIFF // lines now looks something like this [':100644 100644 9a0b08f... 0000000... M tree1.js']
49. A BETTER DIFF lines.map(function(line) { var parts = line.split(' '); // var file = parts.pop().split('\t'); // return [file[1], parts[2].slice(0, -3)]; });
50. A BETTER DIFF lines.map(function(line) { var parts = line.split(' '); var file = parts.pop().split('\t'); // return [file[1], parts[2].slice(0, -3)]; });
51. A BETTER DIFF lines.map(function(line) { var parts = line.split(' '); var file = parts.pop().split('\t'); return [file[1], parts[2].slice(0, -3)]; });
52. A BETTER DIFF // the key parts of our git diff [ [ "tree1.js", "9a0b08f"] ]
53. PARSING // generate an AST from a string of code espree.parse("console.log('UtahJS)");
54. .map(function(files) { var after = fs.readFileSync(files[0]); // var before = child_process.execSync("git show" + files[1]); // return { // filename: files[0], // before: espree.parse(before, options), // after: espree.parse(after, options) // }; })
55. .map(function(files) { var after = fs.readFileSync(files[0]); var before = child_process.execSync("git show" + files[1]); // return { // filename: files[0], // before: espree.parse(before, options), // after: espree.parse(after, options) // }; })
56. .map(function(files) { var after = fs.readFileSync(files[0]); var before = child_process.execSync("git show" + files[1]); return { filename: files[0], before: espree.parse(before, options), after: espree.parse(after, options) }; })
57. [{ filename: "trees1.js", before: { type: "Program", body: [Object] }, after: { type: "Program", body: [Object] } }]
58. FITS ON ONE SLIDE*
59. var lines = fs.readFileSync('/dev/stdin').toString().split('\n'); var trees = lines.map(function(line) { var parts = line.split(' '); var file = parts.pop().split('\t'); return [path.resolve(file[1]), parts[2].slice(0, -3)]; }).filter(function(files) { return files[0].indexOf('.js') > -1; }).map(function(files) { var after = fs.readFileSync(files[0]); var before = child_process.execSync("git show " + files[1]); return { filename: files[0], before: espree.parse(before, options), after: espree.parse(after, options) }; });
62. DIFFING THE TREES // let's see if something changed var different = deepEqual(treeBefore, treeAfter);
63. function deepEqual(a, b) { if (a === b) { return true; } if (!a || !b) { return false; } if (Array.isArray(a)) { return a.every(function(item, i) { return deepEqual(item, b[i]); }); } if (typeof a === 'object') { return Object.keys(a).every(function(key) { return deepEqual(a[key], b[key]); }); } return false; }
64. if (typeof a === 'object') { return Object.keys(a).every(function(key) { return deepEqual(a[key], b[key]); }); } return false;
65. if (typeof a === 'object') { var equal = Object.keys(a).every(function(key) { return deepEqual(a[key], b[key]); }); return equal; } return false;
66. if (typeof a === 'object') { // var equal = Object.keys(a).every(function(key) { // return deepEqual(a[key], b[key]); // }); if (!equal) { console.log('[' + a.type + '] => [' + b.type + ']'); } // return equal; } console.log('"' + a + '" => "' + b + '"'); // return false;
67. git diff --raw | node compare.js "log" => "error" [Identifier] => [Identifier] [MemberExpressio] => [MemberExpression] [CallExpression] => [CallExpression] [ExpressionStatement] => [ExpressionStatement] [Program] => [Program]
71. I CAN'T PROCESS THIS TREE IN MY HEAD
72. export function buildHouse(lot, color, size, bedrooms) { clearLot(lot); let foundation = buildFoundation(size); let walls = buildWalls(bedrooms); let paintedWalls = paintWalls(color, walls); let roof = buildRoof(foundation, walls); let house = foundation + paintedWalls + roof; // house is all done right-away return house; }
73. function getPermits(callback) { setTimeout(callback, 1.0519e10); // 4 months because trees } export function buildHouse(lot, color, size, bedrooms, callback) { getPermits((permits) => { clearLot(permits, lot); let foundation = buildFoundation(size); let walls = buildWalls(bedrooms); let paintedWalls = paintWalls(color, walls); let roof = buildRoof(foundation, walls); let house = foundation + paintedWalls + roof; // house will be ready in about a year callback(house); }); }
76. OUR GOAL git diff --raw | node compare.js house.js 1. The exported `buildHouse` function output went from a return to a callback. 2. The private `getPermits` function was added.
77. AN ARRAY OF TREES [{ filename: "trees1.js", before: { type: "Program", body: [Object] }, after: { type: "Program", body: [Object] } }]
78. VISITING OUR TREES esrecurse.visit(diff.before, { // export function a() {} ExportNamedDeclaration: function(node) { /* ... */ }, // function a() {} FunctionDeclaration: function(node) { /* ... */ } });
79. INSPECTING FUNCTION DECLARATIONS function inspectFunction(node, visiblity) { return { name: node.id.name, // "buildHouse", // params: node.params.map(param => param.name), // ["lot", "color", ...] // visibility: visiblity || "private", // outputType: getOutputType(node) }; }
80. INSPECTING FUNCTION NODES function inspectFunction(node, visiblity) { return { name: node.id.name, // "buildHouse" params: node.params.map(param => param.name), // ["lot", "color", ...] // visibility: visiblity || "private", // outputType: getOutputType(node) }; }
81. INSPECTING FUNCTION DECLARATIONS function inspectFunction(node, visiblity) { return { name: node.id.name, // "buildHouse" params: node.params.map(param => param.name), // ["lot", "color", ...] visibility: visiblity || "private", // outputType: getOutputType(node) }; }
82. INSPECTING FUNCTION DECLARATIONS function inspectFunction(node, visiblity) { return { name: node.id.name, // "buildHouse" params: node.params.map(param => param.name), // ["lot", "color", ...] visibility: visiblity || "private", outputType: getOutputType(node) }; }
83. GETTING OUTPUT TYPE function getOutputType(node) { var params = node.params.map(param => param.name); // is there a callback param? var hasCallback = params[params.length - 1] === 'callback'; // is there a return in the immediate body? var hasReturn = node.body.some((node) => node.type === 'ReturnStatement'); // callback or return or '' return hasCallback ? 'callback' : (hasReturn ? 'return' : ''); }
84. THE ESSENCE OF A FUNCTION { name: "getPermits", visibility: "private", params: [ "callback" ], outputType: "callback" }
85. MORE COMPLEX REPRESENTATION [{ filename: "house.js", functions: { "getPermits": { after: { /* ... */ } }, "buildHouse": { before: { /* ... */ }, after: { /* ... */ } } }]
86. REDUCING COMPLEXITY function getReadableOutput(functions) { return Object.keys(functions).reduce(function(ouput, name, i) { var whatHappened = getWhatHappened(functions[name]); // var visibility = functions[name].after.visibility; // var message = `The ${visibility} ${name} function ${whatHappened}.\n` // return `${output}${i + 1}. ${message}`; }, ""); }
87. HUMAN READABLE FTW! function getWhatHappened(func) { if (!func.before) { return "was added" } if (!func.after) { return "was removed" } if (func.before.outputType !== func.after.outputType) { return "output went from a " + func.before.outputType + " to a " + func.after.outputType; } }
88. `1. The exported `buildHouse` function output went from a return to a callback. 2. The private `getPermits` function was added.`
89. UNROLLING OUR ARRAY .map(function(diff) { return diff.filename + "\n" + getReadableOutput(diff.functions); }).join("\n");
90. A HAPPY ENDING git diff --raw | node compare.js house.js 1. The exported `buildHouse` function output went from a return to a callback. 2. The private `getPermits` function was added.
91. TREES ARE SUPER POWERFUL
92. THE END