NACIS 2016 - Practical Cartography Day

Bash and Make tips for scripting reproducible workflows

Transcript

1. $ echo “Make, etc.” | \ wall -g PCD Seth Fitzsimmons
 Stamen Design, etc. seth@mojodna.net
2. # slides $ open \ http://bit.ly/2eFFvJt
3. # why? # # * repeatable # * self-documenting # * transformations # <> data changes
4. # on Windows 10? # no worries # # “Bash on Ubuntu # on Windows”
5. # shell fundamentals $ /bin/sh -c theory
6. # is a comment $ is a prompt
7. # <cmd> ––help usually # works # man <cmd> is detailed
8. # do this, then that $ this; that
9. # if this, then that $ this && that
10. # that, unless this $ this || that
11. # standard file # descriptors # (stdio)
12. # stdin (fd 0) $ cat > greeting Hi! ⌃d $ cat greeting Hi!
13. # stdout (fd 1) $ echo “Hi PCD!”
14. # stderr $ >&2 echo Error
15. # a data black hole /dev/null
16. # redirection $ echo hi > greeting
17. # append $ echo hey >> greeting
18. # stderr → stdout $ thing 2>&1 logs
19. # pipe # this means everything # can be combined!!! $ cat /etc/passwd | \ grep -i calendar
20. # exit codes $ thing; echo $?
21. # success (0) $ thing && echo $? 0
22. # failure (not 0) $ thing || echo $? 1
23. $ make basics
24. # same input (repeatedly), # same output $ make idempotency Yes $ make idempotency Yes
25. # target this: touch $@
26. # prerequisite this: that # that exists cp that this
27. $ make -j2 vars fns
28. # target this: # outputs “this” echo $@
29. # all prereqs this: that the_other # “that the_other” echo $^
30. # first prereq this: that the_other # “that” echo $<
31. # first prereq this: that the_other # “that” echo $<
32. # nth prereq this: that the_other # “the_other” echo $(word 2, $^)
33. # catch-all %: echo 42 > $@
34. $ ls /bin
35. # explore
36. # who am I? $ whoami seth
37. # where am I? $ pwd /home/seth
38. # what time is it? $ date Tue Oct 18 23:58:42 MDT 2016
39. # what’s here? $ ls -lh total 8 -rw-r--r-- 1 seth wheel …
40. # what’s in that file? $ cat /etc/passwd …
41. # …a page at a time $ less /etc/passwd …
42. # …just the beginning $ head /etc/passwd …
43. # …just the end $ tail /etc/passwd …
44. # what variables are set? $ env HOME=/home/seth …
45. # where’s that file? $ find . -type f -name hi <list of files>
46. # where’d that file go? # (full-text search, macOS) $ mdfind Seattle <list of files>
47. # get help $ man man …
48. # manipulation
49. # create a file $ touch file
50. # copy a file $ cp file file2
51. # move a file $ mv file2 file3
52. # delete a file $ rm file3
53. # create a directory $ mkdir -p my/stuff
54. # remove a directory $ rmdir my/stuff
55. # remove a directory # and everything in it $ rm -r my
56. # find lines in a file $ grep -i name file.txt
57. # find non-matching lines $ grep -v name file.txt
58. # count words, lines, # characters $ wc file.txt
59. # pretty-print JSON $ jq . file.json
60. # extract fields $ jq .name file.json
61. # replace things $ sed 's/this/that/' file
62. # extract columns $ cut -d , -f 1,3 file.csv
63. # extract columns $ cut -d , -f 1,3 file.csv
64. # compression
65. # open a zip file $ unzip file.zip
66. # list zip contents $ unzip -v file.zip
67. # create a zip $ zip file file.zip
68. # open a tarball $ tar zxf file.tar.gz
69. # list a tarball $ tar ztf file.tar.gz
70. # create a tarball $ tar zcf file.tar.gz stuff/
71. # compress with gzip $ gzip file.tar
72. # uncompress with gzip $ gzip -d file.tar.gz
73. # misc
74. # always exit 0 $ true; echo $? 0
75. # always exit non-0 $ false; echo $? 1
76. # fetch and fail # if appropriate $ curl -f nacis.org
77. # download $ wget nacis.org
78. # download $ wget nacis.org
79. # “open” (macOS) $ open nacis.org
80. # display progress $ cat /etc/passwd | \
 pv | wc -l
81. # also write to a file $ echo hi | tee file
82. $ man bash
83. #!/usr/bin/env bash
84. set -eo pipefail
85. set -x
86. NACIS=2016
87. # assignment NACIS=“2016”
88. # capture a command NACIS=$(curl nacis.org)
89. echo $NACIS
90. echo ${NACIS}
91. # set a default value echo ${NACIS:-PCD}
92. # replace echo ${NACIS/2016/2017}
93. # replace all echo ${NACIS//2016/2017}
94. # substring echo ${NACIS:2:2}
95. # remove suffix $ filename=“world.tif” $ echo ${filename%.tif} world
96. # do math $ echo $[2 ** 3] 8
97. if [[ “this” != “that” ]]; then echo Control Flow elif [[ ! -f file ]]; then touch file elif [[ $six -le $five ]]; then false else rm -f file fi
98. while true; do echo Control Flow done
99. for f in $(ls); do echo $f done
100. # man test
101. # misc
102. # extract filename $ basename /etc/passwd passwd
103. # extract directory $ dirname /etc/passwd /etc
104. $ make recipes
105. $ make convert source.json: source.shp ogr2ogr \ -t_srs EPSG:4326 \ -f GeoJSON \ $@ \ $<
106. $ make reproject output.tif: source.tif gdalwarp \ -q \ -t_srs EPSG:3857 \ $< \ $@
107. $ make wilderness data/S_USA.Wilderness.zip: @mkdir -p $$(dirname $@) @curl -sfL http://data.fs.usda.gov/ geodata/edw/edw_resources/shp/ S_USA.Wilderness.zip \ -o $@
108. $ make table db/wilderness: sql/wilderness.sql psql \ -c "\d $(subst db/,,$@)" \ > /dev/null \ 2>&1 || \ psql \ -v ON_ERROR_STOP=1 \ -qX1f \ $<
109. # resources • https://bost.ocks.org/mike/make/ • http://www.gregreda.com/2013/07/15/unix- commands-for-data-science/ • https://google.github.io/styleguide/shell.xml • https://github.com/stamen/toner-carto/blob/ master/Makefile (WARNING!) • http://mojodna.net/2015/01/07/make-for-data- using-make.html