Data Parsing and Visualization

Data Parsing And Visualization Kirby Wu @ g0v.tw

http://comic.sfacg.com/HTML/Naruto/ http://comic.sfacg.com/Utility/4/xxx.js comicCounterID = 4; give link parse  ID parse 
list get js url get imgs

var comicName = “怠蔣敹!"; var nextVolume=“/HTML/Naruto/476/"; var preVolume=“/HTML/Naruto/474/"; var picCount
= 17; var picAy = new Array(); var hosts = ["http://hotpic.sfacg.com","http://hotpic.sfacg. “http://ltpic.sfacg.com"]; picAy[0] = “/Pic/OnlineComic1/Naruto/475/001_1924.png"; picAy[1] = “/Pic/OnlineComic1/Naruto/475/002_1209.png"; picAy[2] = “/Pic/OnlineComic1/Naruto/475/003_17512.png"; picAy[3] = “/Pic/OnlineComic1/Naruto/475/004_13346.png"; picAy[4] = “/Pic/OnlineComic1/Naruto/475/005_6797.png"; picAy[5] = “/Pic/OnlineComic1/Naruto/475/006_16563.png"; picAy[6] = “/Pic/OnlineComic1/Naruto/475/007_4992.png"; picAy[7] = “/Pic/OnlineComic1/Naruto/475/008_5900.png"; picAy[8] = “/Pic/OnlineComic1/Naruto/475/009_30082.png"; picAy[9] = “/Pic/OnlineComic1/Naruto/475/010_18438.png"; picAy[10] = “/Pic/OnlineComic1/Naruto/475/011_19255.png"; picAy[11] = “/Pic/OnlineComic1/Naruto/475/012_17436.png"; picAy[12] = “/Pic/OnlineComic1/Naruto/475/013_14834.png"; picAy[13] = “/Pic/OnlineComic1/Naruto/475/014_16148.png"; . . . . . . (後略)

http://campaign-ﬁnance.g0v.ctiml.tw/

@g0v hackath8n

Data Scraping crawling / 砍材

http://www.ptt.cc/bbs/Gossiping/index.html index4534.html

HTML • html — document with tags <html> <body> <img
src=“blah”/> <h1 id=“main”> Header </h1> <p class=“hot”>hot!</p> <p> this is a book … </p> <a href=“http://…”>back</a> </body> </html>

CSS Selector • html — document with tags • tag
selector: p, div • class selector: .title, .desc • id selector: #content, #dialog • relations: h1 > a, br ~ p, p div • pseudo: a:ﬁrst-letter, h2:ﬁrst-line • attr: a[href=“#”]

.r-ent h1 .r-ent .title a .r-ent date .r-ent author document.querySelectorAll(“.r-ent
h1”)

cheerio jQuery subset in server side cheerio = require(“cheerio”) $
= cheerio.load(YourHtmlSnippet); ! result = $(“.r-ent .hl”).text() npm install cheerio

request simpliﬁed HTTP Client request = require(“request”) ptt = “http://ptt.cc/bbs/Gossiping/index.html”
! request(ptt, function(e,r,b) { console.log(b); }); npm install request

request + cheerio request = require(“request”) cheerio = require(“cheerio”) !
ptt = “http://ptt.cc/bbs/Gossiping/index.html” ! request(ptt, function(e,r,b) { $ = cheerio.load(b); result = $(“.r-ent .hl”).text(); }); npm install cheerio request

request + cheerio result = {}, idx = 1; function
download() { request(ptt(idx), function(e,r,b) { if(e) return; $ = cheerio.load(b); result[idx] = $(“.r-ent .hl”).text(); idx++; setTimeout(download, 0); }); }

http://zbryikt.github.io/visualize/ptt-user/

request + cheerio result = {}, idx = 1; function
download() { request(ptt(idx), function(e,r,b) { if(e) return; $ = cheerio.load(b); result[idx] = $(“.r-ent .hl”).text(); idx++; setTimeout(download, 0); }); }

HTTP request response CLIENT SERVER GET,POST,… Referer Cookie Session Source
IP Frequency Captcha State

https://cris.hpa.gov.tw/ 癌症線上登記系統 curl 'https://cris.hpa.gov.tw/pagepub/Home.aspx?itemNo=cr.q.40' -H 'Cookie: ASP.NET_SessionId=zdmkqgelubtzswj0majv3aiw; _ga=GA1.3.1312976917.1398148250' -H 'Origin:
https://cris.hpa.gov.tw' -H 'Accept-Encoding: gzip,deflate,sdch' -H 'Accept-Language: zh-TW,zh;q=0.8,en-US;q=0.6,en;q=0.4,ja;q=0.2,zh-CN;q=0.2' -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.116 Safari/537.36' -H 'Content-Type: multipart/form-data; boundary=----WebKitFormBoundaryuuaKlSwaG0BKt1db' -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/ *;q=0.8' -H 'Cache-Control: max-age=0' -H 'Referer: https://cris.hpa.gov.tw/ pagepub/Home.aspx?itemNo=cr.q.40' -H 'Connection: keep-alive' --data-binary $'------WebKitFormBoundaryuuaKlSwaG0BKt1db\r\nContent-Disposition: form-data; name="__EVENTTARGET"\r\n\r\n\r\n------WebKitFormBoundaryuuaKlSwaG0BKt1db\r \nContent-Disposition: form-data; name="__EVENTARGUMENT"\r\n\r\n\r\n------ WebKitFormBoundaryuuaKlSwaG0BKt1db\r\nContent-Disposition: form-data; name="__VIEWSTATE_ID"\r\n\r\n871165\r\n------WebKitFormBoundaryuuaKlSwaG0BKt1db \r\nContent-Disposition: form-data; name="__VIEWSTATE"\r\n\r\n\r\n------ WebKitFormBoundaryuuaKlSwaG0BKt1db\r\nContent-Disposition: form-data; name="__EVENTVALIDATION"\r\n\r\nXOBXdAUfZ/ +c9MQLjFuAu4ua2oATUYv7Ky61orP3bnuQBqTBl/he3+CGfCqTF6lo5pzEGdQnpmzPocNVZ +EjiTjjj/0tLHu8PFynQ0i1IVvE3EEyJvHguowAYIThKE4kvfrG7A==\r\n------ WebKitFormBoundaryuuaKlSwaG0BKt1db\r\nContent-Disposition: form-data;

phantomjs a headless WebKit scriptable with a JavaScript API page
= require(“webpage”).create() page.open(SomeURL, function() { comicID = page.evaluate(function() { return comicCounterID; }); page.render(“snapshot.png”); }); http://github.com/hcchien/doh-cancer/

Summary * CSS Selector ! * Cheerio + request !
* HTTP Header *wget / curl ! * PhantomJS ! * Tools like OpenCV

var comicName = “怠蔣敹!"; var nextVolume=“/HTML/Naruto/476/"; var preVolume=“/HTML/Naruto/474/"; var picCount
= 17; var picAy = new Array(); var hosts = ["http://hotpic.sfacg.com","http://hotpic.sfacg. “http://ltpic.sfacg.com"]; picAy[0] = “/Pic/OnlineComic1/Naruto/475/001_1924.png"; picAy[1] = “/Pic/OnlineComic1/Naruto/475/002_1209.png"; picAy[2] = “/Pic/OnlineComic1/Naruto/475/003_17512.png"; picAy[3] = “/Pic/OnlineComic1/Naruto/475/004_13346.png"; picAy[4] = “/Pic/OnlineComic1/Naruto/475/005_6797.png"; picAy[5] = “/Pic/OnlineComic1/Naruto/475/006_16563.png"; picAy[6] = “/Pic/OnlineComic1/Naruto/475/007_4992.png"; picAy[7] = “/Pic/OnlineComic1/Naruto/475/008_5900.png"; picAy[8] = “/Pic/OnlineComic1/Naruto/475/009_30082.png"; picAy[9] = “/Pic/OnlineComic1/Naruto/475/010_18438.png"; picAy[10] = “/Pic/OnlineComic1/Naruto/475/011_19255.png"; picAy[11] = “/Pic/OnlineComic1/Naruto/475/012_17436.png"; picAy[12] = “/Pic/OnlineComic1/Naruto/475/013_14834.png"; picAy[13] = “/Pic/OnlineComic1/Naruto/475/014_16148.png"; . . . . . . (後略)

Search & Replace

Search & Replace [email protected] http://4am.tw 0800-000-123 texts with pattern picAy[1]
= “ … “

123 —> 123 123 —> \d{3} abc —> \w{3} !
B98902123 —> B\d{8}

abc abbc abbbc … ab+c ac abc abbc abbbc …
ab*c ac abc ab?c

aac aBc a:c a-c a1c a2c …. a.c aac abc
acc adc aec …. a[a-z]c aac abc acc adc aec …. ^a[a-z]c$ a[^A-Z]c

aacabcacc (a[a-z]c)+

+ * ? . [ ] ^ $ () \w
\W \d \D \s \S \\ \. \+ \? \* …

gmail.com tkirby [a-zA-Z0-9]+ [^.]+\.[^.]+ [email protected] [a-zA-Z0-9]+@[^.]+\.[^.]+

Regular Expression a sequence of characters that forms a search
pattern, mainly for use in pattern matching with strings… 正規表達式

09xx-xxx-xxx \d+-\d+-\d+ \d{4}-\d{3}-\d{3} \d{4}-?\d{3}-?\d{3} b98902123 b\d{8} <a href=“url”> … <a\s+href\s+=\s+“([^”]+)”

hl f3”>(\d+)< “date”>([^<]+)< “author”>([^<]+)< title“><a href=“([^”]+)”>([^<]+)<

re = { push: ‘hl f3”>(\d+)<‘ title: ‘title“><a href=“([^”]+)”>([^<]+)<‘ date:
‘“date”>([^<]+)<‘ author: ‘“author”>([^<]+)<‘ for(i=0;i<lines.length;i++) { ret = re.push.exec(lines[i]); if(ret) { push = ret[1]; } else … }

grep grep [-rvi] “regexp” [filename] globally search a regular expression
and print command:

sed sed “s/pattern/string/“ stream editor for ﬁltering and transforming text
command:

衛⽣生福利部台灣鄉鎮別罹癌統計

xlsjs XLS (Excel 97-2004) and XML (2003/2004) parser xls2csv [filename]
command: npm install xlsjs

xls —> csv —> json doc —> txt —> json
pdf —> txt —> json html —> txt —> json shp —> json xml —> json …… Always Search for Existing Tools

v/[a-zA-Z0-9]\+@[^.]\+\.[^.]\+/d Vim g/[a-zA-Z0-9]\+@[^.]\+\.[^.]\+/d highly conﬁgurable text editor to enable efﬁcient
text editing

Vim 指令式⽂文字編輯器 h j k l — 左下上
右指令模式 / 編輯模式 a o i <ESC> — 附加新增⼀一⾏行插入退出編輯模式 c d r x — 改寫刪除覆蓋刪掉⼀一個字數字鍵 — 重複次數 y p — 複製貼上

100i2<ESC> — 插入 100 個 2 ddp — 與下⼀一⾏行交換 100dd100jp
— 下⼀一百⾏行移到兩百⾏行之後 d100w — 刪除⼀一百個字 qad100wq — 將「d100w」命名為「a」 @a — 重複剛剛命名為「a」的動作

⾃自動化的⽂文字編輯 q100dd100jp100j 100@a 將0~100, 100~200 ⾏行交換, 200~300, 300~400 ⾏行交換, 400~500,
500~600 ⾏行交換, ……. ⼀一直重複到 10000 ⾏行為⽌止

⾃自動化的⽂文字編輯範例 g0v.tw 聊天室記錄 HTML 檔 ( http://logbot.g0v.tw/ ) 台北市親山步道列表
( http://zbryikt.github.io/visualize/hiking/ )  ( http://www.tcge.taipei.gov.tw/MP_106051.html )

td:nth-of-type(2n) td:nth-of-type(2n+1) tr td:nth-of-type(1) td:nth-of-type(2n+1) tr td:nth-of-type(3)

! 捷運關渡站 1.75K → 約34分雕塑公園 0.4K → 約7分打印台
0.6K → 約16分忠義山打印台 0.8K → 約25分⾏行天宮登山⼜⼝口 ! 捷運忠義站 1.1K → 約15分⾏行天宮登山⼜⼝口 0.8K → 約30分忠義山打印台 1.34K → 約43分中央北路四段30巷

Online Tools • Kimonolabs • ScraperWiki • Web Scrape Master
• Web Scraper

Summary • Regular Expression • grep, sed, sort, … •
xlsjs, xls2csv, … • vim, sublime Text, Emacs • 定義規則, 重複操作, 利⽤用前⼈人的成果

axe.g0v.tw cris.hpa.gov.tw 斧頭幫的挑戰狀

Useful Junk? The Effects of Visual Embellishment on Comprehension and
Memorability of Charts Scott Bateman, Saskatchewan university

1 2 3 4 反⿊黑箱⽩白⾊色正義 (⼈人次)

1 2 3 4 反⿊黑箱⽩白⾊色正義 2 2 y (⼈人次)

50萬反⿊黑箱⽩白⾊色正義 10萬錯誤的⻑⾧長條圖使⽤用⽅方式! 基準點需要從零開始

50萬反⿊黑箱⽩白⾊色正義 0 正確的⻑⾧長條圖使⽤用⽅方式! ...好吧這是個失敗的例⼦子

1000 反⿊黑箱⽩白⾊色正義 0 換個觀點，陳述的事實也不同

1000 反⿊黑箱⽩白⾊色正義 0 萬⼈人響應 ———— 九⼈人到場少數⼈人  不代表主流⺠民意 y:
響應 / 到場

brand / position / 3D

7 6 5 2 8 7 6 5 2 8
7 6 5 2 8 7 6 5 2 8 7 6 5 2 8 7 6 5 2 8 7 6 5 2 8 7 6 5 2 8 7 6 5 2 8 7 6 5 2 8 2 8 7 6 5 2 2 2 8 2

7 6 5 2 8 7 6 5 2 8
7 6 5 2 8 7 6 5 2 8 7 6 5 2 8 7 6 5 2 8 7 6 5 2 8 7 6 5 2 8 7 6 5 2 8 7 6 2 8 2 8 7 6 5 2 2 2 8 2

7 6 5 2 8 7 6 5 2 8
7 6 5 2 8 7 6 5 2 8 7 6 5 2 8 7 6 5 2 8 7 6 5 2 8 7 6 5 2 8 7 6 5 2 8 7 6 2 8 2 8 6 5 2 2 2 8 2 7

Visualization in Browser

Primitive Shapes in CSS

<div></div> width: 10px  height: 10px border-bottom: 20px solid #000 border-left:
20px solid transparent border-right: 20px solid transparent border-radius: 50%

http://zbryikt.github.io/visualize/kirby/

Scalable Vector Graphics SVG

<circle> ! <ellipse> ! <line> ! <path> ! <text> hello
world!

<path d=“....”> M (x,y)+ - Move to (+ lineto) Z
- Close Path L (x,y)+ - Line to C (x1,y1,x2,y2,x,y)- Curve to ctrl1 ctrl2

<path d=“....”> Q (x1,y1,x,y)+ - Quadratic Bezier

<path d=“....”> A (rx,ry,a,b,c,x,y)+ - Quadratic Bezier

<path d=“....”> m (x,y)+ - Move to (+ lineto) z
- Close Path l (x,y)+ - Line to c (x1,y1,x2,y2,x,y)+ - Curve to ctrl1 ctrl2 q (x1,y1,x,y)+ - Quadratic Bezier ! a (rx,ry,a,b,c,x,y)+ - Quadratic Bezier

M75,50.5l5-1.5c-0.1-2.2-0.4-4.3-0.9-6.3l-5.2-0.1c-0.2-0.6-0.4-1.1-0. 6-1.7l4-3.3c-0.9-1.9-2-3.8-3.2-5.5L69.2,34c-0.4-0.5-0.8-0.9-1.2-1.3l 2.4-4.6c-1.6-1.4-3.3-2.7-5.1-3.8l-3.7,3.6c-0.5-0.3-1.1-0.5-1.6-0.8l0.5 -5.2c-2-0.7-4-1.3-6.2-1.6l-2.1,4.8c-0.6-0.1-1.2-0.1-1.8-0.1l-1.5-5c-2. 2,0.1-4.3,0.4-6.3,0.9l-0.1,5.2c-0.6,0.2-1.1,0.4-1.7,0.6l-3.3-4c-1.9,0.9 -3.8,2-5.5,3.2l1.9,4.9c-0.5,0.4-0.9,0.8-1.3,1.2l-4.6-2.4c-1.4,1.6-2.7,3 .3-3.8,5.1l3.6,3.7c-0.3,0.5-0.5,1.1-0.8,1.6l-5.2-0.5c-0.7,2-1.3,4-1.6,6 .2l4.8,2.1c-0.1,0.6-0.1,1.2-0.1,1.8l-5,1.5c0.1,2.2,0.4,4.3,0.9,6.3l5.2, 0.1c0.2,0.6,0.4,1.1,0.6,1.7l-4,3.3c0.9,1.9,2,3.8,3.2,5.5l4.9-1.9c0.4,0. 5,0.8,0.9,1.2,1.3l-2.4,4.6c1.6,1.4,3.3,2.7,5.1,3.8l3.7-3.6c0.5,0.3,1.1,
0.5,1.6,0.8l-0.5,5.2c2,0.7,4,1.3,6.2,1.6l2.1-4.8c0.6,0.1,1.2,0.1,1.8,0. 1l1.5,5c2.2-0.1,4.3-0.4,6.3-0.9l0.1-5.2c0.6-0.2,1.1-0.4,1.7-0.6l3.3,4c 1.9-0.9,3.8-2,5.5-3.2L66,69.2c0.5-0.4,0.9-0.8,1.3-1.2l4.6,2.4c1.4-1.6 ,2.7-3.3,3.8-5.1l-3.6-3.7c0.3-0.5,0.5-1.1,0.8-1.6l5.2,0.5c0.7-2,1.3-4, 1.6-6.2l-4.8-2.1C74.9,51.7,75,51.1,75,50.5zM50,65c-8.3,0-15-6.7-1 5-15c0-8.3,6.7-15,15-15s15,6.7,15,15 C65,58.3,58.3,65,50,65z d=“ ”

http://goo.gl/fPUFgv SVG Editor

<text x=“0” dx=“0” y=“0” dy=“0” text-anchor=“middle" dominant-baseline="central" textLength=“100" rotate=“ 30
60 90 120 150 180 210 240 270 300 330 360”> helloworld! </text>

Coordinate <svg width=“200px” height=“200px” viewBox=“0 0 100 100” preserveAspectRatio=“…”> ……
</svg> preserveAspectRatio=“None” or x{Min,Mid,Max}Y{Min,Mid,Max}

SMIL <path d=“……”> <animate/> <animateTransform/> <animateMotion/> Animation in SVG

<animate/> 屬性變化 ! <animateTransform/> 線性轉換變化 ! <animateMotion/> 路徑

SVG Animation in Practice http://loading.io

Work with HTML <html> <head> </head> <body> <h1>Hello World!</h1> <svg
viewBox=“0 0 100 100”> <circle cx=“50” cy=“50” r=“25”/> </svg> </body> </html>

Tips and Pits * SVG 屬於 XML — 記得 </>
*部分屬性可以寫成 CSS *功能族煩不及備載，包含 *圖⽚片 / Pattern / 漸層 *連結 / 群組

D3JS js library for manipulating documents based on data <script
type=“text/javascript” src=“d3.v3.min.js”></script> <script type=“text/javascript”> d3.json(“data.json”, function(data) { min = d3.min(data); }); </script>

Data Driven ! Document Model Powerful  Visualizing Lib d3js d3js
d3js d3js d3js d3js d3js d3js d3js d3js

data data data data data selection enter exit selection

data data data data selection enter exit selection d3.selectAll(“div”) .data(data)
.enter().append(“div”) .exit().remove() d3.selectAll(“div”)

new  data data new data.onChange enter exit selection

Pack Layout data = { children: [ {value: 1}, {value:
2}, …… ]} pack = d3.layout.pack(); pack.node(data);

pack = d3.layout.pack(); pack.node(data); ! d3.select(“svg”) .selectAll(“circle”) .data(data) .enter() .append(“circle”)
.attr({ “cx”: function(it) { return it.x;}, “cy”: function(it) { return it.y;}, “r”: function(it) { return it.r; } });

http://mbostock.github.io/d3/talk/20111018/collision.html zbryikt.github.io/visualize/dorling/

http://www.informationisbeautiful.net

information is beautiful easel.ly visual.ly piktochart.com stockphoto noun project dafont

Summary * Purpose of Visualization * Various Chart — Pits
and Falls * CSS and SVG Visualization * D3JS basic * Examples * Resources * Infographics? should be 另⼀一⾨門課了! :p

links http://comic.sfacg.com/HTML/Naruto/ http://campaign-ﬁnance.g0v.ctiml.tw/ http://ptt.cc/bbs/Gossiping/ http://zbryikt.github.io/visualize/ptt-user/ https://cris.hpa.gov.tw/ http://g0v.github.io/nsc-projects/v2.html http://g0v.github.io/nsc-projects/index.html  http://zbryikt.github.io/visualize/kirby/ http://mbostock.github.io/d3/talk/20111018/collision.html
http://zbryikt.github.io/visualize/dorling/ http://bl.ocks.org/zbryikt/raw/4696905/ http://bl.ocks.org/zbryikt/raw/4248542 http://zbryikt.github.io/visualize/banana/ http://zbryikt.github.io/visualize/mrt/ http://g0v.github.io/cancer/web/ http://zbryikt.github.io/visualize/crossﬁlter/ http://zbryikt.github.io/visualize/jobless/ http://zbryikt.github.io/visualize/nsc-2/

Data Parsing and Visualization

Data Parsing and Visualization

More Decks by tkirby

Other Decks in Education

Featured

Transcript