Share
Embed
How should we manage experiments?
The slide of talk for ML kitchen
https://www.meetup.com/tokyo-machine-learning-kitchen/events/248982280/
URL of expstock:
https://github.com/chie8842/expstock
chie8842
April 12, 2018
Transcript
18 slides
-
$IJF)BZBTIJEB !DIJF • .BDIJOF-FBSOJOHBOE*OGSBTUSVDUVSF&OHJOFFSBU $PPLQBE*OD • $POUSJCVUPSPG"QBDIF4QBSL )BEPPQ BOETPPO
• (SBEVBUFE0TBLB6OJWBOE/"*45 – &OHBHFEJO7PJDFDPOWFSTJPOSFTFBSDI *DPOUSJCVUFEBSUJDMFTBCPVUTDJFOUJGJD DPNQVUJOHXJUI1ZUIPO *UbTPOTBMF"QSJM -
5IFMJGFDZDMFPGUIFNBDIJOFMFBSOJOHFYQFSJNFOU • 8IFOXFXPSLPONBDIJOFMFBSOJOH XFMPPQGPMMPXJOH TUFQT – %FDJEFFWBMVBUJPOJOEFY – &YFDVUFFYQFSJNFOUT •
'FBUVSF&OHJOFFSJOH • 5SBJONPEFM • &WBMVBUFNPEFM • $IBOHFDPOEJUJPOT – 4VNNBSJ[FBOEWJTVBMJ[FUIFSFTVMUPGFYQFSJFODFTGPSSFQPSUTPS QBQFST Loop many times -
"WBSJFUZPGQBUUFSOT • $SFBUFGFBUVSFEBUBTFUGSPNIVHFEBUB – )PXNVDIEBUBEPXFVTF – 8IBULJOEPGEBUBEPXFVTF BDUJPOMPH
VTFSBOEJUFNDPOUFYUT JNBHFT FUD – )PXUPJNQMFNFOUGFBUVSFFOHJOFFSJOH /PSNBMJ[BUJPO .JTTJOH WBMVFQSPDFTTJOH FUD • 5SZBWBSJFUZPGBMHPSJTNT • 1BSBNFUFS5VOJOH – &HCBUDITJ[F MFBSOJOHSBUF FUD -
%JGGJDVMUZJONBOBHJOHFYQFSJNFOUT • 8FIBWFUPNBOBHFTPNBOZJOGPSNBUJPO – %P[FOTPGFYQFSJNFOUT – .BOZQBSBNFUFST – TPVSDFDPEFWFSTJPO –
&OWJSPONFOUBM*OGPSNBUJPO • 3FTPVSDFTJ[F • 6TF(16 • 'SBNFXPSLWFSTJPOT • &YQFSJNFOUQJQFMJOFPGUFODPOUBJOTTFWFSBMOVNCFSPG TUFQT -
&YJTUJOHXPSLT • 4BDSFE IUUQTHJUIVCDPN*%4*"TBDSFE • '(-BC IUUQTHJUIVCDPN,BJYIJO'(-BC •
5IFZBSFUPPMTUPNBOBHFFYQFSJNFOUMPHT • 5IFZVTF.POHP%#GPSEBUBTUPDLFSBOEQSPWJEFTUIF SJDI6* • *U`TWFSZDPOWFOJFOUGPSTDBMBCMFFYQFSJNFOUT – 4IBSFFYQFSJNFOUTXJUI.VMUJQMFFOWJSPONFOUT NVMUJQMF SFTFBSDIFS FUD • #VUUIFZBSFPWFSTQFDGPSNFʜ – /POFDFTTBSZPGSJDIQMBUGPSN – *EPO`UXBOUUPNBJOUBJOB.POHP%# – *EPO`UMJLFUPDIBOHFNZTDSJQUTGPSMPHNBOBHFNFOU -
&YJTUJOHXPSLT • $PNFUNM IUUQTXXXDPNFUNM – $MPVE4FSWJDFGPSMPHUSBDLJOH – 4IBSFMPHT NFNPSBOEVNGVODUJPO
FBTZUPVTF – -PPLTHPPE #VUʜ – 8FDBO`UVQMPBEBEEJUJPOBMGJMFTTVDIBTUSBJOFENPEFMTPSEBUB – /PDPMMBCPSBUJPOXJUIHJU – 8FIBWFUPUBLFDBSFOPUUPVQMPBEDPOGJEFOUJBMJOGPSNBUJPO -
$PODFQUPGFYQTUPDL • *UIBTWFSZTJNQMFBSDIJUFDUVSF – /PEFQFOEFODZ – /PNBJOUFOBODF • 8FEPO`UOFFEUPDIBOHFPVSTDSJQUT –
0OMZBEEJOHUXPPSUISFFDPNNBOET • *UIBTHPPEMPH"DDFTTJCJMJUZXJUI7JTVBMJ[BUJPO – 8FDBODPNQBSFQBTUFYQFSJNFOUT – 8FDBOBDDFTTFBDIMPHGSPNBCSPXTFS -
6TBHFPOGVODUJPOBMJ[FETDSJQUT *GZPVBMSFBEZIBWFGVODUJPOBMJ[FEUBSHFUQSPDFTT ZPVDBO VTFUIFGPMMPXJOHEFDPSBUPS from expstock import expstock a=1 b=2
e = expstock.ExpStock(exp_name=‘test_experiment’) @expstock.expstock(e) def run(a, b): return a + b e.append_param(a=a, b=b) run(a, b) -
6TBHFXJUI+VQZUFS/PUFCPPL 8JUIJOUFSBDUJWFTDSJQUTFHFYQFSJNFOUTPO+VQZUFS /PUFCPPL ZPVDBOVTFUIFGPMMPXJOHDPNNBOE from expstock import expstock a=1 b=2
e = expstock.ExpStock(exp_name=‘test_experiment’) e.append_param(a=a, b=b) e.pre_stock() a + b e.post_stock() -
%JSFDUPSZ4USVDUVSF -PHTBSFTUPDLFEBUFBDITVCEJSFDUPSZ experiments ᵓᴷᴷ <yyyymmdd_hhmmss>_<experiment_name> ᴹ ᵓᴷᴷ exec_time.txt ᴹ ᵓᴷᴷ
git_diff.txt ᴹ ᵓᴷᴷ git_head.txt ᴹ ᵓᴷᴷ machine_info.txt ᴹ ᵓᴷᴷ memo.txt ᴹ ᵓᴷᴷ params.txt ᴹ ᵓᴷᴷ report.txt ᴹ ᵓᴷᴷ result.txt ᴹ ᵓᴷᴷ stderr.txt ᴹ ᵋᴷᴷ stdout.txt -
4BWFBEEJUJPOBMGJMFT • :PVDBOTBWFBEEJUJPOBMGJMFTBUTBNFEJSFDUPSZ # save keras model from keras.models import
load_model model_file = os.path.join(e.log_dirname, ‘model.h5’) model.save(model_file) # save matplotlib firure plt_png = os.path.join(e.log_dirname, ‘plt.png’) plt.savefig(plt_png)