• Softmax with large units is a bottleneck in the training time • is set to 1 to avoid storing parameters depend on context words ( ) • Empirically, it did not affect the performance 14 O(|V |#contexts) <latexit sha1_base64="GV40hula22RSeszxaEw3x1HTW14=">AAACgXichVHLLgRBFD3ae7wGGwkLMSFsJrdFQtgIGzvPGRLDpLsVOvqV7poJ2mws/YCFFYkgVvyCjR+w8AliSWJj4U5PJ4LgVqrq1Kl7bp2q0j3LDCTRY5VSXVNbV9/QmGhqbmltS7Z3ZAO34BsiY7iW66/oWiAs0xEZaUpLrHi+0GzdEsv6znR5f7ko/MB0nSW554k1W9tyzE3T0CRT+WTP7OBB9mA9zKVytia3fTs0XEeKXRmUSkP5ZIrSFEXvT6DGIIU45tzkBXLYgAsDBdgQcCAZW9AQcFuFCoLH3BpC5nxGZrQvUEKCtQXOEpyhMbvD4xavVmPW4XW5ZhCpDT7F4u6zshf99EBX9EL3dE1P9P5rrTCqUfayx7Ne0Qov33bUtfj2r8rmWWL7U/WnZ4lNjEVeTfbuRUz5FkZFX9w/flkcX+gPB+iMntn/KT3SHd/AKb4a5/Ni4QQJ/gD1+3P/BNnhtEppdX4kNTkVf0UDutGHQX7vUUxiBnPI8LmHuMQNbpVqZUghZbiSqlTFmk58CWXiAx0DlNg=</latexit> <latexit sha1_base64="GV40hula22RSeszxaEw3x1HTW14=">AAACgXichVHLLgRBFD3ae7wGGwkLMSFsJrdFQtgIGzvPGRLDpLsVOvqV7poJ2mws/YCFFYkgVvyCjR+w8AliSWJj4U5PJ4LgVqrq1Kl7bp2q0j3LDCTRY5VSXVNbV9/QmGhqbmltS7Z3ZAO34BsiY7iW66/oWiAs0xEZaUpLrHi+0GzdEsv6znR5f7ko/MB0nSW554k1W9tyzE3T0CRT+WTP7OBB9mA9zKVytia3fTs0XEeKXRmUSkP5ZIrSFEXvT6DGIIU45tzkBXLYgAsDBdgQcCAZW9AQcFuFCoLH3BpC5nxGZrQvUEKCtQXOEpyhMbvD4xavVmPW4XW5ZhCpDT7F4u6zshf99EBX9EL3dE1P9P5rrTCqUfayx7Ne0Qov33bUtfj2r8rmWWL7U/WnZ4lNjEVeTfbuRUz5FkZFX9w/flkcX+gPB+iMntn/KT3SHd/AKb4a5/Ni4QQJ/gD1+3P/BNnhtEppdX4kNTkVf0UDutGHQX7vUUxiBnPI8LmHuMQNbpVqZUghZbiSqlTFmk58CWXiAx0DlNg=</latexit> <latexit sha1_base64="GV40hula22RSeszxaEw3x1HTW14=">AAACgXichVHLLgRBFD3ae7wGGwkLMSFsJrdFQtgIGzvPGRLDpLsVOvqV7poJ2mws/YCFFYkgVvyCjR+w8AliSWJj4U5PJ4LgVqrq1Kl7bp2q0j3LDCTRY5VSXVNbV9/QmGhqbmltS7Z3ZAO34BsiY7iW66/oWiAs0xEZaUpLrHi+0GzdEsv6znR5f7ko/MB0nSW554k1W9tyzE3T0CRT+WTP7OBB9mA9zKVytia3fTs0XEeKXRmUSkP5ZIrSFEXvT6DGIIU45tzkBXLYgAsDBdgQcCAZW9AQcFuFCoLH3BpC5nxGZrQvUEKCtQXOEpyhMbvD4xavVmPW4XW5ZhCpDT7F4u6zshf99EBX9EL3dE1P9P5rrTCqUfayx7Ne0Qov33bUtfj2r8rmWWL7U/WnZ4lNjEVeTfbuRUz5FkZFX9w/flkcX+gPB+iMntn/KT3SHd/AKb4a5/Ni4QQJ/gD1+3P/BNnhtEppdX4kNTkVf0UDutGHQX7vUUxiBnPI8LmHuMQNbpVqZUghZbiSqlTFmk58CWXiAx0DlNg=</latexit> <latexit sha1_base64="GV40hula22RSeszxaEw3x1HTW14=">AAACgXichVHLLgRBFD3ae7wGGwkLMSFsJrdFQtgIGzvPGRLDpLsVOvqV7poJ2mws/YCFFYkgVvyCjR+w8AliSWJj4U5PJ4LgVqrq1Kl7bp2q0j3LDCTRY5VSXVNbV9/QmGhqbmltS7Z3ZAO34BsiY7iW66/oWiAs0xEZaUpLrHi+0GzdEsv6znR5f7ko/MB0nSW554k1W9tyzE3T0CRT+WTP7OBB9mA9zKVytia3fTs0XEeKXRmUSkP5ZIrSFEXvT6DGIIU45tzkBXLYgAsDBdgQcCAZW9AQcFuFCoLH3BpC5nxGZrQvUEKCtQXOEpyhMbvD4xavVmPW4XW5ZhCpDT7F4u6zshf99EBX9EL3dE1P9P5rrTCqUfayx7Ne0Qov33bUtfj2r8rmWWL7U/WnZ4lNjEVeTfbuRUz5FkZFX9w/flkcX+gPB+iMntn/KT3SHd/AKb4a5/Ni4QQJ/gD1+3P/BNnhtEppdX4kNTkVf0UDutGHQX7vUUxiBnPI8LmHuMQNbpVqZUghZbiSqlTFmk58CWXiAx0DlNg=</latexit> From Andriy Mnih, Yee Whye Teh, A Fast and Simple Algorithm for Training Neural Probabilistic Language Models. ICML, 2012. c <latexit sha1_base64="lu/Cf4S7fI2pX3rx5Qu1Ikpy69Q=">AAACZHichVHLSsNAFD2Nr1ofrYogCFIsFVflRgTFVdGNS9taW1CRJE5raJqEJC3U4g/oVnHhSkFE/Aw3/oALf0AQlxXcuPA2DYgW9Q4zc+bMPXfOzKi2obse0VNI6unt6x8ID0aGhkdGo7Gx8S3XqjmayGuWYTlFVXGFoZsi7+meIYq2I5SqaoiCWllr7xfqwnF1y9z0GrbYrSplUy/pmuIxldH2YglKkR/xbiAHIIEgNqzYDXawDwsaaqhCwITH2IACl9s2ZBBs5nbRZM5hpPv7AkeIsLbGWYIzFGYrPJZ5tR2wJq/bNV1frfEpBneHlXEk6ZFuqUUPdEcv9PFrraZfo+2lwbPa0Qp7L3o8lXv/V1Xl2cPBl+pPzx5KWPa96uzd9pn2LbSOvn543sqtZJPNObqiV/Z/SU90zzcw62/adUZkLxDhD5B/Pnc32FpIyZSSM4uJ9GrwFWFMYxbz/N5LSGMdG8jzuQInOMVZ6FkaliakyU6qFAo0E/gW0swnzEyJ4w==</latexit> <latexit sha1_base64="lu/Cf4S7fI2pX3rx5Qu1Ikpy69Q=">AAACZHichVHLSsNAFD2Nr1ofrYogCFIsFVflRgTFVdGNS9taW1CRJE5raJqEJC3U4g/oVnHhSkFE/Aw3/oALf0AQlxXcuPA2DYgW9Q4zc+bMPXfOzKi2obse0VNI6unt6x8ID0aGhkdGo7Gx8S3XqjmayGuWYTlFVXGFoZsi7+meIYq2I5SqaoiCWllr7xfqwnF1y9z0GrbYrSplUy/pmuIxldH2YglKkR/xbiAHIIEgNqzYDXawDwsaaqhCwITH2IACl9s2ZBBs5nbRZM5hpPv7AkeIsLbGWYIzFGYrPJZ5tR2wJq/bNV1frfEpBneHlXEk6ZFuqUUPdEcv9PFrraZfo+2lwbPa0Qp7L3o8lXv/V1Xl2cPBl+pPzx5KWPa96uzd9pn2LbSOvn543sqtZJPNObqiV/Z/SU90zzcw62/adUZkLxDhD5B/Pnc32FpIyZSSM4uJ9GrwFWFMYxbz/N5LSGMdG8jzuQInOMVZ6FkaliakyU6qFAo0E/gW0swnzEyJ4w==</latexit> <latexit sha1_base64="lu/Cf4S7fI2pX3rx5Qu1Ikpy69Q=">AAACZHichVHLSsNAFD2Nr1ofrYogCFIsFVflRgTFVdGNS9taW1CRJE5raJqEJC3U4g/oVnHhSkFE/Aw3/oALf0AQlxXcuPA2DYgW9Q4zc+bMPXfOzKi2obse0VNI6unt6x8ID0aGhkdGo7Gx8S3XqjmayGuWYTlFVXGFoZsi7+meIYq2I5SqaoiCWllr7xfqwnF1y9z0GrbYrSplUy/pmuIxldH2YglKkR/xbiAHIIEgNqzYDXawDwsaaqhCwITH2IACl9s2ZBBs5nbRZM5hpPv7AkeIsLbGWYIzFGYrPJZ5tR2wJq/bNV1frfEpBneHlXEk6ZFuqUUPdEcv9PFrraZfo+2lwbPa0Qp7L3o8lXv/V1Xl2cPBl+pPzx5KWPa96uzd9pn2LbSOvn543sqtZJPNObqiV/Z/SU90zzcw62/adUZkLxDhD5B/Pnc32FpIyZSSM4uJ9GrwFWFMYxbz/N5LSGMdG8jzuQInOMVZ6FkaliakyU6qFAo0E/gW0swnzEyJ4w==</latexit> <latexit sha1_base64="lu/Cf4S7fI2pX3rx5Qu1Ikpy69Q=">AAACZHichVHLSsNAFD2Nr1ofrYogCFIsFVflRgTFVdGNS9taW1CRJE5raJqEJC3U4g/oVnHhSkFE/Aw3/oALf0AQlxXcuPA2DYgW9Q4zc+bMPXfOzKi2obse0VNI6unt6x8ID0aGhkdGo7Gx8S3XqjmayGuWYTlFVXGFoZsi7+meIYq2I5SqaoiCWllr7xfqwnF1y9z0GrbYrSplUy/pmuIxldH2YglKkR/xbiAHIIEgNqzYDXawDwsaaqhCwITH2IACl9s2ZBBs5nbRZM5hpPv7AkeIsLbGWYIzFGYrPJZ5tR2wJq/bNV1frfEpBneHlXEk6ZFuqUUPdEcv9PFrraZfo+2lwbPa0Qp7L3o8lXv/V1Xl2cPBl+pPzx5KWPa96uzd9pn2LbSOvn543sqtZJPNObqiV/Z/SU90zzcw62/adUZkLxDhD5B/Pnc32FpIyZSSM4uJ9GrwFWFMYxbz/N5LSGMdG8jzuQInOMVZ6FkaliakyU6qFAo0E/gW0swnzEyJ4w==</latexit>