n i=1 > w xi ) <latexit sha1_base64="S+Gu7cjvBnYt/ScT9ZZVE+BdpiU=">AAACl3icfVBdaxQxFM2OX3X96seT+BJcBBFdZpZSfWuhKoVSbMFtC812uJO9MxuaZEKSsV2m86P8NeKb/hMzuyvoVrwQOJxzbu69JzNSOB/H3zvRrdt37t5bud998PDR4yera+vHrqwsxyEvZWlPM3AohcahF17iqbEIKpN4kl3stvrJF7ROlPqznxocKSi0yAUHH6h0dZ8Jnac1m4Cnlw1lrjJp3QIFfpJl9YeGXc9F+oZesut0cD6gDNxUmWAWhYLzAcstcDoOv/XifjwrehMkC9AjizpM1zqbbFzySqH2XIJzZ0ls/KgG6wWX2HRZ5dAAv4ACzwLUoNCN6tnVDX0RmDHNSxue9nTG/tlRg3Jhzyw422PcstaS/9QytTTZ5+9GtdCm8qj5fHBeSepL2kZKx8Ii93IaAHArwu6UTyBk4kPw3S57j+E4iwdh0CeDFnxpX9UMbKGEbsKxBXvdov8Z4eq3MaCQc7Kc6k1wPOgnW/2to83ezu4i8RXyjDwnL0lC3pIdskcOyZBw8pV8Iz/Iz+hptB19jPbm1qiz6Nkgf1V09Asi582O</latexit> <latexit sha1_base64="4RjtynnJq4DEhuEWy3woq+SfFWU=">AAACbXicfZDfahNBFMYnq9UaraaKV4qMhtI/aNiVUr0RCvXCG2kLTRvopuHs5CQZMn+WmbNqWHLt03jbPotP0VfoJN2CpuKBgd985xvOnC/LlfQUx79r0Z27S/fuLz+oP3y08vhJY/XpsbeFE9gWVlnXycCjkgbbJElhJ3cIOlN4ko33Zv2Tb+i8tOaIJjl2NQyNHEgBFKRe43U6AuLf+Se+0TlLyea8s3lWvkumvLpOeo1m3IrnxW9DUkGTVXXQW61tp30rCo2GhALvT5M4p24JjqRQOK2nhcccxBiGeBrQgEbfLee7TPlaUPp8YF04hvhc/fNFCdr7ic6CUwON/GJvJv6zl+mFyTT42C2lyQtCI64HDwrFyfJZULwvHQpSkwAgnAx/52IEDgSFOOv19DOG5Rx+DYP2c3RA1m2VKbihlmYalh2mb2f0PyP8uDEGCjkni6nehuP3rWSntXO43dzdqxJfZi/YG7bBEvaB7bIv7IC1mWA/2S92zi5ql9Hz6GX06toa1ao3z9hfFa1fAbBjvBs=</latexit> > ŵ = (X X) 1 > X y inf sup Ekŵ ŵ w 2 d 2 2 wk2 ⇣ n
<latexit sha1_base64="9A8DVNCoJep5zMhNCyALJJT1r74=">AAACjHicfZBvaxNBEMY3578atU31pW8Wg5CIhrsa2oIUAhXxRcUKpin00jC3mSRL98+xOycNR76On8a3Cn4bN8kJmooDCz+eZ4bZebJcSU9x/LMW3bp95+69rfv1Bw8fbe80dh+feVs4gX1hlXXnGXhU0mCfJCk8zx2CzhQOsqvjpT/4gs5Laz7TPMehhqmREymAgjRq9Ob8iA8uT9J8JluDy/LkVbJYcSrGlnyQXlfWXquC5Lodau2326NGM+7Eq+I3Iamgyao6He3WuunYikKjIaHA+4skzmlYgiMpFC7qaeExB3EFU7wIaECjH5arUxf8eVDGfGJdeIb4Sv1zogTt/VxnoVMDzfymtxT/6WV6YzNNDoelNHlBaMR68aRQnCxf5sjH0qEgNQ8Awsnwdy5m4EBQSLteT99iOM7hh7DoY44OyLoXZQpuqqVZhGOn6csl/a8Rrn83Bgo5J5up3oSzvU6y39n/1G32jqvEt9hT9oy1WMIOWI+9Z6eszwT7yr6x7+xHtB11ozfR0bo1qlUzT9hfFb37BUc/xmw=</latexit> y=W 1 2 … x y <latexit sha1_base64="KgDhIumMmgJR5Go00Yuc+VxfdEU=">AAACS3icfZDPSgMxEMaz1Wqtf6tHL4tFEJGyK0U9FurBi6hgreCWMptO29AkuyRZsSx9Aq/6WD6Az+FNPJitFbQVBwI/vvnCzHxhzJk2nvfq5Obm8wuLhaXi8srq2vpGafNGR4mi2KARj9RtCBo5k9gwzHC8jRWCCDk2w0E96zfvUWkWyWszjLEloCdZl1EwVroatjfKXsUblzsL/gTKZFKX7ZJTDToRTQRKQzlofed7sWmloAyjHEfFINEYAx1AD+8sShCoW+l405G7a5WO242UfdK4Y/XnjxSE1kMRWqcA09fTvUz8sxeKqcmme9JKmYwTg5J+De4m3DWRm8XgdphCavjQAlDF7O4u7YMCamxYxWJwivY4hed20EWMCkyk9tMAVE8wObLH9oKDjP4zwsO30ZLN2Z9OdRZuDiv+UeXoqlqu1SeJF8g22SF7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc05kz9b5Ffl8p/VfbNz</latexit> <latexit sha1_base64="2oa+EUQF+8zDjmnZ6Nf+98VygGw=">AAACS3icfZDPSgMxEMaz9X/91+rRy2IRRKTsSqkeC/XgRVSwtdAtMptOazDJLklWWpY+gVd9LB/A5/AmHkzrCtqKA4Ef33xhZr4w5kwbz3t1cnPzC4tLyyv51bX1jc1Ccaupo0RRbNCIR6oVgkbOJDYMMxxbsUIQIceb8L4+7t88oNIsktdmGGNHQF+yHqNgrHQ1uC2UvLI3KXcW/AxKJKvL26JTCboRTQRKQzlo3fa92HRSUIZRjqN8kGiMgd5DH9sWJQjUnXSy6cjds0rX7UXKPmncifrzRwpC66EIrVOAudPTvbH4Zy8UU5NN76STMhknBiX9GtxLuGsidxyD22UKqeFDC0AVs7u79A4UUGPDyueDU7THKTy3gy5iVGAidZAGoPqCyZE9th8cjuk/Iwy+jZZszv50qrPQPCr71XL1qlKq1bPEl8kO2SX7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc052Z9t8qtyC5/TlrNy</latexit> L <latexit sha1_base64="Z9F6XKWw9ZrSkzsAWVfQaQfq1tU=">AAACTnicfZBNSwMxEIaz9avWz+rRy2IRRKTsSlGPhXrwIlawVnBLmU1n22CSXZKsWJb+Ba/6s7z6R7yJZmsFreJA4OGdN8zMGyacaeN5L05hZnZufqG4WFpaXlldWy9vXOk4VRRbNOaxug5BI2cSW4YZjteJQhAhx3Z428j77TtUmsXy0gwT7AjoSxYxCiaXgmTAuusVr+qNy/0N/gQqZFLNbtmpBb2YpgKloRy0vvG9xHQyUIZRjqNSkGpMgN5CH28sShCoO9l42ZG7Y5WeG8XKPmncsfr9RwZC66EIrVOAGejpXi7+2QvF1GQTHXcyJpPUoKSfg6OUuyZ28yTcHlNIDR9aAKqY3d2lA1BAjc2rVApO0B6n8MwOOk9QgYnVXhaA6gsmR/bYfrCf039GuP8yWrI5+9Op/oarg6p/WD28qFXqjUniRbJFtsku8ckRqZNT0iQtQsmAPJBH8uQ8O6/Om/P+aS04kz+b5EcVih9zXbS1</latexit> (W <latexit sha1_base64="6fpeLvjCHtT3PXJ9u3sazCj0s/E=">AAACUXicfZBNSwMxEIZn1+/6rUcvi0UQ0bIroh4FPXhQVLBWcKvMptMammSXJCuWpT/Cq/4sT/4Ub6a1glZxIPDwzhtm5k0ywY0NwzfPHxkdG5+YnCpNz8zOzS8sLl2ZNNeMqiwVqb5O0JDgiqqWW0HXmSaUiaBa0j7s9WsPpA1P1aXtZFSX2FK8yRlaJ9Vqt8XJVtS9WyiHlbBfwW+IBlCGQZ3fLXo7cSNluSRlmUBjbqIws/UCteVMULcU54YyZG1s0Y1DhZJMvejv2w3WnNIImql2T9mgr37/UaA0piMT55Ro781wryf+2Uvk0GTb3K8XXGW5JcU+BzdzEdg06IURNLgmZkXHATLN3e4Bu0eNzLrISqX4iNxxmk7doLOMNNpUbxQx6pbkquuObcWbPfrPiI9fRkcu52g41d9wtV2Jdiu7Fzvlg8NB4pOwAquwDhHswQEcwzlUgUEbnuAZXrxX790H3/+0+t7gzzL8KH/6AwZjs48=</latexit> <latexit sha1_base64="1jCUJB1wsCZJMZRmcibeh66EBBw=">AAACTXicfZBNSwMxEIaz9aNav1o9elksgoiUXRX1KOjBi6horeDWMptOt6FJdkmyYln6E7zqz/LsD/EmYloraCsOBB7eecPMvGHCmTae9+rkJianpvMzs4W5+YXFpWJp+VrHqaJYpTGP1U0IGjmTWDXMcLxJFIIIOdbCzlG/X7tHpVksr0w3wbqASLIWo2CsdFm722kUy17FG5Q7Dv4QymRY542Ssxs0Y5oKlIZy0PrW9xJTz0AZRjn2CkGqMQHagQhvLUoQqOvZYNeeu26VptuKlX3SuAP1548MhNZdEVqnANPWo72++GcvFCOTTeugnjGZpAYl/RrcSrlrYrcfhNtkCqnhXQtAFbO7u7QNCqixcRUKwTHa4xSe2kFnCSowsdrMAlCRYLJnj42CrT79Z4SHb6Mlm7M/muo4XG9X/L3K3sVu+fBomPgMWSVrZIP4ZJ8ckhNyTqqEkog8kify7Lw4b8678/FlzTnDPyvkV+Xyn/RRs/Y=</latexit> L 1 W L 1 <latexit sha1_base64="Z9F6XKWw9ZrSkzsAWVfQaQfq1tU=">AAACTnicfZBNSwMxEIaz9avWz+rRy2IRRKTsSlGPhXrwIlawVnBLmU1n22CSXZKsWJb+Ba/6s7z6R7yJZmsFreJA4OGdN8zMGyacaeN5L05hZnZufqG4WFpaXlldWy9vXOk4VRRbNOaxug5BI2cSW4YZjteJQhAhx3Z428j77TtUmsXy0gwT7AjoSxYxCiaXgmTAuusVr+qNy/0N/gQqZFLNbtmpBb2YpgKloRy0vvG9xHQyUIZRjqNSkGpMgN5CH28sShCoO9l42ZG7Y5WeG8XKPmncsfr9RwZC66EIrVOAGejpXi7+2QvF1GQTHXcyJpPUoKSfg6OUuyZ28yTcHlNIDR9aAKqY3d2lA1BAjc2rVApO0B6n8MwOOk9QgYnVXhaA6gsmR/bYfrCf039GuP8yWrI5+9Op/oarg6p/WD28qFXqjUniRbJFtsku8ckRqZNT0iQtQsmAPJBH8uQ8O6/Om/P+aS04kz+b5EcVih9zXbS1</latexit> (· · · (W <latexit sha1_base64="xBJ4k0CUIqZOE77ZE/R1zKcuK5Q=">AAACTXicfZBNSwMxEIaz9aNaP6tHL4tFEJGyK0U9CnrwoKhoreDWMptOt6FJdkmyYln6E7zqz/LsD/EmYloraCsOBB7eecPMvGHCmTae9+rkJianpvMzs4W5+YXFpeXiyrWOU0WxSmMeq5sQNHImsWqY4XiTKAQRcqyFncN+v3aPSrNYXplugnUBkWQtRsFY6bJ2d9JYLnllb1DuOPhDKJFhnTeKTiVoxjQVKA3loPWt7yWmnoEyjHLsFYJUYwK0AxHeWpQgUNezwa49d8MqTbcVK/ukcQfqzx8ZCK27IrROAaatR3t98c9eKEYmm9Z+PWMySQ1K+jW4lXLXxG4/CLfJFFLDuxaAKmZ3d2kbFFBj4yoUgiO0xyk8tYPOElRgYrWVBaAiwWTPHhsF2336zwgP30ZLNmd/NNVxuN4p+7vl3YtK6eBwmPgMWSPrZJP4ZI8ckGNyTqqEkog8kify7Lw4b8678/FlzTnDP6vkV+XynyPvtA8=</latexit> W L <latexit sha1_base64="Z9F6XKWw9ZrSkzsAWVfQaQfq1tU=">AAACTnicfZBNSwMxEIaz9avWz+rRy2IRRKTsSlGPhXrwIlawVnBLmU1n22CSXZKsWJb+Ba/6s7z6R7yJZmsFreJA4OGdN8zMGyacaeN5L05hZnZufqG4WFpaXlldWy9vXOk4VRRbNOaxug5BI2cSW4YZjteJQhAhx3Z428j77TtUmsXy0gwT7AjoSxYxCiaXgmTAuusVr+qNy/0N/gQqZFLNbtmpBb2YpgKloRy0vvG9xHQyUIZRjqNSkGpMgN5CH28sShCoO9l42ZG7Y5WeG8XKPmncsfr9RwZC66EIrVOAGejpXi7+2QvF1GQTHXcyJpPUoKSfg6OUuyZ28yTcHlNIDR9aAKqY3d2lA1BAjc2rVApO0B6n8MwOOk9QgYnVXhaA6gsmR/bYfrCf039GuP8yWrI5+9Op/oarg6p/WD28qFXqjUniRbJFtsku8ckRqZNT0iQtQsmAPJBH8uQ8O6/Om/P+aS04kz+b5EcVih9zXbS1</latexit> 3 2 1 (W ( (W x)))) · · · ))
y=W W 2 2 … x y <latexit sha1_base64="KgDhIumMmgJR5Go00Yuc+VxfdEU=">AAACS3icfZDPSgMxEMaz1Wqtf6tHL4tFEJGyK0U9FurBi6hgreCWMptO29AkuyRZsSx9Aq/6WD6Az+FNPJitFbQVBwI/vvnCzHxhzJk2nvfq5Obm8wuLhaXi8srq2vpGafNGR4mi2KARj9RtCBo5k9gwzHC8jRWCCDk2w0E96zfvUWkWyWszjLEloCdZl1EwVroatjfKXsUblzsL/gTKZFKX7ZJTDToRTQRKQzlofed7sWmloAyjHEfFINEYAx1AD+8sShCoW+l405G7a5WO242UfdK4Y/XnjxSE1kMRWqcA09fTvUz8sxeKqcmme9JKmYwTg5J+De4m3DWRm8XgdphCavjQAlDF7O4u7YMCamxYxWJwivY4hed20EWMCkyk9tMAVE8wObLH9oKDjP4zwsO30ZLN2Z9OdRZuDiv+UeXoqlqu1SeJF8g22SF7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc05kz9b5Ffl8p/VfbNz</latexit> <latexit sha1_base64="2oa+EUQF+8zDjmnZ6Nf+98VygGw=">AAACS3icfZDPSgMxEMaz9X/91+rRy2IRRKTsSqkeC/XgRVSwtdAtMptOazDJLklWWpY+gVd9LB/A5/AmHkzrCtqKA4Ef33xhZr4w5kwbz3t1cnPzC4tLyyv51bX1jc1Ccaupo0RRbNCIR6oVgkbOJDYMMxxbsUIQIceb8L4+7t88oNIsktdmGGNHQF+yHqNgrHQ1uC2UvLI3KXcW/AxKJKvL26JTCboRTQRKQzlo3fa92HRSUIZRjqN8kGiMgd5DH9sWJQjUnXSy6cjds0rX7UXKPmncifrzRwpC66EIrVOAudPTvbH4Zy8UU5NN76STMhknBiX9GtxLuGsidxyD22UKqeFDC0AVs7u79A4UUGPDyueDU7THKTy3gy5iVGAidZAGoPqCyZE9th8cjuk/Iwy+jZZszv50qrPQPCr71XL1qlKq1bPEl8kO2SX7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc052Z9t8qtyC5/TlrNy</latexit> L 1 W3 <latexit sha1_base64="6fpeLvjCHtT3PXJ9u3sazCj0s/E=">AAACUXicfZBNSwMxEIZn1+/6rUcvi0UQ0bIroh4FPXhQVLBWcKvMptMammSXJCuWpT/Cq/4sT/4Ub6a1glZxIPDwzhtm5k0ywY0NwzfPHxkdG5+YnCpNz8zOzS8sLl2ZNNeMqiwVqb5O0JDgiqqWW0HXmSaUiaBa0j7s9WsPpA1P1aXtZFSX2FK8yRlaJ9Vqt8XJVtS9WyiHlbBfwW+IBlCGQZ3fLXo7cSNluSRlmUBjbqIws/UCteVMULcU54YyZG1s0Y1DhZJMvejv2w3WnNIImql2T9mgr37/UaA0piMT55Ro781wryf+2Uvk0GTb3K8XXGW5JcU+BzdzEdg06IURNLgmZkXHATLN3e4Bu0eNzLrISqX4iNxxmk7doLOMNNpUbxQx6pbkquuObcWbPfrPiI9fRkcu52g41d9wtV2Jdiu7Fzvlg8NB4pOwAquwDhHswQEcwzlUgUEbnuAZXrxX790H3/+0+t7gzzL8KH/6AwZjs48=</latexit> <latexit sha1_base64="1jCUJB1wsCZJMZRmcibeh66EBBw=">AAACTXicfZBNSwMxEIaz9aNav1o9elksgoiUXRX1KOjBi6horeDWMptOt6FJdkmyYln6E7zqz/LsD/EmYloraCsOBB7eecPMvGHCmTae9+rkJianpvMzs4W5+YXFpWJp+VrHqaJYpTGP1U0IGjmTWDXMcLxJFIIIOdbCzlG/X7tHpVksr0w3wbqASLIWo2CsdFm722kUy17FG5Q7Dv4QymRY542Ssxs0Y5oKlIZy0PrW9xJTz0AZRjn2CkGqMQHagQhvLUoQqOvZYNeeu26VptuKlX3SuAP1548MhNZdEVqnANPWo72++GcvFCOTTeugnjGZpAYl/RrcSrlrYrcfhNtkCqnhXQtAFbO7u7QNCqixcRUKwTHa4xSe2kFnCSowsdrMAlCRYLJnj42CrT79Z4SHb6Mlm7M/muo4XG9X/L3K3sVu+fBomPgMWSVrZIP4ZJ8ckhNyTqqEkog8kify7Lw4b8678/FlzTnDPyvkV+Xyn/RRs/Y=</latexit> 3 W 2 1 ···W W W x L 1 <latexit sha1_base64="xBJ4k0CUIqZOE77ZE/R1zKcuK5Q=">AAACTXicfZBNSwMxEIaz9aNaP6tHL4tFEJGyK0U9CnrwoKhoreDWMptOt6FJdkmyYln6E7zqz/LsD/EmYloraCsOBB7eecPMvGHCmTae9+rkJianpvMzs4W5+YXFpeXiyrWOU0WxSmMeq5sQNHImsWqY4XiTKAQRcqyFncN+v3aPSrNYXplugnUBkWQtRsFY6bJ2d9JYLnllb1DuOPhDKJFhnTeKTiVoxjQVKA3loPWt7yWmnoEyjHLsFYJUYwK0AxHeWpQgUNezwa49d8MqTbcVK/ukcQfqzx8ZCK27IrROAaatR3t98c9eKEYmm9Z+PWMySQ1K+jW4lXLXxG4/CLfJFFLDuxaAKmZ3d2kbFFBj4yoUgiO0xyk8tYPOElRgYrWVBaAiwWTPHhsF2336zwgP30ZLNmd/NNVxuN4p+7vl3YtK6eBwmPgMWSPrZJP4ZI8ckGNyTqqEkog8kify7Lw4b8678/FlzTnDP6vkV+XynyPvtA8=</latexit> W L
kyi W n i=1 1 min kyi W 1:L n i=1 2 W xi k L W W L 1 む NN NN ⾒ 1 2 1 1 2 (W , W ) ! 7 ( W , W ) ; e.g. homogeneity <latexit sha1_base64="ylwJ1qcXweHP6Doh9X58PmFiszI=">AAACgHicfVDRShtBFJ2s1tq0atRHX4aGQiwSd0Ow4pOgD76UWjBGcGO4O7mbDM7MLjN3xbDkR/o1fdU/6N90ElNao/TCwOGcc++de5JcSUdh+KsSLC2/WXm7+q76/sPa+kZtc+vSZYUV2BGZyuxVAg6VNNghSQqvcougE4Xd5PZkqnfv0DqZmQsa59jTMDQylQLIU/1au9G9ifZ496a1y2MNuaOMN2LlBwyAz6SYUgsi+su1dvu1etgMZ8VfgmgO6mxe5/3NSjseZKLQaEgocO46CnPqlWBJCoWTalw4zEHcwhCvPTSg0fXK2XkT/skzA55m1j9DfMb+21GCdm6sE+/UQCO3qE3JV7VEL2ym9LBXSpMXhEY8LU4LxX0m0+z4QFoUpMYegLDS/52LEfh0yCdcrcan6I+z+NUv+pajBcrs5zIGO9TSTPyxw3hviv5nhPs/Ro98ztFiqi/BZasZHTQPvrfrxyfzxFfZDvvIGixiX9gxO2PnrMME+8F+sgf2GARBI9gPoidrUJn3bLNnFRz9BhNdwYg=</latexit> 一 非 一 力 ⾒ 3 2 1 2 · · · W W W xi k NN = む 3 2 NN
⇥N1 1 W 2R 2 ,W 2 R N3 ⇥N2 <latexit sha1_base64="I2c8rPXgJiVsk1hZFoxoKID7eqQ=">AAACe3icfZDfShtBFMYna1tt+i/qZW8GQ8GKTXeD/24EQQveSC00ruAky9nJ2Tg4M7vMzIphzWP4NN7qQ/gwBScxhTaWHhj48Z3vcOZ8aSGFdWH4UAvmXrx8Nb/wuv7m7bv3HxqLSyc2Lw3HDs9lbk5TsCiFxo4TTuJpYRBUKjFOL/bH/fgSjRW5/umGBXYVDLTIBAfnpaTx9dtq/JnuUpYZ4FGb2VIlldiNRj3NroeJoF9o3GvHvegqEey6104azbAVToo+h2gKTTKt42SxtsH6OS8VasclWHsWhYXrVmCc4BJHdVZaLIBfwADPPGpQaLvV5LIR/eSVPs1y4592dKL+OVGBsnaoUu9U4M7tbG8s/rOXqpnNLtvpVkIXpUPNnxZnpaQup+PYaF8Y5E4OPQA3wv+d8nPwmTkfbr3ODtAfZ/DIL/peoAGXm7WKgRkooUf+2AFbH9P/jHD12+jJ5xzNpvocTtqtaKu19WOjubc/TXyBfCQrZJVEZJvskUNyTDqEkxtyS+7Ife1X0AzWgvUna1CbziyTvyrYfAR1KsFl</latexit> n {(xi , yi )}i=1 <latexit sha1_base64="A0WnP2aUlxWqCCxSnZ7ftO30sLE=">AAACYXicfZBNS8NAEIa38avWr7biyUuwCCpSEhH1Igh68CIq2CqYGibbaV3c3YTdjbSE/Biv+os8+0fcfghaxYGFh3feZWbeKOFMG897LzhT0zOzc8X50sLi0vJKuVJt6jhVFBs05rG6i0AjZxIbhhmOd4lCEBHH2+jpdNC/fUalWSxvTD/BloCuZB1GwVgpLK8F2VYvZLv9kG0HeZixYz9/sHrNq3vDcn+DP4YaGddVWCnsB+2YpgKloRy0vve9xLQyUIZRjnkpSDUmQJ+gi/cWJQjUrWy4f+5uWqXtdmJlnzTuUP3+IwOhdV9E1inAPOrJ3kD8sxeJicmmc9TKmExSg5KOBndS7prYHYTjtplCanjfAlDF7O4ufQQF1NgIS6XgDO1xCi/soMsEFZhY7WQBqK5gMrfHdoPdAf1nhN6X0ZLN2Z9M9Tc09+r+Qf3ger92cjpOvEjWyQbZIj45JCfknFyRBqEkIy/klbwVPpx5p+xUR1anMP6zSn6Us/4J0564ng==</latexit> む E(W ) = <latexit sha1_base64="rd5oajX48R65mKNFLX36m+8U7g4=">AAACl3icfVDbahRBEO0db3G9bcyT+NK4CCKyzGiIIgQDCRIQY6JuEkjvDj29tZMmfRm6a8IOw3xUvkZ80z+xZ7OCbsSCgsOpU5yqkxVKeozj753o2vUbN2+t3O7euXvv/oPe6sNDb0snYCissu444x6UNDBEiQqOCwdcZwqOsrPtdn50Ds5La75iVcBI89zIqRQcA5X2PrAvMtecvt2kDGGGHisFlPlSp7XcTJqxqVI5S+WYoS0ok4YyzfE0y+rPzbjeS18xlBo83UuTJu3140E8L3oVJAvQJ4vaT1c762xiRanBoFDc+5MkLnBUc4dSKGi6rPRQcHHGczgJ0PBgNarnXzf0aWAmdGpdaIN0zv65UXPtfaWzoGxP9suzlvznLNNLzjh9M6qlKUoEIy6Np6WiaGkbKZ1IBwJVFQAXTobbqTjljgsMwXe7bAfCcw4+BqNPBTiO1j2vGXe5lqYJz+bsRYv+J+Sz38KAQs7JcqpXweHLQbIx2DhY729tLxJfIY/JE/KMJOQ12SK7ZJ8MiSAX5Bv5QX5Gj6J30fto91IadRY7a+Svig5+AeYTzfs=</latexit> ⌃ := む <latexit sha1_base64="Rjxu2609D7TUSzr62u44TakxgT8=">AAACeHicfVBdSxtBFJ2s/bBprdE+9mVoKFWxYTeI9UUQ7IMvpRYbI7hJuDu5WQfnY5m5K4Ylf6K/xlf7L/pX+uQkplBj6YWBwznncuaerFDSUxz/qkVLT54+e778ov7y1crr1cba+qm3pRPYEVZZd5aBRyUNdkiSwrPCIehMYTe7PJzq3St0XlrzncYF9jTkRo6kAArUoLGdfkZFwLv9hO/zFAmqbr896adki430ROYa+MegtoNhc9Boxq14NvwxSOagyeZzPFir7aRDK0qNhoQC78+TuKBeBY6kUDipp6XHAsQl5HgeoAGNvlfNzprw94EZ8pF14RniM/bvjQq092OdBacGuvCL2pT8p5bphWQa7fUqaYqS0Ij74FGpOFk+7YwPpUNBahwACCfD37m4AAeCQrP1eqgwHOfwSwj6WqADsm6rSsHlWppJODZPt6fof0a4/mMMKPScLLb6GJy2W8lua/fbTvPgcN74MnvL3rENlrBP7IAdsWPWYYL9YDfslv2s/Y549CHavLdGtfnOG/ZgovYdeA+//w==</latexit> 1 W = ⌘W 2> Pn > N3 ⇥N1 y x 2 R ; i=1 i i 2 (⌃ 2 W W ) W = ⌘(⌃ 2 n X 1 2 i=1 2 kyi 1 W W xi k 2 Pn <latexit sha1_base64="xjOxvmjR64ptkxcnbTP+RZUHXVA=">AAACfXicfZDbahsxEIblTQ+Je3LSy96ImkApweyGkPQmEEgu2ou2KdRJIHKWWXnsiOiwlWaDzbLP0afpbfsMfZpUdlxondJBgo9/fjGavyi1CpSmP1vJyr37Dx6urrUfPX7y9FlnfeMkuMpL7EunnT8rIKBWFvukSONZ6RFMofG0uDqc9U+v0Qfl7GealjgwMLZqpCRQlPJOJggnFGiqkYtQmbxW+1lzYSe5iudCkCu5wC+Vuubv8vpDnjV5p5v20nnxu5AtoMsWdZyvt3bE0MnKoCWpIYTzLC1pUIMnJTU2bVEFLEFewRjPI1owGAb1fLeGb0ZlyEfOx2uJz9U/X9RgQpiaIjoN0GVY7s3Ef/YKszSZRm8GtbJlRWjl7eBRpTk5PguOD5VHSXoaAaRX8e9cXoIHSTHedlscYVzO4/s46GOJHsj517UAPzbKNnHZsdia0f+MMPltjBRzzpZTvQsn271st7f7aad7cLhIfJW9YC/ZK5axPXbA3rJj1meSfWXf2Hf2o3WTbCZbSe/WmrQWb56zvyrZ+wXtY8T8</latexit> > x x i=1 i i ⌘ IN1 x 1 <latexit sha1_base64="Fy4yLhPQfi77rKUpechpsDSdSpc=">AAACeXicfZBdSxtBFIYnaz9s+hX1sjfThkIqNexasb0pCHrRG9HSxghuEs5OTtbB+VhmzpaGJb/CX9Pb+iv8Ld50NqbQxtIDMzy85x3OnDcrlPQUx9eNaOXe/QcPVx81Hz95+ux5a239xNvSCewJq6w7zcCjkgZ7JEnhaeEQdKawn13s1/3+N3ReWvOVpgUONORGTqQACtKotZUeoCLg/eE2/8hTDNhJv8hcA9+qxf4weVOFazZMyRajVjvuxvPidyFZQJst6ni01thJx1aUGg0JBd6fJXFBgwocSaFw1kxLjwWIC8jxLKABjX5Qzfea8ddBGfOJdeEY4nP1zxcVaO+nOgtODXTul3u1+M9eppcm0+TDoJKmKAmNuB08KRUny+vQ+Fg6FKSmAUA4Gf7OxTk4EBSibTZDhmE5h4dh0FGBDsi6zSoFl2tpZmHZPH1b0/+M8P23MVDIOVlO9S6cbHeT3e7u55323v4i8VX2gr1iHZaw92yPfWLHrMcEu2Q/2E921biJXkadaPPWGjUWbzbYXxW9+wXRS8Ap</latexit> 1 W W )W (η : ) y <latexit sha1_base64="KgDhIumMmgJR5Go00Yuc+VxfdEU=">AAACS3icfZDPSgMxEMaz1Wqtf6tHL4tFEJGyK0U9FurBi6hgreCWMptO29AkuyRZsSx9Aq/6WD6Az+FNPJitFbQVBwI/vvnCzHxhzJk2nvfq5Obm8wuLhaXi8srq2vpGafNGR4mi2KARj9RtCBo5k9gwzHC8jRWCCDk2w0E96zfvUWkWyWszjLEloCdZl1EwVroatjfKXsUblzsL/gTKZFKX7ZJTDToRTQRKQzlofed7sWmloAyjHEfFINEYAx1AD+8sShCoW+l405G7a5WO242UfdK4Y/XnjxSE1kMRWqcA09fTvUz8sxeKqcmme9JKmYwTg5J+De4m3DWRm8XgdphCavjQAlDF7O4u7YMCamxYxWJwivY4hed20EWMCkyk9tMAVE8wObLH9oKDjP4zwsO30ZLN2Z9OdRZuDiv+UeXoqlqu1SeJF8g22SF7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc05kz9b5Ffl8p/VfbNz</latexit> N2 <latexit sha1_base64="NgEKFcYW2o1IZfa5uw8QAVDCf3s=">AAACTXicfZBNS8NAEIY39aNav1o9egkWQURKUkQ9FvTgxS+0KphSJttpXLq7CbsbsYT+BK/6szz7Q7yJuK0RtIoDCw/vvMvMvGHCmTae9+IUJianposzs6W5+YXFpXJl+VLHqaLYpDGP1XUIGjmT2DTMcLxOFIIIOV6Fvf1h/+oOlWaxvDD9BFsCIsm6jIKx0vlxu94uV72aNyr3N/g5VElep+2Ksx10YpoKlIZy0PrG9xLTykAZRjkOSkGqMQHagwhvLEoQqFvZaNeBu26VjtuNlX3SuCP1+48MhNZ9EVqnAHOrx3tD8c9eKMYmm+5eK2MySQ1K+jm4m3LXxO4wCLfDFFLD+xaAKmZ3d+ktKKDGxlUqBQdoj1N4ZAedJKjAxGozC0BFgsmBPTYKtob0nxHuv4yWbM7+eKq/4bJe83dqO2fb1cZ+nvgMWSVrZIP4ZJc0yCE5JU1CSUQeyCN5cp6dV+fNef+0Fpz8zwr5UYXiB+Mhs+0=</latexit> <latexit sha1_base64="2oa+EUQF+8zDjmnZ6Nf+98VygGw=">AAACS3icfZDPSgMxEMaz9X/91+rRy2IRRKTsSqkeC/XgRVSwtdAtMptOazDJLklWWpY+gVd9LB/A5/AmHkzrCtqKA4Ef33xhZr4w5kwbz3t1cnPzC4tLyyv51bX1jc1Ccaupo0RRbNCIR6oVgkbOJDYMMxxbsUIQIceb8L4+7t88oNIsktdmGGNHQF+yHqNgrHQ1uC2UvLI3KXcW/AxKJKvL26JTCboRTQRKQzlo3fa92HRSUIZRjqN8kGiMgd5DH9sWJQjUnXSy6cjds0rX7UXKPmncifrzRwpC66EIrVOAudPTvbH4Zy8UU5NN76STMhknBiX9GtxLuGsidxyD22UKqeFDC0AVs7u79A4UUGPDyueDU7THKTy3gy5iVGAidZAGoPqCyZE9th8cjuk/Iwy+jZZszv50qrPQPCr71XL1qlKq1bPEl8kO2SX7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc052Z9t8qtyC5/TlrNy</latexit> <latexit sha1_base64="haSdbYtsNQyoRe7ykMC/yp4rtBI=">AAACTXicfZBNSwMxEIaz9aNaP6tHL4tFEJGyK0U9FurBi6hordCtMptO19AkuyRZsSz9CV71Z3n2h3gTMa0VtBUHAg/vvGFm3jDhTBvPe3VyU9Mzs/m5+cLC4tLyympx7UrHqaJYpzGP1XUIGjmTWDfMcLxOFIIIOTbCbm3Qb9yj0iyWl6aXYEtAJFmHUTBWumjc+LerJa/sDcudBH8EJTKqs9uiUwnaMU0FSkM5aN30vcS0MlCGUY79QpBqTIB2IcKmRQkCdSsb7tp3t6zSdjuxsk8ad6j+/JGB0LonQusUYO70eG8g/tkLxdhk0zlsZUwmqUFJvwZ3Uu6a2B0E4baZQmp4zwJQxezuLr0DBdTYuAqF4AjtcQpP7KDTBBWYWO1kAahIMNm3x0bB7oD+M8LDt9GSzdkfT3USrvbK/n55/7xSqtZGic+RDbJJtolPDkiVHJMzUieUROSRPJFn58V5c96djy9rzhn9WSe/Kpf/BPCDs/Q=</latexit> W 1 <latexit sha1_base64="aufhfgnOg5Qu4gUpqbQ+th/Tt6I=">AAACTXicfZBNSwMxEIaz9aNav1o9elksgoiUXRH1WNCDF1HR2kK3ltl0ug0m2SXJimXpT/CqP8uzP8SbiGmtoFUcCDy884aZecOEM20878XJTU3PzObn5gsLi0vLK8XS6rWOU0WxRmMeq0YIGjmTWDPMcGwkCkGEHOvh7dGwX79DpVksr0w/wZaASLIuo2CsdFm/2W0Xy17FG5X7G/wxlMm4ztslZy/oxDQVKA3loHXT9xLTykAZRjkOCkGqMQF6CxE2LUoQqFvZaNeBu2mVjtuNlX3SuCP1+48MhNZ9EVqnANPTk72h+GcvFBOTTfewlTGZpAYl/RzcTblrYncYhNthCqnhfQtAFbO7u7QHCqixcRUKwTHa4xSe2kFnCSowsdrOAlCRYHJgj42CnSH9Z4T7L6Mlm7M/mepvuN6t+PuV/Yu9cvVonPgcWScbZIv45IBUyQk5JzVCSUQeyCN5cp6dV+fNef+05pzxnzXyo3L5D/Jqs/U=</latexit> W 2 1> N1 <latexit sha1_base64="rULBMhMqQ9482yCyBwTKgM//9KA=">AAACTXicfZBNSwMxEIaz9aNaP6tHL4tFEJGyK0U9FurBi19otdAtZTadrqFJdkmyYln6E7zqz/LsD/EmYloraBUHAg/vvGFm3jDhTBvPe3FyU9Mzs/m5+cLC4tLyympx7VrHqaJYpzGPVSMEjZxJrBtmODYShSBCjjdhrzbs39yh0iyWV6afYEtAJFmXUTBWujxt++3Vklf2RuX+Bn8MJTKu83bRqQSdmKYCpaEctG76XmJaGSjDKMdBIUg1JkB7EGHTogSBupWNdh24W1bpuN1Y2SeNO1K//8hAaN0XoXUKMLd6sjcU/+yFYmKy6R62MiaT1KCkn4O7KXdN7A6DcDtMITW8bwGoYnZ3l96CAmpsXIVCcIT2OIUndtBZggpMrHayAFQkmBzYY6Ngd0j/GeH+y2jJ5uxPpvobrvfK/n55/6JSqtbGic+RDbJJtolPDkiVHJNzUieUROSBPJIn59l5dd6c909rzhn/WSc/Kpf/AOE6s+w=</latexit> 1 Wnew 2 Wnew <latexit sha1_base64="FnWtJpnmESiWqPWmMXLIvj3Hl/g=">AAACcXicfZDfShtBFMYn6//U2theFW8GgyBawm4R7aWgF70pKhgjuGk4OzlZB2dml5mzaljyBD6Nt+2T+By+gLMxgsbSAwO/+c43nDlfkivpKAwfasHM7Nz8wuJS/cPyx5VPjdXPZy4rrMC2yFRmzxNwqKTBNklSeJ5bBJ0o7CRXB1W/c43Wycyc0jDHrobUyIEUQF7qNTY6v6NeTHhLpcGbEY9TJMe9yLd5fIiKoLr0Gs2wFY6Lv4doAk02qePeam0n7mei0GhIKHDuIgpz6pZgSQqFo3pcOMxBXEGKFx4NaHTdcrzPiG94pc8HmfXHEB+rr1+UoJ0b6sQ7NdClm+5V4j97iZ6aTIMf3VKavCA04nnwoFCcMl6FxfvSoiA19ADCSv93Li7BgiAfab3uA/LLWfzlBx3laIEyu1XGYFMtzcgvm8bfKvqfEW5fjJ58ztF0qu/h7Hsr2m3tnuw09w8miS+yNbbONlnE9tg++8mOWZsJdsfu2R/2t/YYfA14sP5sDWqTN1/Ymwq2nwAwAr4q</latexit> <latexit sha1_base64="32vosqh6Ebm6Jfs/Vp/DHnn+K6g=">AAACcXicfZDRShtBFIYnW7WaVo16VXozGISiEnZFrJeCXnhTqmCM4Kbh7ORkHZyZXWbOtoYlT9Cn8VafpM/RF3A2RtAoHhj45j//cOb8Sa6kozD8Vws+zMzOfZxfqH/6vLi03FhZPXdZYQW2RaYye5GAQyUNtkmSwovcIuhEYSe5Pqz6nd9onczMGQ1z7GpIjRxIAeSlXmOj82unFxPeUGnwz4jHKZLjXuRbPD5CRVBdeo1m2ArHxV9DNIEmm9RJb6W2G/czUWg0JBQ4dxmFOXVLsCSFwlE9LhzmIK4hxUuPBjS6bjneZ8Q3vNLng8z6Y4iP1ecvStDODXXinRroyk33KvHNXqKnJtNgv1tKkxeERjwOHhSKU8arsHhfWhSkhh5AWOn/zsUVWBDkI63XfUB+OYs//KCfOVqgzG6WMdhUSzPyy6bxdkXvGeHmyejJ5xxNp/oaznda0V5r73S3eXA4SXyefWXr7BuL2Hd2wI7ZCWszwf6yW3bH7mv/gy8BD9YfrUFt8maNvahg6wE15r4t</latexit> 1 W + 2 W + W 1 W 2 小 行 力 入 二 4 1 行 行 Saxe, A., McClelland, J., & Ganguli, S. (2014). Exact solutions to the nonlinear dynamics of learning in deep linear neural networks. 力 ] 1 <latexit sha1_base64="z0c8cJk/WbV+FfELQ4+YHKO02PI=">AAACknicfVBdaxNBFJ2s1db40VR968tgEERK2E1LFZ8qFemDrVVMU+gmy93JTTp0ZnaZuSuGZf+Rv8Y30R/jbLoVTYsXLhzOPZdz70lzJR2F4Y9WcGvl9p3Vtbvte/cfPFzvbDw6cVlhBQ5EpjJ7moJDJQ0OSJLC09wi6FThML3Yr+fDL2idzMxnmuc40jAzcioFkKeSzrvhOOKxNDzWQOdpWn6qxuVR0o9JanT8KImqLT4c92/QbP/R9Kuk0w174aL4dRA1oMuaOk42WjvxJBOFRkNCgXNnUZjTqARLUiis2nHhMAdxATM889CAtxqVi4cr/swzEz7NrG9DfMH+vVGCdm6uU6+sT3bLs5q8cZbqJWeavhqV0uQFoRGXxtNCccp4nSafSIuC1NwDEFb627k4BwuCfObtdvwW/XMWD73RhxwtUGZflDHYmZam8s/O4q0a/U8IX6+EHvmco+VUr4OTfi/a7e1+3Onu7TeJr7FN9pQ9ZxF7yfbYATtmAybYN/ad/WS/gifB6+BN0GiDVrPzmP1TwfvfBADKbA==</latexit> 4 入 2 📄 2 N3 <latexit sha1_base64="EMusQFr7bl0LES0y9LepKT4756Y=">AAACTXicfZBNSwMxEIaz9aNav1o9elksgoiUXRX1KOjBi19oq9AtZTadbkOT7JJkxbL0J3jVn+XZH+JNxLRW0CoOBB7eecPMvGHCmTae9+LkJianpvMzs4W5+YXFpWJpuabjVFGs0pjH6jYEjZxJrBpmON4mCkGEHG/C7tGgf3OHSrNYXptegg0BkWRtRsFY6eqsudMslr2KNyz3N/gjKJNRXTRLzm7QimkqUBrKQeu67yWmkYEyjHLsF4JUYwK0CxHWLUoQqBvZcNe+u26VltuOlX3SuEP1+48MhNY9EVqnANPR472B+GcvFGOTTfugkTGZpAYl/RzcTrlrYncQhNtiCqnhPQtAFbO7u7QDCqixcRUKwTHa4xSe2kHnCSowsdrMAlCRYLJvj42CrQH9Z4T7L6Mlm7M/nupvqG1X/L3K3uVu+fBolPgMWSVrZIP4ZJ8ckhNyQaqEkog8kEfy5Dw7r86b8/5pzTmjPyvkR+XyH+UIs+4=</latexit>
⇥N1 1 W 2R 2 ,W 2 R N3 ⇥N2 <latexit sha1_base64="I2c8rPXgJiVsk1hZFoxoKID7eqQ=">AAACe3icfZDfShtBFMYna1tt+i/qZW8GQ8GKTXeD/24EQQveSC00ruAky9nJ2Tg4M7vMzIphzWP4NN7qQ/gwBScxhTaWHhj48Z3vcOZ8aSGFdWH4UAvmXrx8Nb/wuv7m7bv3HxqLSyc2Lw3HDs9lbk5TsCiFxo4TTuJpYRBUKjFOL/bH/fgSjRW5/umGBXYVDLTIBAfnpaTx9dtq/JnuUpYZ4FGb2VIlldiNRj3NroeJoF9o3GvHvegqEey6104azbAVToo+h2gKTTKt42SxtsH6OS8VasclWHsWhYXrVmCc4BJHdVZaLIBfwADPPGpQaLvV5LIR/eSVPs1y4592dKL+OVGBsnaoUu9U4M7tbG8s/rOXqpnNLtvpVkIXpUPNnxZnpaQup+PYaF8Y5E4OPQA3wv+d8nPwmTkfbr3ODtAfZ/DIL/peoAGXm7WKgRkooUf+2AFbH9P/jHD12+jJ5xzNpvocTtqtaKu19WOjubc/TXyBfCQrZJVEZJvskUNyTDqEkxtyS+7Ife1X0AzWgvUna1CbziyTvyrYfAR1KsFl</latexit> n {(xi , yi )}i=1 <latexit sha1_base64="A0WnP2aUlxWqCCxSnZ7ftO30sLE=">AAACYXicfZBNS8NAEIa38avWr7biyUuwCCpSEhH1Igh68CIq2CqYGibbaV3c3YTdjbSE/Biv+os8+0fcfghaxYGFh3feZWbeKOFMG897LzhT0zOzc8X50sLi0vJKuVJt6jhVFBs05rG6i0AjZxIbhhmOd4lCEBHH2+jpdNC/fUalWSxvTD/BloCuZB1GwVgpLK8F2VYvZLv9kG0HeZixYz9/sHrNq3vDcn+DP4YaGddVWCnsB+2YpgKloRy0vve9xLQyUIZRjnkpSDUmQJ+gi/cWJQjUrWy4f+5uWqXtdmJlnzTuUP3+IwOhdV9E1inAPOrJ3kD8sxeJicmmc9TKmExSg5KOBndS7prYHYTjtplCanjfAlDF7O4ufQQF1NgIS6XgDO1xCi/soMsEFZhY7WQBqK5gMrfHdoPdAf1nhN6X0ZLN2Z9M9Tc09+r+Qf3ger92cjpOvEjWyQbZIj45JCfknFyRBqEkIy/klbwVPpx5p+xUR1anMP6zSn6Us/4J0564ng==</latexit> む E(W ) = <latexit sha1_base64="rd5oajX48R65mKNFLX36m+8U7g4=">AAACl3icfVDbahRBEO0db3G9bcyT+NK4CCKyzGiIIgQDCRIQY6JuEkjvDj29tZMmfRm6a8IOw3xUvkZ80z+xZ7OCbsSCgsOpU5yqkxVKeozj753o2vUbN2+t3O7euXvv/oPe6sNDb0snYCissu444x6UNDBEiQqOCwdcZwqOsrPtdn50Ds5La75iVcBI89zIqRQcA5X2PrAvMtecvt2kDGGGHisFlPlSp7XcTJqxqVI5S+WYoS0ok4YyzfE0y+rPzbjeS18xlBo83UuTJu3140E8L3oVJAvQJ4vaT1c762xiRanBoFDc+5MkLnBUc4dSKGi6rPRQcHHGczgJ0PBgNarnXzf0aWAmdGpdaIN0zv65UXPtfaWzoGxP9suzlvznLNNLzjh9M6qlKUoEIy6Np6WiaGkbKZ1IBwJVFQAXTobbqTjljgsMwXe7bAfCcw4+BqNPBTiO1j2vGXe5lqYJz+bsRYv+J+Sz38KAQs7JcqpXweHLQbIx2DhY729tLxJfIY/JE/KMJOQ12SK7ZJ8MiSAX5Bv5QX5Gj6J30fto91IadRY7a+Svig5+AeYTzfs=</latexit> ⌃ := む Pn > N3 ⇥N1 y x 2 R ; i=1 i i d 1 2> 2 1 ⌧ W = W (⌃ W W ) dt d 2 2 1 1> ⌧ W = (⌃ W W )W dt n X 1 2 i=1 2 kyi 1 W W xi k 2 Pn <latexit sha1_base64="xjOxvmjR64ptkxcnbTP+RZUHXVA=">AAACfXicfZDbahsxEIblTQ+Je3LSy96ImkApweyGkPQmEEgu2ou2KdRJIHKWWXnsiOiwlWaDzbLP0afpbfsMfZpUdlxondJBgo9/fjGavyi1CpSmP1vJyr37Dx6urrUfPX7y9FlnfeMkuMpL7EunnT8rIKBWFvukSONZ6RFMofG0uDqc9U+v0Qfl7GealjgwMLZqpCRQlPJOJggnFGiqkYtQmbxW+1lzYSe5iudCkCu5wC+Vuubv8vpDnjV5p5v20nnxu5AtoMsWdZyvt3bE0MnKoCWpIYTzLC1pUIMnJTU2bVEFLEFewRjPI1owGAb1fLeGb0ZlyEfOx2uJz9U/X9RgQpiaIjoN0GVY7s3Ef/YKszSZRm8GtbJlRWjl7eBRpTk5PguOD5VHSXoaAaRX8e9cXoIHSTHedlscYVzO4/s46GOJHsj517UAPzbKNnHZsdia0f+MMPltjBRzzpZTvQsn271st7f7aad7cLhIfJW9YC/ZK5axPXbA3rJj1meSfWXf2Hf2o3WTbCZbSe/WmrQWb56zvyrZ+wXtY8T8</latexit> > x x i=1 i i ⌘ IN1 y <latexit sha1_base64="aoewlSBH5BQHYAHU0t//VBdVAGs=">AAACf3icfVBdaxNBFJ2sX238Su1jXwaDWEXDbixVHwqF+uCLWNE0hW4S7s7e3Q6dj2XmrhiW/SH+Gl/1J/hvnKQRNBUvDBzOOZcz92SVkp7i+Gcnunb9xs1bG5vd23fu3rvf23pw4m3tBI6EVdadZuBRSYMjkqTwtHIIOlM4zi6OFvr4MzovrflE8wonGkojCymAAjXrvUgJap4WDkSTt01OLR9PE37Am/F02E5TstVu+lGWGvjzoAyD+GTW68eDeDn8KkhWoM9Wczzb6uyluRW1RkNCgfdnSVzRpAFHUihsu2ntsQJxASWeBWhAo580y+ta/igwOS+sC88QX7J/bjSgvZ/rLDg10Llf1xbkP7VMryVT8WrSSFPVhEZcBhe14mT5ojqeS4eC1DwAEE6Gv3NxDqE5CgV3u+kbDMc5fBeC3lfogKx72qTgSi1NG44t02cL9D8jfPltDCj0nKy3ehWcDAfJ/mD/w17/8GjV+AbbYQ/ZLkvYS3bI3rJjNmKCfWXf2Hf2I+pEj6NBFF9ao85qZ5v9NdHrX7oqwlQ=</latexit> <latexit sha1_base64="eTyooeVi8+L+NaNjtnmt/dC42d8=">AAACf3icfVDbbtNAEN24XEq4peWRlxURoiCI7FIVeECqVB54QRRBmkp1Eo3XY3fVvVi7Y0Rk+UP4Gl7hE/gb1mmQIEWMtKuzZ87o7JysUtJTHP/sRRtXrl67vnmjf/PW7Tt3B1vbx97WTuBYWGXdSQYelTQ4JkkKTyqHoDOFk+z8sOtPPqPz0ppPtKhwqqE0spACKFDzwfOUoOZp4UA0edvk1PLJbJe/5jvpR1lq4M+692SWPG7C1c5SstV8MIxH8bL4ZZCswJCt6mi+1dtLcytqjYaEAu9Pk7iiaQOOpFDY9tPaYwXiHEo8DdCARj9tltu1/GFgcl5YF44hvmT/nGhAe7/QWVBqoDO/3uvIf/YyveZMxctpI01VExpxYVzUipPlXXQ8lw4FqUUAIJwMf+fiDEJyFALu99M3GJZz+C4Yva/QAVn3pEnBlVqaNixbpk879D8hfPktDCjknKynehkc746S/dH+h73hweEq8U12nz1gOyxhL9gBe8uO2JgJ9pV9Y9/Zj6gXPYpGUXwhjXqrmXvsr4pe/QKoBsJU</latexit> x <latexit sha1_base64="KgDhIumMmgJR5Go00Yuc+VxfdEU=">AAACS3icfZDPSgMxEMaz1Wqtf6tHL4tFEJGyK0U9FurBi6hgreCWMptO29AkuyRZsSx9Aq/6WD6Az+FNPJitFbQVBwI/vvnCzHxhzJk2nvfq5Obm8wuLhaXi8srq2vpGafNGR4mi2KARj9RtCBo5k9gwzHC8jRWCCDk2w0E96zfvUWkWyWszjLEloCdZl1EwVroatjfKXsUblzsL/gTKZFKX7ZJTDToRTQRKQzlofed7sWmloAyjHEfFINEYAx1AD+8sShCoW+l405G7a5WO242UfdK4Y/XnjxSE1kMRWqcA09fTvUz8sxeKqcmme9JKmYwTg5J+De4m3DWRm8XgdphCavjQAlDF7O4u7YMCamxYxWJwivY4hed20EWMCkyk9tMAVE8wObLH9oKDjP4zwsO30ZLN2Z9OdRZuDiv+UeXoqlqu1SeJF8g22SF7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc05kz9b5Ffl8p/VfbNz</latexit> N2 <latexit sha1_base64="NgEKFcYW2o1IZfa5uw8QAVDCf3s=">AAACTXicfZBNS8NAEIY39aNav1o9egkWQURKUkQ9FvTgxS+0KphSJttpXLq7CbsbsYT+BK/6szz7Q7yJuK0RtIoDCw/vvMvMvGHCmTae9+IUJianposzs6W5+YXFpXJl+VLHqaLYpDGP1XUIGjmT2DTMcLxOFIIIOV6Fvf1h/+oOlWaxvDD9BFsCIsm6jIKx0vlxu94uV72aNyr3N/g5VElep+2Ksx10YpoKlIZy0PrG9xLTykAZRjkOSkGqMQHagwhvLEoQqFvZaNeBu26VjtuNlX3SuCP1+48MhNZ9EVqnAHOrx3tD8c9eKMYmm+5eK2MySQ1K+jm4m3LXxO4wCLfDFFLD+xaAKmZ3d+ktKKDGxlUqBQdoj1N4ZAedJKjAxGozC0BFgsmBPTYKtob0nxHuv4yWbM7+eKq/4bJe83dqO2fb1cZ+nvgMWSVrZIP4ZJc0yCE5JU1CSUQeyCN5cp6dV+fNef+0Fpz8zwr5UYXiB+Mhs+0=</latexit> <latexit sha1_base64="2oa+EUQF+8zDjmnZ6Nf+98VygGw=">AAACS3icfZDPSgMxEMaz9X/91+rRy2IRRKTsSqkeC/XgRVSwtdAtMptOazDJLklWWpY+gVd9LB/A5/AmHkzrCtqKA4Ef33xhZr4w5kwbz3t1cnPzC4tLyyv51bX1jc1Ccaupo0RRbNCIR6oVgkbOJDYMMxxbsUIQIceb8L4+7t88oNIsktdmGGNHQF+yHqNgrHQ1uC2UvLI3KXcW/AxKJKvL26JTCboRTQRKQzlo3fa92HRSUIZRjqN8kGiMgd5DH9sWJQjUnXSy6cjds0rX7UXKPmncifrzRwpC66EIrVOAudPTvbH4Zy8UU5NN76STMhknBiX9GtxLuGsidxyD22UKqeFDC0AVs7u79A4UUGPDyueDU7THKTy3gy5iVGAidZAGoPqCyZE9th8cjuk/Iwy+jZZszv50qrPQPCr71XL1qlKq1bPEl8kO2SX7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc052Z9t8qtyC5/TlrNy</latexit> (τ = η −1) <latexit sha1_base64="haSdbYtsNQyoRe7ykMC/yp4rtBI=">AAACTXicfZBNSwMxEIaz9aNaP6tHL4tFEJGyK0U9FurBi6hordCtMptO19AkuyRZsSz9CV71Z3n2h3gTMa0VtBUHAg/vvGFm3jDhTBvPe3VyU9Mzs/m5+cLC4tLyympx7UrHqaJYpzGP1XUIGjmTWDfMcLxOFIIIOTbCbm3Qb9yj0iyWl6aXYEtAJFmHUTBWumjc+LerJa/sDcudBH8EJTKqs9uiUwnaMU0FSkM5aN30vcS0MlCGUY79QpBqTIB2IcKmRQkCdSsb7tp3t6zSdjuxsk8ad6j+/JGB0LonQusUYO70eG8g/tkLxdhk0zlsZUwmqUFJvwZ3Uu6a2B0E4baZQmp4zwJQxezuLr0DBdTYuAqF4AjtcQpP7KDTBBWYWO1kAahIMNm3x0bB7oD+M8LDt9GSzdkfT3USrvbK/n55/7xSqtZGic+RDbJJtolPDkiVHJMzUieUROSRPJFn58V5c96djy9rzhn9WSe/Kpf/BPCDs/Q=</latexit> W 1 <latexit sha1_base64="aufhfgnOg5Qu4gUpqbQ+th/Tt6I=">AAACTXicfZBNSwMxEIaz9aNav1o9elksgoiUXRH1WNCDF1HR2kK3ltl0ug0m2SXJimXpT/CqP8uzP8SbiGmtoFUcCDy884aZecOEM20878XJTU3PzObn5gsLi0vLK8XS6rWOU0WxRmMeq0YIGjmTWDPMcGwkCkGEHOvh7dGwX79DpVksr0w/wZaASLIuo2CsdFm/2W0Xy17FG5X7G/wxlMm4ztslZy/oxDQVKA3loHXT9xLTykAZRjkOCkGqMQF6CxE2LUoQqFvZaNeBu2mVjtuNlX3SuCP1+48MhNZ9EVqnANPTk72h+GcvFBOTTfewlTGZpAYl/RzcTblrYncYhNthCqnhfQtAFbO7u7QHCqixcRUKwTHa4xSe2kFnCSowsdrOAlCRYHJgj42CnSH9Z4T7L6Mlm7M/mepvuN6t+PuV/Yu9cvVonPgcWScbZIv45IBUyQk5JzVCSUQeyCN5cp6dV+fNef+05pzxnzXyo3L5D/Jqs/U=</latexit> W 2 N1 <latexit sha1_base64="rULBMhMqQ9482yCyBwTKgM//9KA=">AAACTXicfZBNSwMxEIaz9aNaP6tHL4tFEJGyK0U9FurBi19otdAtZTadrqFJdkmyYln6E7zqz/LsD/EmYloraBUHAg/vvGFm3jDhTBvPe3FyU9Mzs/m5+cLC4tLyympx7VrHqaJYpzGPVSMEjZxJrBtmODYShSBCjjdhrzbs39yh0iyWV6afYEtAJFmXUTBWujxt++3Vklf2RuX+Bn8MJTKu83bRqQSdmKYCpaEctG76XmJaGSjDKMdBIUg1JkB7EGHTogSBupWNdh24W1bpuN1Y2SeNO1K//8hAaN0XoXUKMLd6sjcU/+yFYmKy6R62MiaT1KCkn4O7KXdN7A6DcDtMITW8bwGoYnZ3l96CAmpsXIVCcIT2OIUndtBZggpMrHayAFQkmBzYY6Ngd0j/GeH+y2jJ5uxPpvobrvfK/n55/6JSqtbGic+RDbJJtolPDkiVHJNzUieUROSBPJIn59l5dd6c909rzhn/WSc/Kpf/AOE6s+w=</latexit> 1 Wnew 2 Wnew <latexit sha1_base64="FnWtJpnmESiWqPWmMXLIvj3Hl/g=">AAACcXicfZDfShtBFMYn6//U2theFW8GgyBawm4R7aWgF70pKhgjuGk4OzlZB2dml5mzaljyBD6Nt+2T+By+gLMxgsbSAwO/+c43nDlfkivpKAwfasHM7Nz8wuJS/cPyx5VPjdXPZy4rrMC2yFRmzxNwqKTBNklSeJ5bBJ0o7CRXB1W/c43Wycyc0jDHrobUyIEUQF7qNTY6v6NeTHhLpcGbEY9TJMe9yLd5fIiKoLr0Gs2wFY6Lv4doAk02qePeam0n7mei0GhIKHDuIgpz6pZgSQqFo3pcOMxBXEGKFx4NaHTdcrzPiG94pc8HmfXHEB+rr1+UoJ0b6sQ7NdClm+5V4j97iZ6aTIMf3VKavCA04nnwoFCcMl6FxfvSoiA19ADCSv93Li7BgiAfab3uA/LLWfzlBx3laIEyu1XGYFMtzcgvm8bfKvqfEW5fjJ58ztF0qu/h7Hsr2m3tnuw09w8miS+yNbbONlnE9tg++8mOWZsJdsfu2R/2t/YYfA14sP5sDWqTN1/Ymwq2nwAwAr4q</latexit> <latexit sha1_base64="32vosqh6Ebm6Jfs/Vp/DHnn+K6g=">AAACcXicfZDRShtBFIYnW7WaVo16VXozGISiEnZFrJeCXnhTqmCM4Kbh7ORkHZyZXWbOtoYlT9Cn8VafpM/RF3A2RtAoHhj45j//cOb8Sa6kozD8Vws+zMzOfZxfqH/6vLi03FhZPXdZYQW2RaYye5GAQyUNtkmSwovcIuhEYSe5Pqz6nd9onczMGQ1z7GpIjRxIAeSlXmOj82unFxPeUGnwz4jHKZLjXuRbPD5CRVBdeo1m2ArHxV9DNIEmm9RJb6W2G/czUWg0JBQ4dxmFOXVLsCSFwlE9LhzmIK4hxUuPBjS6bjneZ8Q3vNLng8z6Y4iP1ecvStDODXXinRroyk33KvHNXqKnJtNgv1tKkxeERjwOHhSKU8arsHhfWhSkhh5AWOn/zsUVWBDkI63XfUB+OYs//KCfOVqgzG6WMdhUSzPyy6bxdkXvGeHmyejJ5xxNp/oaznda0V5r73S3eXA4SXyefWXr7BuL2Hd2wI7ZCWszwf6yW3bH7mv/gy8BD9YfrUFt8maNvahg6wE15r4t</latexit> 1 W + 2 W + W 1 W 2 小 行 力 入 二 4 1 行 行 Saxe, A., McClelland, J., & Ganguli, S. (2014). Exact solutions to the nonlinear dynamics of learning in deep linear neural networks. 力 ] 1 <latexit sha1_base64="z0c8cJk/WbV+FfELQ4+YHKO02PI=">AAACknicfVBdaxNBFJ2s1db40VR968tgEERK2E1LFZ8qFemDrVVMU+gmy93JTTp0ZnaZuSuGZf+Rv8Y30R/jbLoVTYsXLhzOPZdz70lzJR2F4Y9WcGvl9p3Vtbvte/cfPFzvbDw6cVlhBQ5EpjJ7moJDJQ0OSJLC09wi6FThML3Yr+fDL2idzMxnmuc40jAzcioFkKeSzrvhOOKxNDzWQOdpWn6qxuVR0o9JanT8KImqLT4c92/QbP/R9Kuk0w174aL4dRA1oMuaOk42WjvxJBOFRkNCgXNnUZjTqARLUiis2nHhMAdxATM889CAtxqVi4cr/swzEz7NrG9DfMH+vVGCdm6uU6+sT3bLs5q8cZbqJWeavhqV0uQFoRGXxtNCccp4nSafSIuC1NwDEFb627k4BwuCfObtdvwW/XMWD73RhxwtUGZflDHYmZam8s/O4q0a/U8IX6+EHvmco+VUr4OTfi/a7e1+3Onu7TeJr7FN9pQ9ZxF7yfbYATtmAybYN/ad/WS/gifB6+BN0GiDVrPzmP1TwfvfBADKbA==</latexit> 5 入 2 📄 2 N3 <latexit sha1_base64="EMusQFr7bl0LES0y9LepKT4756Y=">AAACTXicfZBNSwMxEIaz9aNav1o9elksgoiUXRX1KOjBi19oq9AtZTadbkOT7JJkxbL0J3jVn+XZH+JNxLRW0CoOBB7eecPMvGHCmTae9+LkJianpvMzs4W5+YXFpWJpuabjVFGs0pjH6jYEjZxJrBpmON4mCkGEHG/C7tGgf3OHSrNYXptegg0BkWRtRsFY6eqsudMslr2KNyz3N/gjKJNRXTRLzm7QimkqUBrKQeu67yWmkYEyjHLsF4JUYwK0CxHWLUoQqBvZcNe+u26VltuOlX3SuEP1+48MhNY9EVqnANPR472B+GcvFGOTTfugkTGZpAYl/RzcTrlrYncQhNtiCqnhPQtAFbO7u7QDCqixcRUKwTHa4xSe2kHnCSowsdrMAlCRYLJvj42CrQH9Z4T7L6Mlm7M/nupvqG1X/L3K3uVu+fBolPgMWSVrZIP4ZJ8ckhNyQaqEkog8kEfy5Dw7r86b8/5pzTmjPyvkR+XyH+UIs+4=</latexit>
> = N1 X > s ↵ u↵ v ↵ ; ↵=1 N 1 N3 <latexit sha1_base64="tdzJLIFIDwXliskwx2dHNvWnIis=">AAACVnicfZDbSgMxEIaz67meql56EyyCiJRdFfVS0Atv1Aq2FdxSZtNpDU2yS5IVy9LX8FYfS19GTA+CVnEg8PHPH2bmj1PBjQ2Cd8+fmp6ZnZtfKCwuLa+sFtfWaybJNMMqS0Si72IwKLjCquVW4F2qEWQssB53zwb9+iNqwxN1a3spNiR0FG9zBtZJ0VUzpJFAetU8oM1iKSgHw6K/IRxDiYyr0lzzDqNWwjKJyjIBxtyHQWobOWjLmcB+IcoMpsC60MF7hwokmkY+XLpPt53Sou1Eu6csHarff+QgjenJ2Dkl2Acz2RuIf/ZiOTHZtk8aOVdpZlGx0eB2JqhN6CAR2uIamRU9B8A0d7tT9gAamHW5FQrRObrjNF66QdcparCJ3s0j0B3JVd8d24n2BvSfEZ6+jI5czuFkqr+htl8Oj8pHN4el07Nx4vNkk2yRHRKSY3JKLkiFVAkjKXkmL+TVe/M+/Bl/bmT1vfGfDfKj/OInnU60tQ==</latexit> (wlog) む d 1 2> ⌧ W = W (⌃ dt <latexit sha1_base64="aoewlSBH5BQHYAHU0t//VBdVAGs=">AAACf3icfVBdaxNBFJ2sX238Su1jXwaDWEXDbixVHwqF+uCLWNE0hW4S7s7e3Q6dj2XmrhiW/SH+Gl/1J/hvnKQRNBUvDBzOOZcz92SVkp7i+Gcnunb9xs1bG5vd23fu3rvf23pw4m3tBI6EVdadZuBRSYMjkqTwtHIIOlM4zi6OFvr4MzovrflE8wonGkojCymAAjXrvUgJap4WDkSTt01OLR9PE37Am/F02E5TstVu+lGWGvjzoAyD+GTW68eDeDn8KkhWoM9Wczzb6uyluRW1RkNCgfdnSVzRpAFHUihsu2ntsQJxASWeBWhAo580y+ta/igwOS+sC88QX7J/bjSgvZ/rLDg10Llf1xbkP7VMryVT8WrSSFPVhEZcBhe14mT5ojqeS4eC1DwAEE6Gv3NxDqE5CgV3u+kbDMc5fBeC3lfogKx72qTgSi1NG44t02cL9D8jfPltDCj0nKy3ehWcDAfJ/mD/w17/8GjV+AbbYQ/ZLkvYS3bI3rJjNmKCfWXf2Hf2I+pEj6NBFF9ao85qZ5v9NdHrX7oqwlQ=</latexit> d 2 ⌧ W = (⌃ dt <latexit sha1_base64="eTyooeVi8+L+NaNjtnmt/dC42d8=">AAACf3icfVDbbtNAEN24XEq4peWRlxURoiCI7FIVeECqVB54QRRBmkp1Eo3XY3fVvVi7Y0Rk+UP4Gl7hE/gb1mmQIEWMtKuzZ87o7JysUtJTHP/sRRtXrl67vnmjf/PW7Tt3B1vbx97WTuBYWGXdSQYelTQ4JkkKTyqHoDOFk+z8sOtPPqPz0ppPtKhwqqE0spACKFDzwfOUoOZp4UA0edvk1PLJbJe/5jvpR1lq4M+692SWPG7C1c5SstV8MIxH8bL4ZZCswJCt6mi+1dtLcytqjYaEAu9Pk7iiaQOOpFDY9tPaYwXiHEo8DdCARj9tltu1/GFgcl5YF44hvmT/nGhAe7/QWVBqoDO/3uvIf/YyveZMxctpI01VExpxYVzUipPlXXQ8lw4FqUUAIJwMf+fiDEJyFALu99M3GJZz+C4Yva/QAVn3pEnBlVqaNixbpk879D8hfPktDCjknKynehkc746S/dH+h73hweEq8U12nz1gOyxhL9gBe8uO2JgJ9pV9Y9/Zj6gXPYpGUXwhjXqrmXvsr4pe/QKoBsJU</latexit> 2 1 W W ) 1 <latexit sha1_base64="lADZqvD1jq4EQoZ3DU7noDJGtks=">AAACZnicfVBdSxtBFJ2stbWp1WgphfZlaCiUUsJuEfVFEPTBF6mFJhHcGO5ObuLgfCwzd8WwBPw1fdW/4z/wZzgbU2ijeGHgzLnnzpl7slxJT3F8W4sWXiy+fLX0uv5m+e3KamNtveNt4QS2hVXWHWfgUUmDbZKk8Dh3CDpT2M3O96p+9wKdl9b8pnGOPQ0jI4dSAAWq3/jYPU34Dk9tEFVv8OreOU3J5v1GM27F0+KPQTIDTTaro/5abSMdWFFoNCQUeH+SxDn1SnAkhcJJPS085iDOYYQnARrQ6HvldIkJ/xKYAR9aF44hPmX/nShBez/WWVBqoDM/36vIJ3uZnnOm4XavlCYvCI14MB4WipPlVUJ8IB0KUuMAQDgZ/s7FGTgQFHKs19N9DMs5PAxGP3N0QNZ9K1NwIy3NJCw7Sr9X6DkhXP4VBhRyTuZTfQw6P1rJZmvz10Zzd2+W+BL7xD6zryxhW2yXHbAj1maCXbE/7Jrd1O6ileh99OFBGtVmM+/YfxXxe5l4udQ=</latexit> 2 d 1 2> ⌧ W = W (S dt W2 = UW <latexit sha1_base64="vtqRQqNagU2/CqmTU+eODBmQnrE=">AAACoXicfVFNbxMxEHWWrxI+msKRi0WEVBBEu1VVuCBVKgc4IMJHmkpxGs16Z1OrXntlzyKi1f4yfglHrvAn8KZBalLESJae3nvj8TynpVae4vhHJ7p2/cbNW1u3u3fu3ru/3dt5cOxt5SSOpNXWnaTgUSuDI1Kk8aR0CEWqcZyeH7X6+Cs6r6z5QosSpwXMjcqVBArUrDcSBJXIHcg6a+qMGmGDvb2Nj08T/prXl4m95lSQLfnuZ/6CrwlrbU9nvX48iJfFr4JkBfpsVcPZTmdfZFZWBRqSGryfJHFJ0xocKamx6YrKYwnyHOY4CdBAgX5aL/dv+JPAZDy3LhxDfMle7qih8H5RpMFZAJ35Ta0l/6mlxcZkyl9Na2XKitDIi8F5pTlZ3obLM+VQkl4EANKp8HYuzyCES+ELul3xBsNyDt+HQR9KdEDWPasFuHmhTBOWnYvnLfqfEb79NQYUck42U70KjvcGycHg4ON+//BolfgWe8Qes12WsJfskL1lQzZikn1nP9kv9jvqR++iYfTpwhp1Vj0P2VpFkz+0sNEO</latexit> d 2 ⌧ W = (S dt 1 W W )W <latexit sha1_base64="nxrvmp5jtcwtpIwUs1pWzkUlXgE=">AAACYHicfVBdSxtBFJ2stY3R1kR9al8Gg1BKCbsi1hdB0AdfSlMwiZCN4e7kJg7OxzIzK8Yl/8VX/Ue++kucTVKoUbwwcDj33Dn3niQV3LowfCwFSx+WP34qr1RW1z5/Wa/WNtpWZ4Zhi2mhzXkCFgVX2HLcCTxPDYJMBHaSq+Oi37lGY7lWZ26cYk/CSPEhZ+A81a9udS526SFtxdqrik+oJ/rVetgIp0Vfg2gO6mRezX6ttBcPNMskKscEWNuNwtT1cjCOM4GTSpxZTIFdwQi7HiqQaHv5dP0J3fHMgA618U85OmX/n8hBWjuWiVdKcJd2sVeQb/YSueDshge9nKs0c6jYzHiYCeo0LbKhA26QOTH2AJjhfnfKLsEAcz7BSiU+QX+cwd/e6E+KBpw2P/IYzEhyNfHHjuKfBXpPCDf/hB75nKPFVF+D9m4j2m/s/92rHx3PEy+Tb2SbfCcR+UWOyClpkhZh5JbckXvyUHoKysF6UJtJg9J8ZpO8qODrM2pot2w=</latexit> W1 = W V > 1> x N2 <latexit sha1_base64="haSdbYtsNQyoRe7ykMC/yp4rtBI=">AAACTXicfZBNSwMxEIaz9aNaP6tHL4tFEJGyK0U9FurBi6hordCtMptO19AkuyRZsSz9CV71Z3n2h3gTMa0VtBUHAg/vvGFm3jDhTBvPe3VyU9Mzs/m5+cLC4tLyympx7UrHqaJYpzGP1XUIGjmTWDfMcLxOFIIIOTbCbm3Qb9yj0iyWl6aXYEtAJFmHUTBWumjc+LerJa/sDcudBH8EJTKqs9uiUwnaMU0FSkM5aN30vcS0MlCGUY79QpBqTIB2IcKmRQkCdSsb7tp3t6zSdjuxsk8ad6j+/JGB0LonQusUYO70eG8g/tkLxdhk0zlsZUwmqUFJvwZ3Uu6a2B0E4baZQmp4zwJQxezuLr0DBdTYuAqF4AjtcQpP7KDTBBWYWO1kAahIMNm3x0bB7oD+M8LDt9GSzdkfT3USrvbK/n55/7xSqtZGic+RDbJJtolPDkiVHJMzUieUROSRPJFn58V5c96djy9rzhn9WSe/Kpf/BPCDs/Q=</latexit> W 2 y <latexit sha1_base64="KgDhIumMmgJR5Go00Yuc+VxfdEU=">AAACS3icfZDPSgMxEMaz1Wqtf6tHL4tFEJGyK0U9FurBi6hgreCWMptO29AkuyRZsSx9Aq/6WD6Az+FNPJitFbQVBwI/vvnCzHxhzJk2nvfq5Obm8wuLhaXi8srq2vpGafNGR4mi2KARj9RtCBo5k9gwzHC8jRWCCDk2w0E96zfvUWkWyWszjLEloCdZl1EwVroatjfKXsUblzsL/gTKZFKX7ZJTDToRTQRKQzlofed7sWmloAyjHEfFINEYAx1AD+8sShCoW+l405G7a5WO242UfdK4Y/XnjxSE1kMRWqcA09fTvUz8sxeKqcmme9JKmYwTg5J+De4m3DWRm8XgdphCavjQAlDF7O4u7YMCamxYxWJwivY4hed20EWMCkyk9tMAVE8wObLH9oKDjP4zwsO30ZLN2Z9OdRZuDiv+UeXoqlqu1SeJF8g22SF7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc05kz9b5Ffl8p/VfbNz</latexit> <latexit sha1_base64="NgEKFcYW2o1IZfa5uw8QAVDCf3s=">AAACTXicfZBNS8NAEIY39aNav1o9egkWQURKUkQ9FvTgxS+0KphSJttpXLq7CbsbsYT+BK/6szz7Q7yJuK0RtIoDCw/vvMvMvGHCmTae9+IUJianposzs6W5+YXFpXJl+VLHqaLYpDGP1XUIGjmT2DTMcLxOFIIIOV6Fvf1h/+oOlWaxvDD9BFsCIsm6jIKx0vlxu94uV72aNyr3N/g5VElep+2Ksx10YpoKlIZy0PrG9xLTykAZRjkOSkGqMQHagwhvLEoQqFvZaNeBu26VjtuNlX3SuCP1+48MhNZ9EVqnAHOrx3tD8c9eKMYmm+5eK2MySQ1K+jm4m3LXxO4wCLfDFFLD+xaAKmZ3d+ktKKDGxlUqBQdoj1N4ZAedJKjAxGozC0BFgsmBPTYKtob0nxHuv4yWbM7+eKq/4bJe83dqO2fb1cZ+nvgMWSVrZIP4ZJc0yCE5JU1CSUQeyCN5cp6dV+fNef+0Fpz8zwr5UYXiB+Mhs+0=</latexit> <latexit sha1_base64="2oa+EUQF+8zDjmnZ6Nf+98VygGw=">AAACS3icfZDPSgMxEMaz9X/91+rRy2IRRKTsSqkeC/XgRVSwtdAtMptOazDJLklWWpY+gVd9LB/A5/AmHkzrCtqKA4Ef33xhZr4w5kwbz3t1cnPzC4tLyyv51bX1jc1Ccaupo0RRbNCIR6oVgkbOJDYMMxxbsUIQIceb8L4+7t88oNIsktdmGGNHQF+yHqNgrHQ1uC2UvLI3KXcW/AxKJKvL26JTCboRTQRKQzlo3fa92HRSUIZRjqN8kGiMgd5DH9sWJQjUnXSy6cjds0rX7UXKPmncifrzRwpC66EIrVOAudPTvbH4Zy8UU5NN76STMhknBiX9GtxLuGsidxyD22UKqeFDC0AVs7u79A4UUGPDyueDU7THKTy3gy5iVGAidZAGoPqCyZE9th8cjuk/Iwy+jZZszv50qrPQPCr71XL1qlKq1bPEl8kO2SX7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc052Z9t8qtyC5/TlrNy</latexit> 1 <latexit sha1_base64="aufhfgnOg5Qu4gUpqbQ+th/Tt6I=">AAACTXicfZBNSwMxEIaz9aNav1o9elksgoiUXRH1WNCDF1HR2kK3ltl0ug0m2SXJimXpT/CqP8uzP8SbiGmtoFUcCDy884aZecOEM20878XJTU3PzObn5gsLi0vLK8XS6rWOU0WxRmMeq0YIGjmTWDPMcGwkCkGEHOvh7dGwX79DpVksr0w/wZaASLIuo2CsdFm/2W0Xy17FG5X7G/wxlMm4ztslZy/oxDQVKA3loHXT9xLTykAZRjkOCkGqMQF6CxE2LUoQqFvZaNeBu2mVjtuNlX3SuCP1+48MhNZ9EVqnANPTk72h+GcvFBOTTfewlTGZpAYl/RzcTblrYncYhNthCqnhfQtAFbO7u7QHCqixcRUKwTHa4xSe2kFnCSowsdrOAlCRYHJgj42CnSH9Z4T7L6Mlm7M/mepvuN6t+PuV/Yu9cvVonPgcWScbZIv45IBUyQk5JzVCSUQeyCN5cp6dV+fNef+05pzxnzXyo3L5D/Jqs/U=</latexit> W 2 <latexit sha1_base64="JvAcOgrCKAA5nIOCF0i0oH2858o=">AAACoXicfVFNbxMxEHWWrxI+msKRi0WEVBBEu1VVuCBVKgc4IMJHmkpxGs16Z1OrXntlzyKi1f4yfglHrvAn8KZBalLESJbevHnj8TynpVae4vhHJ7p2/cbNW1u3u3fu3ru/3dt5cOxt5SSOpNXWnaTgUSuDI1Kk8aR0CEWqcZyeH7X18Vd0XlnzhRYlTguYG5UrCRSoWW8kCCqRO5B11tQZNcIGeXsbH5/u8dd89zN/wdfIy0nytF5Lm1NBtuSzXj8exMvgV0GyAn22iuFsp7MvMiurAg1JDd5PkrikaQ2OlNTYdEXlsQR5DnOcBGigQD+tl/s3/ElgMp5bF44hvmQvd9RQeL8o0qAsgM78Zq0l/1lLi43JlL+a1sqUFaGRF4PzSnOyvDWXZ8qhJL0IAKRT4e1cnkEwl8IXdLviDYblHL4Pgz6U6ICse1YLcPNCmSYsOxfPW/Q/IXz7Kwwo+JxsunoVHO8NkoPBwcf9/uHRyvEt9og9ZrssYS/ZIXvLhmzEJPvOfrJf7HfUj95Fw+jThTTqrHoesrWIJn8AllfRDg==</latexit> 2 1 W W ) 2 1 W W )W 1> N1 <latexit sha1_base64="rULBMhMqQ9482yCyBwTKgM//9KA=">AAACTXicfZBNSwMxEIaz9aNaP6tHL4tFEJGyK0U9FurBi19otdAtZTadrqFJdkmyYln6E7zqz/LsD/EmYloraBUHAg/vvGFm3jDhTBvPe3FyU9Mzs/m5+cLC4tLyympx7VrHqaJYpzGPVSMEjZxJrBtmODYShSBCjjdhrzbs39yh0iyWV6afYEtAJFmXUTBWujxt++3Vklf2RuX+Bn8MJTKu83bRqQSdmKYCpaEctG76XmJaGSjDKMdBIUg1JkB7EGHTogSBupWNdh24W1bpuN1Y2SeNO1K//8hAaN0XoXUKMLd6sjcU/+yFYmKy6R62MiaT1KCkn4O7KXdN7A6DcDtMITW8bwGoYnZ3l96CAmpsXIVCcIT2OIUndtBZggpMrHayAFQkmBzYY6Ngd0j/GeH+y2jJ5uxPpvobrvfK/n55/6JSqtbGic+RDbJJtolPDkiVHJNzUieUROSBPJIn59l5dd6c909rzhn/WSc/Kpf/AOE6s+w=</latexit> 力 入 行 4 1 行 行 力 入 力 Saxe, A., McClelland, J., & Ganguli, S. (2014). Exact solutions to the nonlinear dynamics of learning in deep linear neural networks. 6 入 2 📄 [Saxe-McClelland-Ganguli N3 <latexit sha1_base64="EMusQFr7bl0LES0y9LepKT4756Y=">AAACTXicfZBNSwMxEIaz9aNav1o9elksgoiUXRX1KOjBi19oq9AtZTadbkOT7JJkxbL0J3jVn+XZH+JNxLRW0CoOBB7eecPMvGHCmTae9+LkJianpvMzs4W5+YXFpWJpuabjVFGs0pjH6jYEjZxJrBpmON4mCkGEHG/C7tGgf3OHSrNYXptegg0BkWRtRsFY6eqsudMslr2KNyz3N/gjKJNRXTRLzm7QimkqUBrKQeu67yWmkYEyjHLsF4JUYwK0CxHWLUoQqBvZcNe+u26VltuOlX3SuEP1+48MhNY9EVqnANPR472B+GcvFGOTTfugkTGZpAYl/RzcTrlrYncQhNtiCqnhPQtAFbO7u7QDCqixcRUKwTHa4xSe2kHnCSowsdrMAlCRYLJvj42CrQH9Z4T7L6Mlm7M/nupvqG1X/L3K3uVu+fBolPgMWSVrZIP4ZJ8ckhNyQaqEkog8kEfy5Dw7r86b8/5pzTmjPyvkR+XyH+UIs+4=</latexit>
2 ⌧ W = (⌃ dt <latexit sha1_base64="aoewlSBH5BQHYAHU0t//VBdVAGs=">AAACf3icfVBdaxNBFJ2sX238Su1jXwaDWEXDbixVHwqF+uCLWNE0hW4S7s7e3Q6dj2XmrhiW/SH+Gl/1J/hvnKQRNBUvDBzOOZcz92SVkp7i+Gcnunb9xs1bG5vd23fu3rvf23pw4m3tBI6EVdadZuBRSYMjkqTwtHIIOlM4zi6OFvr4MzovrflE8wonGkojCymAAjXrvUgJap4WDkSTt01OLR9PE37Am/F02E5TstVu+lGWGvjzoAyD+GTW68eDeDn8KkhWoM9Wczzb6uyluRW1RkNCgfdnSVzRpAFHUihsu2ntsQJxASWeBWhAo580y+ta/igwOS+sC88QX7J/bjSgvZ/rLDg10Llf1xbkP7VMryVT8WrSSFPVhEZcBhe14mT5ojqeS4eC1DwAEE6Gv3NxDqE5CgV3u+kbDMc5fBeC3lfogKx72qTgSi1NG44t02cL9D8jfPltDCj0nKy3ehWcDAfJ/mD/w17/8GjV+AbbYQ/ZLkvYS3bI3rJjNmKCfWXf2Hf2I+pEj6NBFF9ao85qZ5v9NdHrX7oqwlQ=</latexit> 1 2 1 W W ) 2 > d 1 2> ⌧ W = W (S dt 2 W = UW d 2 ⌧ W = (S dt <latexit sha1_base64="vtqRQqNagU2/CqmTU+eODBmQnrE=">AAACoXicfVFNbxMxEHWWrxI+msKRi0WEVBBEu1VVuCBVKgc4IMJHmkpxGs16Z1OrXntlzyKi1f4yfglHrvAn8KZBalLESJae3nvj8TynpVae4vhHJ7p2/cbNW1u3u3fu3ru/3dt5cOxt5SSOpNXWnaTgUSuDI1Kk8aR0CEWqcZyeH7X6+Cs6r6z5QosSpwXMjcqVBArUrDcSBJXIHcg6a+qMGmGDvb2Nj08T/prXl4m95lSQLfnuZ/6CrwlrbU9nvX48iJfFr4JkBfpsVcPZTmdfZFZWBRqSGryfJHFJ0xocKamx6YrKYwnyHOY4CdBAgX5aL/dv+JPAZDy3LhxDfMle7qih8H5RpMFZAJ35Ta0l/6mlxcZkyl9Na2XKitDIi8F5pTlZ3obLM+VQkl4EANKp8HYuzyCES+ELul3xBsNyDt+HQR9KdEDWPasFuHmhTBOWnYvnLfqfEb79NQYUck42U70KjvcGycHg4ON+//BolfgWe8Qes12WsJfskL1lQzZikn1nP9kv9jvqR++iYfTpwhp1Vj0P2VpFkz+0sNEO</latexit> 1 W W )W <latexit sha1_base64="nxrvmp5jtcwtpIwUs1pWzkUlXgE=">AAACYHicfVBdSxtBFJ2stY3R1kR9al8Gg1BKCbsi1hdB0AdfSlMwiZCN4e7kJg7OxzIzK8Yl/8VX/Ue++kucTVKoUbwwcDj33Dn3niQV3LowfCwFSx+WP34qr1RW1z5/Wa/WNtpWZ4Zhi2mhzXkCFgVX2HLcCTxPDYJMBHaSq+Oi37lGY7lWZ26cYk/CSPEhZ+A81a9udS526SFtxdqrik+oJ/rVetgIp0Vfg2gO6mRezX6ttBcPNMskKscEWNuNwtT1cjCOM4GTSpxZTIFdwQi7HiqQaHv5dP0J3fHMgA618U85OmX/n8hBWjuWiVdKcJd2sVeQb/YSueDshge9nKs0c6jYzHiYCeo0LbKhA26QOTH2AJjhfnfKLsEAcz7BSiU+QX+cwd/e6E+KBpw2P/IYzEhyNfHHjuKfBXpPCDf/hB75nKPFVF+D9m4j2m/s/92rHx3PEy+Tb2SbfCcR+UWOyClpkhZh5JbckXvyUHoKysF6UJtJg9J8ZpO8qODrM2pot2w=</latexit> W =W V 1> 2 <latexit sha1_base64="JvAcOgrCKAA5nIOCF0i0oH2858o=">AAACoXicfVFNbxMxEHWWrxI+msKRi0WEVBBEu1VVuCBVKgc4IMJHmkpxGs16Z1OrXntlzyKi1f4yfglHrvAn8KZBalLESJbevHnj8TynpVae4vhHJ7p2/cbNW1u3u3fu3ru/3dt5cOxt5SSOpNXWnaTgUSuDI1Kk8aR0CEWqcZyeH7X18Vd0XlnzhRYlTguYG5UrCRSoWW8kCCqRO5B11tQZNcIGeXsbH5/u8dd89zN/wdfIy0nytF5Lm1NBtuSzXj8exMvgV0GyAn22iuFsp7MvMiurAg1JDd5PkrikaQ2OlNTYdEXlsQR5DnOcBGigQD+tl/s3/ElgMp5bF44hvmQvd9RQeL8o0qAsgM78Zq0l/1lLi43JlL+a1sqUFaGRF4PzSnOyvDWXZ8qhJL0IAKRT4e1cnkEwl8IXdLviDYblHL4Pgz6U6ICse1YLcPNCmSYsOxfPW/Q/IXz7Kwwo+JxsunoVHO8NkoPBwcf9/uHRyvEt9og9ZrssYS/ZIXvLhmzEJPvOfrJf7HfUj95Fw+jThTTqrHoesrWIJn8AllfRDg==</latexit> 2 1 W W ) 2 1 W W )W ↵ 1> ↵ N2 b 2R <latexit sha1_base64="DHLyS4uYdl0Lg3kV3tIdx5ykrr4=">AAACaXicfZDfSiMxFMbT8c+6dXXb9UYUIVgWFpEyI+LupeBeeONfrApOLSfpaRtMMkOSWSzDXPk0e6tPs8/gS5ipFdYqHgj8+M4XzjkfS6WwLgz/VYKp6ZnZT3Ofq/NfFha/1urfzm2SGY4tnsjEXDKwKIXGlhNO4mVqEBSTeMFu9sr+xR80ViT6zA1TbCvoa9ETHJyXOrU1dh2DTAdAY6FprMANGMtPi+v8sLNVdGqNsBmOir6FaAwNMq7jTr2yHXcTninUjkuw9ioKU9fOwTjBJRbVOLOYAr+BPl551KDQtvPRHQX97pUu7SXGP+3oSP3/Rw7K2qFi3lkuaid7pfhuj6mJya73q50LnWYONX8e3MskdQktQ6JdYZA7OfQA3Ai/O+UDMMCdj7JajX+jP87ggR90lKIBl5iNPAbTV0IX/th+vFnSR0a4fTF68jlHk6m+hfOtZrTT3DnZbuzujROfI6tknfwgEflJdsk+OSYtwskd+UvuyUPlMagHy8HKszWojP8skVcVNJ4AZ0C8JA==</latexit> N2 a 2R <latexit sha1_base64="DdEbmgm8rji84Qf/R9oPSY2wd5c=">AAACaXicfZDbSiNBEIY742HdeNhEb0QRGoMgImFGRL0U3AtvPGJUcGKo6VSSxu6eobtHDMNc+TR7q0+zz+BL2BMjaBQLGj7++puq+qNEcGN9/3/JGxufmPw19bs8PTM796dSnb80caoZNlgsYn0dgUHBFTYstwKvE40gI4FX0d1B0b+6R214rC5sP8GmhK7iHc7AOqlVWYHbEETSAxpyRUMJthdF2Xl+mx23tvJWpebX/UHRrxAMoUaGddqqlrbDdsxSicoyAcbcBH5imxloy5nAvBymBhNgd9DFG4cKJJpmNrgjp2tOadNOrN1Tlg7Ujz8ykMb0ZeScxaJmtFeI3/YiOTLZdvaaGVdJalGxt8GdVFAb0yIk2uYamRV9B8A0d7tT1gMNzLooy+XwL7rjNB65QScJarCx3shC0F3JVe6O7YabBf1khId3oyOXczCa6le43KoHO/Wds+3a/sEw8SmyTFbJOgnILtknh+SUNAgjj+QfeSLPpRev6i16S29WrzT8s0A+lVd7BWU9vCM=</latexit> x 1 <latexit sha1_base64="2N2HhVHrFmNqTj/W7r8zUXVj4gk=">AAACWHicfVBdSxtREJ2sbaOxtkYffbk0CKVI2BWxvgiCffCl1IIxghtl9mYSL7kfy713xbDkf/ha/5X+Gu/GCDWKAwOHc84wMyfLpXA+ju9r0cKHj5/qi0uN5c8rX76uNtdOnSkspw430tizDB1JoanjhZd0lltClUnqZqPDSu9ek3XC6BM/zqmncKjFQHD0gbpITRCrWda92N6/XG3F7Xha7DVIZqAFszq+bNZ20r7hhSLtuUTnzpM4970SrRdc0qSRFo5y5CMc0nmAGhW5Xjk9e8I2A9NnA2NDa8+m7P8TJSrnxioLToX+ys1rFfmmlqm5zX6w1yuFzgtPmj8tHhSSecOqTFhfWOJejgNAbkW4nfErtMh9SK7RSH9ReM7S77DoT04WvbE/yhTtUAk9Cc8O060KvWfEm2djQCHnZD7V1+B0u53stnf/7rQODmeJL8IGfIPvkMBPOIAjOIYOcLBwC//grvYQQVSPlp6sUW02sw4vKlp7BBtXtYM=</latexit> N2 <latexit sha1_base64="NgEKFcYW2o1IZfa5uw8QAVDCf3s=">AAACTXicfZBNS8NAEIY39aNav1o9egkWQURKUkQ9FvTgxS+0KphSJttpXLq7CbsbsYT+BK/6szz7Q7yJuK0RtIoDCw/vvMvMvGHCmTae9+IUJianposzs6W5+YXFpXJl+VLHqaLYpDGP1XUIGjmT2DTMcLxOFIIIOV6Fvf1h/+oOlWaxvDD9BFsCIsm6jIKx0vlxu94uV72aNyr3N/g5VElep+2Ksx10YpoKlIZy0PrG9xLTykAZRjkOSkGqMQHagwhvLEoQqFvZaNeBu26VjtuNlX3SuCP1+48MhNZ9EVqnAHOrx3tD8c9eKMYmm+5eK2MySQ1K+jm4m3LXxO4wCLfDFFLD+xaAKmZ3d+ktKKDGxlUqBQdoj1N4ZAedJKjAxGozC0BFgsmBPTYKtob0nxHuv4yWbM7+eKq/4bJe83dqO2fb1cZ+nvgMWSVrZIP4ZJc0yCE5JU1CSUQeyCN5cp6dV+fNef+0Fpz8zwr5UYXiB+Mhs+0=</latexit> W = 2 N3 <latexit sha1_base64="EMusQFr7bl0LES0y9LepKT4756Y=">AAACTXicfZBNSwMxEIaz9aNav1o9elksgoiUXRX1KOjBi19oq9AtZTadbkOT7JJkxbL0J3jVn+XZH+JNxLRW0CoOBB7eecPMvGHCmTae9+LkJianpvMzs4W5+YXFpWJpuabjVFGs0pjH6jYEjZxJrBpmON4mCkGEHG/C7tGgf3OHSrNYXptegg0BkWRtRsFY6eqsudMslr2KNyz3N/gjKJNRXTRLzm7QimkqUBrKQeu67yWmkYEyjHLsF4JUYwK0CxHWLUoQqBvZcNe+u26VltuOlX3SuEP1+48MhNY9EVqnANPR472B+GcvFGOTTfugkTGZpAYl/RzcTrlrYncQhNtiCqnhPQtAFbO7u7QDCqixcRUKwTHa4xSe2kHnCSowsdrMAlCRYLJvj42CrQH9Z4T7L6Mlm7M/nupvqG1X/L3K3uVu+fBolPgMWSVrZIP4ZJ8ckhNyQaqEkog8kEfy5Dw7r86b8/5pzTmjPyvkR+XyH+UIs+4=</latexit> W = y <latexit sha1_base64="KgDhIumMmgJR5Go00Yuc+VxfdEU=">AAACS3icfZDPSgMxEMaz1Wqtf6tHL4tFEJGyK0U9FurBi6hgreCWMptO29AkuyRZsSx9Aq/6WD6Az+FNPJitFbQVBwI/vvnCzHxhzJk2nvfq5Obm8wuLhaXi8srq2vpGafNGR4mi2KARj9RtCBo5k9gwzHC8jRWCCDk2w0E96zfvUWkWyWszjLEloCdZl1EwVroatjfKXsUblzsL/gTKZFKX7ZJTDToRTQRKQzlofed7sWmloAyjHEfFINEYAx1AD+8sShCoW+l405G7a5WO242UfdK4Y/XnjxSE1kMRWqcA09fTvUz8sxeKqcmme9JKmYwTg5J+De4m3DWRm8XgdphCavjQAlDF7O4u7YMCamxYxWJwivY4hed20EWMCkyk9tMAVE8wObLH9oKDjP4zwsO30ZLN2Z9OdRZuDiv+UeXoqlqu1SeJF8g22SF7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc05kz9b5Ffl8p/VfbNz</latexit> N2 <latexit sha1_base64="NgEKFcYW2o1IZfa5uw8QAVDCf3s=">AAACTXicfZBNS8NAEIY39aNav1o9egkWQURKUkQ9FvTgxS+0KphSJttpXLq7CbsbsYT+BK/6szz7Q7yJuK0RtIoDCw/vvMvMvGHCmTae9+IUJianposzs6W5+YXFpXJl+VLHqaLYpDGP1XUIGjmT2DTMcLxOFIIIOV6Fvf1h/+oOlWaxvDD9BFsCIsm6jIKx0vlxu94uV72aNyr3N/g5VElep+2Ksx10YpoKlIZy0PrG9xLTykAZRjkOSkGqMQHagwhvLEoQqFvZaNeBu26VjtuNlX3SuCP1+48MhNZ9EVqnAHOrx3tD8c9eKMYmm+5eK2MySQ1K+jm4m3LXxO4wCLfDFFLD+xaAKmZ3d+ktKKDGxlUqBQdoj1N4ZAedJKjAxGozC0BFgsmBPTYKtob0nxHuv4yWbM7+eKq/4bJe83dqO2fb1cZ+nvgMWSVrZIP4ZJc0yCE5JU1CSUQeyCN5cp6dV+fNef+0Fpz8zwr5UYXiB+Mhs+0=</latexit> <latexit sha1_base64="2oa+EUQF+8zDjmnZ6Nf+98VygGw=">AAACS3icfZDPSgMxEMaz9X/91+rRy2IRRKTsSqkeC/XgRVSwtdAtMptOazDJLklWWpY+gVd9LB/A5/AmHkzrCtqKA4Ef33xhZr4w5kwbz3t1cnPzC4tLyyv51bX1jc1Ccaupo0RRbNCIR6oVgkbOJDYMMxxbsUIQIceb8L4+7t88oNIsktdmGGNHQF+yHqNgrHQ1uC2UvLI3KXcW/AxKJKvL26JTCboRTQRKQzlo3fa92HRSUIZRjqN8kGiMgd5DH9sWJQjUnXSy6cjds0rX7UXKPmncifrzRwpC66EIrVOAudPTvbH4Zy8UU5NN76STMhknBiX9GtxLuGsidxyD22UKqeFDC0AVs7u79A4UUGPDyueDU7THKTy3gy5iVGAidZAGoPqCyZE9th8cjuk/Iwy+jZZszv50qrPQPCr71XL1qlKq1bPEl8kO2SX7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc052Z9t8qtyC5/TlrNy</latexit> <latexit sha1_base64="eC1WdBSeIwB8HNfOlRvzX95FnW0=">AAACWHicfVDbSiNBEK3Mrrd410dfmg2CiIQZEd2XBUEffBEVjBGcKDWdSmzSl6G7Z9kw5D/21f2r3a/ZnhhBo1hQcDjnFFV1slwK5+P4by368nVmdm5+ob64tLyyura+ceNMYTm1uJHG3mboSApNLS+8pNvcEqpMUjsbnFR6+ydZJ4y+9sOcOgr7WvQERx+o+9QEsZpl7fvkx8NaI27G42LvQTIBDZjU5cN67SDtGl4o0p5LdO4uiXPfKdF6wSWN6mnhKEc+wD7dBahRkeuU47NHbDswXdYzNrT2bMy+nihROTdUWXAq9I9uWqvID7VMTW32ve+dUui88KT58+JeIZk3rMqEdYUl7uUwAORWhNsZf0SL3Ifk6vX0lMJzls7DooucLHpjd8sUbV8JPQrP9tO9Cn1mxF8vxoBCzsl0qu/BzX4zOWweXh00jk8mic/DFnyDHUjgCI7hDC6hBRws/IYn+FP7F0E0Fy08W6PaZGYT3lS08R8Zb7WC</latexit> ] <latexit sha1_base64="eTyooeVi8+L+NaNjtnmt/dC42d8=">AAACf3icfVDbbtNAEN24XEq4peWRlxURoiCI7FIVeECqVB54QRRBmkp1Eo3XY3fVvVi7Y0Rk+UP4Gl7hE/gb1mmQIEWMtKuzZ87o7JysUtJTHP/sRRtXrl67vnmjf/PW7Tt3B1vbx97WTuBYWGXdSQYelTQ4JkkKTyqHoDOFk+z8sOtPPqPz0ppPtKhwqqE0spACKFDzwfOUoOZp4UA0edvk1PLJbJe/5jvpR1lq4M+692SWPG7C1c5SstV8MIxH8bL4ZZCswJCt6mi+1dtLcytqjYaEAu9Pk7iiaQOOpFDY9tPaYwXiHEo8DdCARj9tltu1/GFgcl5YF44hvmT/nGhAe7/QWVBqoDO/3uvIf/YyveZMxctpI01VExpxYVzUipPlXXQ8lw4FqUUAIJwMf+fiDEJyFALu99M3GJZz+C4Yva/QAVn3pEnBlVqaNixbpk879D8hfPktDCjknKynehkc746S/dH+h73hweEq8U12nz1gOyxhL9gBe8uO2JgJ9pV9Y9/Zj6gXPYpGUXwhjXqrmXvsr4pe/QKoBsJU</latexit> 1 <latexit sha1_base64="lADZqvD1jq4EQoZ3DU7noDJGtks=">AAACZnicfVBdSxtBFJ2stbWp1WgphfZlaCiUUsJuEfVFEPTBF6mFJhHcGO5ObuLgfCwzd8WwBPw1fdW/4z/wZzgbU2ijeGHgzLnnzpl7slxJT3F8W4sWXiy+fLX0uv5m+e3KamNtveNt4QS2hVXWHWfgUUmDbZKk8Dh3CDpT2M3O96p+9wKdl9b8pnGOPQ0jI4dSAAWq3/jYPU34Dk9tEFVv8OreOU3J5v1GM27F0+KPQTIDTTaro/5abSMdWFFoNCQUeH+SxDn1SnAkhcJJPS085iDOYYQnARrQ6HvldIkJ/xKYAR9aF44hPmX/nShBez/WWVBqoDM/36vIJ3uZnnOm4XavlCYvCI14MB4WipPlVUJ8IB0KUuMAQDgZ/s7FGTgQFHKs19N9DMs5PAxGP3N0QNZ9K1NwIy3NJCw7Sr9X6DkhXP4VBhRyTuZTfQw6P1rJZmvz10Zzd2+W+BL7xD6zryxhW2yXHbAj1maCXbE/7Jrd1O6ileh99OFBGtVmM+/YfxXxe5l4udQ=</latexit> <latexit sha1_base64="haSdbYtsNQyoRe7ykMC/yp4rtBI=">AAACTXicfZBNSwMxEIaz9aNaP6tHL4tFEJGyK0U9FurBi6hordCtMptO19AkuyRZsSz9CV71Z3n2h3gTMa0VtBUHAg/vvGFm3jDhTBvPe3VyU9Mzs/m5+cLC4tLyympx7UrHqaJYpzGP1XUIGjmTWDfMcLxOFIIIOTbCbm3Qb9yj0iyWl6aXYEtAJFmHUTBWumjc+LerJa/sDcudBH8EJTKqs9uiUwnaMU0FSkM5aN30vcS0MlCGUY79QpBqTIB2IcKmRQkCdSsb7tp3t6zSdjuxsk8ad6j+/JGB0LonQusUYO70eG8g/tkLxdhk0zlsZUwmqUFJvwZ3Uu6a2B0E4baZQmp4zwJQxezuLr0DBdTYuAqF4AjtcQpP7KDTBBWYWO1kAahIMNm3x0bB7oD+M8LDt9GSzdkfT3USrvbK/n55/7xSqtZGic+RDbJJtolPDkiVHJMzUieUROSRPJFn58V5c96djy9rzhn9WSe/Kpf/BPCDs/Q=</latexit> W 1 <latexit sha1_base64="aufhfgnOg5Qu4gUpqbQ+th/Tt6I=">AAACTXicfZBNSwMxEIaz9aNav1o9elksgoiUXRH1WNCDF1HR2kK3ltl0ug0m2SXJimXpT/CqP8uzP8SbiGmtoFUcCDy884aZecOEM20878XJTU3PzObn5gsLi0vLK8XS6rWOU0WxRmMeq0YIGjmTWDPMcGwkCkGEHOvh7dGwX79DpVksr0w/wZaASLIuo2CsdFm/2W0Xy17FG5X7G/wxlMm4ztslZy/oxDQVKA3loHXT9xLTykAZRjkOCkGqMQF6CxE2LUoQqFvZaNeBu2mVjtuNlX3SuCP1+48MhNZ9EVqnANPTk72h+GcvFBOTTfewlTGZpAYl/RzcTblrYncYhNthCqnhfQtAFbO7u7QHCqixcRUKwTHa4xSe2kFnCSowsdrOAlCRYHJgj42CnSH9Z4T7L6Mlm7M/mepvuN6t+PuV/Yu9cvVonPgcWScbZIv45IBUyQk5JzVCSUQeyCN5cp6dV+fNef+05pzxnzXyo3L5D/Jqs/U=</latexit> W 2 N2 <latexit sha1_base64="NgEKFcYW2o1IZfa5uw8QAVDCf3s=">AAACTXicfZBNS8NAEIY39aNav1o9egkWQURKUkQ9FvTgxS+0KphSJttpXLq7CbsbsYT+BK/6szz7Q7yJuK0RtIoDCw/vvMvMvGHCmTae9+IUJianposzs6W5+YXFpXJl+VLHqaLYpDGP1XUIGjmT2DTMcLxOFIIIOV6Fvf1h/+oOlWaxvDD9BFsCIsm6jIKx0vlxu94uV72aNyr3N/g5VElep+2Ksx10YpoKlIZy0PrG9xLTykAZRjkOSkGqMQHagwhvLEoQqFvZaNeBu26VjtuNlX3SuCP1+48MhNZ9EVqnAHOrx3tD8c9eKMYmm+5eK2MySQ1K+jm4m3LXxO4wCLfDFFLD+xaAKmZ3d+ktKKDGxlUqBQdoj1N4ZAedJKjAxGozC0BFgsmBPTYKtob0nxHuv4yWbM7+eKq/4bJe83dqO2fb1cZ+nvgMWSVrZIP4ZJc0yCE5JU1CSUQeyCN5cp6dV+fNef+0Fpz8zwr5UYXiB+Mhs+0=</latexit> N1 <latexit sha1_base64="rULBMhMqQ9482yCyBwTKgM//9KA=">AAACTXicfZBNSwMxEIaz9aNaP6tHL4tFEJGyK0U9FurBi19otdAtZTadrqFJdkmyYln6E7zqz/LsD/EmYloraBUHAg/vvGFm3jDhTBvPe3FyU9Mzs/m5+cLC4tLyympx7VrHqaJYpzGPVSMEjZxJrBtmODYShSBCjjdhrzbs39yh0iyWV6afYEtAJFmXUTBWujxt++3Vklf2RuX+Bn8MJTKu83bRqQSdmKYCpaEctG76XmJaGSjDKMdBIUg1JkB7EGHTogSBupWNdh24W1bpuN1Y2SeNO1K//8hAaN0XoXUKMLd6sjcU/+yFYmKy6R62MiaT1KCkn4O7KXdN7A6DcDtMITW8bwGoYnZ3l96CAmpsXIVCcIT2OIUndtBZggpMrHayAFQkmBzYY6Ngd0j/GeH+y2jJ5uxPpvobrvfK/n55/6JSqtbGic+RDbJJtolPDkiVHJNzUieUROSBPJIn59l5dd6c909rzhn/WSc/Kpf/AOE6s+w=</latexit> N1 <latexit sha1_base64="rULBMhMqQ9482yCyBwTKgM//9KA=">AAACTXicfZBNSwMxEIaz9aNaP6tHL4tFEJGyK0U9FurBi19otdAtZTadrqFJdkmyYln6E7zqz/LsD/EmYloraBUHAg/vvGFm3jDhTBvPe3FyU9Mzs/m5+cLC4tLyympx7VrHqaJYpzGPVSMEjZxJrBtmODYShSBCjjdhrzbs39yh0iyWV6afYEtAJFmXUTBWujxt++3Vklf2RuX+Bn8MJTKu83bRqQSdmKYCpaEctG76XmJaGSjDKMdBIUg1JkB7EGHTogSBupWNdh24W1bpuN1Y2SeNO1K//8hAaN0XoXUKMLd6sjcU/+yFYmKy6R62MiaT1KCkn4O7KXdN7A6DcDtMITW8bwGoYnZ3l96CAmpsXIVCcIT2OIUndtBZggpMrHayAFQkmBzYY6Ngd0j/GeH+y2jJ5uxPpvobrvfK/n55/6JSqtbGic+RDbJJtolPDkiVHJNzUieUROSBPJIn59l5dd6c909rzhn/WSc/Kpf/AOE6s+w=</latexit> d ↵ ⌧ a = (s↵ dt <latexit sha1_base64="AfJpY4yJMF2bPyXsWRiLbIFyJ0o=">AAAC5nicfVFNaxRBEO0Zv+L6tYlHL42LEiVuZiQkXoRAPHgRI7hJIL0ONb01s026e4buHnEZ5i94E8GTf8t/Y8/sLOomWNDweO9VVVdVWkphXRT9CsJr12/cvLVxe3Dn7r37D4abWye2qAzHCS9kYc5SsCiFxokTTuJZaRBUKvE0vThq9dPPaKwo9Ee3KHGqINciExycp5LhD+agoiwzwOtZU89cA58YyHIO9DXdtkmPX1AmQecS6UreoenKyEwnPfvDeLutVFKzHJQCppEuhaa1dNzV9XppWS8ZjqJx1AW9DOIejEgfx8lmsMdmBa8UasclWHseR6Wb1mCc4BKbAasslsAvIMdzDzUotNO6W2JDn3hmRrPC+Kcd7di/M2pQ1i5U6p0K3Nyuay15pZaqtc4uezWthS4rh5ovG2eVpK6g7YXoTBjkTi48AG6E/zvlc/AHcv6OgwF7g344g+98o/clGnCFeV4zMLkSuvHD5mynRf8zwpeV0SO/53h9q5fByctxvD/e/7A3OjzqN75BHpHHZJvE5IAckrfkmEwID0jwNNgNonAefg2/hd+X1jDocx6SfyL8+RshW+iw</latexit> ↵ ↵ ha , b i)b ↵ X 6=↵ d ↵ ⌧ b = (s↵ dt <latexit sha1_base64="PJ2wG72ScPWOCIlChp32K9sl1Os=">AAAC5nicfVFda9RAFJ3Er7p+bfXRl8FFqVK3iZTqi1CoD76IFdy20FnDzeQmO3RmEmYm4hLyF3wTwSf/lv/GSTYW3RYvDBzOOffeufemlRTWRdGvILxy9dr1Gxs3R7du37l7b7x5/8iWteE446UszUkKFqXQOHPCSTypDIJKJR6nZwedfvwZjRWl/uiWFc4VFFrkgoPzVDL+wRzUlOUGeJO1Teba9BMDWS2AvqZbNhnwc8ok6EIihUHepudGZnrp6bnW2W2tkoYVoBQwjXQltF1Sz11eb5BW9ZLxJJpGfdCLIB7AhAxxmGwGuywrea1QOy7B2tM4qty8AeMEl9iOWG2xAn4GBZ56qEGhnTf9Elv62DMZzUvjn3a0Z//OaEBZu1SpdypwC7uudeSlWqrWOrv81bwRuqodar5qnNeSupJ2F6KZMMidXHoA3Aj/d8oX4A/k/B1HI/YG/XAG3/lG7ys04ErzrGFgCiV064ct2HaH/meEL3+MHvk9x+tbvQiOXkzjveneh93J/sGw8Q3ykDwiWyQmL8k+eUsOyYzwgARPgp0gChfh1/Bb+H1lDYMh5wH5J8KfvwEhluiw</latexit> ↵ b ha , b i ↵ ↵ ha , b i)a ↵ X 6=↵ ↵ b ha , b i 4 1 Saxe, A., McClelland, J., & Ganguli, S. (2014). Exact solutions to the nonlinear dynamics of learning in deep linear neural networks. 7 2 📄 [Saxe-McClelland-Ganguli N3 <latexit sha1_base64="EMusQFr7bl0LES0y9LepKT4756Y=">AAACTXicfZBNSwMxEIaz9aNav1o9elksgoiUXRX1KOjBi19oq9AtZTadbkOT7JJkxbL0J3jVn+XZH+JNxLRW0CoOBB7eecPMvGHCmTae9+LkJianpvMzs4W5+YXFpWJpuabjVFGs0pjH6jYEjZxJrBpmON4mCkGEHG/C7tGgf3OHSrNYXptegg0BkWRtRsFY6eqsudMslr2KNyz3N/gjKJNRXTRLzm7QimkqUBrKQeu67yWmkYEyjHLsF4JUYwK0CxHWLUoQqBvZcNe+u26VltuOlX3SuEP1+48MhNY9EVqnANPR472B+GcvFGOTTfugkTGZpAYl/RzcTrlrYncQhNtiCqnhPQtAFbO7u7QDCqixcRUKwTHa4xSe2kHnCSowsdrMAlCRYLJvj42CrQH9Z4T7L6Mlm7M/nupvqG1X/L3K3uVu+fBolPgMWSVrZIP4ZJ8ckhNyQaqEkog8kEfy5Dw7r86b8/5pzTmjPyvkR+XyH+UIs+4=</latexit>
, b↵ i)b↵ ↵ 6=↵ N2 a 2R <latexit sha1_base64="DdEbmgm8rji84Qf/R9oPSY2wd5c=">AAACaXicfZDbSiNBEIY742HdeNhEb0QRGoMgImFGRL0U3AtvPGJUcGKo6VSSxu6eobtHDMNc+TR7q0+zz+BL2BMjaBQLGj7++puq+qNEcGN9/3/JGxufmPw19bs8PTM796dSnb80caoZNlgsYn0dgUHBFTYstwKvE40gI4FX0d1B0b+6R214rC5sP8GmhK7iHc7AOqlVWYHbEETSAxpyRUMJthdF2Xl+mx23tvJWpebX/UHRrxAMoUaGddqqlrbDdsxSicoyAcbcBH5imxloy5nAvBymBhNgd9DFG4cKJJpmNrgjp2tOadNOrN1Tlg7Ujz8ykMb0ZeScxaJmtFeI3/YiOTLZdvaaGVdJalGxt8GdVFAb0yIk2uYamRV9B8A0d7tT1gMNzLooy+XwL7rjNB65QScJarCx3shC0F3JVe6O7YabBf1khId3oyOXczCa6le43KoHO/Wds+3a/sEw8SmyTFbJOgnILtknh+SUNAgjj+QfeSLPpRev6i16S29WrzT8s0A+lVd7BWU9vCM=</latexit> <latexit sha1_base64="eC1WdBSeIwB8HNfOlRvzX95FnW0=">AAACWHicfVDbSiNBEK3Mrrd410dfmg2CiIQZEd2XBUEffBEVjBGcKDWdSmzSl6G7Z9kw5D/21f2r3a/ZnhhBo1hQcDjnFFV1slwK5+P4by368nVmdm5+ob64tLyyura+ceNMYTm1uJHG3mboSApNLS+8pNvcEqpMUjsbnFR6+ydZJ4y+9sOcOgr7WvQERx+o+9QEsZpl7fvkx8NaI27G42LvQTIBDZjU5cN67SDtGl4o0p5LdO4uiXPfKdF6wSWN6mnhKEc+wD7dBahRkeuU47NHbDswXdYzNrT2bMy+nihROTdUWXAq9I9uWqvID7VMTW32ve+dUui88KT58+JeIZk3rMqEdYUl7uUwAORWhNsZf0SL3Ifk6vX0lMJzls7DooucLHpjd8sUbV8JPQrP9tO9Cn1mxF8vxoBCzsl0qu/BzX4zOWweXh00jk8mic/DFnyDHUjgCI7hDC6hBRws/IYn+FP7F0E0Fy08W6PaZGYT3lS08R8Zb7WC</latexit> 1 d ↵ ⌧ b = (s↵ dt <latexit sha1_base64="PJ2wG72ScPWOCIlChp32K9sl1Os=">AAAC5nicfVFda9RAFJ3Er7p+bfXRl8FFqVK3iZTqi1CoD76IFdy20FnDzeQmO3RmEmYm4hLyF3wTwSf/lv/GSTYW3RYvDBzOOffeufemlRTWRdGvILxy9dr1Gxs3R7du37l7b7x5/8iWteE446UszUkKFqXQOHPCSTypDIJKJR6nZwedfvwZjRWl/uiWFc4VFFrkgoPzVDL+wRzUlOUGeJO1Teba9BMDWS2AvqZbNhnwc8ok6EIihUHepudGZnrp6bnW2W2tkoYVoBQwjXQltF1Sz11eb5BW9ZLxJJpGfdCLIB7AhAxxmGwGuywrea1QOy7B2tM4qty8AeMEl9iOWG2xAn4GBZ56qEGhnTf9Elv62DMZzUvjn3a0Z//OaEBZu1SpdypwC7uudeSlWqrWOrv81bwRuqodar5qnNeSupJ2F6KZMMidXHoA3Aj/d8oX4A/k/B1HI/YG/XAG3/lG7ys04ErzrGFgCiV064ct2HaH/meEL3+MHvk9x+tbvQiOXkzjveneh93J/sGw8Q3ykDwiWyQmL8k+eUsOyYzwgARPgp0gChfh1/Bb+H1lDYMh5wH5J8KfvwEhluiw</latexit> b ha↵ , b i ha↵ , b↵ i)a↵ ↵ N2 b 2R <latexit sha1_base64="DHLyS4uYdl0Lg3kV3tIdx5ykrr4=">AAACaXicfZDfSiMxFMbT8c+6dXXb9UYUIVgWFpEyI+LupeBeeONfrApOLSfpaRtMMkOSWSzDXPk0e6tPs8/gS5ipFdYqHgj8+M4XzjkfS6WwLgz/VYKp6ZnZT3Ofq/NfFha/1urfzm2SGY4tnsjEXDKwKIXGlhNO4mVqEBSTeMFu9sr+xR80ViT6zA1TbCvoa9ETHJyXOrU1dh2DTAdAY6FprMANGMtPi+v8sLNVdGqNsBmOir6FaAwNMq7jTr2yHXcTninUjkuw9ioKU9fOwTjBJRbVOLOYAr+BPl551KDQtvPRHQX97pUu7SXGP+3oSP3/Rw7K2qFi3lkuaid7pfhuj6mJya73q50LnWYONX8e3MskdQktQ6JdYZA7OfQA3Ai/O+UDMMCdj7JajX+jP87ggR90lKIBl5iNPAbTV0IX/th+vFnSR0a4fTF68jlHk6m+hfOtZrTT3DnZbuzujROfI6tknfwgEflJdsk+OSYtwskd+UvuyUPlMagHy8HKszWojP8skVcVNJ4AZ0C8JA==</latexit> N2 <latexit sha1_base64="2N2HhVHrFmNqTj/W7r8zUXVj4gk=">AAACWHicfVBdSxtREJ2sbaOxtkYffbk0CKVI2BWxvgiCffCl1IIxghtl9mYSL7kfy713xbDkf/ha/5X+Gu/GCDWKAwOHc84wMyfLpXA+ju9r0cKHj5/qi0uN5c8rX76uNtdOnSkspw430tizDB1JoanjhZd0lltClUnqZqPDSu9ek3XC6BM/zqmncKjFQHD0gbpITRCrWda92N6/XG3F7Xha7DVIZqAFszq+bNZ20r7hhSLtuUTnzpM4970SrRdc0qSRFo5y5CMc0nmAGhW5Xjk9e8I2A9NnA2NDa8+m7P8TJSrnxioLToX+ys1rFfmmlqm5zX6w1yuFzgtPmj8tHhSSecOqTFhfWOJejgNAbkW4nfErtMh9SK7RSH9ReM7S77DoT04WvbE/yhTtUAk9Cc8O060KvWfEm2djQCHnZD7V1+B0u53stnf/7rQODmeJL8IGfIPvkMBPOIAjOIYOcLBwC//grvYQQVSPlp6sUW02sw4vKlp7BBtXtYM=</latexit> <latexit sha1_base64="NgEKFcYW2o1IZfa5uw8QAVDCf3s=">AAACTXicfZBNS8NAEIY39aNav1o9egkWQURKUkQ9FvTgxS+0KphSJttpXLq7CbsbsYT+BK/6szz7Q7yJuK0RtIoDCw/vvMvMvGHCmTae9+IUJianposzs6W5+YXFpXJl+VLHqaLYpDGP1XUIGjmT2DTMcLxOFIIIOV6Fvf1h/+oOlWaxvDD9BFsCIsm6jIKx0vlxu94uV72aNyr3N/g5VElep+2Ksx10YpoKlIZy0PrG9xLTykAZRjkOSkGqMQHagwhvLEoQqFvZaNeBu26VjtuNlX3SuCP1+48MhNZ9EVqnAHOrx3tD8c9eKMYmm+5eK2MySQ1K+jm4m3LXxO4wCLfDFFLD+xaAKmZ3d+ktKKDGxlUqBQdoj1N4ZAedJKjAxGozC0BFgsmBPTYKtob0nxHuv4yWbM7+eKq/4bJe83dqO2fb1cZ+nvgMWSVrZIP4ZJc0yCE5JU1CSUQeyCN5cp6dV+fNef+0Fpz8zwr5UYXiB+Mhs+0=</latexit> W = 2 X 6=↵ b ha↵ , b i N3 <latexit sha1_base64="EMusQFr7bl0LES0y9LepKT4756Y=">AAACTXicfZBNSwMxEIaz9aNav1o9elksgoiUXRX1KOjBi19oq9AtZTadbkOT7JJkxbL0J3jVn+XZH+JNxLRW0CoOBB7eecPMvGHCmTae9+LkJianpvMzs4W5+YXFpWJpuabjVFGs0pjH6jYEjZxJrBpmON4mCkGEHG/C7tGgf3OHSrNYXptegg0BkWRtRsFY6eqsudMslr2KNyz3N/gjKJNRXTRLzm7QimkqUBrKQeu67yWmkYEyjHLsF4JUYwK0CxHWLUoQqBvZcNe+u26VltuOlX3SuEP1+48MhNY9EVqnANPR472B+GcvFGOTTfugkTGZpAYl/RzcTrlrYncQhNtiCqnhPQtAFbO7u7QDCqixcRUKwTHa4xSe2kHnCSowsdrMAlCRYLJvj42CrQH9Z4T7L6Mlm7M/nupvqG1X/L3K3uVu+fBolPgMWSVrZIP4ZJ8ckhNyQaqEkog8kEfy5Dw7r86b8/5pzTmjPyvkR+XyH+UIs+4=</latexit> W = N2 y <latexit sha1_base64="NgEKFcYW2o1IZfa5uw8QAVDCf3s=">AAACTXicfZBNS8NAEIY39aNav1o9egkWQURKUkQ9FvTgxS+0KphSJttpXLq7CbsbsYT+BK/6szz7Q7yJuK0RtIoDCw/vvMvMvGHCmTae9+IUJianposzs6W5+YXFpXJl+VLHqaLYpDGP1XUIGjmT2DTMcLxOFIIIOV6Fvf1h/+oOlWaxvDD9BFsCIsm6jIKx0vlxu94uV72aNyr3N/g5VElep+2Ksx10YpoKlIZy0PrG9xLTykAZRjkOSkGqMQHagwhvLEoQqFvZaNeBu26VjtuNlX3SuCP1+48MhNZ9EVqnAHOrx3tD8c9eKMYmm+5eK2MySQ1K+jm4m3LXxO4wCLfDFFLD+xaAKmZ3d+ktKKDGxlUqBQdoj1N4ZAedJKjAxGozC0BFgsmBPTYKtob0nxHuv4yWbM7+eKq/4bJe83dqO2fb1cZ+nvgMWSVrZIP4ZJc0yCE5JU1CSUQeyCN5cp6dV+fNef+0Fpz8zwr5UYXiB+Mhs+0=</latexit> N1 <latexit sha1_base64="rULBMhMqQ9482yCyBwTKgM//9KA=">AAACTXicfZBNSwMxEIaz9aNaP6tHL4tFEJGyK0U9FurBi19otdAtZTadrqFJdkmyYln6E7zqz/LsD/EmYloraBUHAg/vvGFm3jDhTBvPe3FyU9Mzs/m5+cLC4tLyympx7VrHqaJYpzGPVSMEjZxJrBtmODYShSBCjjdhrzbs39yh0iyWV6afYEtAJFmXUTBWujxt++3Vklf2RuX+Bn8MJTKu83bRqQSdmKYCpaEctG76XmJaGSjDKMdBIUg1JkB7EGHTogSBupWNdh24W1bpuN1Y2SeNO1K//8hAaN0XoXUKMLd6sjcU/+yFYmKy6R62MiaT1KCkn4O7KXdN7A6DcDtMITW8bwGoYnZ3l96CAmpsXIVCcIT2OIUndtBZggpMrHayAFQkmBzYY6Ngd0j/GeH+y2jJ5uxPpvobrvfK/n55/6JSqtbGic+RDbJJtolPDkiVHJNzUieUROSBPJIn59l5dd6c909rzhn/WSc/Kpf/AOE6s+w=</latexit> x <latexit sha1_base64="KgDhIumMmgJR5Go00Yuc+VxfdEU=">AAACS3icfZDPSgMxEMaz1Wqtf6tHL4tFEJGyK0U9FurBi6hgreCWMptO29AkuyRZsSx9Aq/6WD6Az+FNPJitFbQVBwI/vvnCzHxhzJk2nvfq5Obm8wuLhaXi8srq2vpGafNGR4mi2KARj9RtCBo5k9gwzHC8jRWCCDk2w0E96zfvUWkWyWszjLEloCdZl1EwVroatjfKXsUblzsL/gTKZFKX7ZJTDToRTQRKQzlofed7sWmloAyjHEfFINEYAx1AD+8sShCoW+l405G7a5WO242UfdK4Y/XnjxSE1kMRWqcA09fTvUz8sxeKqcmme9JKmYwTg5J+De4m3DWRm8XgdphCavjQAlDF7O4u7YMCamxYxWJwivY4hed20EWMCkyk9tMAVE8wObLH9oKDjP4zwsO30ZLN2Z9OdRZuDiv+UeXoqlqu1SeJF8g22SF7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc05kz9b5Ffl8p/VfbNz</latexit> N2 <latexit sha1_base64="NgEKFcYW2o1IZfa5uw8QAVDCf3s=">AAACTXicfZBNS8NAEIY39aNav1o9egkWQURKUkQ9FvTgxS+0KphSJttpXLq7CbsbsYT+BK/6szz7Q7yJuK0RtIoDCw/vvMvMvGHCmTae9+IUJianposzs6W5+YXFpXJl+VLHqaLYpDGP1XUIGjmT2DTMcLxOFIIIOV6Fvf1h/+oOlWaxvDD9BFsCIsm6jIKx0vlxu94uV72aNyr3N/g5VElep+2Ksx10YpoKlIZy0PrG9xLTykAZRjkOSkGqMQHagwhvLEoQqFvZaNeBu26VjtuNlX3SuCP1+48MhNZ9EVqnAHOrx3tD8c9eKMYmm+5eK2MySQ1K+jm4m3LXxO4wCLfDFFLD+xaAKmZ3d+ktKKDGxlUqBQdoj1N4ZAedJKjAxGozC0BFgsmBPTYKtob0nxHuv4yWbM7+eKq/4bJe83dqO2fb1cZ+nvgMWSVrZIP4ZJc0yCE5JU1CSUQeyCN5cp6dV+fNef+0Fpz8zwr5UYXiB+Mhs+0=</latexit> <latexit sha1_base64="2oa+EUQF+8zDjmnZ6Nf+98VygGw=">AAACS3icfZDPSgMxEMaz9X/91+rRy2IRRKTsSqkeC/XgRVSwtdAtMptOazDJLklWWpY+gVd9LB/A5/AmHkzrCtqKA4Ef33xhZr4w5kwbz3t1cnPzC4tLyyv51bX1jc1Ccaupo0RRbNCIR6oVgkbOJDYMMxxbsUIQIceb8L4+7t88oNIsktdmGGNHQF+yHqNgrHQ1uC2UvLI3KXcW/AxKJKvL26JTCboRTQRKQzlo3fa92HRSUIZRjqN8kGiMgd5DH9sWJQjUnXSy6cjds0rX7UXKPmncifrzRwpC66EIrVOAudPTvbH4Zy8UU5NN76STMhknBiX9GtxLuGsidxyD22UKqeFDC0AVs7u79A4UUGPDyueDU7THKTy3gy5iVGAidZAGoPqCyZE9th8cjuk/Iwy+jZZszv50qrPQPCr71XL1qlKq1bPEl8kO2SX7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc052Z9t8qtyC5/TlrNy</latexit> <latexit sha1_base64="haSdbYtsNQyoRe7ykMC/yp4rtBI=">AAACTXicfZBNSwMxEIaz9aNaP6tHL4tFEJGyK0U9FurBi6hordCtMptO19AkuyRZsSz9CV71Z3n2h3gTMa0VtBUHAg/vvGFm3jDhTBvPe3VyU9Mzs/m5+cLC4tLyympx7UrHqaJYpzGP1XUIGjmTWDfMcLxOFIIIOTbCbm3Qb9yj0iyWl6aXYEtAJFmHUTBWumjc+LerJa/sDcudBH8EJTKqs9uiUwnaMU0FSkM5aN30vcS0MlCGUY79QpBqTIB2IcKmRQkCdSsb7tp3t6zSdjuxsk8ad6j+/JGB0LonQusUYO70eG8g/tkLxdhk0zlsZUwmqUFJvwZ3Uu6a2B0E4baZQmp4zwJQxezuLr0DBdTYuAqF4AjtcQpP7KDTBBWYWO1kAahIMNm3x0bB7oD+M8LDt9GSzdkfT3USrvbK/n55/7xSqtZGic+RDbJJtolPDkiVHJMzUieUROSRPJFn58V5c96djy9rzhn9WSe/Kpf/BPCDs/Q=</latexit> W む <latexit sha1_base64="cZg78mnPPwKJ3E/xzhV3bFjVIFE=">AAAC6XicfVFda9RAFJ2kftT1a6uPvgwuQqt1SWKpvhQKVfBFreC2hc5uuJm9yQ47mYSZibiE/AjfRPDJX+W/cbIbod0WLwwczrkfc89NSimMDYI/nr9x4+at25t3enfv3X/wsL/16MQUleY44oUs9FkCBqVQOLLCSjwrNUKeSDxN5ketfvoVtRGF+mIXJY5zyJRIBQfrqLj/6x09oCzVwMM6Yhaqhpkqj2sGspzBQdhM6o9x2GybeMXQl5RJUJlECpMVtZt0gOmlsDOJ6IvLPenFpkwhS9BCc22nVukaTaK4PwiGwTLoVRB2YEC6OI63vD02LXiVo7JcgjHnYVDacQ3aCi6x6bHKYAl8DhmeO6ggRzOulz429JljpjQttHvK0iV7saKG3JhFnrjMHOzMrGstea2W5GuTbfpmXAtVVhYVXw1OK0ltQdsj0anQyK1cOABcC/d3ymfgDLXulL0ee4tuOY0f3KBPJWqwhX7u3NVZLlTjls3Ybov+lwjf/iU65HwO1129Ck6iYbg/3P+8Nzg86hzfJE/IU7JNQvKaHJL35JiMCPc2vB0v8l75c/+7/8P/uUr1va7mMbkU/u+/CUjqVQ==</latexit> E= N1 X 1 2⌧ ↵=1 (s↵ X 1 ↵ ↵ 2 ↵ 2 ha , b i) + ha , b i 2⌧ 1 <latexit sha1_base64="aufhfgnOg5Qu4gUpqbQ+th/Tt6I=">AAACTXicfZBNSwMxEIaz9aNav1o9elksgoiUXRH1WNCDF1HR2kK3ltl0ug0m2SXJimXpT/CqP8uzP8SbiGmtoFUcCDy884aZecOEM20878XJTU3PzObn5gsLi0vLK8XS6rWOU0WxRmMeq0YIGjmTWDPMcGwkCkGEHOvh7dGwX79DpVksr0w/wZaASLIuo2CsdFm/2W0Xy17FG5X7G/wxlMm4ztslZy/oxDQVKA3loHXT9xLTykAZRjkOCkGqMQF6CxE2LUoQqFvZaNeBu2mVjtuNlX3SuCP1+48MhNZ9EVqnANPTk72h+GcvFBOTTfewlTGZpAYl/RzcTblrYncYhNthCqnhfQtAFbO7u7QHCqixcRUKwTHa4xSe2kFnCSowsdrOAlCRYHJgj42CnSH9Z4T7L6Mlm7M/mepvuN6t+PuV/Yu9cvVonPgcWScbZIv45IBUyQk5JzVCSUQeyCN5cp6dV+fNef+05pzxnzXyo3L5D/Jqs/U=</latexit> W 2 N1 <latexit sha1_base64="rULBMhMqQ9482yCyBwTKgM//9KA=">AAACTXicfZBNSwMxEIaz9aNaP6tHL4tFEJGyK0U9FurBi19otdAtZTadrqFJdkmyYln6E7zqz/LsD/EmYloraBUHAg/vvGFm3jDhTBvPe3FyU9Mzs/m5+cLC4tLyympx7VrHqaJYpzGPVSMEjZxJrBtmODYShSBCjjdhrzbs39yh0iyWV6afYEtAJFmXUTBWujxt++3Vklf2RuX+Bn8MJTKu83bRqQSdmKYCpaEctG76XmJaGSjDKMdBIUg1JkB7EGHTogSBupWNdh24W1bpuN1Y2SeNO1K//8hAaN0XoXUKMLd6sjcU/+yFYmKy6R62MiaT1KCkn4O7KXdN7A6DcDtMITW8bwGoYnZ3l96CAmpsXIVCcIT2OIUndtBZggpMrHayAFQkmBzYY6Ngd0j/GeH+y2jJ5uxPpvobrvfK/n55/6JSqtbGic+RDbJJtolPDkiVHJNzUieUROSBPJIn59l5dd6c909rzhn/WSc/Kpf/AOE6s+w=</latexit> ↵6= sα 見 4 1 行 目 Saxe, A., McClelland, J., & Ganguli, S. (2014). Exact solutions to the nonlinear dynamics of learning in deep linear neural networks. 目 ] 2 <latexit sha1_base64="AfJpY4yJMF2bPyXsWRiLbIFyJ0o=">AAAC5nicfVFNaxRBEO0Zv+L6tYlHL42LEiVuZiQkXoRAPHgRI7hJIL0ONb01s026e4buHnEZ5i94E8GTf8t/Y8/sLOomWNDweO9VVVdVWkphXRT9CsJr12/cvLVxe3Dn7r37D4abWye2qAzHCS9kYc5SsCiFxokTTuJZaRBUKvE0vThq9dPPaKwo9Ee3KHGqINciExycp5LhD+agoiwzwOtZU89cA58YyHIO9DXdtkmPX1AmQecS6UreoenKyEwnPfvDeLutVFKzHJQCppEuhaa1dNzV9XppWS8ZjqJx1AW9DOIejEgfx8lmsMdmBa8UasclWHseR6Wb1mCc4BKbAasslsAvIMdzDzUotNO6W2JDn3hmRrPC+Kcd7di/M2pQ1i5U6p0K3Nyuay15pZaqtc4uezWthS4rh5ovG2eVpK6g7YXoTBjkTi48AG6E/zvlc/AHcv6OgwF7g344g+98o/clGnCFeV4zMLkSuvHD5mynRf8zwpeV0SO/53h9q5fByctxvD/e/7A3OjzqN75BHpHHZJvE5IAckrfkmEwID0jwNNgNonAefg2/hd+X1jDocx6SfyL8+RshW+iw</latexit> 8 2 📄 [Saxe-McClelland-Ganguli N3 <latexit sha1_base64="EMusQFr7bl0LES0y9LepKT4756Y=">AAACTXicfZBNSwMxEIaz9aNav1o9elksgoiUXRX1KOjBi19oq9AtZTadbkOT7JJkxbL0J3jVn+XZH+JNxLRW0CoOBB7eecPMvGHCmTae9+LkJianpvMzs4W5+YXFpWJpuabjVFGs0pjH6jYEjZxJrBpmON4mCkGEHG/C7tGgf3OHSrNYXptegg0BkWRtRsFY6eqsudMslr2KNyz3N/gjKJNRXTRLzm7QimkqUBrKQeu67yWmkYEyjHLsF4JUYwK0CxHWLUoQqBvZcNe+u26VltuOlX3SuEP1+48MhNY9EVqnANPR472B+GcvFGOTTfugkTGZpAYl/RzcTrlrYncQhNtiCqnhPQtAFbO7u7QDCqixcRUKwTHa4xSe2kHnCSowsdrMAlCRYLJvj42CrQH9Z4T7L6Mlm7M/nupvqG1X/L3K3uVu+fBolPgMWSVrZIP4ZJ8ckhNyQaqEkog8kEfy5Dw7r86b8/5pzTmjPyvkR+XyH+UIs+4=</latexit>
dt X <latexit sha1_base64="AfJpY4yJMF2bPyXsWRiLbIFyJ0o=">AAAC5nicfVFNaxRBEO0Zv+L6tYlHL42LEiVuZiQkXoRAPHgRI7hJIL0ONb01s026e4buHnEZ5i94E8GTf8t/Y8/sLOomWNDweO9VVVdVWkphXRT9CsJr12/cvLVxe3Dn7r37D4abWye2qAzHCS9kYc5SsCiFxokTTuJZaRBUKvE0vThq9dPPaKwo9Ee3KHGqINciExycp5LhD+agoiwzwOtZU89cA58YyHIO9DXdtkmPX1AmQecS6UreoenKyEwnPfvDeLutVFKzHJQCppEuhaa1dNzV9XppWS8ZjqJx1AW9DOIejEgfx8lmsMdmBa8UasclWHseR6Wb1mCc4BKbAasslsAvIMdzDzUotNO6W2JDn3hmRrPC+Kcd7di/M2pQ1i5U6p0K3Nyuay15pZaqtc4uezWthS4rh5ovG2eVpK6g7YXoTBjkTi48AG6E/zvlc/AHcv6OgwF7g344g+98o/clGnCFeV4zMLkSuvHD5mynRf8zwpeV0SO/53h9q5fByctxvD/e/7A3OjzqN75BHpHHZJvE5IAckrfkmEwID0jwNNgNonAefg2/hd+X1jDocx6SfyL8+RshW+iw</latexit> ha↵ , b↵ i)b↵ 6=↵ d ↵ ⌧ b = (s↵ dt <latexit sha1_base64="PJ2wG72ScPWOCIlChp32K9sl1Os=">AAAC5nicfVFda9RAFJ3Er7p+bfXRl8FFqVK3iZTqi1CoD76IFdy20FnDzeQmO3RmEmYm4hLyF3wTwSf/lv/GSTYW3RYvDBzOOffeufemlRTWRdGvILxy9dr1Gxs3R7du37l7b7x5/8iWteE446UszUkKFqXQOHPCSTypDIJKJR6nZwedfvwZjRWl/uiWFc4VFFrkgoPzVDL+wRzUlOUGeJO1Teba9BMDWS2AvqZbNhnwc8ok6EIihUHepudGZnrp6bnW2W2tkoYVoBQwjXQltF1Sz11eb5BW9ZLxJJpGfdCLIB7AhAxxmGwGuywrea1QOy7B2tM4qty8AeMEl9iOWG2xAn4GBZ56qEGhnTf9Elv62DMZzUvjn3a0Z//OaEBZu1SpdypwC7uudeSlWqrWOrv81bwRuqodar5qnNeSupJ2F6KZMMidXHoA3Aj/d8oX4A/k/B1HI/YG/XAG3/lG7ys04ErzrGFgCiV064ct2HaH/meEL3+MHvk9x+tbvQiOXkzjveneh93J/sGw8Q3ykDwiWyQmL8k+eUsOyYzwgARPgp0gChfh1/Bb+H1lDYMh5wH5J8KfvwEhluiw</latexit> b ha↵ , b i ha↵ , b↵ i)a↵ X 6=↵ b ha↵ , b i む <latexit sha1_base64="hl5vUVhi53mZqpAaL6x/cLPiFWA=">AAACinicfVBdSxtBFJ2s1dr40dg+9mVoKKhI2BWxSmkR9KEvpQpGBSeGu5ObzeDM7HbmrjQs+Tf9NX1tX/pvOvkoaJReGDiccy5n7kkLrTzF8Z9atPBscen58ov6yura+svGxqsLn5dOYlvmOndXKXjUymKbFGm8KhyCSTVeprfHY/3yDp1XuT2nYYEdA5lVfSWBAtVtfBIabKaRw40AXQxgh6c3IkUCLtxU+chjLr6V0OObUwsXFvnEs9VtNONWPBn+GCQz0GSzOe1u1PZEL5elQUtSg/fXSVxQpwJHSmoc1UXpsQB5CxleB2jBoO9Uk0NH/F1geryfu/As8Ql7f6MC4/3QpMFpgAZ+XhuTT2qpmUum/kGnUrYoCa2cBvdLzSnn4xZ5TzmUpIcBgHQq/J3LATiQFLqu18UJhuMcfglBXwt0QLnbrgS4zCg7CsdmYmeM/meE7/+MAYWek/lWH4OL3Vay39o/22seHc8aX2Zv2Fu2yRL2nh2xz+yUtZlkP9hP9ov9jlaj3egw+jC1RrXZzmv2YKKTv43gxqg=</latexit> ↵ ha , b i = 0 (↵ 6= ) ↵ ↵ ↵ ↵ ha , b i = s↵ or a = b = 0 <latexit sha1_base64="Ea5kdNpN4nwsHMUFsRxlEiHjU5c=">AAACn3icfZFdaxNBFIYn61eNX2l76c1oEERK2C2lelMotFBv1IimrXRjODs52Qydj2XmrDQsyf/qT/HKW/0XTpKtH6l4YOHhPe/hzHk3K5T0FMdfG9GNm7du31m727x3/8HDR631jWNvSyewJ6yy7jQDj0oa7JEkhaeFQ9CZwpPs/GDeP/mCzktrPtKkwL6G3MiRFEBBGrQ+pApMrpDD5xRUMYYtntXEU7ds7XE/uJIIL6iazaybzaa/ZoIj+43xoNWOO/Gi+HVIamizurqD9cZOOrSi1GhIKPD+LIkL6lfgSAqF02ZaeixAnEOOZwENaPT9anH9lD8LypCPrAufIb5Q/5yoQHs/0VlwaqCxX+3NxX/2Mr2ymUav+pU0RUloxHLxqFScLJ9Hy4fSoSA1CQDCyfB2LsbgQFD4Ac1meojhOIdvwqJ3BTog615UKbhcSzMNx+bp1pz+Z4SLK2OgkHOymup1ON7uJLud3fc77f2DOvE19pg9Zc9Zwl6yffaadVmPCXbJvrHv7Ef0JDqK3kbdpTVq1DOb7K+KPv0Ec+vQJg==</latexit> x <latexit sha1_base64="4TKeSCbBf6RPl17MrVhAL1AoEqc=">AAACjXicfVDRahNBFJ1stdaobVoffRkMgkgJu6W2fWilUkFfxAqmLXTS5e7kZjN0ZnaZuSuGJd/j1/iq+DdOshE0Lb0wcO4553LnnqzUylMc/25FK/furz5Ye9h+9PjJ+kZnc+vMF5WT2JeFLtxFBh61stgnRRovSodgMo3n2fXJTD//is6rwn6hSYkDA7lVIyWBApV23goNNtfI4UqALsewzbMrkSEBF65RjrhPG42LIWqCtG7auW2adrpxL54XvwmSBeiyRZ2mm61dMSxkZdCS1OD9ZRKXNKjBkZIap21ReSxBXkOOlwFaMOgH9fzWKX8RmCEfFS48S3zO/jtRg/F+YrLgNEBjv6zNyFu1zCxtptHBoFa2rAitbBaPKs2p4LMg+VA5lKQnAYB0KvydyzE4kBTibrfFOwzHOfwYFn0q0QEV7lUIzuVG2Wk4NhfbM3SXEb79NQYUck6WU70JznZ6yV5v7/Nu9/hkkfgae8aes5csYfvsmH1gp6zPJPvOfrCf7Fe0Eb2ODqM3jTVqLWaesv8qev8HxqXJoA==</latexit> ↵ N2 <latexit sha1_base64="LEHExU3FME6f2R8noD0EWlfCYRM=">AAACi3icfZDfahNBFMYna9Uarab20pvBIIhI2G1LldJioRZ6Y61gmkI3Wc5OTpKh84+Z2WJY9nH6NN7qRd+mk2QLmooHBn585zucOV9uBHc+jm8a0YOVh48erz5pPn229vxFa/3lmdOFZdhlWmh7noNDwRV2PfcCz41FkLnAXn55OOv3rtA6rtV3PzXYlzBWfMQZ+CBlrU+9wSbtDRK6T1NXyKxMQZgJ7CfVoDzJkoq6bKHQ4g6uahikXpus1Y478bzofUhqaJO6TrP1xnY61KyQqDwT4NxFEhvfL8F6zgRWzbRwaIBdwhgvAiqQ6Prl/NKKvgnKkI60DU95Olf/nChBOjeVeXBK8BO33JuJ/+zlcmmzH33sl1yZwqNii8WjQlCv6SxGOuQWmRfTAMAsD3+nbAIWmA9hN5vpZwzHWfwSFn01aMFr+y5ka8eSqyocO07fz+h/RvhxZwwUck6WU70PZ5udZKez8227fXBYJ75KXpHX5C1JyAdyQI7JKekSRq7JT/KL/I7Woq1oN9pbWKNGPbNB/qro6BZtgcgf</latexit> 2 1 W W = N1 X > s ↵ u↵ v ↵ <latexit sha1_base64="haSdbYtsNQyoRe7ykMC/yp4rtBI=">AAACTXicfZBNSwMxEIaz9aNaP6tHL4tFEJGyK0U9FurBi6hordCtMptO19AkuyRZsSz9CV71Z3n2h3gTMa0VtBUHAg/vvGFm3jDhTBvPe3VyU9Mzs/m5+cLC4tLyympx7UrHqaJYpzGP1XUIGjmTWDfMcLxOFIIIOTbCbm3Qb9yj0iyWl6aXYEtAJFmHUTBWumjc+LerJa/sDcudBH8EJTKqs9uiUwnaMU0FSkM5aN30vcS0MlCGUY79QpBqTIB2IcKmRQkCdSsb7tp3t6zSdjuxsk8ad6j+/JGB0LonQusUYO70eG8g/tkLxdhk0zlsZUwmqUFJvwZ3Uu6a2B0E4baZQmp4zwJQxezuLr0DBdTYuAqF4AjtcQpP7KDTBBWYWO1kAahIMNm3x0bB7oD+M8LDt9GSzdkfT3USrvbK/n55/7xSqtZGic+RDbJJtolPDkiVHJMzUieUROSRPJFn58V5c96djy9rzhn9WSe/Kpf/BPCDs/Q=</latexit> W ha , b i = s↵ ↵ む y <latexit sha1_base64="KgDhIumMmgJR5Go00Yuc+VxfdEU=">AAACS3icfZDPSgMxEMaz1Wqtf6tHL4tFEJGyK0U9FurBi6hgreCWMptO29AkuyRZsSx9Aq/6WD6Az+FNPJitFbQVBwI/vvnCzHxhzJk2nvfq5Obm8wuLhaXi8srq2vpGafNGR4mi2KARj9RtCBo5k9gwzHC8jRWCCDk2w0E96zfvUWkWyWszjLEloCdZl1EwVroatjfKXsUblzsL/gTKZFKX7ZJTDToRTQRKQzlofed7sWmloAyjHEfFINEYAx1AD+8sShCoW+l405G7a5WO242UfdK4Y/XnjxSE1kMRWqcA09fTvUz8sxeKqcmme9JKmYwTg5J+De4m3DWRm8XgdphCavjQAlDF7O4u7YMCamxYxWJwivY4hed20EWMCkyk9tMAVE8wObLH9oKDjP4zwsO30ZLN2Z9OdRZuDiv+UeXoqlqu1SeJF8g22SF7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc05kz9b5Ffl8p/VfbNz</latexit> <latexit sha1_base64="NgEKFcYW2o1IZfa5uw8QAVDCf3s=">AAACTXicfZBNS8NAEIY39aNav1o9egkWQURKUkQ9FvTgxS+0KphSJttpXLq7CbsbsYT+BK/6szz7Q7yJuK0RtIoDCw/vvMvMvGHCmTae9+IUJianposzs6W5+YXFpXJl+VLHqaLYpDGP1XUIGjmT2DTMcLxOFIIIOV6Fvf1h/+oOlWaxvDD9BFsCIsm6jIKx0vlxu94uV72aNyr3N/g5VElep+2Ksx10YpoKlIZy0PrG9xLTykAZRjkOSkGqMQHagwhvLEoQqFvZaNeBu26VjtuNlX3SuCP1+48MhNZ9EVqnAHOrx3tD8c9eKMYmm+5eK2MySQ1K+jm4m3LXxO4wCLfDFFLD+xaAKmZ3d+ktKKDGxlUqBQdoj1N4ZAedJKjAxGozC0BFgsmBPTYKtob0nxHuv4yWbM7+eKq/4bJe83dqO2fb1cZ+nvgMWSVrZIP4ZJc0yCE5JU1CSUQeyCN5cp6dV+fNef+0Fpz8zwr5UYXiB+Mhs+0=</latexit> <latexit sha1_base64="2oa+EUQF+8zDjmnZ6Nf+98VygGw=">AAACS3icfZDPSgMxEMaz9X/91+rRy2IRRKTsSqkeC/XgRVSwtdAtMptOazDJLklWWpY+gVd9LB/A5/AmHkzrCtqKA4Ef33xhZr4w5kwbz3t1cnPzC4tLyyv51bX1jc1Ccaupo0RRbNCIR6oVgkbOJDYMMxxbsUIQIceb8L4+7t88oNIsktdmGGNHQF+yHqNgrHQ1uC2UvLI3KXcW/AxKJKvL26JTCboRTQRKQzlo3fa92HRSUIZRjqN8kGiMgd5DH9sWJQjUnXSy6cjds0rX7UXKPmncifrzRwpC66EIrVOAudPTvbH4Zy8UU5NN76STMhknBiX9GtxLuGsidxyD22UKqeFDC0AVs7u79A4UUGPDyueDU7THKTy3gy5iVGAidZAGoPqCyZE9th8cjuk/Iwy+jZZszv50qrPQPCr71XL1qlKq1bPEl8kO2SX7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc052Z9t8qtyC5/TlrNy</latexit> 1 <latexit sha1_base64="aufhfgnOg5Qu4gUpqbQ+th/Tt6I=">AAACTXicfZBNSwMxEIaz9aNav1o9elksgoiUXRH1WNCDF1HR2kK3ltl0ug0m2SXJimXpT/CqP8uzP8SbiGmtoFUcCDy884aZecOEM20878XJTU3PzObn5gsLi0vLK8XS6rWOU0WxRmMeq0YIGjmTWDPMcGwkCkGEHOvh7dGwX79DpVksr0w/wZaASLIuo2CsdFm/2W0Xy17FG5X7G/wxlMm4ztslZy/oxDQVKA3loHXT9xLTykAZRjkOCkGqMQF6CxE2LUoQqFvZaNeBu2mVjtuNlX3SuCP1+48MhNZ9EVqnANPTk72h+GcvFBOTTfewlTGZpAYl/RzcTblrYncYhNthCqnhfQtAFbO7u7QHCqixcRUKwTHa4xSe2kFnCSowsdrOAlCRYHJgj42CnSH9Z4T7L6Mlm7M/mepvuN6t+PuV/Yu9cvVonPgcWScbZIv45IBUyQk5JzVCSUQeyCN5cp6dV+fNef+05pzxnzXyo3L5D/Jqs/U=</latexit> W 2 N1 <latexit sha1_base64="rULBMhMqQ9482yCyBwTKgM//9KA=">AAACTXicfZBNSwMxEIaz9aNaP6tHL4tFEJGyK0U9FurBi19otdAtZTadrqFJdkmyYln6E7zqz/LsD/EmYloraBUHAg/vvGFm3jDhTBvPe3FyU9Mzs/m5+cLC4tLyympx7VrHqaJYpzGPVSMEjZxJrBtmODYShSBCjjdhrzbs39yh0iyWV6afYEtAJFmXUTBWujxt++3Vklf2RuX+Bn8MJTKu83bRqQSdmKYCpaEctG76XmJaGSjDKMdBIUg1JkB7EGHTogSBupWNdh24W1bpuN1Y2SeNO1K//8hAaN0XoXUKMLd6sjcU/+yFYmKy6R62MiaT1KCkn4O7KXdN7A6DcDtMITW8bwGoYnZ3l96CAmpsXIVCcIT2OIUndtBZggpMrHayAFQkmBzYY6Ngd0j/GeH+y2jJ5uxPpvobrvfK/n55/6JSqtbGic+RDbJJtolPDkiVHJNzUieUROSBPJIn59l5dd6c909rzhn/WSc/Kpf/AOE6s+w=</latexit> ↵=1 大 4 1 行 十 目 目 Saxe, A., McClelland, J., & Ganguli, S. (2014). Exact solutions to the nonlinear dynamics of learning in deep linear neural networks. 9 2 📄 saddle N3 <latexit sha1_base64="EMusQFr7bl0LES0y9LepKT4756Y=">AAACTXicfZBNSwMxEIaz9aNav1o9elksgoiUXRX1KOjBi19oq9AtZTadbkOT7JJkxbL0J3jVn+XZH+JNxLRW0CoOBB7eecPMvGHCmTae9+LkJianpvMzs4W5+YXFpWJpuabjVFGs0pjH6jYEjZxJrBpmON4mCkGEHG/C7tGgf3OHSrNYXptegg0BkWRtRsFY6eqsudMslr2KNyz3N/gjKJNRXTRLzm7QimkqUBrKQeu67yWmkYEyjHLsF4JUYwK0CxHWLUoQqBvZcNe+u26VltuOlX3SuEP1+48MhNY9EVqnANPR472B+GcvFGOTTfugkTGZpAYl/RzcTrlrYncQhNtiCqnhPQtAFbO7u7QDCqixcRUKwTHa4xSe2kHnCSowsdrMAlCRYLJvj42CrQH9Z4T7L6Mlm7M/nupvqG1X/L3K3uVu+fBolPgMWSVrZIP4ZJ8ckhNyQaqEkog8kEfy5Dw7r86b8/5pzTmjPyvkR+XyH+UIs+4=</latexit>
X <latexit sha1_base64="AfJpY4yJMF2bPyXsWRiLbIFyJ0o=">AAAC5nicfVFNaxRBEO0Zv+L6tYlHL42LEiVuZiQkXoRAPHgRI7hJIL0ONb01s026e4buHnEZ5i94E8GTf8t/Y8/sLOomWNDweO9VVVdVWkphXRT9CsJr12/cvLVxe3Dn7r37D4abWye2qAzHCS9kYc5SsCiFxokTTuJZaRBUKvE0vThq9dPPaKwo9Ee3KHGqINciExycp5LhD+agoiwzwOtZU89cA58YyHIO9DXdtkmPX1AmQecS6UreoenKyEwnPfvDeLutVFKzHJQCppEuhaa1dNzV9XppWS8ZjqJx1AW9DOIejEgfx8lmsMdmBa8UasclWHseR6Wb1mCc4BKbAasslsAvIMdzDzUotNO6W2JDn3hmRrPC+Kcd7di/M2pQ1i5U6p0K3Nyuay15pZaqtc4uezWthS4rh5ovG2eVpK6g7YXoTBjkTi48AG6E/zvlc/AHcv6OgwF7g344g+98o/clGnCFeV4zMLkSuvHD5mynRf8zwpeV0SO/53h9q5fByctxvD/e/7A3OjzqN75BHpHHZJvE5IAckrfkmEwID0jwNNgNonAefg2/hd+X1jDocx6SfyL8+RshW+iw</latexit> ha↵ , b↵ i)b↵ 6=↵ d ↵ ⌧ b = (s↵ dt <latexit sha1_base64="PJ2wG72ScPWOCIlChp32K9sl1Os=">AAAC5nicfVFda9RAFJ3Er7p+bfXRl8FFqVK3iZTqi1CoD76IFdy20FnDzeQmO3RmEmYm4hLyF3wTwSf/lv/GSTYW3RYvDBzOOffeufemlRTWRdGvILxy9dr1Gxs3R7du37l7b7x5/8iWteE446UszUkKFqXQOHPCSTypDIJKJR6nZwedfvwZjRWl/uiWFc4VFFrkgoPzVDL+wRzUlOUGeJO1Teba9BMDWS2AvqZbNhnwc8ok6EIihUHepudGZnrp6bnW2W2tkoYVoBQwjXQltF1Sz11eb5BW9ZLxJJpGfdCLIB7AhAxxmGwGuywrea1QOy7B2tM4qty8AeMEl9iOWG2xAn4GBZ56qEGhnTf9Elv62DMZzUvjn3a0Z//OaEBZu1SpdypwC7uudeSlWqrWOrv81bwRuqodar5qnNeSupJ2F6KZMMidXHoA3Aj/d8oX4A/k/B1HI/YG/XAG3/lG7ys04ErzrGFgCiV064ct2HaH/meEL3+MHvk9x+tbvQiOXkzjveneh93J/sGw8Q3ykDwiWyQmL8k+eUsOyYzwgARPgp0gChfh1/Bb+H1lDYMh5wH5J8KfvwEhluiw</latexit> b ha↵ , b i : a (0) = a↵ (0)r , b (0) = b↵ (0)r ↵ <latexit sha1_base64="Nf2/J4Ynmmon5e39QAN5JUha0Ko=">AAACkXicfVBdSxtBFJ2s/dDYj0QffRkaBFsk7EqI+iAEoyCUooVGA9k03J3cJENmZpeZWTEs+UX+mj5W/4yTZEWNpRcGzj3nXO7cEyWCG+v7fwveypu3796vrhXXP3z89LlU3rg0caoZtlgsYt2OwKDgCluWW4HtRCPISOBVNG7O9Ktr1IbH6pedJNiVMFR8wBlYR/VKp/A7BJGMYMf/So8o9J46nSu7NHrhiZ48j5ZeqeJX/XnR1yDIQYXkddErF2phP2apRGWZAGM6gZ/YbgbaciZwWgxTgwmwMQyx46ACiaabze+d0m3H9Okg1u4pS+fs84kMpDETGTmnBDsyy9qM/KcWyaXNdnDQzbhKUouKLRYPUkFtTGdh0j7XyKyYOABMc/d3ykaggVkXebEYnqA7TuMPt+g8QQ021t+yEPRQcjV1xw7D3Rn6nxFuHo0OuZyD5VRfg8u9alCv1n/WKo1mnvgq2SJfyA4JyD5pkDNyQVqEkVvyh9yRe2/TO/Qa3vHC6hXymU3yorzvD1CnyC0=</latexit> d ↵ ⌧ a = (s↵ dt ↵ X <latexit sha1_base64="AfJpY4yJMF2bPyXsWRiLbIFyJ0o=">AAAC5nicfVFNaxRBEO0Zv+L6tYlHL42LEiVuZiQkXoRAPHgRI7hJIL0ONb01s026e4buHnEZ5i94E8GTf8t/Y8/sLOomWNDweO9VVVdVWkphXRT9CsJr12/cvLVxe3Dn7r37D4abWye2qAzHCS9kYc5SsCiFxokTTuJZaRBUKvE0vThq9dPPaKwo9Ee3KHGqINciExycp5LhD+agoiwzwOtZU89cA58YyHIO9DXdtkmPX1AmQecS6UreoenKyEwnPfvDeLutVFKzHJQCppEuhaa1dNzV9XppWS8ZjqJx1AW9DOIejEgfx8lmsMdmBa8UasclWHseR6Wb1mCc4BKbAasslsAvIMdzDzUotNO6W2JDn3hmRrPC+Kcd7di/M2pQ1i5U6p0K3Nyuay15pZaqtc4uezWthS4rh5ovG2eVpK6g7YXoTBjkTi48AG6E/zvlc/AHcv6OgwF7g344g+98o/clGnCFeV4zMLkSuvHD5mynRf8zwpeV0SO/53h9q5fByctxvD/e/7A3OjzqN75BHpHHZJvE5IAckrfkmEwID0jwNNgNonAefg2/hd+X1jDocx6SfyL8+RshW+iw</latexit> ha↵ , b↵ i)b↵ 6=↵ ↵ ↵ <latexit sha1_base64="ekaTd1bJ58JDE7/+e9JBZ1zDKbI=">AAAChHicfZBdSxtBFIYna/1orDbay94MDYKUEHatXzeKoBe9ES00KrgxnJ2cbAZnZpeZs2JY8lf8Nb1t7/03nU1SaGPxwMDDed/DmfMmuZKOwvC5Fiy8WVxaXnlbX323tv6+sbF55bLCCuyITGX2JgGHShrskCSFN7lF0InC6+T+tNKvH9A6mZnvNMqxqyE1ciAFkG/1GoexApMq5PYuBpUPoVVRggQ8tlPliMd9VAS9cuqYqONeoxm2w0nxlxDNoMlmddnbqO3G/UwUGg0JBc7dRmFO3RIsSaFwXI8LhzmIe0jx1qMBja5bTk4c8y3f6fNBZv0zxCfdvydK0M6NdOKdGmjo5rWq+V8t0XObaXDYLaXJC0IjposHheKU8So/3pcWBamRBxBW+r9zMQQLgnzK9Xp8hv44i+d+0UWOFiizn31wNtXSjP2xadyq6DUjPP4xevI5R/OpvoSrnXa0397/tts8OZ0lvsI+sk9sm0XsgJ2wr+ySdZhgT+wH+8l+BUtBK/gS7E2tQW0284H9U8Hxb58mxhM=</latexit> ha↵ , b↵ i)a↵ ↵ hr , r i = d ↵ ⌧ b = (s↵ dt b ha↵ , b i X b ha↵ , b i X b ha↵ , b i 6=↵ ↵ <latexit sha1_base64="PJ2wG72ScPWOCIlChp32K9sl1Os=">AAAC5nicfVFda9RAFJ3Er7p+bfXRl8FFqVK3iZTqi1CoD76IFdy20FnDzeQmO3RmEmYm4hLyF3wTwSf/lv/GSTYW3RYvDBzOOffeufemlRTWRdGvILxy9dr1Gxs3R7du37l7b7x5/8iWteE446UszUkKFqXQOHPCSTypDIJKJR6nZwedfvwZjRWl/uiWFc4VFFrkgoPzVDL+wRzUlOUGeJO1Teba9BMDWS2AvqZbNhnwc8ok6EIihUHepudGZnrp6bnW2W2tkoYVoBQwjXQltF1Sz11eb5BW9ZLxJJpGfdCLIB7AhAxxmGwGuywrea1QOy7B2tM4qty8AeMEl9iOWG2xAn4GBZ56qEGhnTf9Elv62DMZzUvjn3a0Z//OaEBZu1SpdypwC7uudeSlWqrWOrv81bwRuqodar5qnNeSupJ2F6KZMMidXHoA3Aj/d8oX4A/k/B1HI/YG/XAG3/lG7ys04ErzrGFgCiV064ct2HaH/meEL3+MHvk9x+tbvQiOXkzjveneh93J/sGw8Q3ykDwiWyQmL8k+eUsOyYzwgARPgp0gChfh1/Bb+H1lDYMh5wH5J8KfvwEhluiw</latexit> ha↵ , b↵ i)a↵ 6=↵ decoupled d ⌧ a = b(s dt d ⌧ b = a(s dt <latexit sha1_base64="e30gZbrWQ0ow6F5zp0ZEP3MnZhU=">AAACaHicfZDBShxBEIZ7JybRVZM1HkT00LgENCTLTBDjJSDowYtowFXBWZaanpq1sbtn6K4JWYa5+DRe49v4CnmK9K4r6BpS0PDx119U158USjoKw/tG8Grm9Zu3s3PN+YXFd+9bSx/OXF5agV2Rq9xeJOBQSYNdkqTworAIOlF4nlzvj/rnP9E6mZtTGhbY0zAwMpMCyEv91npMUPI4syCqtK5SqoF/58mm+wLJVr/VDjvhuPhLiCbQZpM66S81tuM0F6VGQ0KBc5dRWFCvAktSKKybcemwAHENA7z0aECj61XjM2r+0Sspz3LrnyE+Vp9OVKCdG+rEOzXQlZvujcR/9hI9tZmy3V4lTVESGvGwOCsVp5yPMuKptChIDT2AsNL/nYsr8BGRT7LZjA/QH2fxyC86LtAC5fZTFYMdaGlqf+wg/jyi/xnh16PRk885mk71JZx97UQ7nZ0f2+29/Unis2yNbbBNFrFvbI8dshPWZYLdsFv2m901/gStYCVYfbAGjcnMMntWwcZfUqG7Gg==</latexit> <latexit sha1_base64="n5jjzTsvJdOimLjKJySjsUXqeSM=">AAACaHicfZDBShxBEIZ7JybRVZM1HkT00LgENCTLTBDjJSDowYtowFXBWZaanpq1sbtn6K4JWYa5+DRe49v4CnmK9K4r6BpS0PDx119U158USjoKw/tG8Grm9Zu3s3PN+YXFd+9bSx/OXF5agV2Rq9xeJOBQSYNdkqTworAIOlF4nlzvj/rnP9E6mZtTGhbY0zAwMpMCyEv91npMUPI4syCqtK5SqhP+ncOm+wLJVr/VDjvhuPhLiCbQZpM66S81tuM0F6VGQ0KBc5dRWFCvAktSKKybcemwAHENA7z0aECj61XjM2r+0Sspz3LrnyE+Vp9OVKCdG+rEOzXQlZvujcR/9hI9tZmy3V4lTVESGvGwOCsVp5yPMuKptChIDT2AsNL/nYsr8BGRT7LZjA/QH2fxyC86LtAC5fZTFYMdaGlqf+wg/jyi/xnh16PRk885mk71JZx97UQ7nZ0f2+29/Unis2yNbbBNFrFvbI8dshPWZYLdsFv2m901/gStYCVYfbAGjcnMMntWwcZfUqW7Gg==</latexit> ab) 1 <latexit sha1_base64="bDK/i3gH15rrQq6tjoo/A6Erpbc=">AAACgnicfVBdSxtBFJ1sbbWpbWN97MtgKIhI2E0lFlpBsGBfihaaRHBjuDu5iYPzsczcLYYl/8Rf09f2D/TfdDZG0Fh6YeBwzrn3zj1ZrqSnOP5Ti56sPH22uva8/mL95avXjY03PW8LJ7ArrLLuLAOPShrskiSFZ7lD0JnCfnZ1VOn9H+i8tOY7TXMcaJgYOZYCKFDDRqd/kfADfpza4KqG8Iro7fL0Y0DtIHXvS21+fJGSzYeNZtyK58Ufg2QBmmxRp8ON2l46sqLQaEgo8P48iXMalOBICoWzelp4zEFcwQTPAzSg0Q/K+YEz/i4wIz62LjxDfM7e7yhBez/VWXBqoEu/rFXkP7VML22m8YdBKU1eEBpxu3hcKE6WV+nxkXQoSE0DAOFk+DsXl+BAUMi4Xk8/YzjO4dew6CRHB2TdTpmCm2hpZuHYSbpbof8Z4frOGFDIOVlO9THotVtJp9X5ttc8PFokvsbesi22zRK2zw7ZF3bKukywG/aT/WK/o5VoJ0qi97fWqLbo2WQPKvr0F3Izwh4=</latexit> 1 む 2 2 W = GW V, W = U W G 2 <latexit sha1_base64="lZRa2sOHegu1hDN7u2W7PW3biFw=">AAACeHicfVBdSxtBFJ2s/dD0K9ZHX4aGUi0SdoNYXwqCPvSlaKFJBDcJdyc3cXA+lpm7xbDkT/hrfNV/4V/pU2djBBtLLwycOffcOXNPlivpKY7vatHKs+cvXq6u1V+9fvP2XWP9fdfbwgnsCKusO83Ao5IGOyRJ4WnuEHSmsJddHFb93i90Xlrzk6Y59jVMjBxLARSoYWOnN2j3Bgn/yjtbqQ3K6iEeyMeXZLs7SMnmw0YzbsXz4k9BsgBNtqiT4XptNx1ZUWg0JBR4f5bEOfVLcCSFwlk9LTzmIC5ggmcBGtDo++V8rRn/GJgRH1sXjiE+Zx9PlKC9n+osKDXQuV/uVeQ/e5lecqbxfr+UJi8Ijbg3HheKk+VVZnwkHQpS0wBAOBn+zsU5OBAUkq3X0yMMyzn8HoyOc3RA1n0uU3ATLc0sLDtJdyr0PyFcPggDCjkny6k+Bd12K9lr7f3YbR4cLhJfZZvsA9tiCfvCDtg3dsI6TLArds1u2G3td8SjT9H2vTSqLWY22F8Vtf8A15HAvA==</latexit> 1 ; W 2 W 1 = U (W W )V > ab) (G 2 R > <latexit sha1_base64="1s/ZqHsgtJaGz/ba32ItsUZHe4I=">AAACa3icfZDfShtBFMYnq2019k/UOxUcDIKUEnZFbC8FBXvjX4wKbgxnJydxcGZ2mTlbGpa99Gl6Wx+mD+E7OBsjtLF4YODHd87wnfMlmZKOwvBPLZiafvP23cxsfe79h4+fGvML5y7NrcC2SFVqLxNwqKTBNklSeJlZBJ0ovEhud6v+xQ+0TqbmjIYZdjQMjOxLAeSlbmN1P5Ym1kA3SVKcltfFYXeTxyQ1Ou6x7DaaYSscFX8J0RiabFzH3fnaVtxLRa7RkFDg3FUUZtQpwJIUCst6nDvMQNzCAK88GvBWnWJ0ScnXvdLj/dT6Z4iP1L9/FKCdG+rET1Y7u8leJf63l+gJZ+p/6xTSZDmhEU/G/VxxSnkVE+9Ji4LU0AMIK/3uXNyABUE+zHo93kN/nMUDb3SUoQVK7eciBjvQ0pT+2EH8paLXBuHn86Ann3M0mepLON9sRdut7ZOt5s7uOPEZtszW2AaL2Fe2w76zY9Zmgt2xX+w3u689BIvBUrDyNBrUxn8W2T8VrD8Cwy28ug==</latexit> N2 ⇥N2 : ) ⌃ = U SV > <latexit sha1_base64="gqsoMc39D4bwJRNI9IXCMxyR9zQ=">AAACXXicfZDfShtBFMYna602tTaxF73wZjAUpJSwW0S9EQR74U2pYhMFN4azk5N1cP4sM2fVsORRvNVn8qqv0klMoY3FAwM/vvMN55wvK5T0FMePtWjh1eLrpeU39bcr71bfN5prXW9LJ7AjrLLuLAOPShrskCSFZ4VD0JnC0+zqYNI/vUbnpTU/aVRgT0Nu5FAKoCD1G830ROYa+B7vnHQvUrJFv9GK2/G0+HNIZtBiszrqN2tb6cCKUqMhocD78yQuqFeBIykUjutp6bEAcQU5ngc0oNH3qunuY/4pKAM+tC48Q3yq/v2jAu39SGfBqYEu/XxvIv63l+m5yTTc7VXSFCWhEU+Dh6XiZPkkGD6QDgWpUQAQTobdubgEB4JCfPV6+g3DcQ6/h0E/CnRA1n2uUnC5lmYcjs3TLxN6yQi3f4yBQs7JfKrPofu1nWy3t4+3WvsHs8SX2TrbYJssYTtsnx2yI9Zhgt2wO3bPHmq/osVoJVp9ska12Z8P7J+KPv4Gphi2ow==</latexit> 行 力 入 4 1 行 目 目 Saxe, A., McClelland, J., & Ganguli, S. (2014). Exact solutions to the nonlinear dynamics of learning in deep linear neural networks. 0 3 📄 : ]
b = a(s dt <latexit sha1_base64="e30gZbrWQ0ow6F5zp0ZEP3MnZhU=">AAACaHicfZDBShxBEIZ7JybRVZM1HkT00LgENCTLTBDjJSDowYtowFXBWZaanpq1sbtn6K4JWYa5+DRe49v4CnmK9K4r6BpS0PDx119U158USjoKw/tG8Grm9Zu3s3PN+YXFd+9bSx/OXF5agV2Rq9xeJOBQSYNdkqTworAIOlF4nlzvj/rnP9E6mZtTGhbY0zAwMpMCyEv91npMUPI4syCqtK5SqoF/58mm+wLJVr/VDjvhuPhLiCbQZpM66S81tuM0F6VGQ0KBc5dRWFCvAktSKKybcemwAHENA7z0aECj61XjM2r+0Sspz3LrnyE+Vp9OVKCdG+rEOzXQlZvujcR/9hI9tZmy3V4lTVESGvGwOCsVp5yPMuKptChIDT2AsNL/nYsr8BGRT7LZjA/QH2fxyC86LtAC5fZTFYMdaGlqf+wg/jyi/xnh16PRk885mk71JZx97UQ7nZ0f2+29/Unis2yNbbBNFrFvbI8dshPWZYLdsFv2m901/gStYCVYfbAGjcnMMntWwcZfUqG7Gg==</latexit> <latexit sha1_base64="n5jjzTsvJdOimLjKJySjsUXqeSM=">AAACaHicfZDBShxBEIZ7JybRVZM1HkT00LgENCTLTBDjJSDowYtowFXBWZaanpq1sbtn6K4JWYa5+DRe49v4CnmK9K4r6BpS0PDx119U158USjoKw/tG8Grm9Zu3s3PN+YXFd+9bSx/OXF5agV2Rq9xeJOBQSYNdkqTworAIOlF4nlzvj/rnP9E6mZtTGhbY0zAwMpMCyEv91npMUPI4syCqtK5SqhP+ncOm+wLJVr/VDjvhuPhLiCbQZpM66S81tuM0F6VGQ0KBc5dRWFCvAktSKKybcemwAHENA7z0aECj61XjM2r+0Sspz3LrnyE+Vp9OVKCdG+rEOzXQlZvujcR/9hI9tZmy3V4lTVESGvGwOCsVp5yPMuKptChIDT2AsNL/nYsr8BGRT7LZjA/QH2fxyC86LtAC5fZTFYMdaGlqf+wg/jyi/xnh16PRk885mk71JZx97UQ7nZ0f2+29/Unis2yNbbBNFrFvbI8dshPWZYLdsFv2m901/gStYCVYfbAGjcnMMntWwcZfUqW7Gg==</latexit> ab) 1 E(a, b) = 2⌧ (s む <latexit sha1_base64="GLKH/1hpZny1m7QVE1xHGjqHC00=">AAACdXicfZDRShtBFIYna7U21Tbay1IYTJWkpOluEO1NQbCF3pRaaFRw03B2cjYOzswuM2fFsOQZ+jTe6nP4JL3tJKagsXhg4OM//+HM+ZNcSUdheFMJFp4sLj1dflZ9vrL64mVtbf3QZYUV2BWZyuxxAg6VNNglSQqPc4ugE4VHydn+pH90jtbJzPykUY49DUMjUymAvNSvNb80oJU0+SceE16Qo5FCHqcWRFR2YoJi3HDvIWn+6vRr9bAdTos/hGgGdTarg/5aZTseZKLQaEgocO4kCnPqlWBJCoXjalw4zEGcwRBPPBrQ6Hrl9KYx3/TKgKeZ9c8Qn6p3J0rQzo104p0a6NTN9ybif3uJnttM6cdeKU1eEBpxuzgtFKeMTwLjA2lRkBp5AGGl/zsXp+ADIh9rtRp/Rn+cxW9+0fccLVBm35Ux2KGWZuyPHcatCT1mhIt/Rk8+52g+1Ydw2GlHO+2dH9v1vf1Z4svsNdtgDRaxXbbHvrID1mWC/WaX7IpdV/4Eb4K3wdatNajMZl6xexV8+Av/Cr9v</latexit> s = ab 2 a <latexit sha1_base64="zxeL66RFJEXuUZn7DMsZnD7/Hdk=">AAACS3icfZDPSgMxEMaz9V+tf6tHL4tFEJGyK0U9FurBi6hga6FbZDadtqFJdkmyYln6BF71sXwAn8ObeDDbVtAqDgR+fPOFmfnCmDNtPO/Vyc3NLywu5ZcLK6tr6xubxa2GjhJFsU4jHqlmCBo5k1g3zHBsxgpBhBxvw0Et69/eo9IskjdmGGNbQE+yLqNgrHQNd5slr+yNy/0N/hRKZFpXd0WnEnQimgiUhnLQuuV7sWmnoAyjHEeFINEYAx1AD1sWJQjU7XS86cjds0rH7UbKPmncsfr9RwpC66EIrVOA6evZXib+2QvFzGTTPW2nTMaJQUkng7sJd03kZjG4HaaQGj60AFQxu7tL+6CAGhtWoRCcoT1O4YUddBmjAhOpgzQA1RNMjuyxveAwo/+M8PBltGRz9mdT/Q2No7J/XD6+rpSqtWniebJDdsk+8ckJqZJzckXqhBIkj+SJPDsvzpvz7nxMrDln+meb/Kjcwien1bNb</latexit> = b ab) ab) s = ab む <latexit sha1_base64="PW3tAs1QCudVRa9RYJQKp+evktU=">AAACUXicfZBNSwMxEIZn1+/6rUcvi0UQ0bJbRD0KevAiKlgruFVm02kNTbJLkhXL0h/hVX+WJ3+KN9NaQas4EHh45w0z8yaZ4MaG4Zvnj41PTE5Nz5Rm5+YXFpeWV65MmmtGNZaKVF8naEhwRTXLraDrTBPKRFA96Rz1+/UH0oan6tJ2M2pIbCve4gytk+p4W91Jbqt3S+WwEg4q+A3REMowrPO7ZW83bqYsl6QsE2jMTRRmtlGgtpwJ6pXi3FCGrINtunGoUJJpFIN9e8GGU5pBK9XuKRsM1O8/CpTGdGXinBLtvRnt9cU/e4kcmWxbB42Cqyy3pNjn4FYuApsG/TCCJtfErOg6QKa52z1g96iRWRdZqRQfkztO06kbdJaRRpvqrSJG3ZZc9dyx7Xi7T/8Z8fHL6MjlHI2m+huuqpVor7J3sVs+PBomPg1rsA6bEME+HMIJnEMNGHTgCZ7hxXv13n3w/U+r7w3/rMKP8mc/AH6Vs0g=</latexit> a 2 b a <latexit sha1_base64="i3ODyEEUl87ObflhTOKxPnEzK24=">AAACTnicfZBNSwMxEIaz9avWz+rRy2IRRKTsiqgXoVAPXsQKVgvdIrPptA0m2SXJimXpX/CqP8urf8SbaLauoFUcCDy884aZecOYM20878UpTE3PzM4V50sLi0vLK6vltSsdJYpik0Y8Uq0QNHImsWmY4diKFYIIOV6Ht/Wsf32HSrNIXpphjB0Bfcl6jILJJH0M4c1qxat643J/g59DheTVuCk7+0E3oolAaSgHrdu+F5tOCsowynFUChKNMdBb6GPbogSBupOOlx25W1bpur1I2SeNO1a//0hBaD0UoXUKMAM92cvEP3uhmJhsekedlMk4MSjp5+Bewl0TuVkSbpcppIYPLQBVzO7u0gEooMbmVSoFJ2iPU3hmB53HqMBEaicNQPUFkyN7bD/Yzeg/I9x/GS3ZnP3JVH/D1V7VP6geXOxXavU88SLZIJtkm/jkkNTIKWmQJqFkQB7II3lynp1X5815/7QWnPzPOvlRheIHI0+0iw==</latexit> 2 <latexit sha1_base64="Rhwfy4KFyngwypMReK/iGk+an1E=">AAACh3icfZDbahRBEIZ7x1OyHrLRS28aF0FE15kQ1kAQIvHCGzGCmw1sT5aenprZJn0YumuCyzAP49N4q5e+jb2bEXQjFjR89VcV1fVnlZIe4/hnL7px89btO1vb/bv37j/YGew+PPW2dgImwirrzjLuQUkDE5So4KxywHWmYJpdHK/q00twXlrzGZcVpJqXRhZScAzSfHDICsdFk7dNji1TUOBsep404bXnDG1FX9KQ7HVJIOZkucCUvqHxfDCMR/E66HVIOhiSLk7mu719lltRazAoFPd+lsQVpg13KIWCts9qDxUXF7yEWUDDNfi0WV/Z0qdByWlhXXgG6Vr9c6Lh2vulzkKn5rjwm7WV+M9apjc2Y3GQNtJUNYIRV4uLWlG0dGUhzaUDgWoZgAsnw9+pWPBgIwaj+332DsJxDj6ERR8rcByte94w7kotTRuOLdmLFf2vkX/53Rgo+JxsunodTvdGyXg0/rQ/PDruHN8ij8kT8owk5DU5Iu/JCZkQQb6Sb+Q7+RFtR6+icXRw1Rr1uplH5K+I3v4CXQ3Grg==</latexit> <latexit sha1_base64="C7WIyqWowAWoqVB4i9nLF/HUqIg=">AAACfHicfZBdaxNBFIYn61eNX6leejMYhfqRsLuU2Buh0F54I1YwbaGbhjOzZ9OhM7PLzFlpWPZv+Gu81f/gn5FO0giaigcGHt7zHs6cV1RaeYrjn53oxs1bt+9s3O3eu//g4aPe5uNDX9ZO4liWunTHAjxqZXFMijQeVw7BCI1H4nxv0T/6gs6r0n6meYUTAzOrCiWBgjTtxRkVDmSTt01O7RacpgNxmr7k73gKWV4SBz7gqViiCGo87fXjYbwsfh2SFfTZqg6mm53tMC1rg5akBu9PkriiSQOOlNTYdrPaYwXyHGZ4EtCCQT9plqe1/EVQcl6ULjxLfKn+OdGA8X5uRHAaoDO/3luI/+wJs7aZip1Jo2xVE1p5tbioNaeSL3LjuXIoSc8DgHQq/J3LMwjRUUi32832MRzn8ENY9LFCB1S6V00GbmaUbcOxs+zNgv5nhIvfxkAh52Q91etwmA6T0XD0abu/u7dKfIM9Zc/YFkvYW7bL3rMDNmaSfWXf2Hf2o/Mreh69jgZX1qizmnnC/qpodAn+h8EE</latexit> 2 d dt (a 2 b ) = 2aȧ 2bḃ = 0 d h 1 1> W W dt matrix balancing W 2> i W2 = 0 d ⌧ u = 2u(s dt <latexit sha1_base64="MnNkMwQbfToD1Bjp1o6pBovfJjY=">AAACZ3icfZDPShxBEMZ7R5OYzR9XAyJ46WQJmJAsMyImF0HQgxfRQFYFZ1lqemrWxu6eobs6ZBnmkKfJVR8nj+Bb2LtuIFlDChp+fPUV1fVllZKO4vhXK1pYfPT4ydLT9rPnL14ud1ZWT13prcC+KFVpzzNwqKTBPklSeF5ZBJ0pPMuu9if9s29onSzNVxpXONAwMrKQAihIw85GSuDTwoKo86bOqfF8l2/5TffRvxt2unEvnhZ/CMkMumxWJ8OV1naal8JrNCQUOHeRxBUNarAkhcKmnXqHFYgrGOFFQAMa3aCeXtHwt0HJeVHa8AzxqfrnRA3aubHOglMDXbr53kT8Zy/Tc5up+Dyopak8oRH3iwuvOJV8EhHPpUVBahwAhJXh71xcQoiIQpDtdnqA4TiLR2HRcYUWqLTv6xTsSEvThGNH6YcJ/c8I338bA4Wck/lUH8LpVi/Z6e182e7u7c8SX2Ib7A3bZAn7xPbYITthfSbYD/aTXbOb1m20HK1F6/fWqDWbecX+quj1HQEIuvs=</latexit> むa = b u ⌘ ab <latexit sha1_base64="9zU9Q5ZXCuYTOa3DLx3d5fhQbQ8=">AAACTXicfZBNSwMxEIaz9aO1flU9elksgoiUXRH1Igj14EVUtFboljKbTtfQJLskWbEs/Qle9Wd59od4EzGtFbQVBwIP77xhZt4w4Uwbz3t1clPTM7P5wlxxfmFxabm0snqj41RRrNGYx+o2BI2cSawZZjjeJgpBhBzrYbc66NfvUWkWy2vTS7ApIJKswygYK13BUdgqlb2KNyx3EvwRlMmoLlorzl7QjmkqUBrKQeuG7yWmmYEyjHLsF4NUYwK0CxE2LEoQqJvZcNe+u2mVttuJlX3SuEP1548MhNY9EVqnAHOnx3sD8c9eKMYmm85hM2MySQ1K+jW4k3LXxO4gCLfNFFLDexaAKmZ3d+kdKKDGxlUsBidoj1N4ZgedJ6jAxGo7C0BFgsm+PTYKdgb0nxEevo2WbM7+eKqTcLNb8fcr+5d75ePqKPECWScbZIv45IAck1NyQWqEkog8kify7Lw4b8678/FlzTmjP2vkV+XynyH7tA4=</latexit> <latexit sha1_base64="QqO10Uz6UXy375kIMx9DT0GF/Sk=">AAACVHicfZDPSgMxEMazq/VP1drq0UuwCCJSdkXUo6AHL6KCtaJbZDadtqFJdk2yYln6Fl71sQTfxYNpraBVHAj8+OYLM/PFqeDGBsGb509NF2Zm5+aLC4tLpeVyZeXKJJlmWGeJSPR1DAYFV1i33Aq8TjWCjAU24t7RsN94QG14oi5tP8WmhI7ibc7AOukmi/A+4w8U4rtyNagFo6K/IRxDlYzr/K7i7UathGUSlWUCjLkNg9Q2c9CWM4GDYpQZTIH1oIO3DhVINM18tPKAbjilRduJdk9ZOlK//8hBGtOXsXNKsF0z2RuKf/ZiOTHZtg+aOVdpZlGxz8HtTFCb0GEetMU1Miv6DoBp7nanrAsamHWpFYvRMbrjNJ66QWcparCJ3soj0B3J1cAd24m2h/SfER6/jI5czuFkqr/haqcW7tX2Lnarh0fjxOfIGlknmyQk++SQnJBzUieMKPJEnsmL9+q9+1N+4dPqe+M/q+RH+aUPaXu1NA==</latexit> u(t) = se e2st/⌧ u) Riccati <latexit sha1_base64="Nk9PZApyQlyQJmnxHqZ7ccbUQvI=">AAACinicfZBdSxtBFIYn2w9tamusl70ZDIXYj7gbxFZKi6AXvSm10KjgpuHs5GwcnJldZs6WhmH/TX9Nb+uN/8ZJjGBj6YGBZ97zDmfOm5VKOorjy0Z07/6Dh0vLj5qPV548XW2tPTtyRWUF9kWhCnuSgUMlDfZJksKT0iLoTOFxdr4/7R//QOtkYb7RpMSBhrGRuRRAQRq2PlYd2uQfeJpbEN7hd99ztJUSVHXtb9/4G57wVze+2ledeLOuh6123I1nxe9CMoc2m9fhcK2xnY4KUWk0JBQ4d5rEJQ08WJJCYd1MK4cliHMY42lAAxrdwM8WrfmLoIx4XthwDPGZevuFB+3cRGfBqYHO3GJvKv6zl+mFyZS/G3hpyorQiOvBeaU4FXyaIh9Ji4LUJAAIK8PfuTiDEA2FrJvN9ADDchY/h0FfSrRAhX3pU7BjLU0dlh2nr6f0PyP8vDEGCjkni6nehaNeN9np7nzdbu/tzxNfZs/ZBuuwhL1le+wTO2R9Jtgv9pv9YRfRStSLdqP319aoMX+zzv6q6OAKzuHHTw==</latexit> 2st/⌧ s 1 + u(0) s 一 行 行 方 目 目 一 Saxe, A., McClelland, J., & Ganguli, S. (2014). Exact solutions to the nonlinear dynamics of learning in deep linear neural networks. 1 一 3 📄 b <latexit sha1_base64="067UFn5yTNcR3dGAVXVkIrV15gk=">AAACS3icfZDPSgMxEMaz9V+tf6tHL4tFEJGyK0U9FurBi6hga6FbZDadtqFJdkmyYln6BF71sXwAn8ObeDDbVtAqDgR+fPOFmfnCmDNtPO/Vyc3NLywu5ZcLK6tr6xubxa2GjhJFsU4jHqlmCBo5k1g3zHBsxgpBhBxvw0Et69/eo9IskjdmGGNbQE+yLqNgrHQd3m2WvLI3Lvc3+FMokWld3RWdStCJaCJQGspB65bvxaadgjKMchwVgkRjDHQAPWxZlCBQt9PxpiN3zyodtxsp+6Rxx+r3HykIrYcitE4Bpq9ne5n4Zy8UM5NN97SdMhknBiWdDO4m3DWRm8XgdphCavjQAlDF7O4u7YMCamxYhUJwhvY4hRd20GWMCkykDtIAVE8wObLH9oLDjP4zwsOX0ZLN2Z9N9Tc0jsr+cfn4ulKq1qaJ58kO2SX7xCcnpErOyRWpE0qQPJIn8uy8OG/Ou/Mxseac6Z9t8qNyC5+pvLNc</latexit>
= a(s dt <latexit sha1_base64="e30gZbrWQ0ow6F5zp0ZEP3MnZhU=">AAACaHicfZDBShxBEIZ7JybRVZM1HkT00LgENCTLTBDjJSDowYtowFXBWZaanpq1sbtn6K4JWYa5+DRe49v4CnmK9K4r6BpS0PDx119U158USjoKw/tG8Grm9Zu3s3PN+YXFd+9bSx/OXF5agV2Rq9xeJOBQSYNdkqTworAIOlF4nlzvj/rnP9E6mZtTGhbY0zAwMpMCyEv91npMUPI4syCqtK5SqoF/58mm+wLJVr/VDjvhuPhLiCbQZpM66S81tuM0F6VGQ0KBc5dRWFCvAktSKKybcemwAHENA7z0aECj61XjM2r+0Sspz3LrnyE+Vp9OVKCdG+rEOzXQlZvujcR/9hI9tZmy3V4lTVESGvGwOCsVp5yPMuKptChIDT2AsNL/nYsr8BGRT7LZjA/QH2fxyC86LtAC5fZTFYMdaGlqf+wg/jyi/xnh16PRk885mk71JZx97UQ7nZ0f2+29/Unis2yNbbBNFrFvbI8dshPWZYLdsFv2m901/gStYCVYfbAGjcnMMntWwcZfUqG7Gg==</latexit> <latexit sha1_base64="n5jjzTsvJdOimLjKJySjsUXqeSM=">AAACaHicfZDBShxBEIZ7JybRVZM1HkT00LgENCTLTBDjJSDowYtowFXBWZaanpq1sbtn6K4JWYa5+DRe49v4CnmK9K4r6BpS0PDx119U158USjoKw/tG8Grm9Zu3s3PN+YXFd+9bSx/OXF5agV2Rq9xeJOBQSYNdkqTworAIOlF4nlzvj/rnP9E6mZtTGhbY0zAwMpMCyEv91npMUPI4syCqtK5SqhP+ncOm+wLJVr/VDjvhuPhLiCbQZpM66S81tuM0F6VGQ0KBc5dRWFCvAktSKKybcemwAHENA7z0aECj61XjM2r+0Sspz3LrnyE+Vp9OVKCdG+rEOzXQlZvujcR/9hI9tZmy3V4lTVESGvGwOCsVp5yPMuKptChIDT2AsNL/nYsr8BGRT7LZjA/QH2fxyC86LtAC5fZTFYMdaGlqf+wg/jyi/xnh16PRk885mk71JZx97UQ7nZ0f2+29/Unis2yNbbBNFrFvbI8dshPWZYLdsFv2m901/gStYCVYfbAGjcnMMntWwcZfUqW7Gg==</latexit> ab) 1 E(a, b) = 2⌧ (s む <latexit sha1_base64="GLKH/1hpZny1m7QVE1xHGjqHC00=">AAACdXicfZDRShtBFIYna7U21Tbay1IYTJWkpOluEO1NQbCF3pRaaFRw03B2cjYOzswuM2fFsOQZ+jTe6nP4JL3tJKagsXhg4OM//+HM+ZNcSUdheFMJFp4sLj1dflZ9vrL64mVtbf3QZYUV2BWZyuxxAg6VNNglSQqPc4ugE4VHydn+pH90jtbJzPykUY49DUMjUymAvNSvNb80oJU0+SceE16Qo5FCHqcWRFR2YoJi3HDvIWn+6vRr9bAdTos/hGgGdTarg/5aZTseZKLQaEgocO4kCnPqlWBJCoXjalw4zEGcwRBPPBrQ6Hrl9KYx3/TKgKeZ9c8Qn6p3J0rQzo104p0a6NTN9ybif3uJnttM6cdeKU1eEBpxuzgtFKeMTwLjA2lRkBp5AGGl/zsXp+ADIh9rtRp/Rn+cxW9+0fccLVBm35Ux2KGWZuyPHcatCT1mhIt/Rk8+52g+1Ydw2GlHO+2dH9v1vf1Z4svsNdtgDRaxXbbHvrID1mWC/WaX7IpdV/4Eb4K3wdatNajMZl6xexV8+Av/Cr9v</latexit> ab) ab) s = ab 2 a <latexit sha1_base64="zxeL66RFJEXuUZn7DMsZnD7/Hdk=">AAACS3icfZDPSgMxEMaz9V+tf6tHL4tFEJGyK0U9FurBi6hga6FbZDadtqFJdkmyYln6BF71sXwAn8ObeDDbVtAqDgR+fPOFmfnCmDNtPO/Vyc3NLywu5ZcLK6tr6xubxa2GjhJFsU4jHqlmCBo5k1g3zHBsxgpBhBxvw0Et69/eo9IskjdmGGNbQE+yLqNgrHQNd5slr+yNy/0N/hRKZFpXd0WnEnQimgiUhnLQuuV7sWmnoAyjHEeFINEYAx1AD1sWJQjU7XS86cjds0rH7UbKPmncsfr9RwpC66EIrVOA6evZXib+2QvFzGTTPW2nTMaJQUkng7sJd03kZjG4HaaQGj60AFQxu7tL+6CAGhtWoRCcoT1O4YUddBmjAhOpgzQA1RNMjuyxveAwo/+M8PBltGRz9mdT/Q2No7J/XD6+rpSqtWniebJDdsk+8ckJqZJzckXqhBIkj+SJPDsvzpvz7nxMrDln+meb/Kjcwien1bNb</latexit> s = ab <latexit sha1_base64="i3ODyEEUl87ObflhTOKxPnEzK24=">AAACTnicfZBNSwMxEIaz9avWz+rRy2IRRKTsiqgXoVAPXsQKVgvdIrPptA0m2SXJimXpX/CqP8urf8SbaLauoFUcCDy884aZecOYM20878UpTE3PzM4V50sLi0vLK6vltSsdJYpik0Y8Uq0QNHImsWmY4diKFYIIOV6Ht/Wsf32HSrNIXpphjB0Bfcl6jILJJH0M4c1qxat643J/g59DheTVuCk7+0E3oolAaSgHrdu+F5tOCsowynFUChKNMdBb6GPbogSBupOOlx25W1bpur1I2SeNO1a//0hBaD0UoXUKMAM92cvEP3uhmJhsekedlMk4MSjp5+Bewl0TuVkSbpcppIYPLQBVzO7u0gEooMbmVSoFJ2iPU3hmB53HqMBEaicNQPUFkyN7bD/Yzeg/I9x/GS3ZnP3JVH/D1V7VP6geXOxXavU88SLZIJtkm/jkkNTIKWmQJqFkQB7II3lynp1X5815/7QWnPzPOvlRheIHI0+0iw==</latexit> b 2 <latexit sha1_base64="Rhwfy4KFyngwypMReK/iGk+an1E=">AAACh3icfZDbahRBEIZ7x1OyHrLRS28aF0FE15kQ1kAQIvHCGzGCmw1sT5aenprZJn0YumuCyzAP49N4q5e+jb2bEXQjFjR89VcV1fVnlZIe4/hnL7px89btO1vb/bv37j/YGew+PPW2dgImwirrzjLuQUkDE5So4KxywHWmYJpdHK/q00twXlrzGZcVpJqXRhZScAzSfHDICsdFk7dNji1TUOBsep404bXnDG1FX9KQ7HVJIOZkucCUvqHxfDCMR/E66HVIOhiSLk7mu719lltRazAoFPd+lsQVpg13KIWCts9qDxUXF7yEWUDDNfi0WV/Z0qdByWlhXXgG6Vr9c6Lh2vulzkKn5rjwm7WV+M9apjc2Y3GQNtJUNYIRV4uLWlG0dGUhzaUDgWoZgAsnw9+pWPBgIwaj+332DsJxDj6ERR8rcByte94w7kotTRuOLdmLFf2vkX/53Rgo+JxsunodTvdGyXg0/rQ/PDruHN8ij8kT8owk5DU5Iu/JCZkQQb6Sb+Q7+RFtR6+icXRw1Rr1uplH5K+I3v4CXQ3Grg==</latexit> <latexit sha1_base64="C7WIyqWowAWoqVB4i9nLF/HUqIg=">AAACfHicfZBdaxNBFIYn61eNX6leejMYhfqRsLuU2Buh0F54I1YwbaGbhjOzZ9OhM7PLzFlpWPZv+Gu81f/gn5FO0giaigcGHt7zHs6cV1RaeYrjn53oxs1bt+9s3O3eu//g4aPe5uNDX9ZO4liWunTHAjxqZXFMijQeVw7BCI1H4nxv0T/6gs6r0n6meYUTAzOrCiWBgjTtxRkVDmSTt01O7RacpgNxmr7k73gKWV4SBz7gqViiCGo87fXjYbwsfh2SFfTZqg6mm53tMC1rg5akBu9PkriiSQOOlNTYdrPaYwXyHGZ4EtCCQT9plqe1/EVQcl6ULjxLfKn+OdGA8X5uRHAaoDO/3luI/+wJs7aZip1Jo2xVE1p5tbioNaeSL3LjuXIoSc8DgHQq/J3LMwjRUUi32832MRzn8ENY9LFCB1S6V00GbmaUbcOxs+zNgv5nhIvfxkAh52Q91etwmA6T0XD0abu/u7dKfIM9Zc/YFkvYW7bL3rMDNmaSfWXf2Hf2o/Mreh69jgZX1qizmnnC/qpodAn+h8EE</latexit> む a 6= b <latexit sha1_base64="RKPK5FAsv81nSo+OocCdWGdaIEU=">AAACUHicfZBBSxtBFMffxtrqqm2ix16GhoKIhF2RtMeAHnoRFUwUsiG8nbzE0ZnZZWZWDEu+g1f9WL35TbzZSYxgY+mDYX783394b/5pLoV1UfQYVJY+LH/8tLIarq1vfP5SrW12bFYYTm2eycxcpGhJCk1tJ5yki9wQqlTSeXp9MO2f35CxItNnbpxTT+FIi6Hg6LzUwUQTS/vVetSIZsXeQzyHOszrpF8L9pNBxgtF2nGJ1nbjKHe9Eo0TXNIkTApLOfJrHFHXo0ZFtlfO1p2w714ZsGFm/NGOzdS3L0pU1o5V6p0K3aVd7E3Ff/ZStTDZDX/2SqHzwpHmL4OHhWQuY9Ms2EAY4k6OPSA3wu/O+CUa5M4nFobJIfnPGTryg45zMugys1MmaEZK6In/7CjZndL/jHj7avTkc44XU30Pnb1G3Gw0T/frrYN54ivwFb7BNsTwA1rwC06gDRyu4A7u4SH4HTwFz5Xgxfp6wxb8VZXwD37qtD8=</latexit> 2 b ) = 2aȧ : 2bḃ = 0 <latexit sha1_base64="oP8v2VHPU5MGbyKPFKzgr5IvwlM=">AAACgHicfVBdSxtBFJ2strXpV9THvgyGwqaEuCvBFqEg2Ie+iBYaFdwQ7k5uksH52M7cLYYlf6S/xtf2H/TfdBIjaCy9MHA451zO3JMXSnpKkj+1aG39ydNnG8/rL16+ev2msbl15m3pBPaEVdZd5OBRSYM9kqTwonAIOld4nl8dzfXzH+i8tOYbTQvsaxgbOZICKFCDRjeGdt7in3jmvzuqxCCZxZmwfhJnNEGC3b1WO2jS3CNag0Yz6SSL4Y9BugRNtpzTwWatmw2tKDUaEgq8v0yTgvoVOJJC4ayelR4LEFcwxssADWj0/Wpx3oy/C8yQj6wLzxBfsPc3KtDeT3UenBpo4le1OflPLdcryTT62K+kKUpCI26DR6XiZPm8Oz6UDgWpaQAgnAx/52ICDgSFhuv17DOG4xweh6CTAh2Qde+rDNxYSzMLx46z9hz9zwjXd8aAQs/paquPwdleJ93v7H/tNg+Plo1vsLdsh8UsZR/YIfvCTlmPCfaT3bBf7HcURXG0G6W31qi23NlmDyY6+Av8HsH2</latexit> (a, b) = d✓ ⌧ =s dt p W 2> c0 (cosh(✓/2), sinh(✓/2)) ( i W2 = 0 c0 2 d dt (a d h 1 1> W W dt matrix balancing 2 = a 2 b む <latexit sha1_base64="PW3tAs1QCudVRa9RYJQKp+evktU=">AAACUXicfZBNSwMxEIZn1+/6rUcvi0UQ0bJbRD0KevAiKlgruFVm02kNTbJLkhXL0h/hVX+WJ3+KN9NaQas4EHh45w0z8yaZ4MaG4Zvnj41PTE5Nz5Rm5+YXFpeWV65MmmtGNZaKVF8naEhwRTXLraDrTBPKRFA96Rz1+/UH0oan6tJ2M2pIbCve4gytk+p4W91Jbqt3S+WwEg4q+A3REMowrPO7ZW83bqYsl6QsE2jMTRRmtlGgtpwJ6pXi3FCGrINtunGoUJJpFIN9e8GGU5pBK9XuKRsM1O8/CpTGdGXinBLtvRnt9cU/e4kcmWxbB42Cqyy3pNjn4FYuApsG/TCCJtfErOg6QKa52z1g96iRWRdZqRQfkztO06kbdJaRRpvqrSJG3ZZc9dyx7Xi7T/8Z8fHL6MjlHI2m+huuqpVor7J3sVs+PBomPg1rsA6bEME+HMIJnEMNGHTgCZ7hxXv13n3w/U+r7w3/rMKP8mc/AH6Vs0g=</latexit> 2 − ) <latexit sha1_base64="NvMEsf3MWLINwP/dQEVikUeR0oA=">AAACeHicfVDLShxBFK1p89DJa9RlNkWGkAdm6BYxbgKCWbgRDWRUsIbhdvXtmcKq6qbqdnBo+ifyNW6Tv/BXXFnzEJIx5ELB4ZxzOXVPWmrlKY5vWtHKo8dPnq6utZ89f/HyVWd949QXlZPYl4Uu3HkKHrWy2CdFGs9Lh2BSjWfp5cFUP/uBzqvCfqdJiQMDI6tyJYECNexsCYKKi9yBrDNBYyRo6owa/oV7/onLYSy8suO5Mux04148G/4QJAvQZYs5Ga63dkRWyMqgJanB+4skLmlQgyMlNTZtUXksQV7CCC8CtGDQD+rZWQ1/G5iM54ULzxKfsX9u1GC8n5g0OA3Q2C9rU/KfWmqWkinfG9TKlhWhlfPgvNKcCj7tjGfKoSQ9CQCkU+HvXI4hVEah2XZbfMVwnMOjEHRcogMq3MdagBsZZZtw7EhsTdH/jHB1bwwo9Jwst/oQnG73kt3e7red7v7BovFV9pq9Ye9Zwj6zfXbITlifSfaTXbNf7HfrNuLRu+jD3Bq1Fjub7K+Jtu8AfC/CBw==</latexit> c0 sinh ✓ ; 一 一 行 目 目 一 Saxe, A., McClelland, J., & Ganguli, S. (2014). Exact solutions to the nonlinear dynamics of learning in deep linear neural networks. 3 3 1 a 📄 b <latexit sha1_base64="067UFn5yTNcR3dGAVXVkIrV15gk=">AAACS3icfZDPSgMxEMaz9V+tf6tHL4tFEJGyK0U9FurBi6hga6FbZDadtqFJdkmyYln6BF71sXwAn8ObeDDbVtAqDgR+fPOFmfnCmDNtPO/Vyc3NLywu5ZcLK6tr6xubxa2GjhJFsU4jHqlmCBo5k1g3zHBsxgpBhBxvw0Et69/eo9IskjdmGGNbQE+yLqNgrHQd3m2WvLI3Lvc3+FMokWld3RWdStCJaCJQGspB65bvxaadgjKMchwVgkRjDHQAPWxZlCBQt9PxpiN3zyodtxsp+6Rxx+r3HykIrYcitE4Bpq9ne5n4Zy8UM5NN97SdMhknBiWdDO4m3DWRm8XgdphCavjQAlDF7O4u7YMCamxYhUJwhvY4hRd20GWMCkykDtIAVE8wObLH9oLDjP4zwsOX0ZLN2Z9N9Tc0jsr+cfn4ulKq1qaJ58kO2SX7xCcnpErOyRWpE0qQPJIn8uy8OG/Ou/Mxseac6Z9t8qNyC5+pvLNc</latexit>
) = <latexit sha1_base64="rd5oajX48R65mKNFLX36m+8U7g4=">AAACl3icfVDbahRBEO0db3G9bcyT+NK4CCKyzGiIIgQDCRIQY6JuEkjvDj29tZMmfRm6a8IOw3xUvkZ80z+xZ7OCbsSCgsOpU5yqkxVKeozj753o2vUbN2+t3O7euXvv/oPe6sNDb0snYCissu444x6UNDBEiQqOCwdcZwqOsrPtdn50Ds5La75iVcBI89zIqRQcA5X2PrAvMtecvt2kDGGGHisFlPlSp7XcTJqxqVI5S+WYoS0ok4YyzfE0y+rPzbjeS18xlBo83UuTJu3140E8L3oVJAvQJ4vaT1c762xiRanBoFDc+5MkLnBUc4dSKGi6rPRQcHHGczgJ0PBgNarnXzf0aWAmdGpdaIN0zv65UXPtfaWzoGxP9suzlvznLNNLzjh9M6qlKUoEIy6Np6WiaGkbKZ1IBwJVFQAXTobbqTjljgsMwXe7bAfCcw4+BqNPBTiO1j2vGXe5lqYJz+bsRYv+J+Sz38KAQs7JcqpXweHLQbIx2DhY729tLxJfIY/JE/KMJOQ12SK7ZJ8MiSAX5Bv5QX5Gj6J30fto91IadRY7a+Svig5+AeYTzfs=</latexit> ⌃ := む <latexit sha1_base64="rsRj1eWWlrKUfhxnz29b60rvjQc=">AAACY3icfZDRShtBFIYnq7Y2ahttwQspDIZCkRp2RdSbgqAX3kgVjRHcEM5OTjaDM7PLzFkxLHkab/WBfADfw0lMQWPpgYGP//wzZ86f5Eo6CsPHSjAzO/fh4/yn6sLi0ucvteWVC5cVVmBTZCqzlwk4VNJgkyQpvMwtgk4UtpLrg1G/dYPWycyc0yDHtobUyJ4UQF7q1FZjgiLuZsRb/DePz2SqgW/yVqdWDxvhuPh7iCZQZ5M66SxXtv0zotBoSChw7ioKc2qXYEkKhcNqXDjMQVxDilceDWh07XK8wZD/8EqX9zLrjyE+Vl/fKEE7N9CJd2qgvpvujcR/9hI9NZl6e+1SmrwgNOJlcK9QnDI+iod3pUVBauABhJX+71z0wYIgH2K1Gh+iX87isR/0J0cLlNmNMgabammGftk0/jWi/xnh9q/Rk885mk71PVxsNaKdxs7pdn3/YJL4PFtj6+wni9gu22dH7IQ1mWBDdsfu2UPlKVgIVoJvL9agMrnzlb2p4Pszf1q4WA==</latexit> ⌧ Ẇ = ⌃ [Saxe-McClelland-Ganguli ⾒ y = Wx <latexit sha1_base64="CGge8VvQCkNFBMLs8IE7JzFPFiQ=">AAACUHicfZBBSxtBFMffptqmW63aHnsZDIVSJOyK2F4KgfTQSzGCSQR3kbeTt3F0ZnaZmS2GJd+h1/qxvPlNvNVJjNBG8cEwP/7vP7w3/6yUwroougkaL1ZWX75qvg7frK2/3djcejewRWU49XkhC3OcoSUpNPWdcJKOS0OoMknD7KI76w9/kbGi0EduUlKqcKxFLjg6Lw0m7BsbXp5utqJ2NC/2GOIFtGBRvdOtYC8ZFbxSpB2XaO1JHJUurdE4wSVNw6SyVCK/wDGdeNSoyKb1fN0p++iVEcsL4492bK7++6JGZe1EZd6p0J3Z5d5MfLKXqaXJLv+a1kKXlSPN7wfnlWSuYLMs2EgY4k5OPCA3wu/O+Bka5M4nFobJd/KfM/TTDzooyaArzOc6QTNWQk/9Z8fJzoyeM+Llg9GTzzleTvUxDHbb8X57/3Cv1ekuEm/CB9iGTxDDF+jAD+hBHzicw2/4A1fBdXAb/G0E99aHG97Df9UI7wDrArPy</latexit> <latexit sha1_base64="0wRcr1oa3UexjVoLV2D7uLrF3k0=">AAACdnicfZDRShtBFIYnq1WbtjbqpSBDg9RKG3aDqDeCoEJvRAuNEZy4nJ2cTQZnZpeZWTGseQefxtv6Gr6Jl53ECG0sPTDw8Z//cOb8SS6FdWH4WAlmZt/MzS+8rb57/2HxY21p+cxmheHY4pnMzHkCFqXQ2HLCSTzPDYJKJLaTq4NRv32NxopM/3SDHDsKelqkgoPzUlzbPNpof6F7lKUGeNRktlBxKfai4aVmt4NY0G+0fRMLdnvZjGv1sBGOi76GaAJ1MqnTeKmyxboZLxRqxyVYexGFueuUYJzgEodVVljMgV9BDy88alBoO+X4qCFd90qXppnxTzs6Vv+cKEFZO1CJdypwfTvdG4n/7CVqarNLdzul0HnhUPPnxWkhqcvoKDHaFQa5kwMPwI3wf6e8Dz4u53OtVtkh+uMMHvtFJzkacJnZLBmYnhJ66I/tsa8j+p8Rbl6MnnzO0XSqr+Gs2Yi2G9s/tur7B5PEF8gq+UQ2SER2yD75Tk5Ji3ByR+7JL/JQeQrWgvXg87M1qExmVshfFYS/Aarzv70=</latexit> W Pn > N3 ⇥N1 y x 2 R ; i=1 i i n X 1 2 i=1 2 kyi W xi k Pn <latexit sha1_base64="xjOxvmjR64ptkxcnbTP+RZUHXVA=">AAACfXicfZDbahsxEIblTQ+Je3LSy96ImkApweyGkPQmEEgu2ou2KdRJIHKWWXnsiOiwlWaDzbLP0afpbfsMfZpUdlxondJBgo9/fjGavyi1CpSmP1vJyr37Dx6urrUfPX7y9FlnfeMkuMpL7EunnT8rIKBWFvukSONZ6RFMofG0uDqc9U+v0Qfl7GealjgwMLZqpCRQlPJOJggnFGiqkYtQmbxW+1lzYSe5iudCkCu5wC+Vuubv8vpDnjV5p5v20nnxu5AtoMsWdZyvt3bE0MnKoCWpIYTzLC1pUIMnJTU2bVEFLEFewRjPI1owGAb1fLeGb0ZlyEfOx2uJz9U/X9RgQpiaIjoN0GVY7s3Ef/YKszSZRm8GtbJlRWjl7eBRpTk5PguOD5VHSXoaAaRX8e9cXoIHSTHedlscYVzO4/s46GOJHsj517UAPzbKNnHZsdia0f+MMPltjBRzzpZTvQsn271st7f7aad7cLhIfJW9YC/ZK5axPXbA3rJj1meSfWXf2Hf2o3WTbCZbSe/WmrQWb56zvyrZ+wXtY8T8</latexit> ⌃ = U SV > > x x i=1 i i ⌘ IN1 > <latexit sha1_base64="NXupUhNl5crBqocmcqCFCVvbpy4=">AAACYHicfVBdSxtBFJ2stU1jWxP1qX0ZGgRbStgtor4Igj74UmqhMYKbhruTmzg4H8vMXTEu+S99rf/I1/4SZ2MKNZYeGDhz7rnce0+WK+kpju9q0dKz5ecv6i8bK69ev1ltttZOvS2cwK6wyrqzDDwqabBLkhSe5Q5BZwp72eVhVe9dofPSmu80ybGvYWzkSAqgIA2aG7BFH/g+L36kZHPeq35Xg2Y77sQz8KckmZM2m+Nk0Kptp0MrCo2GhALvz5M4p34JjqRQOG2khcccxCWM8TxQAxp9v5ytP+WbQRnykXXhGeIz9e+OErT3E50Fpwa68Iu1SvxnLdMLk2m01y+lyQtCIx4GjwrFyfIqGz6UDgWpSSAgnAy7c3EBDgSFBBuN9AjDcQ6/hEFfc3RA1n0sU3BjLc00HDtOP1Xsf0a4/mMMLOScLKb6lJx+7iQ7nZ1v2+2Dw3nidfaOvWdbLGG77IAdsxPWZYLdsJ/sF7ut/Y7q0WrUerBGtXnPOnuE6O09Yg224w==</latexit> <latexit sha1_base64="gqsoMc39D4bwJRNI9IXCMxyR9zQ=">AAACXXicfZDfShtBFMYna602tTaxF73wZjAUpJSwW0S9EQR74U2pYhMFN4azk5N1cP4sM2fVsORRvNVn8qqv0klMoY3FAwM/vvMN55wvK5T0FMePtWjh1eLrpeU39bcr71bfN5prXW9LJ7AjrLLuLAOPShrskCSFZ4VD0JnC0+zqYNI/vUbnpTU/aVRgT0Nu5FAKoCD1G830ROYa+B7vnHQvUrJFv9GK2/G0+HNIZtBiszrqN2tb6cCKUqMhocD78yQuqFeBIykUjutp6bEAcQU5ngc0oNH3qunuY/4pKAM+tC48Q3yq/v2jAu39SGfBqYEu/XxvIv63l+m5yTTc7VXSFCWhEU+Dh6XiZPkkGD6QDgWpUQAQTobdubgEB4JCfPV6+g3DcQ6/h0E/CnRA1n2uUnC5lmYcjs3TLxN6yQi3f4yBQs7JfKrPofu1nWy3t4+3WvsHs8SX2TrbYJssYTtsnx2yI9Zhgt2wO3bPHmq/osVoJVp9ska12Z8P7J+KPv4Gphi2ow==</latexit> a(t) = u W (t)v <latexit sha1_base64="aU52D6XpcGvichH7RskOtBpIihM=">AAACZHicfZDRShtBFIYna61pqjUaBKEgQ4OgRcOuiHpTEPSiN0ULRgU3hLOTk2RwZnaZOSuGJT6Nt/V9+gI+h5MYQWPpgYGP//wzZ86fZEo6CsO/pWDmw+zHufKnyuf5hS+L1aXlc5fmVmBTpCq1lwk4VNJgkyQpvMwsgk4UXiTXR6P+xQ1aJ1NzRoMMWxp6RnalAPJSu7oaE+RxJyUOG7TJf3DHt8fYrtbDRjgu/h6iCdTZpE7bS6Vd/47INRoSCpy7isKMWgVYkkLhsBLnDjMQ19DDK48GNLpWMV5hyNe90uHd1PpjiI/V1zcK0M4NdOKdGqjvpnsj8Z+9RE9Npu5Bq5AmywmNeB7czRWnlI/y4R1pUZAaeABhpf87F32wIMinWKnEx+iXs/jLDzrJ0AKl9nsRg+1paYZ+2V68NaL/GeH2xejJ5xxNp/oeznca0V5j7/du/fBokniZfWXf2AaL2D47ZD/ZKWsywe7YPfvDHkqPwXxQC1aerUFpcqfG3lSw9gQ25rgm</latexit> ⌧ ȧ(t) = s a(t) ODE <latexit sha1_base64="yeBAXjFLPW/2a0rC3dgKmWFSri8=">AAACbHicfZDRShtBFIYnW1s12hq1d1IYDJakatwVsb0pCPbCG1HBqOCm4ezkJA7OzC4zZ4thyW2fxtv6Lr6Ez+AkRrCx9MDAx3/+w5nzJ5mSjsLwvhS8mXr7bnpmtjw3//7DQmVx6cyluRXYFKlK7UUCDpU02CRJCi8yi6AThefJ9f6wf/4LrZOpOaV+hi0NPSO7UgB5qV3hUKM6/84dX+c1qIV1vsldHX8Wm7QVE+SDdqUaNsJR8dcQjaHKxnXcXiztxJ1U5BoNCQXOXUZhRq0CLEmhcFCOc4cZiGvo4aVHAxpdqxidMuBrXunwbmr9M8RH6suJArRzfZ14pwa6cpO9ofjPXqInNlP3W6uQJssJjXha3M0Vp5QPc+IdaVGQ6nsAYaX/OxdXYEGQT7Ncjn+gP87ioV90lKEFSu2XIgbb09IM/LG9eGNI/zPCzbPRk885mkz1NZxtN6Ldxu7JTnVvf5z4DFthq6zGIvaV7bEDdsyaTLDf7Jb9YXelh+BjsBJ8erIGpfHMMvurgs+Pumi6KQ==</latexit> a(t) = s + (a(0) s)e t/⌧ 小 目 行 比 力 入 二 4 1 行 力 Saxe, A., McClelland, J., & Ganguli, S. (2014). Exact solutions to the nonlinear dynamics of learning in deep linear neural networks. 4 入 3 一 📄 2 ]
7 自 3 自 n X <latexit sha1_base64="zd0qs9YnZx3rv+u8W9SoszuY7LY=">AAACh3icfVBdSxtBFJ2s/dC01WgffRkahCg27oqkghQUS+lLUaFRwYnL3cndODgzu8zMimHZH9Nf09f20X/jbEyhRumFgcO553LmnCSXwrowvGsEcy9evno9v9B88/bd4lJreeXUZoXh2OeZzMx5Ahal0Nh3wkk8zw2CSiSeJdeH9f7sBo0Vmf7hxjkOFIy0SAUH56m4tceU0HGZMqGZAnfFQdKvFWWpAR5pZgsVl+JzVF1q2hnHgn6kaec2Fuvr9HI7brXDbjgZ+hREU9Am0zmOlxs7bJjxQqF2XIK1F1GYu0EJxgkusWqywmIO/BpGeOGhBoV2UE5SVnTNM0OaZsY/7eiE/feiBGXtWCVeWSexs7uafHaXqBlnl+4OSqHzwqHmD8ZpIanLaF0hHQqD3MmxB8CN8H+n/Ap8Yc4X3WyyL+jDGfzujY5yNOAys1EyMCNfdeXDjthmjf4nhNu/Qo98z9Fsq0/B6XY36nV7Jzvt/cNp4/NklXwgHRKRT2SffCPHpE84+Ul+kd/kT7AQbAW9YPdBGjSmN+/JowkO7gGTOMXH</latexit> n X <latexit sha1_base64="k8IjcVVg0ced+68bAA+pKfkHlGg=">AAACfXicfZDbahRBEIZ7x1NcTxu99KZxCUQJy4yExBshEC+8ESO4SSC9DjW9NbNN+jB018guwz6HT+OtPoNPo72bEXQjFjR8/PUX1fUXtVaB0vRHL7lx89btO1t3+/fuP3j4aLD9+DS4xkscS6edPy8goFYWx6RI43ntEUyh8ay4PF71zz6jD8rZj7SocWKgsqpUEihK+SATBua5oBkScFF6kJkVoTF5q15ny09WaFfxujPsznP1PB8M01G6Ln4dsg6GrKuTfLu3L6ZONgYtSQ0hXGRpTZMWPCmpcdkXTcAa5CVUeBHRgsEwade3LflOVKa8dD4+S3yt/jnRgglhYYroNECzsNlbif/sFWZjM5WvJq2ydUNo5dXistGcHF8Fx6fKoyS9iADSq/h3LmcQA6MYb78v3mA8zuO7uOh9jR7I+RetAF8ZZZfx2Ersreh/Rpj/NkaKOWebqV6H05ej7GB08GF/eHTcJb7FnrJnbJdl7JAdsbfshI2ZZF/YV/aNfe/9THaSvWR0ZU163cwT9lclh78A1prEcw==</latexit> f (xi )) 2 xi <latexit sha1_base64="h8lWl2VcAk8a9cUwBnalCUZxLC4=">AAACTXicfZBNSwMxEIaz9aNav1o9elksgoiUXRH1KNSDF1HR2kK3lNl0uoYm2SXJimXpT/CqP8uzP8SbiGmtoK04EHh45w0z84YJZ9p43quTm5mdm88vLBaWlldW14ql9Vsdp4pijcY8Vo0QNHImsWaY4dhIFIIIOdbDXnXYr9+j0iyWN6afYEtAJFmXUTBWun5os3ax7FW8UbnT4I+hTMZ12S45B0EnpqlAaSgHrZu+l5hWBsowynFQCFKNCdAeRNi0KEGgbmWjXQfutlU6bjdW9knjjtSfPzIQWvdFaJ0CzJ2e7A3FP3uhmJhsusetjMkkNSjp1+Buyl0Tu8Mg3A5TSA3vWwCqmN3dpXeggBobV6EQnKI9TuG5HXSRoAITq90sABUJJgf22CjYG9J/Rnj4NlqyOfuTqU7D7X7FP6wcXh2UT6rjxBfIJtkiO8QnR+SEnJFLUiOUROSRPJFn58V5c96djy9rzhn/2SC/Kpf/BJwLtE4=</latexit> ⾒ si <latexit sha1_base64="roEqQMiNvGE//dX6FbSz1oYgNug=">AAACTXicfZBNSwMxEIaz9aO1flU9elksgoiUXRH1KNSDF1HRWqFbymw63YYm2SXJimXpT/CqP8uzP8SbiGmtoK04EHh45w0z84YJZ9p43quTm5mdm88XFoqLS8srq6W19Vsdp4pijcY8VnchaORMYs0ww/EuUQgi5FgPe9Vhv36PSrNY3ph+gk0BkWQdRsFY6Vq3WKtU9ireqNxp8MdQJuO6bK05B0E7pqlAaSgHrRu+l5hmBsowynFQDFKNCdAeRNiwKEGgbmajXQfutlXabidW9knjjtSfPzIQWvdFaJ0CTFdP9obin71QTEw2neNmxmSSGpT0a3An5a6J3WEQbpsppIb3LQBVzO7u0i4ooMbGVSwGp2iPU3huB10kqMDEajcLQEWCyYE9Ngr2hvSfER6+jZZszv5kqtNwu1/xDyuHVwflk+o48QLZJFtkh/jkiJyQM3JJaoSSiDySJ/LsvDhvzrvz8WXNOeM/G+RX5fKfkn60SQ==</latexit> yi <latexit sha1_base64="BbkNUAiU7AyoEFOqIMluMR6TRus=">AAACTXicfZBNSwMxEIaz9aO1flU9elksgoiUXRH1KNSDF1HRWqFbymw63YYm2SXJimXpT/CqP8uzP8SbiGmtoK04EHh45w0z84YJZ9p43quTm5mdm88XFoqLS8srq6W19Vsdp4pijcY8VnchaORMYs0ww/EuUQgi5FgPe9Vhv36PSrNY3ph+gk0BkWQdRsFY6brfYq1S2at4o3KnwR9DmYzrsrXmHATtmKYCpaEctG74XmKaGSjDKMdBMUg1JkB7EGHDogSBupmNdh2421Zpu51Y2SeNO1J//shAaN0XoXUKMF092RuKf/ZCMTHZdI6bGZNJalDSr8GdlLsmdodBuG2mkBretwBUMbu7S7uggBobV7EYnKI9TuG5HXSRoAITq90sABUJJgf22CjYG9J/Rnj4NlqyOfuTqU7D7X7FP6wcXh2UT6rjxAtkk2yRHeKTI3JCzsglqRFKIvJInsiz8+K8Oe/Ox5c154z/bJBflct/Ap30tE8=</latexit> 1 max log p✓ (xi ) ✓ n i=1 yi
<latexit sha1_base64="zT4jofQAeInRAyLCg8ASv/1Qtug=">AAACVHicfVDBahRBEO2ZJBpHjZvk4MFL4xIQkaHHjZs9BvWQiyQBNwnsLEtNb82mk57uobsnZBnma7zG3xH8Fw/p2WwgilrQ8HjvVXXVy0oprGPsZxCurK49erz+JHr67PnGi87m1onVleE45Fpqc5aBRSkUDp1wEs9Kg1BkEk+zy0+tfnqFxgqtvrp5ieMCZkrkgoPz1KTzMtVebrvrtAB3nuX1ddNMLiadLotZr9/r7VIW99iALcAeY4MPCU1itqguWdbRZDPYSaeaVwUqxyVYO0pY6cY1GCe4xCZKK4sl8EuY4chDBQXacb24oKE7npnSXBv/lKML9mFHDYW18yLzznZL+6fWkn/TRpXLB+NaqLJyqPjdR3klqdO0jYNOhUHu5NwD4Eb4XSk/BwPc+dCiKP2M/hiDX/zgwxINOG3e1imYWSFU44+bpe9a9D8jXN8bPYp8sPfp0X+Dk/dx0o/7x7vd/Y/LiNfJK/KavCEJ2SP75IAckSHhpCHfyA35HvwIfoUr4dqdNQyWPdvktwo3bgGhVLVr</latexit> <latexit sha1_base64="zT4jofQAeInRAyLCg8ASv/1Qtug=">AAACVHicfVDBahRBEO2ZJBpHjZvk4MFL4xIQkaHHjZs9BvWQiyQBNwnsLEtNb82mk57uobsnZBnma7zG3xH8Fw/p2WwgilrQ8HjvVXXVy0oprGPsZxCurK49erz+JHr67PnGi87m1onVleE45Fpqc5aBRSkUDp1wEs9Kg1BkEk+zy0+tfnqFxgqtvrp5ieMCZkrkgoPz1KTzMtVebrvrtAB3nuX1ddNMLiadLotZr9/r7VIW99iALcAeY4MPCU1itqguWdbRZDPYSaeaVwUqxyVYO0pY6cY1GCe4xCZKK4sl8EuY4chDBQXacb24oKE7npnSXBv/lKML9mFHDYW18yLzznZL+6fWkn/TRpXLB+NaqLJyqPjdR3klqdO0jYNOhUHu5NwD4Eb4XSk/BwPc+dCiKP2M/hiDX/zgwxINOG3e1imYWSFU44+bpe9a9D8jXN8bPYp8sPfp0X+Dk/dx0o/7x7vd/Y/LiNfJK/KavCEJ2SP75IAckSHhpCHfyA35HvwIfoUr4dqdNQyWPdvktwo3bgGhVLVr</latexit> <latexit sha1_base64="zT4jofQAeInRAyLCg8ASv/1Qtug=">AAACVHicfVDBahRBEO2ZJBpHjZvk4MFL4xIQkaHHjZs9BvWQiyQBNwnsLEtNb82mk57uobsnZBnma7zG3xH8Fw/p2WwgilrQ8HjvVXXVy0oprGPsZxCurK49erz+JHr67PnGi87m1onVleE45Fpqc5aBRSkUDp1wEs9Kg1BkEk+zy0+tfnqFxgqtvrp5ieMCZkrkgoPz1KTzMtVebrvrtAB3nuX1ddNMLiadLotZr9/r7VIW99iALcAeY4MPCU1itqguWdbRZDPYSaeaVwUqxyVYO0pY6cY1GCe4xCZKK4sl8EuY4chDBQXacb24oKE7npnSXBv/lKML9mFHDYW18yLzznZL+6fWkn/TRpXLB+NaqLJyqPjdR3klqdO0jYNOhUHu5NwD4Eb4XSk/BwPc+dCiKP2M/hiDX/zgwxINOG3e1imYWSFU44+bpe9a9D8jXN8bPYp8sPfp0X+Dk/dx0o/7x7vd/Y/LiNfJK/KavCEJ2SP75IAckSHhpCHfyA35HvwIfoUr4dqdNQyWPdvktwo3bgGhVLVr</latexit> <latexit sha1_base64="zT4jofQAeInRAyLCg8ASv/1Qtug=">AAACVHicfVDBahRBEO2ZJBpHjZvk4MFL4xIQkaHHjZs9BvWQiyQBNwnsLEtNb82mk57uobsnZBnma7zG3xH8Fw/p2WwgilrQ8HjvVXXVy0oprGPsZxCurK49erz+JHr67PnGi87m1onVleE45Fpqc5aBRSkUDp1wEs9Kg1BkEk+zy0+tfnqFxgqtvrp5ieMCZkrkgoPz1KTzMtVebrvrtAB3nuX1ddNMLiadLotZr9/r7VIW99iALcAeY4MPCU1itqguWdbRZDPYSaeaVwUqxyVYO0pY6cY1GCe4xCZKK4sl8EuY4chDBQXacb24oKE7npnSXBv/lKML9mFHDYW18yLzznZL+6fWkn/TRpXLB+NaqLJyqPjdR3klqdO0jYNOhUHu5NwD4Eb4XSk/BwPc+dCiKP2M/hiDX/zgwxINOG3e1imYWSFU44+bpe9a9D8jXN8bPYp8sPfp0X+Dk/dx0o/7x7vd/Y/LiNfJK/KavCEJ2SP75IAckSHhpCHfyA35HvwIfoUr4dqdNQyWPdvktwo3bgGhVLVr</latexit> x j x j x j x xjj ] zi Encoder <latexit sha1_base64="ZOTBYZ4IDYySEXx7pqSQY6rin/I=">AAACR3icfZDRShtBFIZnY6vpaq3Ry94MDYJICbslxuQuqBfeiAqNCtkQzk7OJkNmZ5eZWTEueQ1v9XV8BJ/Cu9JLZ5MIbakeGPj4zz8z5/xhKrg2nvfklJY+fFxeKX9yV9c+r3/ZqGxe6CRTDDssEYm6CkGj4BI7hhuBV6lCiEOBl+H4sOhfXqPSPJE/zSTFXgxDySPOwFgpCGIwozDKb6d93t+oerWW57f29ukcmvUFNFrUr3mzqpJFnfUrznYwSFgWozRMgNZd30tNLwdlOBM4dYNMYwpsDEPsWpQQo+7ls6GndNsqAxolyh5p6Ez980YOsdaTOLTOYkj9b68Q/9frZiZq9nIu08ygZPOPokxQk9AiATrgCpkREwvAFLezUjYCBczYnFw3OEK7jMIT+/BpigpMonbzANQw5nJqlxsG3wt6zwg3r0ZLrg32NT36Nlz8qPmNWuO8Xm0fLCIuk6/kG9khPtknbXJMzkiHMJKSO3JPHpxH59n55fyeW0vO4s4W+atKzgsJ67I5</latexit> zj <latexit sha1_base64="g841n+aIva7pMukoNWeQg2B5H4A=">AAACR3icfZDLbhMxFIY9oUCYcklgycZqFKmqUDRThVx2EbBgUzWVmjRSJorOOGcSE49nZHuqhlFegy28Do/AU7CrWOLJRSoVcCRLn/7z2z7nD1PBtfG8H07pwcHDR4/LT9zDp8+ev6hUXw51kimGA5aIRI1C0Ci4xIHhRuAoVQhxKPAqXL4v+lfXqDRP5KVZpTiJYS55xBkYKwVBDGYRRvnn9fTTtFLzGl3P775t0y10mjtodanf8DZVI7vqT6tOPZglLItRGiZA67HvpWaSgzKcCVy7QaYxBbaEOY4tSohRT/LN0Gtat8qMRomyRxq6Ue/eyCHWehWH1lkMqe/3CvFvvXFmos4k5zLNDEq2/SjKBDUJLRKgM66QGbGyAExxOytlC1DAjM3JdYMPaJdReGYfPk9RgUnUSR6Amsdcru1y8+BNQf8zws3eaMm1we7To/+G4WnDbzVaF81a790u4jJ5TY7IMfFJm/TIR9InA8JISr6Qr+Sb89356dw6v7bWkrO784r8USXnNwvDsjo=</latexit> <latexit sha1_base64="nD1veWt0VjbSwlag1ylwwCGzRQA=">AAACVHicfVDBahRBEO2ZJBpHjRs9eMilcQmIyNDjxs0eg+bgJSQBNwnsLEtNb82mk57uobsnZB3ma7zq7wT8Fw/2bDYQRS1oeLz3qrrqZaUU1jH2IwhXVtcePFx/FD1+8nTjWWfz+YnVleE45Fpqc5aBRSkUDp1wEs9Kg1BkEk+zy4+tfnqFxgqtPrt5ieMCZkrkgoPz1KTzMtVebrvrtAB3nuX1l6aZXEw6XRazXr/X26Es7rEBW4BdxgbvE5rEbFFdsqyjyWawnU41rwpUjkuwdpSw0o1rME5wiU2UVhZL4Jcww5GHCgq043pxQUO3PTOluTb+KUcX7P2OGgpr50Xmne2W9k+tJf+mjSqXD8a1UGXlUPHbj/JKUqdpGwedCoPcybkHwI3wu1J+Dga486FFUbqP/hiDB37wYYkGnDZv6hTMrBCq8cfN0rct+p8Rru+MHkU+2Lv06L/Bybs46cf9453u3odlxOtki7wir0lCdske+USOyJBw0pCv5Bv5HtwEP8OVcO3WGgbLnhfktwo3fgGlDLVt</latexit> <latexit sha1_base64="nD1veWt0VjbSwlag1ylwwCGzRQA=">AAACVHicfVDBahRBEO2ZJBpHjRs9eMilcQmIyNDjxs0eg+bgJSQBNwnsLEtNb82mk57uobsnZB3ma7zq7wT8Fw/2bDYQRS1oeLz3qrrqZaUU1jH2IwhXVtcePFx/FD1+8nTjWWfz+YnVleE45Fpqc5aBRSkUDp1wEs9Kg1BkEk+zy4+tfnqFxgqtPrt5ieMCZkrkgoPz1KTzMtVebrvrtAB3nuX1l6aZXEw6XRazXr/X26Es7rEBW4BdxgbvE5rEbFFdsqyjyWawnU41rwpUjkuwdpSw0o1rME5wiU2UVhZL4Jcww5GHCgq043pxQUO3PTOluTb+KUcX7P2OGgpr50Xmne2W9k+tJf+mjSqXD8a1UGXlUPHbj/JKUqdpGwedCoPcybkHwI3wu1J+Dga486FFUbqP/hiDB37wYYkGnDZv6hTMrBCq8cfN0rct+p8Rru+MHkU+2Lv06L/Bybs46cf9453u3odlxOtki7wir0lCdske+USOyJBw0pCv5Bv5HtwEP8OVcO3WGgbLnhfktwo3fgGlDLVt</latexit> <latexit sha1_base64="nD1veWt0VjbSwlag1ylwwCGzRQA=">AAACVHicfVDBahRBEO2ZJBpHjRs9eMilcQmIyNDjxs0eg+bgJSQBNwnsLEtNb82mk57uobsnZB3ma7zq7wT8Fw/2bDYQRS1oeLz3qrrqZaUU1jH2IwhXVtcePFx/FD1+8nTjWWfz+YnVleE45Fpqc5aBRSkUDp1wEs9Kg1BkEk+zy4+tfnqFxgqtPrt5ieMCZkrkgoPz1KTzMtVebrvrtAB3nuX1l6aZXEw6XRazXr/X26Es7rEBW4BdxgbvE5rEbFFdsqyjyWawnU41rwpUjkuwdpSw0o1rME5wiU2UVhZL4Jcww5GHCgq043pxQUO3PTOluTb+KUcX7P2OGgpr50Xmne2W9k+tJf+mjSqXD8a1UGXlUPHbj/JKUqdpGwedCoPcybkHwI3wu1J+Dga486FFUbqP/hiDB37wYYkGnDZv6hTMrBCq8cfN0rct+p8Rru+MHkU+2Lv06L/Bybs46cf9453u3odlxOtki7wir0lCdske+USOyJBw0pCv5Bv5HtwEP8OVcO3WGgbLnhfktwo3fgGlDLVt</latexit> <latexit sha1_base64="nD1veWt0VjbSwlag1ylwwCGzRQA=">AAACVHicfVDBahRBEO2ZJBpHjRs9eMilcQmIyNDjxs0eg+bgJSQBNwnsLEtNb82mk57uobsnZB3ma7zq7wT8Fw/2bDYQRS1oeLz3qrrqZaUU1jH2IwhXVtcePFx/FD1+8nTjWWfz+YnVleE45Fpqc5aBRSkUDp1wEs9Kg1BkEk+zy4+tfnqFxgqtPrt5ieMCZkrkgoPz1KTzMtVebrvrtAB3nuX1l6aZXEw6XRazXr/X26Es7rEBW4BdxgbvE5rEbFFdsqyjyWawnU41rwpUjkuwdpSw0o1rME5wiU2UVhZL4Jcww5GHCgq043pxQUO3PTOluTb+KUcX7P2OGgpr50Xmne2W9k+tJf+mjSqXD8a1UGXlUPHbj/JKUqdpGwedCoPcybkHwI3wu1J+Dga486FFUbqP/hiDB37wYYkGnDZv6hTMrBCq8cfN0rct+p8Rru+MHkU+2Lv06L/Bybs46cf9453u3odlxOtki7wir0lCdske+USOyJBw0pCv5Bv5HtwEP8OVcO3WGgbLnhfktwo3fgGlDLVt</latexit> <latexit sha1_base64="nD1veWt0VjbSwlag1ylwwCGzRQA=">AAACVHicfVDBahRBEO2ZJBpHjRs9eMilcQmIyNDjxs0eg+bgJSQBNwnsLEtNb82mk57uobsnZB3ma7zq7wT8Fw/2bDYQRS1oeLz3qrrqZaUU1jH2IwhXVtcePFx/FD1+8nTjWWfz+YnVleE45Fpqc5aBRSkUDp1wEs9Kg1BkEk+zy4+tfnqFxgqtPrt5ieMCZkrkgoPz1KTzMtVebrvrtAB3nuX1l6aZXEw6XRazXr/X26Es7rEBW4BdxgbvE5rEbFFdsqyjyWawnU41rwpUjkuwdpSw0o1rME5wiU2UVhZL4Jcww5GHCgq043pxQUO3PTOluTb+KUcX7P2OGgpr50Xmne2W9k+tJf+mjSqXD8a1UGXlUPHbj/JKUqdpGwedCoPcybkHwI3wu1J+Dga486FFUbqP/hiDB37wYYkGnDZv6hTMrBCq8cfN0rct+p8Rru+MHkU+2Lv06L/Bybs46cf9453u3odlxOtki7wir0lCdske+USOyJBw0pCv5Bv5HtwEP8OVcO3WGgbLnhfktwo3fgGlDLVt</latexit> z j z j z j z zjj 0 2 Chen, T., Kornblith, S., Norouzi, M., & Hinton, G. (2020). A simple framework for contrastive learning of visual representations. 0 4 📄 (contrastive learning)
StopGrad( r L rW L 0 2 x )k2 ⇢ ⇢W ⇥ > > > 1 0 > r L= r E x W W x SG( x ) W x 2⇥ ⇤ > > > 0 > = E W W xx W xx <latexit sha1_base64="bYmQXGhvQmVRqMqW14vBkrQLrzk=">AAAEj3iclVNtb9MwEHZDgBHeVvjIF4sK1vFSNdW08QVUMRAggSiCrpPmrnJcJ7VmO5HtolVRfhC/hq/s3+CkXWm6tRqWLD2+e+7uudM5SDjTptk8qzjX3Os3bm7c8m7fuXvv/mb1wYGOx4rQLol5rA4DrClnknYNM5weJopiEXDaC072c3/vJ1WaxfKHmSS0L3AkWcgINtY0qDpvUUAjJlPMWSTpMPOQxAHHgxQJbEZBCFFnxLLiQTCHn72nryEKFSZ+azUzCNL3GeI0NEfeuff0GJk4WeSWDL3iBefPReI8BXw5JSiRfv+Q1S/lbG2X865N9H+61pf1kGLRyPQhQsWQVg2idwV9ZXELakvBeci/PrbKQUty6v5zpFkk8HFr+wpKVpWcOj1E5XC+MoPNWrPRLA68CPwZqIHZ6QyqlR00jMlYUGkIx1of+c3E9FOsDCOc2h0ca5pgcoIjemShxILqflpsewafWMsQhrGyVxpYWBcjUiy0nojAMnPNetmXGy/1BWKpsglf9VMmk7GhkkwLh2MOTQzzrwSHTFFi+MQCTBSz2iEZYfs1jP1wnofeUducol9soa8JVdjE6lmKsIoEk5ltNkIvcrSOiE/PiRbZOfvLU70IDloNf7ex+22n1t6fTXwDPAKPQR34YA+0wUfQAV1AnF/Ob+ePc+ZW3T33jdueUp3KLOYhKB3301+DlIMC</latexit> = (1 + 2 > )W W W x > ⇤ > > 0 W SG( x ) > 1 2 Tian, Y., Chen, X., & Ganguli, S. (2021). Understanding self-supervised learning dynamics without contrastive pairs. 1 5 📄 1 L( , W) = EkW x 2 [Tian-Chen-Ganguli <latexit sha1_base64="fnvw+Uj/IFxh4VrNHQKHz3Vlq7Y=">AAACvXicfZFrb9MwFIadcBthQAeCL3yJqCY2NKqkQoMviImL4AOIIug6qe4qxz1prTl2ZJ9AqywSf5M/wO/AabNpF8SRLD16z3t8eZ3kUliMot+ef+Xqtes31m4Gt9Zv37nb2ri3b3VhOPS5ltocJMyCFAr6KFDCQW6AZYmEQXL0tu4PfoCxQqvvuMhhlLGpEqngDJ00bv2iGcMZZ7L8VG0tOUlpbyZ2Gg4H269oahiPuyslKd9X9Pi0e2bkRJs/owhzLL+hzj8YNjm376npyTY9HncPu+NWO+pEywovQ9xAmzTVG294m3SieZGBQi6ZtcM4ynFUMoOCS6gCWljIGT9iUxg6VCwDOyqXUVXhplMmYaqNWwrDpXp2omSZtYsscc76pvZirxb/1RsWmL4clULlBYLiq4PSQoaowzr3cCIMcJQLB4wb4e4a8hlzwaL7nSCg78A9xsBnt/GXHAxDbZ6WlJlpJlTlHjelOzX9z8jmJ0ZHgQs2vhjjZdjvduLdzu7X5+29N03Ea+QReUy2SExekD3ykfRIn3Dyx1v3HngP/dc++NJXK6vvNTP3ybnyf/4FFEvaEQ==</latexit> ]
x, StopGrad( x )i 0 kW xkkStopGrad( x )k む Taking derivative wrt normalizer makes gradient complicated む Solution: high-dimensional limit norm of random vector concentrates on a hypersphere 5 6 :
x i=1 i i <latexit sha1_base64="jCNnjMpT3TZMYK6q9miJ72KmV9k=">AAAConicfZHfahQxFMaz4786/tvqpTfBtdhqWWZ6UUUobK2oCGKLblvYbIcz2TPb0EkyJJnSZZhH8Yn6BL6Bt3qpN2Z2V1hb8UDgx3e+JCdf0iIX1kXR11Zw5eq16zeWboa3bt+5e6+9fH/f6tJw7HOda3OYgsVcKOw74XI8LAyCTHM8SE92mv7BKRortPrsJgUOJYyVyAQH56WkjUyCO04z9hakhJdblGUGeFxtqJrZUiaV2IrrI7U6t9GzRFQL/KQ+Yk4X9BldFBd42l5L2p2oG02LXoZ4Dp3eWvTm248v27vJcmuFjTQvJSrHc7B2EEeFG1ZgnOA51iErLRbAT2CMA48KJNphNc2jpiteGdFMG7+Uo1N1cUcF0tqJTL2zmdRe7DXiv3qD0mUvhpVQRelQ8dlFWZlTp2kTLh0Jg9zlEw/AjfCzUn4MPlDnvyAM2Wv0jzH4wR/8sUADTpunFQMzlkLV/nFjtt7Q/4xw9sfoKfTBxhdjvAz7G914s7u5F3d6r8islshD8oiskpg8Jz3yjuySPuHknHwnP8mv4HHwPtgLPs2sQWu+5wH5qwL2G1eq1Pg=</latexit> 1 := 2n ] Pn 0> 0 > (x x + x i xi ) i=1 i i 3 2 行 行 己 力 己 力 Simon, J. B., Knutins, M., Ziyin, L., Geisz, D., Fetterman, A. J., & Albrecht, J. (2023). On the stepwise nature of self-supervised learning. 9 自 6 自 入 📄 入 ⾒ (revisited)
(X X) 1 ( > ) <latexit sha1_base64="z2Dc08s5xAXt8BWmvZ1QQES6++M=">AAACZXicfZDRShtBFIYnq1aN2kZbBOmFQ4NoSw27IraXgl54oWihUcEN4ezkZB2cmV1mzpqGJfRpvNXn8Qn6Gp3ECG0sPTDw8Z//cOb8Sa6kozB8rART0zOvZufmqwuLS6/f1JZXzl1WWIFNkanMXibgUEmDTZKk8DK3CDpReJHcHAz7F7donczMd+rn2NKQGtmVAshL7dpabzNOkRzvbcdIEBtIFPDjrd7Hdq0eNsJR8ZcQjaHOxnXWXq7sxp1MFBoNCQXOXUVhTq0SLEmhcFCNC4c5iBtI8cqjAY2uVY5uGPANr3R4N7P+GeIj9c+JErRzfZ14pwa6dpO9ofjPXqInNlP3a6uUJi8IjXha3C0Up4wPA+IdaVGQ6nsAYaX/OxfXYEGQj7FajQ/RH2fxxC86zdECZfZTGYNNtTQDf2wafx7S/4zw49noyeccTab6Es53GtFeY+/bbn3/YJz4HHvPPrAtFrEvbJ8dsTPWZIL9ZHfsnj1UfgVLwbtg9ckaVMYzb9lfFaz/BkxmubM=</latexit> w X y 行 [ 6 7 : む 0 w ⌘rL(w)
xi i <latexit sha1_base64="Ogz5WUyidBxj6A6C8p6HFpWIfhI=">AAACgXicfZDNSuRAEMd74ve46uge99I4CK7IkIhf4EXQw15kFRwVzBAqPZXY2N0J3R11CHkSn8br+gS+jZ1xhHUUCwp+XfUvqusf54Ib6/svDW9icmp6ZnauOf9jYXGptbxyYbJCM+yyTGT6KgaDgivsWm4FXuUaQcYCL+Pbo7p/eYfa8Eyd20GOPQmp4glnYF0pau2EAlQqMJRgb+KkvK+ijc1BxN/fD5VjPZTQMK0TpISo1fY7/jDoZwhG0CajOI2WG9thP2OFRGWZAGOuAz+3vRK05Uxg1QwLgzmwW0jx2qECiaZXDu+r6Jqr9GmSaZfK0mH1/4kSpDEDGTtl/W0z3quLX/ZiObbZJvu9kqu8sKjY2+KkENRmtDaP9rlGZsXAATDN3d8puwENzDqLm83wGN1xGk/cor85arCZ3ihD0KnkqnLHpuFmTd8J4eFd6Mj5HIy7+hkutjrBbmf3bLt9eDRyfJb8IqtknQRkjxySP+SUdAkjj+SJ/CPP3oT32/O9rTep1xjN/CQfwjt4Be1Gxds=</latexit> n X i=1 `(m) = e <latexit sha1_base64="U94f3z6l97Xyl6EofN8Tlv7UmvY=">AAACVHicfZBNSwMxEIaz61et33r0EiyCiJRdEfUkBT14ERWsLbpFZtPpGppklyQrlqX/wqv+LMH/4sG0VtAqDgQe3nnDzLxxJrixQfDm+ROTU9Mzpdny3PzC4tLyyuq1SXPNsM5SkepmDAYFV1i33ApsZhpBxgIbcfd40G88oDY8VVe2l2FLQqJ4hzOwTrqJEpAS6BEN7pYrQTUYFv0N4QgqZFQXdyveXtROWS5RWSbAmNswyGyrAG05E9gvR7nBDFgXErx1qECiaRXDlft00ylt2km1e8rSofr9RwHSmJ6MnVOCvTfjvYH4Zy+WY5Nt57BVcJXlFhX7HNzJBbUpHeRB21wjs6LnAJjmbnfK7kEDsy61cjk6QXecxjM36DxDDTbV20UEOpFc9d2xSbQzoP+M8PhldORyDsdT/Q3Xu9Vwv7p/uVepHY8SL5F1skG2SEgOSI2ckgtSJ4wo8kSeyYv36r37E/7Up9X3Rn/WyI/yFz8A2bq0Yw==</latexit> >0 `(hw, yi xi i) ; m <latexit sha1_base64="syXruHIyJZx4DWoGWWlcpIJU6N0=">AAACXHicfZBNSwMxEIbT9avWr6rgxUuwCCpadkXUiyDowYtYwVbBrWU2nbbBJLsk2WJZ+k+86n/y4m8xrRW0igOBh3feMDNvlAhurO+/5byJyanpmfxsYW5+YXGpuLxSM3GqGVZZLGJ9F4FBwRVWLbcC7xKNICOBt9Hj2aB/20VteKxubC/BuoS24i3OwDqpUSyGKMSW3KYnFB+yPdlvFEt+2R8W/Q3BCEpkVJXGcu4gbMYslagsE2DMfeAntp6BtpwJ7BfC1GAC7BHaeO9QgURTz4ar9+mmU5q0FWv3lKVD9fuPDKQxPRk5pwTbMeO9gfhnL5Jjk23ruJ5xlaQWFfsc3EoFtTEd5EKbXCOzoucAmOZud8o6oIFZl16hEJ6jO07jpRt0laAGG+udLATdllz13bHtcHdA/xnh6cvoyOUcjKf6G2r75eCwfHh9UDo9GyWeJ+tkg2yRgByRU3JBKqRKGOmSZ/JCXnPv3qQ35y18Wr3c6M8q+VHe2gcyLrXt</latexit> <latexit sha1_base64="CARv2nJ+nhBeOVEQJmaph73E5fs=">AAACfnicfVBNSxxBEO2dmKibr1WPXposCWuImxkR9RIQzMGDIQayKjjLUtNTszZ29wzdNSbLMP/DX+PV/IX8m/SsK5g1pKDg8d4rquolhZKOwvB3K3iy8PTZ4tJy+/mLl69ed1ZWT1xeWoEDkavcniXgUEmDA5Kk8KywCDpReJpcHjT66RVaJ3PznSYFDjWMjcykAPLUqLMVZxZElcYa6CLJ+I8ebdRVSjX/xDdjA4kCftR7qG6MOt2wH06LPwbRDHTZrI5HK63tOM1FqdGQUODceRQWNKzAkhQK63ZcOixAXMIYzz00oNENq+lzNX/rmZRnufVtiE/ZhxMVaOcmOvHO5ko3rzXkP7VEz22mbG9YSVOUhEbcLc5KxSnnTXI8lRYFqYkHIKz0t3NxAT498vm22/Fn9M9Z/OIXfS3QAuX2fRWDHWtpav/sOP7QoP8Z4ee90SOfczSf6mNwstWPdvo737a7+wezxJfYOnvDeixiu2yfHbJjNmCCXbMbdst+BSx4F2wGH++sQWs2s8b+qmDvD5bBwlI=</latexit> dw(t) = dt む ( rL(w(t)) ) : <latexit sha1_base64="RG5XaoGbDSP8VCyzHoRXbyj69Lw=">AAACsHicfZFdaxQxFIaz41ddv7Z66U1wEarIMlNK7Y1QqBfeiBXcdqFZhjPZM7uxSWZIzlSX6fxB/4H/wlu9MrMdsd2KBwJv3vOEk7zJSq08xfH3XnTj5q3bdzbu9u/df/Dw0WDz8ZEvKidxLAtduEkGHrWyOCZFGielQzCZxuPs9KDtH5+h86qwn2hZ4tTA3KpcSaBgpYOZ0MqkNQkqhLI5LRuRO5C1MECLLOdftuhFU4vzy3tx3vA3vOMWQH/hZoVetQKdDobxKF4Vvy6STgxZV4fpZm9HzApZGbQkNXh/ksQlTWtwpKTGpi8qjyXIU5jjSZAWDPppvYqj4c+DM+N54cKyxFfu5RM1GO+XJgtke0u/3mvNf/YyszaZ8r1prWxZEVp5MTivNKeCt1nzmXIoSS+DAOlUuDuXCwixUfiRfl+8xfA4h+/DoA8lOqDCvawFuLlRtgmPnYtXrfofCF//gEGFnJP1VK+Lo+1Rsjva/bgz3D/oEt9gT9kztsUS9prts3fskI2ZZN/YD/aT/Yq2o0mURnCBRr3uzBN2paLPvwF3GNoo</latexit> <latexit sha1_base64="BlYTNRSSw5Mn+3FsC7sLJk53Ec4=">AAACzXicfVFda9RAFJ3Er7pau1XQB18GF0GkhE0ptfhUqA++SKu4baGzDZPZu+nQySTM3NQN2QR88jf6C/wbTnZTP7bihYFzzzmXe+feOFfS4nD43fNv3b5z997a/d6Dh+uPNvqbj49tVhgBI5GpzJzG3IKSGkYoUcFpboCnsYKT+PKg1U+uwFiZ6c9Y5jBOeaLlVAqOjor6XxnCDC2WClgqdVSxlONFPKVfmNR0mcTVp/p8UrP5b21+vk0XlVXT2ACDpqnZNDNcKSrfUqa4ThTQX/6tMpLXycxB0+kJ0DDqD4bBcBH0Jgg7MCBdHEWb3g6bZKJIQaNQ3NqzcJjjuOIGpVBQ91hhIefikidw5qDmKdhxtdhVTV86ZkLdrO5ppAv2z4qKp9aWaeyc7cB2VWvJf2pxutIZp3vjSuq8QNBi2XhaKIoZbQ9BJ9KAQFU6wIWRbnYqLrjhAt25ej32DtznDHxwjQ5zMBwz87pi3CTuTLX7bMK2WvQ/I59dGx1yew5Xt3oTHG8H4W6w+3FnsH/QbXyNPCcvyCsSkjdkn7wnR2REBPnhrXtPvWf+oV/4c79ZWn2vq3lC/gr/209H+uI+</latexit> ŵ <latexit sha1_base64="Zw/0V8Yw0bx42w0Ht/geiS1KiM0=">AAACW3icfZDBSiNBEIY7o646665R8eSlMSzIsoQZEdejoAcvosJGBSeEmk5N0tjdM3TXrIYhT7LX9aE8+C72xAhuXCxo+Pjrb6rqTwslHUXRYyOYm1/4tLi0HH5e+fJ1tbm2funy0grsiFzl9joFh0oa7JAkhdeFRdCpwqv09qjuX/1G62RuftGowK6GgZGZFEBe6jVXkyFQlWigYZrxu3Gv2Yra0aT4e4in0GLTOu+tNfaSfi5KjYaEAudu4qigbgWWpFA4DpPSYQHiFgZ449GARtetJpuP+Tev9HmWW/8M8Yn69kcF2rmRTr2zXtHN9mrxv71Uz0ym7KBbSVOUhEa8DM5KxSnndSy8Ly0KUiMPIKz0u3MxBAuCfHhhmByjP87iqR90VqAFyu33KgE70NKM/bGD5EdNHxnh/tXoyeccz6b6Hi532/F+e/9ir3V4NE18iW2xbbbDYvaTHbITds46TLCS/WF/2UPjKZgLwmDlxRo0pn822D8VbD4DluO2sA==</latexit> ŵ w(t) lim = t!1 kw(t)k kŵk minw2Rd kwk 2 s.t. 8i : hw, yi xi i 1 一 大 二 8 1 Soudry, D., Hoffer, E., Nacson, M., Gunasekar, S., & Srebro, N. (2018). The implicit bias of gradient descent on separable data. 3 8 📄 [Srebro+ ]
ŵ w(t) lim = t!1 kw(t)k kŵk minw2Rd kwk 2 s.t. 8i : hw, yi xi i 1 yi x i <latexit sha1_base64="DtGYJwzS8y9N9JprVZsArKmi2KE=">AAACW3icfZBNS8NAEIa38avGj1bFk5dgEUSkJFLUo1APXkQFq4IpYbKd1KW7m7C7EUvoL/GqP8qD/8VNraBVHFh4eOddZuaNM8608f23ijMzOze/UF10l5ZXVmv1tfUbneaKYoemPFV3MWjkTGLHMMPxLlMIIuZ4Gw/aZf/2EZVmqbw2wwy7AvqSJYyCsVJUrw0jFgowD3FSPI0iFtUbftMfl/cbggk0yKQuo7VKK+ylNBcoDeWg9X3gZ6ZbgDKMchy5Ya4xAzqAPt5blCBQd4vx5iNvxyo9L0mVfdJ4Y/X7jwKE1kMRW2e5pJ7uleKfvVhMTTbJcbdgMssNSvo5OMm5Z1KvjMXrMYXU8KEFoIrZ3T36AAqoseG5bniK9jiF53bQRYYKTKr2ihBUXzA5ssf2w/2S/jPC05fRks05mE71N9wcNIPD5uFVq3HSniReJVtkm+ySgByRE3JGLkmHUJKTZ/JCXivvzozjOsufVqcy+bNBfpSz+QEyUrcB</latexit> ŵ <latexit sha1_base64="b9lHQB0WiDlL52SNoShtYtOjOvY=">AAACW3icfVBRSxtBEN6crdqr1VjpU18WQ6GUEu6CxORNsA++SC00KnghzG3mkiW7e8funDYc+SV9bX+UD/4X92KEVloHBj6+7xtm5ksLJR1F0W0jWHvxcn1j81X4euvN9k5z9+25y0srcCByldvLFBwqaXBAkhReFhZBpwov0tlxrV9co3UyN99pXuBQw8TITAogT42aO8kUqEo00DTN+M1i1GxF7chXt8trEPei2IN+v9fp9Hm8lKKoxVZ1NtptHCTjXJQaDQkFzl3FUUHDCixJoXARJqXDAsQMJnjloQGNblgtL1/wD54Z8yy3vg3xJfvnRAXaublOvbM+0T3VavKfWqqfbKasN6ykKUpCIx4WZ6XilPM6Fj6WFgWpuQcgrPS3czEFC4J8eGGYfEH/nMVTv+hrgRYot5+qBOxES7Pwz06SzzV6zgg/Ho0e+Zwfw+T/B+eddtxtd78dtI6OV4lvsvdsn31kMTtkR+yEnbEBE6xkP9kv9rtxF6wFYbD1YA0aq5k99lcF7+4BBc627Q==</latexit> 大 8 1 Soudry, D., Hoffer, E., Nacson, M., Gunasekar, S., & Srebro, N. (2018). The implicit bias of gradient descent on separable data. 4 8 📄 [Srebro+ ]
i) <latexit sha1_base64="ebrEJfSGKzxfK5aTUbND9tbsZfo=">AAACg3icfVBdaxNBFJ2sVmv8aKqPvgwGoX4QdktofQkU6kMfFCuYttCJy93J3c3Qmdll5q4Y1/wUf42v9Qf4b5xNI2gqXrhwOOdc7r0nq7TyFMc/O9GNmxu3bm/e6d69d//BVm/74YkvaydxLEtdurMMPGplcUyKNJ5VDsFkGk+zi8NWP/2EzqvSfqB5hRMDhVW5kkCBSnv7ozc79IwLjTmJr8LXJm3UKFl8tAJ0NYNUBVkYoFmW8y+pEk4Vs+BMe/14EC+LXwfJCvTZqo7T7c5QTEtZG7QkNXh/nsQVTRpwpKTGRVfUHiuQF1DgeYAWDPpJs/xwwZ8GZsrz0oW2xJfsnxMNGO/nJgvO9la/rrXkP7XMrG2m/NWkUbaqCa28WpzXmlPJ2/j4VDmUpOcBgHQq3M7lDBxICiF3u+I1huccvg2L3lXogEr3vBHgCqPsIjxbiJct+p8RPv82BhRyTtZTvQ5OdgfJ3mDv/bB/cLhKfJM9Zk/YDkvYPjtgR+yYjZlk39h3dsl+RBvRi2g3Gl5Zo85q5hH7q6LRL7qAxbQ=</latexit> krL(t)k = n X e hw(t),zi i zi = L(t) i=1 n X L̇(t) = e hw(t),zi i i=1 <latexit sha1_base64="Uz3PxTUp6gaxbfxTmD0LQGPodm4=">AAACdnicfVDRShtBFJ2sbdXY1qiPggwNopUadkXUR0EffKjUglHBTcPdyc06ODO7zNwVw5J/6Nf0VX/DP/Gxs0kKbSxeGDhzzrlz554kV9JRGD7Vgpk3b9/Nzs3XF95/+LjYWFq+cFlhBbZFpjJ7lYBDJQ22SZLCq9wi6EThZXJ7VOmXd2idzMw5DXLsaEiN7EsB5KluY2v76yZ9/rHDY4U87mXEq/voth2noDV4bWzpNpphKxwVfwmiCWiySZ11l2q7/kVRaDQkFDh3HYU5dUqwJIXCYT0uHOYgbiHFaw8NaHSdcrTUkK97psf7mfXHEB+xf3eUoJ0b6MQ7NdCNm9Yq8r9aoqcmU/+gU0qTF4RGjAf3C8Up41VivCctClIDD0BY6f/OxQ1YEORzrdfjY/TLWTz1g77laIEyu1XGYFMtzdAvm8ZfKvSaEe7/GD3yOUfTqb4EFzutaK+19323eXg0SXyOrbJPbJNFbJ8dshN2xtpMsJ/sF3tgj7XnYC1YDzbG1qA26Vlh/1QQ/gaGKb4h</latexit> 2 2 言 一 言 0 L(t) L̇(t) e j z = L(t) i hw(t),z i j =: αi(t) n X ↵i (t)zi i=1 γ ≤ ∥Zα∥ ≤ 1 |`0 (·)| . `(·) <latexit sha1_base64="FKnMI/NRfyI3fAHsoYUydug33L0=">AAACb3icfVDbahRBFOydeInrLYkPPgjSuIiJyNAj2Y2+BeJDXoIR3CSQWcKZ3rObJn0Zus+ELJP9AL/GV/0UP8M/sGczAQ1qQUNRpw7Vp4pSq0BC/OgkS7du37m7fK97/8HDR49XVtcOgqu8xKF02vmjAgJqZXFIijQelR7BFBoPi7OdZn54jj4oZz/TrMSRgalVEyWBonSy0rvMUetX67kcO9q45LnGEIIyvJFbNbpE2n8fscVFOhBCZP2WiD7PUrFAj7XYP1ntbOZjJyuDlqSGEI4zUdKoBk9Kapx38ypgCfIMpngcqQWDYVQvrpnzl1EZ84nz8VniC/X3jRpMCDNTRKcBOg03Z43411lhbiTT5N2oVrasCK28Cp5UmpPjTVV8rDxK0rNIQHoV/87lKXiQFAvtdvMPGI/zuBeDPpbogZx/Xefgp0bZeTx2mr9p2P+McHFtjCz2fF0m/zc5eJtmg3TwabO3vdM2vsyesRdsnWVsi22zXbbPhkyyL+wr+8a+d34mT5PnCb+yJp125wn7A8nGL1w4vmQ=</latexit> hẇ(t), zi i = L(t) e P hw(t),zi i : self-bounding self-bounding property <latexit sha1_base64="aI1SRW4FfkgIkUf2hYJq03lQpY8=">AAAC1XicfVHbahRBEO0Zb3G9bRSfBG1chCjJMhNC4ksgEB98UIzgZgPp3aWmt2bSpLtn6O6JrpN5E8Env8+v8BfsmWyCboIFDaerTtWpS1JIYV0U/QrCa9dv3Ly1dLtz5+69+w+6yw/3bV4ajgOey9wcJGBRCo0DJ5zEg8IgqETiMDnebeLDEzRW5PqTmxU4UpBpkQoOzrsm3R9smjv6bsW9pNt0jdlSTSqxHddjTXFcrTEJOpNImQJ3lKT0syeunn++TgQzbby+4Plq1QW5bspeQW+lTpmGREKrzU7H65NuL+pHrdHLIJ6DHpnb3mQ52PByvFSoHZdg7WEcFW5UgXGC+546rLRYAD+GDA891KDQjqp2ZzV94T1TmubGP+1o6/07owJl7Uwlntn0bxdjjfPKWKIWlF36elQJXZQONT8TTktJXU6bg9CpMMidnHkA3AjfO+VHYIA7f7ZOh71BP5zB917oQ4EGXG5eVQxMpoSu/bAZW23Q/4jw5Zzokd9zvLjVy2B/vR9v9jc/bvR2ducbXyJPyHOyQmKyRXbIW7JHBoST38Hj4GnwLByGdfgt/H5GDYN5ziPyj4U//wB8NeF3</latexit> n X i=1 L krLk L <latexit sha1_base64="FRiVxzXG0ouxmD/o+wjDZiJZVw4=">AAACa3icfZDfShtBFMYna7Wa+ieaO1twaBBEJOwWUS+F9MILSy0YE3BCODs52QyZmV1mZsWw5tKn8bY+jA/hO3QSU2ijeGDgN9/5hjPnizMprAvDp1Kw8GFx6ePySvnT6tr6RmVz68qmueHY5KlMTTsGi1JobDrhJLYzg6Biia142Jj0WzdorEj1pRtl2FGQaNEXHJyXupUdloBSQM8pk0jZHdMQS39ld1PhvFuphfVwWvQ1RDOokVlddDdLh6yX8lyhdlyCtddRmLlOAcYJLnFcZrnFDPgQErz2qEGh7RTTTcZ01ys92k+NP9rRqfrviwKUtSMVe6cCN7DzvYn4Zi9Wc5Nd/6RTCJ3lDjV/GdzPJXUpncREe8Igd3LkAbgR/u+UD8AAdz7Mcpl9R7+cwR9+0M8MDbjU7BcMTKKEHvtlE3YwofeMcPvX6MnnHM2n+hquvtWjo/rRr8PaaWOW+DL5TL6SPRKRY3JKzsgFaRJO7skD+U0eS89BNdgOvrxYg9LsTZX8V8HuH9k2u8E=</latexit> 7 8 <latexit sha1_base64="7UuNyefxwLyJB6CgK0pFR3Lpe5I=">AAACj3icfVBdSxtBFJ1sbbXph0l97MvQIMRiw66I9sUSSB8stNSCUcFJl7uTu+vgzOwyM9sal/yg/hpf9d84G2NpY+mBgTPnnsu99ySFFNaF4U0jeLT0+MnyytPms+cvXq622q+ObF4ajkOey9ycJGBRCo1DJ5zEk8IgqETicXI+qOvHP9BYketDNylwpCDTIhUcnJfi1uBz123QPUqZLVVcib1o+l0zvCi675gEnUmkTIE7S1L60zs3f/8uY8HMzLARtzphL5yBPiTRnHTIHAdxu7HNxjkvFWrHJVh7GoWFG1VgnOASp01WWiyAn0OGp55qUGhH1ezaKV33ypimufFPOzpT/+yoQFk7UYl31rvaxVot/rOWqIXJLn0/qoQuSoea3w1OS0ldTuso6VgY5E5OPAFuhN+d8jMwwJ0PvNlkH9EfZ/CLH/S1QAMuN28rBiZTQk/9sRnbrNn/jHBxb/TM5xwtpvqQHG31op3ezrftTn8wT3yFvCZvSJdEZJf0yT45IEPCyS9yRa7JTdAOdoMPQf/OGjTmPWvkLwSfbgEM2sit</latexit> 2 2 krL(t)k <latexit sha1_base64="RM6GLvk33BZPeHaWeGtUn79D8SA=">AAACV3icfZBRSxtBEMf3zqoxrdXoY1+WhkIUSe9E1BdB0Ic+KFpoVPAOmdtM4uLu3rk7J4Yjn8PX9mP5aXQTI9hYOrDw4z//ZWb+WaGkoyh6DMKZD7Nz87WF+sdPi5+XlhsrZy4vrcCOyFVuLzJwqKTBDklSeFFYBJ0pPM9uDkb98zu0TubmFw0KTDX0jexJAeSl9KhFa3yPn7Ti77R2tdyM2tG4+HuIJ9Bkkzq9agRbSTcXpUZDQoFzl3FUUFqBJSkUDutJ6bAAcQN9vPRoQKNLq/HWQ/7NK13ey61/hvhYffujAu3cQGfeqYGu3XRvJP6zl+mpydTbTStpipLQiJfBvVJxyvkoEt6VFgWpgQcQVvrdubgGC4J8cPV6coj+OIvHftBJgRYot+tVAravpRn6Y/vJxoj+Z4T7V6Mnn3M8nep7ONtsx9vt7Z9bzf2DSeI19oV9ZS0Wsx22z36wU9Zhgt2yB/ab/Qkeg6dwLqy9WMNg8meV/VVh4xk237R2</latexit> L(t) = O(1/t) (Step )
i) <latexit sha1_base64="ebrEJfSGKzxfK5aTUbND9tbsZfo=">AAACg3icfVBdaxNBFJ2sVmv8aKqPvgwGoX4QdktofQkU6kMfFCuYttCJy93J3c3Qmdll5q4Y1/wUf42v9Qf4b5xNI2gqXrhwOOdc7r0nq7TyFMc/O9GNmxu3bm/e6d69d//BVm/74YkvaydxLEtdurMMPGplcUyKNJ5VDsFkGk+zi8NWP/2EzqvSfqB5hRMDhVW5kkCBSnv7ozc79IwLjTmJr8LXJm3UKFl8tAJ0NYNUBVkYoFmW8y+pEk4Vs+BMe/14EC+LXwfJCvTZqo7T7c5QTEtZG7QkNXh/nsQVTRpwpKTGRVfUHiuQF1DgeYAWDPpJs/xwwZ8GZsrz0oW2xJfsnxMNGO/nJgvO9la/rrXkP7XMrG2m/NWkUbaqCa28WpzXmlPJ2/j4VDmUpOcBgHQq3M7lDBxICiF3u+I1huccvg2L3lXogEr3vBHgCqPsIjxbiJct+p8RPv82BhRyTtZTvQ5OdgfJ3mDv/bB/cLhKfJM9Zk/YDkvYPjtgR+yYjZlk39h3dsl+RBvRi2g3Gl5Zo85q5hH7q6LRL7qAxbQ=</latexit> krL(t)k = n X ẇ(t) = e hw(t),zi i zi = L(t) i=1 rL(t) <latexit sha1_base64="d27rWqTcGStGblc62IeRjh5FZ44=">AAACbXicfVBdSxtBFJ1srdXY1tjiU0WmDaUqbdgtor4UBH3oQ0WFRgU3hLuTu3FwZnaZuasNS579Nb62v6W/on/B2RihRvHCwJlzz/06Sa6kozD8WwueTT2ffjEzW597+er1fGPhzZHLCiuwLTKV2ZMEHCppsE2SFJ7kFkEnCo+T850qf3yB1snM/KRBjh0NfSNTKYA81W28j3sZlbEGOktSfjlcoVX+jX+JDSQK+A//7TaaYSscBX8IojFosnEcdBdq676pKDQaEgqcO43CnDolWJJC4bAeFw5zEOfQx1MPDWh0nXJ0y5B/9EyPp5n1zxAfsf9XlKCdG+jEK6ul3WSuIh/NJXpiMqVbnVKavCA04nZwWihOGa+M4j1pUZAaeADCSr87F2dgQZC3s16Pd9EfZ3HPD9rP0QJldq2Mwfa1NEN/bD/+XKGnhPDrTuiR9zmadPUhOPraijZaG4frze2dseMz7B37wFZYxDbZNvvODlibCXbFrtlv9qf2L1gMloLlW2lQG9e8Zfci+HQDW2S8aA==</latexit> n X i=1 hw(t),zi i e P e j z = L(t) i hw(t),z i j =: αi(t) <latexit sha1_base64="g19DYhnvpUMDXiE2nziN4Qedxzc=">AAACg3icfVBdSxtBFJ1sa6vR1qiPvgwNBT9K2JWgfREEfehLqYJRwQ3h7uRuHJyZXWbutoZlf0p/ja/6A/w3nY0RNEovDJw599yvk+RKOgrDh0bw7v3ch4/zC83FpU+fl1srq2cuK6zAnshUZi8ScKikwR5JUniRWwSdKDxPrg/r/PlvtE5m5pTGOfY1jIxMpQDy1KC1Fw8zKmMNdJWk/E+1QZt8n9ckt09swbf957muqAatdtgJJ8Ffg2gK2mwax4OVRte3EIVGQ0KBc5dRmFO/BEtSKKyaceEwB3ENI7z00IBG1y8nF1b8q2eGPM2sf4b4hH1eUYJ2bqwTr6xXdLO5mnwzl+iZyZR+75fS5AWhEY+D00JxynhtHx9Ki4LU2AMQVvrdubgCC4K8yc1mfIT+OIs//aBfOVqgzG6VMdiRlqbyx47ibzX6nxBunoQeeZ+jWVdfg7OdTrTb2T3ptg8Op47Ps3X2hW2wiO2xA/aDHbMeE+wvu2V37D6YC7aDnaD7KA0a05o19iKC/X9j3MUA</latexit> <latexit sha1_base64="ceY1klkK67Ll+4yb87pgAaBH7vk=">AAACrHicfVFdaxQxFM2OX3X96FYffQkuQpW6zEipvhQK9cEXsUK3LWym453snWlokhmSO+o67N/zP/gffNVnM9st6LZ4IXBy7rmcm5O81spTHP/oRTdu3rp9Z+1u/979Bw/XBxuPjnzVOIljWenKneTgUSuLY1Kk8aR2CCbXeJyf73f948/ovKrsIc1qTA2UVhVKAgUqG3zaFb4xWat2k/mpxdP2pdBgS41cGKCzvOBfNun51uXlW6aEW/Tnq7rmel02GMajeFH8KkiWYMiWdZBt9LbFtJKNQUtSg/eTJK4pbcGRksG3LxqPNchzKHESoAWDPm0XUcz5s8BMeVG5cCzxBfv3RAvG+5nJg7Jb1a/2OvLaXm5WnKl4k7bK1g2hlRfGRaM5VbzLmU+VQ0l6FgBIp8LuXJ6BA0nhN/p98RbD4xy+D0YfanRAlXvRCnClUXYeHluKrQ79TwhfL4UBhZyT1VSvgqNXo2RntPNxe7i3v0x8jT1hT9kmS9hrtsfesQM2ZpJ9Zz/ZL/Y7GkWH0SRKL6RRbznzmP1TUfEHyAnWRw==</latexit> = n X i=1 <latexit sha1_base64="H28B7NZxjlH3pQP51/Hqla2B7n8=">AAACg3icfVBdaxNBFJ2sVmv8SuujL4NBqFrCbgmtL4VCffBBsYJpi50Q7k7ubobOxzJzVxqW/BR/ja/6A/w3zqYRNBUvDHM491zuPSevtAqUpj87ya3bG3fubt7r3n/w8NHj3tb2aXC1lziSTjt/nkNArSyOSJHG88ojmFzjWX553PbPvqAPytlPNK9wbKC0qlASKFKT3sHhux16ITTYUiMXBmiWF7yO3C7/LHKnp2Fu4idAVzNopX4pnfT66SBdFr8JshXos1WdTLY6QzF1sjZoSWoI4SJLKxo34ElJjYuuqANWIC+hxIsILRgM42bpcMGfR2bKC+fjs8SX7J8TDZjQHhqVrYWw3mvJf/Zys7aZitfjRtmqJrTyenFRa06Ot/HxqfIoSc8jAOlVvJ3LGXiQFEPudsUbjOY8vo+LPlTogZx/2QjwpVF2Ec2WYrdF/xPC1W9hRDHnbD3Vm+B0b5DtD/Y/DvtHx6vEN9lT9oztsIwdsCP2lp2wEZPsK/vGvrMfyUbyKtlLhtfSpLOaecL+quTwFxr+xOU=</latexit> ↵i (t)zi i=1 hu(t), rL(t)i ṙ(t) = ẇ(t) = ṙu + ru̇ hu, ẇi = ṙ + rhu, u̇i n X γ ≤ ∥Zα∥ ≤ 1 <latexit sha1_base64="1halCfcGzBByYr9cJKDTh3/GJEc=">AAACgnicfVBdaxNBFJ2sVWv8aKqPvgwNQi017LYl9kGhUB98aLGCaQudEO5O7m6HzswuM3eLYck/8df4qn/Af+NskkKbihcGzpx7Lufek5ZaeYrjP63owcrDR49Xn7SfPnv+Yq2z/vLUF5WTOJCFLtx5Ch61sjggRRrPS4dgUo1n6dVh0z+7RudVYb/RpMShgdyqTEmgQI06fTEuiLtNess/8ndCg801cmGALtOMV4Hf5sJCqoEfhY9wc8Go04178az4fZAsQJct6mS03toLTrIyaElq8P4iiUsa1uBISY3Ttqg8liCvIMeLAC0Y9MN6duCUvwnMmGeFC88Sn7G3J2ow3k9MGpTN5n6515D/7KVmyZmy/WGtbFkRWjk3zirNqeBNenysHErSkwBAOhV25/ISHEgKGbfb4hOG4xweB6MvJTqgwm3VAlxulJ2GY3Ox3aD/CeH7jTCgkHOynOp9cLrTS/q9/te97sHhIvFV9pptsE2WsPfsgH1mJ2zAJPvBfrJf7He0Em1FSbQ7l0atxcwrdqeiD38B0aPCyA==</latexit> <latexit sha1_base64="q+7ncnZrRwYGFSWoOE6wUV89GWg=">AAACq3ichVFNbxMxEHW2fJSUj7QcuViNKiEo0S6qCpdKlcqBC6IVpK2oo2jWmd1atb0re5YSrfLz+iP6G7jCHW8+pJIiOpKl55n3PDPPaamVpzi+bkUr9+4/eLj6qL32+MnTZ531jWNfVE5iXxa6cKcpeNTKYp8UaTwtHYJJNZ6kFwdN/eQ7Oq8K+5XGJQ4M5FZlSgKF1LAzFBpsrpELA3SeZrza5mJUUL24X06EmzH2pgXu+Gvu7lJVC9Ww04178TT4bZDMQZfN43C43toJL8nKoCWpwfuzJC5pUIMjJTVO2qLyWIK8gBzPArRg0A/qqRMTvhUyI54VLhxLfJq9qajBeD82aWA2k/rlWpP8Zy01S50pez+olS0rQitnjbNKcyp4YzMfKYeS9DgAkE6F2bk8BweSwme02+IDhuUcfgqNPpfogAr3qhbgcqPsJCybi+0G/Y8IPxbEgILPybKrt8Hx216y29s92unuH8wdX2Uv2CZ7yRL2ju2zj+yQ9ZlkV+wn+8V+R2+iL9G3SMyoUWuuec7+igj/AJQJ1U4=</latexit> e hw(t),zi i hu(t), zi i = L(t)hu(t), Z↵(t)i 1 <latexit sha1_base64="RYJ/+JLtDWjbLEB/6j1ZlmO6AJo=">AAACaHicfVBNSyNBEO2Mu37Er+gelkUPjWFBFw0zIupR0IOHlXXBqOCEUNOpxMbunqG7RgxDLvtrvLr/xr/gr7AnRtC4bEHD61ev+nW9JFPSURg+VoKJT58np6ZnqrNz8wuLtaXlc5fmVmBTpCq1lwk4VNJgkyQpvMwsgk4UXiQ3h2X/4hatk6k5o36GLQ09I7tSAHmqXVvd+rlOGzxWyONOSty+3kq6XauHjXBY/COIRqDORnXaXqrs+FdErtGQUODcVRRm1CrAkhQKB9U4d5iBuIEeXnloQKNrFcM1Bvy7Zzq8m1p/DPEh+3aiAO1cXydeqYGu3XivJP/ZS/SYM3X3W4U0WU5oxItxN1ecUl5mxDvSoiDV9wCElf7vXFyDBUE+yWo1PkK/nMUTb/QrQwuU2h9FDLanpRn4ZXvxZon+J4S7V6FHPudoPNWP4Hy7Ee02dn/v1A8OR4lPsxW2xtZZxPbYATtmp6zJBPvD7tkD+1t5CmrB1+DbizSojGa+sHcVrD0D5mC5Xw==</latexit> 8 8 <latexit sha1_base64="7UuNyefxwLyJB6CgK0pFR3Lpe5I=">AAACj3icfVBdSxtBFJ1sbbXph0l97MvQIMRiw66I9sUSSB8stNSCUcFJl7uTu+vgzOwyM9sal/yg/hpf9d84G2NpY+mBgTPnnsu99ySFFNaF4U0jeLT0+MnyytPms+cvXq622q+ObF4ajkOey9ycJGBRCo1DJ5zEk8IgqETicXI+qOvHP9BYketDNylwpCDTIhUcnJfi1uBz123QPUqZLVVcib1o+l0zvCi675gEnUmkTIE7S1L60zs3f/8uY8HMzLARtzphL5yBPiTRnHTIHAdxu7HNxjkvFWrHJVh7GoWFG1VgnOASp01WWiyAn0OGp55qUGhH1ezaKV33ypimufFPOzpT/+yoQFk7UYl31rvaxVot/rOWqIXJLn0/qoQuSoea3w1OS0ldTuso6VgY5E5OPAFuhN+d8jMwwJ0PvNlkH9EfZ/CLH/S1QAMuN28rBiZTQk/9sRnbrNn/jHBxb/TM5xwtpvqQHG31op3ezrftTn8wT3yFvCZvSJdEZJf0yT45IEPCyS9yRa7JTdAOdoMPQf/OGjTmPWvkLwSfbgEM2sit</latexit> L(t) ṙ(t) L(t) <latexit sha1_base64="CfNqES2E21yVkH2Ja8Pg7riXN/I=">AAACYXicfZBdSxtBFIYna1tj+pUoXnkzNBRiKWG3iHojCPbCG6kFo4IbwtnJyWbIfCwzZ0vDkh/jrf4ir/tHnMQU2lh6YODhPe9wznmzQklPcfxQi9ZevHy1Xt9ovH7z9t37Zmvz0tvSCewJq6y7zsCjkgZ7JEnhdeEQdKbwKpuczPtXP9B5ac0FTQvsa8iNHEkBFKRBc9t1aJcf8fRijASdVNmc0+6g2Y678aL4c0iW0GbLOh+0anvp0IpSoyGhwPubJC6oX4EjKRTOGmnpsQAxgRxvAhrQ6PvVYv8Z/xiUIR9ZF54hvlD//FGB9n6qs+DUQGO/2puL/+xlemUyjQ77lTRFSWjE0+BRqThZPg+HD6VDQWoaAISTYXcuxuBAUIiw0Ui/YjjO4VkY9K1AB2TdpyoFl2tpZuHYPP08p/8Z4edvY6CQc7Ka6nO4/NJN9rv73/faxyfLxOtsh31gHZawA3bMTtk56zHBKnbL7th97Ve0ETWjzSdrVFv+2WJ/VbTzCKTlt38=</latexit> r(t) = ⇥(log t) (Step )
w2Rd 2 1 1 2 Lagrangian L := kwk + 2 i=1 <latexit sha1_base64="XcXzG+l8Ay+nJsPqQ6yDt6b7HpQ=">AAACeHicfVDRShtBFJ1stdVU21gf+zIYilYk7EqIvghCfOhLaQSjgWy63J3c3QzOzC4zs9p0yU/4Nb62f+Gv9KmzMUIbSy8MHM49d869J84FN9b3H2rei5XVl6/W1uuvNzbfvG1svbs0WaEZ9lkmMj2IwaDgCvuWW4GDXCPIWOBVfN2t+lc3qA3P1IWd5jiSkCqecAbWUVHjIJRgJ3FCb+kJDU0ho5KfBLOvKhTukzFE/EnwPeJRo+m3/HnR5yBYgCZZVC/aqrXDccYKicoyAcYMAz+3oxK05UzgrB4WBnNg15Di0EEFEs2onJ81ox8cM6ZJpt1Tls7ZPydKkMZMZeyU1Y5muVeR/+zFcsnZJsejkqu8sKjYo3FSCGozWmVGx1wjs2LqADDN3e6UTUADsy7Zej08Q3ecxs/O6EuOGmym98sQdCq5mrlj0/CgQv8TwrcnoUMu52A51efg8rAVdFqd83bztLtIfI28JztkjwTkiJyST6RH+oSRO3JPfpCftV8e9Xa9j49Sr7aY2SZ/lXf4G558wh4=</latexit> w= n X i zi hw, zi i) i (1 <latexit sha1_base64="Yn2t8qmi2c1sfMiD+1bEqImZlBY=">AAACW3icfZDPSsNAEMa38X+stiqevASLUEVKIqIeBT14ERWsFppSJttpXLq7CbsbsYQ+iVd9KA++i9saQas4sPDjm2+ZmS9KOdPG999Kzszs3PzC4pK7XF5ZrVTX1u90kimKTZrwRLUi0MiZxKZhhmMrVQgi4ngfDc7G/ftHVJol8tYMU+wIiCXrMwrGSt1qpR5y6+5Bl4Ux+rvdas1v+JPyfkNQQI0Udd1dKx2GvYRmAqWhHLRuB35qOjkowyjHkRtmGlOgA4ixbVGCQN3JJ5uPvB2r9Lx+ouyTxpuo33/kILQeisg6BZgHPd0bi3/2IjE12fRPOjmTaWZQ0s/B/Yx7JvHGsXg9ppAaPrQAVDG7u0cfQAE1NjzXDc/RHqfw0g66SlGBSdReHoKKBZMje2wc7o/pPyM8fRkt2ZyD6VR/w91BIzhqHN0c1k7PisQXyRbZJnUSkGNySi7INWkSSjLyTF7Ia+ndmXFcp/xpdUrFnw3yo5zNDxWyteg=</latexit> ( i 0) i=1 <latexit sha1_base64="PYfuZ308YuDxv5HJsWTGVXk7/qc=">AAACh3icfZDRShtBFIYnW2s1tRrby94MhkIRSXdFUkEsil70RrTQqOCE5ezk7GZwZnaZmdWmSx7Gp/HWXvo2nU0itLH0wMDHf/7hnPMnhRTWheFjI3ix8HLx1dJy8/XKm9W11vrbc5uXhmOP5zI3lwlYlEJjzwkn8bIwCCqReJFcH9X9ixs0VuT6uxsV2FeQaZEKDs5LcWuPSW8eQCzoFxpSJtKUeklnEilT4IZJSm+3nuhnLJiZNvdpFLfaYSecFH0O0QzaZFZn8Xpjhw1yXirUjkuw9ioKC9evwDjBJY6brLRYAL+GDK88alBo+9XkyjH94JUBTXPjn3Z0ov75owJl7Ugl3lmva+d7tfjPXqLmJrt0t18JXZQONZ8OTktJXU7rCOlAGOROjjwAN8LvTvkQDHDng2422TH64wye+EGnBRpwudmsGJhMCT32x2Zsq6b/GeHHk9GTzzmaT/U5nG93om6n+22nfXA0S3yJvCcb5COJyGdyQL6SM9IjnNyRe/JAfgXLwaegG+xOrUFj9ucd+auCw98r0sWq</latexit> () hw, zi i = 1 i >0 課 <latexit sha1_base64="skSM7VSREQBWmr+GOZy4KcQnWuw=">AAACv3icfVFtaxQxEM6ub9fz7aof/RI8hNbWY1dKFYpQqR/8Ilbw2kJzPWZzs9vQJLsks+q53A/tr9Hs9SrnVRwIPHmemczMk6zSylOSXEbxrdt37t7rrHXvP3j46HFv/cmRL2sncShLXbqTDDxqZXFIijSeVA7BZBqPs4uDVj/+hs6r0n6laYUjA4VVuZJAgRr3vDBA51nOv2/QVropCiTPl7hNvsWFr824Ue/S2ZkVSCD2OJ41r1xQhQZbaPxTUQdu+/ryc6yEm+szsbdEjnv9ZJDMg98E6QL02SIOx+vRjpiUsjZoSWrw/jRNKho14EjJ8HpX1B4rkBdQ4GmAFgz6UTN3Z8ZfBGbC89KFY4nP2eWKBoz3U5OFzHZGv6q15D+1zKx0pvztqFG2qgmtvGqc15pTyVvr+UQ5lKSnAYB0KszO5Tk4kBQ+qNsVHzAs5/BTaPS5QgdUupeNAFcYZWdh2UJst+h/ifDjOjGg4HO66upNcPR6kO4Odr/s9PcPFo532DP2nG2wlL1h++wjO2RDJtkl+xV1orX4fVzENq6uUuNoUfOU/RXx9Dcnv9oV</latexit> w(t + 1) w(t) + n X r(t) → ∞ <latexit sha1_base64="QzvY3r0hmwy5ytRxREh7qUSl4P4=">AAACmnicfZDfShwxFMazY/3TtepqL9uL4FIoIsuMiIogSO1FS2lVcFUwy3Ame2YMJpkhyYjrsG/Vl2lv2wdpZneldZUeCPw433c4OV9SSGFdGP5oBDMvZufmF142F18tLa+0VtfObV4ajl2ey9xcJmBRCo1dJ5zEy8IgqETiRXJzVOsXt2isyPWZGxTYU5BpkQoOzrfi1jdmSxVXYp8yCTqTSJkCd52ktIw3Nh/4PhbMjOUDyjJQCob1gEr64KW/rrjVDjvhqOhTiCbQJpM6iVcb26yf81KhdlyCtVdRWLheBcYJLnHYZKXFAvgNZHjlUYNC26tGhw/pO9/p0zQ3/mlHR91/JypQ1g5U4p31H+20Vjef1RI1tdmle71K6KJ0qPl4cVpK6nJap0r7wiB3cuABuBH+75RfgwHufPbNJvuI/jiDX/2i4wINuNxsVAxMpoQe+mMztlnT/4xw92D05HOOplN9CudbnWins3O63T48miS+QN6QdfKeRGSXHJJP5IR0CSffyU/yi/wO3gYfgs/Bl7E1aExmXpNHFZz9Acluzrk=</latexit> ⌘e r(t)hu(t),zi i zi KKT : u(t) ! 9u⇤ 小 大 <latexit sha1_base64="UY0P+omkPXXD1yHCEdKFFxOnbGg=">AAACbnicfVBdaxNREL1Ztdb0w7SCLyJeGoS0lLBbSutjID74IlYwbaEbwuzNbHrp/VjunS0NS979Nb7qX/Ff+BO8m0bRtHRg4HDmDGfmZIWSnuL4ZyN69PjJytPVZ8219Y3N562t7VNvSydwIKyy7jwDj0oaHJAkheeFQ9CZwrPsql/Pz67ReWnNF5oWONQwMTKXAihQo9ZOqoEus5yXHdrlKdkUb4Kr/0uP9katdtyN58XvgmQB2mxRJ6OtxmE6tqLUaEgo8P4iiQsaVuBICoWzZlp6LEBcwQQvAjSg0Q+r+TMz/jYwY55bF9oQn7P/blSgvZ/qLCjrG/3yrCbvnWV6yZnyd8NKmqIkNOLWOC8VJ8vrpPhYOhSkpgGAcDLczsUlOBAU8mw20/cYnnP4MRh9KtABWbdXpeAmWppZeHaS7tfoISHc/BEGFHJOllO9C04PuslR9+jzYbvXXyS+yl6xHdZhCTtmPfaBnbABE+wr+8a+sx+NX9HL6HX05lYaNRY7L9h/FXV+A1Uwvec=</latexit> ⟨u(t), zi⟩ X i:hu⇤ ,zi i= i=1 0 9 n X <latexit sha1_base64="7xwyPjZ48Y5cbEBNCCinvYAXi48=">AAACs3icfVFdaxQxFM1O/WjXr2197EtwEarWZWapVYRCoT74oFjBbQub7XAnc2cammSGJKNup/sT+wP8Hb4qmJluRbfihcDJOfdybk6SUgrrwvBbJ1i6cfPW7eWV7p279+4/6K2uHdiiMhxHvJCFOUrAohQaR044iUelQVCJxMPkdK/RDz+jsaLQn9y0xImCXItMcHCeins5U+BOOEj6jr7eoSwzwKMhO2/pJKNf2PnxkD6jzFYqrsVONDvWlElvkEIsNiL6vLnpXCL9PbJ5hc5iwUwrPol7/XAQtkWvg2gO+mRe+/FqZ4ulBa8UasclWDuOwtJNajBOcImzLqsslsBPIcexhxoU2kndJjKjjz2T0qww/mhHW/bPiRqUtVOV+M5mV7uoNeQ/tUQtOLvs1aQWuqwcan5pnFWSuoI2cdNUGOROTj0AboTfnfIT8CE7/yndLnuD/nEG33ujDyUacIV5WjMwuRJ65h+bs80G/a8Rvl41euRzjhZTvQ4OhoNoe7D9cau/uzdPfJmsk0dkg0TkJdklb8k+GRFOLsh38oP8DF4E4yAJ0svWoDOfeUj+qkD9AoTs1rM=</latexit> <latexit sha1_base64="pWdS+Qxgl2MvWyQ02yYAPkSMCeE=">AAACx3icfVFNb9QwEHXCR8vytVuOXCxWSAhVUbKqCuJUqRzggCiIbSvV22jinaRWHSeynXaXNCvxMznyT3B2Uz62iJFsPb95oxm/SUopjA3D755/6/aduxub93r3Hzx89Lg/2Do0RaU5jnkhC32cgEEpFI6tsBKPS42QJxKPkvP9Nn90gdqIQn2x8xInOWRKpIKDdVTcr1kuVOxusGdJSi+ZUHT1SOrPzem0YakGHo3Y1W/J1emIMoszWy8WJrDBYuFUhQYpqXhDmQSVSaS/9NvX6GssmO6SGdIo7g/DIFwGvQmiDgxJFwfxwNth04JXOSrLJRhzEoWlndSgreASmx6rDJbAzyHDEwcV5Ggm9dKlhj53zJS6Qd1Rli7ZPytqyI2Z54lTtgOb9VxL/jOX5Gudbfp6UgtVVhYVXzVOK0ltQdsV0KnQyK2cOwBcCzc75WfgbLZuUb0ee4vucxo/uEYfS9RgC/2yZqAzt6rGfTZj2y36nxBm10KHnM/Ruqs3weEoiHaD3U87w739zvFN8pQ8Iy9IRF6RPfKOHJAx4eSHt+H1vYH/3i/8C3+2kvpeV/OE/BX+t5/Nzd7M</latexit> i.e. i zi
(Edge of stability) <latexit sha1_base64="5FSkUlokt7wtfo9S2WxLMEMRtnw=">AAACinicfZDPahRBEMZ7J1HjGnWjx1waFyFKWGZCiIoogeTgRYzoJoH0stT01mya9D+6e4zLMG/j03iNl7xNeiYb0Y1Y0PSPqq+oqi+3UviQppedZGn5zt17K/e7D1YfPnrcW3ty6E3pOA65kcYd5+BRCo3DIILEY+sQVC7xKD/ba+pH39B5YfTXMLM4UjDVohAcQkyNe+9ZjgHoO8p8accVUxBO84Ke12yzZWPbz6nqyyk4q9H7euO36sW4108HaRv0NmRz6JN5HIzXOttsYnipUAcuwfuTLLVhVIELgkusu6z0aIGfwRRPImpQ6EdVe2hNn8fMhBbGxacDbbN/dlSgvJ+pPCqbFf1irUn+s5arhcmheD2qhLZlQM2vBxelpMHQxkU6EQ55kLMIwJ2Iu1Me/QEeotfdLtvHeJzDj3HQJ4sOgnEvKwZuqoSu47FTttnQ/4Tw/UYYKfqcLbp6Gw63BtnOYOfzdn93b+74Clknz8gGycgrsks+kAMyJJz8ID/JBfmVrCZbyZvk7bU06cx7npK/Itm/ApCIyS0=</latexit> = sup Sharpness(w) w 2 max (r L(w))
(Edge of stability) <latexit sha1_base64="5FSkUlokt7wtfo9S2WxLMEMRtnw=">AAACinicfZDPahRBEMZ7J1HjGnWjx1waFyFKWGZCiIoogeTgRYzoJoH0stT01mya9D+6e4zLMG/j03iNl7xNeiYb0Y1Y0PSPqq+oqi+3UviQppedZGn5zt17K/e7D1YfPnrcW3ty6E3pOA65kcYd5+BRCo3DIILEY+sQVC7xKD/ba+pH39B5YfTXMLM4UjDVohAcQkyNe+9ZjgHoO8p8accVUxBO84Ke12yzZWPbz6nqyyk4q9H7euO36sW4108HaRv0NmRz6JN5HIzXOttsYnipUAcuwfuTLLVhVIELgkusu6z0aIGfwRRPImpQ6EdVe2hNn8fMhBbGxacDbbN/dlSgvJ+pPCqbFf1irUn+s5arhcmheD2qhLZlQM2vBxelpMHQxkU6EQ55kLMIwJ2Iu1Me/QEeotfdLtvHeJzDj3HQJ4sOgnEvKwZuqoSu47FTttnQ/4Tw/UYYKfqcLbp6Gw63BtnOYOfzdn93b+74Clknz8gGycgrsks+kAMyJJz8ID/JBfmVrCZbyZvk7bU06cx7npK/Itm/ApCIyS0=</latexit> = sup Sharpness(w) w 2 max (r L(w)) stay on edge e v i s g s n e i r n g e o p r r P a sh <latexit sha1_base64="FOpfjo1PL0jpMKhD7AEsUswjgyo=">AAACY3icfVBdaxNBFJ2sH62ptWkVfBBhMAhSJGy2Malvhfrgi1jBtIVuCHcnd9Oh88XMXWlY8mt81R/kD/B/ONumoEU9MHA491zO3FM4JQOl6Y9Wcufuvftr6w/aGw83H211tneOg628wLGwyvrTAgIqaXBMkhSeOo+gC4UnxcVhMz/5gj5Iaz7TwuFEw9zIUgqgKE07T/MCCXJwztvLnEoPIsujMu10097b/WH2JuNpL01H2d6wIdlokO3xflQadNkKR9Pt1iCfWVFpNCQUhHDWTx1NavAkhcJlO68COhAXMMezSA1oDJP66oIlfxmVGS+tj88Qv1J/36hBh7DQRXRqoPNwe9aIf50V+lYylfuTWhpXERpxHVxWipPlTT18Jj0KUotIQHgZ/87FOcRWKJbYbufvMB7n8UMM+ujQA1m/W+fg51qaZTx2nr9u2P+McHljjCz2fFMm/zc5znr9YW/4adA9OFw1vs6esRfsFeuzETtg79kRGzPBluwr+8a+t34mG8lO8uTamrRWO4/ZH0ie/wKE+bpw</latexit> 2 ⇡⌘
<latexit sha1_base64="syXruHIyJZx4DWoGWWlcpIJU6N0=">AAACXHicfZBNSwMxEIbT9avWr6rgxUuwCCpadkXUiyDowYtYwVbBrWU2nbbBJLsk2WJZ+k+86n/y4m8xrRW0igOBh3feMDNvlAhurO+/5byJyanpmfxsYW5+YXGpuLxSM3GqGVZZLGJ9F4FBwRVWLbcC7xKNICOBt9Hj2aB/20VteKxubC/BuoS24i3OwDqpUSyGKMSW3KYnFB+yPdlvFEt+2R8W/Q3BCEpkVJXGcu4gbMYslagsE2DMfeAntp6BtpwJ7BfC1GAC7BHaeO9QgURTz4ar9+mmU5q0FWv3lKVD9fuPDKQxPRk5pwTbMeO9gfhnL5Jjk23ruJ5xlaQWFfsc3EoFtTEd5EKbXCOzoucAmOZud8o6oIFZl16hEJ6jO07jpRt0laAGG+udLATdllz13bHtcHdA/xnh6cvoyOUcjKf6G2r75eCwfHh9UDo9GyWeJ+tkg2yRgByRU3JBKqRKGOmSZ/JCXnPv3qQ35y18Wr3c6M8q+VHe2gcyLrXt</latexit> `(m) = e む hw⇤ , yi xi i <latexit sha1_base64="Ogz5WUyidBxj6A6C8p6HFpWIfhI=">AAACgXicfZDNSuRAEMd74ve46uge99I4CK7IkIhf4EXQw15kFRwVzBAqPZXY2N0J3R11CHkSn8br+gS+jZ1xhHUUCwp+XfUvqusf54Ib6/svDW9icmp6ZnauOf9jYXGptbxyYbJCM+yyTGT6KgaDgivsWm4FXuUaQcYCL+Pbo7p/eYfa8Eyd20GOPQmp4glnYF0pau2EAlQqMJRgb+KkvK+ijc1BxN/fD5VjPZTQMK0TpISo1fY7/jDoZwhG0CajOI2WG9thP2OFRGWZAGOuAz+3vRK05Uxg1QwLgzmwW0jx2qECiaZXDu+r6Jqr9GmSaZfK0mH1/4kSpDEDGTtl/W0z3quLX/ZiObbZJvu9kqu8sKjY2+KkENRmtDaP9rlGZsXAATDN3d8puwENzDqLm83wGN1xGk/cor85arCZ3ihD0KnkqnLHpuFmTd8J4eFd6Mj5HIy7+hkutjrBbmf3bLt9eDRyfJb8IqtknQRkjxySP+SUdAkjj+SJ/CPP3oT32/O9rTep1xjN/CQfwjt4Be1Gxds=</latexit> <latexit sha1_base64="U94f3z6l97Xyl6EofN8Tlv7UmvY=">AAACVHicfZBNSwMxEIaz61et33r0EiyCiJRdEfUkBT14ERWsLbpFZtPpGppklyQrlqX/wqv+LMH/4sG0VtAqDgQe3nnDzLxxJrixQfDm+ROTU9Mzpdny3PzC4tLyyuq1SXPNsM5SkepmDAYFV1i33ApsZhpBxgIbcfd40G88oDY8VVe2l2FLQqJ4hzOwTrqJEpAS6BEN7pYrQTUYFv0N4QgqZFQXdyveXtROWS5RWSbAmNswyGyrAG05E9gvR7nBDFgXErx1qECiaRXDlft00ylt2km1e8rSofr9RwHSmJ6MnVOCvTfjvYH4Zy+WY5Nt57BVcJXlFhX7HNzJBbUpHeRB21wjs6LnAJjmbnfK7kEDsy61cjk6QXecxjM36DxDDTbV20UEOpFc9d2xSbQzoP+M8PhldORyDsdT/Q3Xu9Vwv7p/uVepHY8SL5F1skG2SEgOSI2ckgtSJ4wo8kSeyYv36r37E/7Up9X3Rn/WyI/yFz8A2bq0Yw==</latexit> >0 `(hw, yi xi i) m <latexit sha1_base64="HVN1/W6stNVI4xeh2RL3r2DciNM=">AAACY3icfZDfShtBFMYnq60aa41/wAsRhgYhVg27IuqNIOiFN6JCo4KbhrOTk3Vw/iwzs2JY9mm81QfyAfoencQIbSw9MPDjO9/hzPmSTHDrwvC1EkxMfvo8NT1Tnf0y93W+trB4ZXVuGLaYFtrcJGBRcIUtx53Am8wgyETgdXJ/POhfP6CxXKsfrp9hW0KqeI8zcF7q1FZojEI05MZhLHTaiDbxZ7Ety41OrR42w2HRjxCNoE5GddFZqOzGXc1yicoxAdbeRmHm2gUYx5nAshrnFjNg95DirUcFEm27GF5Q0nWvdGlPG/+Uo0P1z4kCpLV9mXinBHdnx3sD8Z+9RI5tdr2DdsFVljtU7G1xLxfUaTqIh3a5QeZE3wMww/3fKbsDA8z5EKvV+AT9cQbP/KLzDA04bb4XMZhUclX6Y9N4a0D/M8Lju9GTzzkaT/UjXO00o73m3uVu/eh4lPg0WSXfSINEZJ8ckVNyQVqEkZI8kWfyUvkVzAaLwfKbNaiMZpbIXxWs/QaCurhe</latexit> <latexit sha1_base64="UqUu31KDfKZfPPqHtylbqfTZtOY=">AAACiHicfVBNaxRBEO0djSajSTbJ0UvjIsR8LDMSkggGgvHgQTGCmwQyy1LTW7Np0t0zdNeoyzB/xl+Tq978N/ZsVthsxIKCx3uvqKqXFko6iqLfreDBw4VHjxeXwidPl1dW22vrZy4vrcCeyFVuL1JwqKTBHklSeFFYBJ0qPE+vTxr9/CtaJ3PzhcYF9jWMjMykAPLUoP0m0UBXaVZ9qwcVbcc1P+IzFPFdniBBYiBVwD9szmovB+1O1I0mxe+DeAo6bFqng7XWXjLMRanRkFDg3GUcFdSvwJIUCuswKR0WIK5hhJceGtDo+tXkzZq/8MyQZ7n1bYhP2NmJCrRzY516Z3Olm9ca8p9aquc2U3bYr6QpSkIjbhdnpeKU8yZDPpQWBamxByCs9LdzcQUWBPmkwzB5h/45ix/9ok8FWqDcblUJ2JGWpvbPjpKdBv3PCN//Gj3yOcfzqd4HZ6+68X53//Ne5/hkmvgie8aes00WswN2zN6zU9Zjgv1gN+wn+xWEQRQcBK9vrUFrOrPB7lTw9g8vaMeY</latexit> wt+1 = wt `(m) = log(1 + e m ) ⌘rL(wt ) ; ⌘>0 <latexit sha1_base64="VJVFbPAq1jx6rJvWLoecIV5GI7c=">AAACUHicfZBRSxtBEMfnola92qr10ZfDUBAp4U5EfRJBH3wpVTBR8ILMbSZxze7esTsnDUe+g6/2Y/XNb+Jbu4kR2igOLPvjP/9lZv9ZoaTjOH4MajOzcx/mFxbDj0ufPi+vrH5puby0gpoiV7m9zNCRkoaaLFnRZWEJdaboIusfjfoXd2SdzM05Dwpqa+wZ2ZUC2UutlBgP4uuVetyIxxW9hmQCdZjU6fVqsJN2clFqMiwUOneVxAW3K7QshaJhmJaOChR97NGVR4OaXLsarzuMvnqlE3Vz64/haKz++6JC7dxAZ96pkW/cdG8kvtnL9NRk7u63K2mKksmI58HdUkWcR6Msoo60JFgNPKCw0u8eiRu0KNgnFobpMfnPWfruB/0oyCLndqtK0fa0NEP/2V76bUTvGfHni9GTzzmZTvU1tLYbyW5j92ynfng0SXwB1mEDNiGBPTiEEziFJgi4hXt4gF/B7+Ap+FMLnq0vN6zBf1UL/wJke7Qx</latexit> 大 二 4 2 0 2 4 Wu, J., Bartlett, P. L., Telgarsky, M., & Yu, B. (2024). Large Stepsize Gradient Descent for Logistic Loss: Non-Monotonicity of the Loss Improves Optimization Efficiency. 2 1 📄 [Wu+ ]
<latexit sha1_base64="syXruHIyJZx4DWoGWWlcpIJU6N0=">AAACXHicfZBNSwMxEIbT9avWr6rgxUuwCCpadkXUiyDowYtYwVbBrWU2nbbBJLsk2WJZ+k+86n/y4m8xrRW0igOBh3feMDNvlAhurO+/5byJyanpmfxsYW5+YXGpuLxSM3GqGVZZLGJ9F4FBwRVWLbcC7xKNICOBt9Hj2aB/20VteKxubC/BuoS24i3OwDqpUSyGKMSW3KYnFB+yPdlvFEt+2R8W/Q3BCEpkVJXGcu4gbMYslagsE2DMfeAntp6BtpwJ7BfC1GAC7BHaeO9QgURTz4ar9+mmU5q0FWv3lKVD9fuPDKQxPRk5pwTbMeO9gfhnL5Jjk23ruJ5xlaQWFfsc3EoFtTEd5EKbXCOzoucAmOZud8o6oIFZl16hEJ6jO07jpRt0laAGG+udLATdllz13bHtcHdA/xnh6cvoyOUcjKf6G2r75eCwfHh9UDo9GyWeJ+tkg2yRgByRU3JBKqRKGOmSZ/JCXnPv3qQ35y18Wr3c6M8q+VHe2gcyLrXt</latexit> `(m) = e む hw⇤ , yi xi i <latexit sha1_base64="Ogz5WUyidBxj6A6C8p6HFpWIfhI=">AAACgXicfZDNSuRAEMd74ve46uge99I4CK7IkIhf4EXQw15kFRwVzBAqPZXY2N0J3R11CHkSn8br+gS+jZ1xhHUUCwp+XfUvqusf54Ib6/svDW9icmp6ZnauOf9jYXGptbxyYbJCM+yyTGT6KgaDgivsWm4FXuUaQcYCL+Pbo7p/eYfa8Eyd20GOPQmp4glnYF0pau2EAlQqMJRgb+KkvK+ijc1BxN/fD5VjPZTQMK0TpISo1fY7/jDoZwhG0CajOI2WG9thP2OFRGWZAGOuAz+3vRK05Uxg1QwLgzmwW0jx2qECiaZXDu+r6Jqr9GmSaZfK0mH1/4kSpDEDGTtl/W0z3quLX/ZiObbZJvu9kqu8sKjY2+KkENRmtDaP9rlGZsXAATDN3d8puwENzDqLm83wGN1xGk/cor85arCZ3ihD0KnkqnLHpuFmTd8J4eFd6Mj5HIy7+hkutjrBbmf3bLt9eDRyfJb8IqtknQRkjxySP+SUdAkjj+SJ/CPP3oT32/O9rTep1xjN/CQfwjt4Be1Gxds=</latexit> <latexit sha1_base64="U94f3z6l97Xyl6EofN8Tlv7UmvY=">AAACVHicfZBNSwMxEIaz61et33r0EiyCiJRdEfUkBT14ERWsLbpFZtPpGppklyQrlqX/wqv+LMH/4sG0VtAqDgQe3nnDzLxxJrixQfDm+ROTU9Mzpdny3PzC4tLyyuq1SXPNsM5SkepmDAYFV1i33ApsZhpBxgIbcfd40G88oDY8VVe2l2FLQqJ4hzOwTrqJEpAS6BEN7pYrQTUYFv0N4QgqZFQXdyveXtROWS5RWSbAmNswyGyrAG05E9gvR7nBDFgXErx1qECiaRXDlft00ylt2km1e8rSofr9RwHSmJ6MnVOCvTfjvYH4Zy+WY5Nt57BVcJXlFhX7HNzJBbUpHeRB21wjs6LnAJjmbnfK7kEDsy61cjk6QXecxjM36DxDDTbV20UEOpFc9d2xSbQzoP+M8PhldORyDsdT/Q3Xu9Vwv7p/uVepHY8SL5F1skG2SEgOSI2ckgtSJ4wo8kSeyYv36r37E/7Up9X3Rn/WyI/yFz8A2bq0Yw==</latexit> >0 `(hw, yi xi i) m <latexit sha1_base64="HVN1/W6stNVI4xeh2RL3r2DciNM=">AAACY3icfZDfShtBFMYnq60aa41/wAsRhgYhVg27IuqNIOiFN6JCo4KbhrOTk3Vw/iwzs2JY9mm81QfyAfoencQIbSw9MPDjO9/hzPmSTHDrwvC1EkxMfvo8NT1Tnf0y93W+trB4ZXVuGLaYFtrcJGBRcIUtx53Am8wgyETgdXJ/POhfP6CxXKsfrp9hW0KqeI8zcF7q1FZojEI05MZhLHTaiDbxZ7Ety41OrR42w2HRjxCNoE5GddFZqOzGXc1yicoxAdbeRmHm2gUYx5nAshrnFjNg95DirUcFEm27GF5Q0nWvdGlPG/+Uo0P1z4kCpLV9mXinBHdnx3sD8Z+9RI5tdr2DdsFVljtU7G1xLxfUaTqIh3a5QeZE3wMww/3fKbsDA8z5EKvV+AT9cQbP/KLzDA04bb4XMZhUclX6Y9N4a0D/M8Lju9GTzzkaT/UjXO00o73m3uVu/eh4lPg0WSXfSINEZJ8ckVNyQVqEkZI8kWfyUvkVzAaLwfKbNaiMZpbIXxWs/QaCurhe</latexit> <latexit sha1_base64="UqUu31KDfKZfPPqHtylbqfTZtOY=">AAACiHicfVBNaxRBEO0djSajSTbJ0UvjIsR8LDMSkggGgvHgQTGCmwQyy1LTW7Np0t0zdNeoyzB/xl+Tq978N/ZsVthsxIKCx3uvqKqXFko6iqLfreDBw4VHjxeXwidPl1dW22vrZy4vrcCeyFVuL1JwqKTBHklSeFFYBJ0qPE+vTxr9/CtaJ3PzhcYF9jWMjMykAPLUoP0m0UBXaVZ9qwcVbcc1P+IzFPFdniBBYiBVwD9szmovB+1O1I0mxe+DeAo6bFqng7XWXjLMRanRkFDg3GUcFdSvwJIUCuswKR0WIK5hhJceGtDo+tXkzZq/8MyQZ7n1bYhP2NmJCrRzY516Z3Olm9ca8p9aquc2U3bYr6QpSkIjbhdnpeKU8yZDPpQWBamxByCs9LdzcQUWBPmkwzB5h/45ix/9ok8FWqDcblUJ2JGWpvbPjpKdBv3PCN//Gj3yOcfzqd4HZ6+68X53//Ne5/hkmvgie8aes00WswN2zN6zU9Zjgv1gN+wn+xWEQRQcBK9vrUFrOrPB7lTw9g8vaMeY</latexit> wt+1 = wt `(m) = log(1 + e m ) ⌘rL(wt ) ; ⌘>0 <latexit sha1_base64="VJVFbPAq1jx6rJvWLoecIV5GI7c=">AAACUHicfZBRSxtBEMfnola92qr10ZfDUBAp4U5EfRJBH3wpVTBR8ILMbSZxze7esTsnDUe+g6/2Y/XNb+Jbu4kR2igOLPvjP/9lZv9ZoaTjOH4MajOzcx/mFxbDj0ufPi+vrH5puby0gpoiV7m9zNCRkoaaLFnRZWEJdaboIusfjfoXd2SdzM05Dwpqa+wZ2ZUC2UutlBgP4uuVetyIxxW9hmQCdZjU6fVqsJN2clFqMiwUOneVxAW3K7QshaJhmJaOChR97NGVR4OaXLsarzuMvnqlE3Vz64/haKz++6JC7dxAZ96pkW/cdG8kvtnL9NRk7u63K2mKksmI58HdUkWcR6Msoo60JFgNPKCw0u8eiRu0KNgnFobpMfnPWfruB/0oyCLndqtK0fa0NEP/2V76bUTvGfHni9GTzzmZTvU1tLYbyW5j92ynfng0SXwB1mEDNiGBPTiEEziFJgi4hXt4gF/B7+Ap+FMLnq0vN6zBf1UL/wJke7Qx</latexit> : EoS Stage <latexit sha1_base64="B55qkJQFu/wxuKIQtEn5qJEykyM=">AAAClnicfVBdaxNBFJ2sXzV+NLUvBV8Gg5CKhl0p1RdLoYoiSiOYttCJy+zs3c2Qmdll5q5tWPY/+WsEn/SnOJumoKl4YeDcc8/lzjlJqaTDMPzRCa5dv3Hz1trt7p279+6v9zYeHLmisgLGolCFPUm4AyUNjFGigpPSAteJguNkdtDOj7+CdbIwn3FewkTz3MhMCo6einvvWWa5iJC5Ssf17FXYfKnxWdR8GDDNcZpk9VkTz7aZAsrOZAooVQr00PcZDi53rcynuB33+uEwXBS9CqIl6JNljeKNzg5LC1FpMCgUd+40Ckuc1NyiFAqaLqsclFzMeA6nHhquwU3qhemGPvZMSrPC+meQLtg/N2qunZvrxCtbJ2511pL/nCV65TJmLye1NGWFYMTF4axSFAvaJkpTaUGgmnvAhZX+71RMuU8Gfe7dLnsN3pyFj/7QYQmWY2Gf1IzbXEvTeLM5e9qi/wn5+aXQI59ztJrqVXD0fBjtDnc/7fT3D5aJr5GH5BEZkIi8IPvkHRmRMRHkG/lOfpJfwVawF7wJ3l5Ig85yZ5P8VcHoN/11zZQ=</latexit> 1 t t 1 X k=0 ✓ ◆ 1 e L(wk ) O t 大 二 4 2 0 1 2 5 Wu, J., Bartlett, P. L., Telgarsky, M., & Yu, B. (2024). Large Stepsize Gradient Descent for Logistic Loss: Non-Monotonicity of the Loss Improves Optimization Efficiency. 2 1 📄 [Wu+ ]
<latexit sha1_base64="syXruHIyJZx4DWoGWWlcpIJU6N0=">AAACXHicfZBNSwMxEIbT9avWr6rgxUuwCCpadkXUiyDowYtYwVbBrWU2nbbBJLsk2WJZ+k+86n/y4m8xrRW0igOBh3feMDNvlAhurO+/5byJyanpmfxsYW5+YXGpuLxSM3GqGVZZLGJ9F4FBwRVWLbcC7xKNICOBt9Hj2aB/20VteKxubC/BuoS24i3OwDqpUSyGKMSW3KYnFB+yPdlvFEt+2R8W/Q3BCEpkVJXGcu4gbMYslagsE2DMfeAntp6BtpwJ7BfC1GAC7BHaeO9QgURTz4ar9+mmU5q0FWv3lKVD9fuPDKQxPRk5pwTbMeO9gfhnL5Jjk23ruJ5xlaQWFfsc3EoFtTEd5EKbXCOzoucAmOZud8o6oIFZl16hEJ6jO07jpRt0laAGG+udLATdllz13bHtcHdA/xnh6cvoyOUcjKf6G2r75eCwfHh9UDo9GyWeJ+tkg2yRgByRU3JBKqRKGOmSZ/JCXnPv3qQ35y18Wr3c6M8q+VHe2gcyLrXt</latexit> `(m) = e む hw⇤ , yi xi i <latexit sha1_base64="Ogz5WUyidBxj6A6C8p6HFpWIfhI=">AAACgXicfZDNSuRAEMd74ve46uge99I4CK7IkIhf4EXQw15kFRwVzBAqPZXY2N0J3R11CHkSn8br+gS+jZ1xhHUUCwp+XfUvqusf54Ib6/svDW9icmp6ZnauOf9jYXGptbxyYbJCM+yyTGT6KgaDgivsWm4FXuUaQcYCL+Pbo7p/eYfa8Eyd20GOPQmp4glnYF0pau2EAlQqMJRgb+KkvK+ijc1BxN/fD5VjPZTQMK0TpISo1fY7/jDoZwhG0CajOI2WG9thP2OFRGWZAGOuAz+3vRK05Uxg1QwLgzmwW0jx2qECiaZXDu+r6Jqr9GmSaZfK0mH1/4kSpDEDGTtl/W0z3quLX/ZiObbZJvu9kqu8sKjY2+KkENRmtDaP9rlGZsXAATDN3d8puwENzDqLm83wGN1xGk/cor85arCZ3ihD0KnkqnLHpuFmTd8J4eFd6Mj5HIy7+hkutjrBbmf3bLt9eDRyfJb8IqtknQRkjxySP+SUdAkjj+SJ/CPP3oT32/O9rTep1xjN/CQfwjt4Be1Gxds=</latexit> <latexit sha1_base64="U94f3z6l97Xyl6EofN8Tlv7UmvY=">AAACVHicfZBNSwMxEIaz61et33r0EiyCiJRdEfUkBT14ERWsLbpFZtPpGppklyQrlqX/wqv+LMH/4sG0VtAqDgQe3nnDzLxxJrixQfDm+ROTU9Mzpdny3PzC4tLyyuq1SXPNsM5SkepmDAYFV1i33ApsZhpBxgIbcfd40G88oDY8VVe2l2FLQqJ4hzOwTrqJEpAS6BEN7pYrQTUYFv0N4QgqZFQXdyveXtROWS5RWSbAmNswyGyrAG05E9gvR7nBDFgXErx1qECiaRXDlft00ylt2km1e8rSofr9RwHSmJ6MnVOCvTfjvYH4Zy+WY5Nt57BVcJXlFhX7HNzJBbUpHeRB21wjs6LnAJjmbnfK7kEDsy61cjk6QXecxjM36DxDDTbV20UEOpFc9d2xSbQzoP+M8PhldORyDsdT/Q3Xu9Vwv7p/uVepHY8SL5F1skG2SEgOSI2ckgtSJ4wo8kSeyYv36r37E/7Up9X3Rn/WyI/yFz8A2bq0Yw==</latexit> >0 `(hw, yi xi i) m <latexit sha1_base64="HVN1/W6stNVI4xeh2RL3r2DciNM=">AAACY3icfZDfShtBFMYnq60aa41/wAsRhgYhVg27IuqNIOiFN6JCo4KbhrOTk3Vw/iwzs2JY9mm81QfyAfoencQIbSw9MPDjO9/hzPmSTHDrwvC1EkxMfvo8NT1Tnf0y93W+trB4ZXVuGLaYFtrcJGBRcIUtx53Am8wgyETgdXJ/POhfP6CxXKsfrp9hW0KqeI8zcF7q1FZojEI05MZhLHTaiDbxZ7Ety41OrR42w2HRjxCNoE5GddFZqOzGXc1yicoxAdbeRmHm2gUYx5nAshrnFjNg95DirUcFEm27GF5Q0nWvdGlPG/+Uo0P1z4kCpLV9mXinBHdnx3sD8Z+9RI5tdr2DdsFVljtU7G1xLxfUaTqIh3a5QeZE3wMww/3fKbsDA8z5EKvV+AT9cQbP/KLzDA04bb4XMZhUclX6Y9N4a0D/M8Lju9GTzzkaT/UjXO00o73m3uVu/eh4lPg0WSXfSINEZJ8ckVNyQVqEkZI8kWfyUvkVzAaLwfKbNaiMZpbIXxWs/QaCurhe</latexit> <latexit sha1_base64="UqUu31KDfKZfPPqHtylbqfTZtOY=">AAACiHicfVBNaxRBEO0djSajSTbJ0UvjIsR8LDMSkggGgvHgQTGCmwQyy1LTW7Np0t0zdNeoyzB/xl+Tq978N/ZsVthsxIKCx3uvqKqXFko6iqLfreDBw4VHjxeXwidPl1dW22vrZy4vrcCeyFVuL1JwqKTBHklSeFFYBJ0qPE+vTxr9/CtaJ3PzhcYF9jWMjMykAPLUoP0m0UBXaVZ9qwcVbcc1P+IzFPFdniBBYiBVwD9szmovB+1O1I0mxe+DeAo6bFqng7XWXjLMRanRkFDg3GUcFdSvwJIUCuswKR0WIK5hhJceGtDo+tXkzZq/8MyQZ7n1bYhP2NmJCrRzY516Z3Olm9ca8p9aquc2U3bYr6QpSkIjbhdnpeKU8yZDPpQWBamxByCs9LdzcQUWBPmkwzB5h/45ix/9ok8FWqDcblUJ2JGWpvbPjpKdBv3PCN//Gj3yOcfzqd4HZ6+68X53//Ne5/hkmvgie8aes00WswN2zN6zU9Zjgv1gN+wn+xWEQRQcBK9vrUFrOrPB7lTw9g8vaMeY</latexit> wt+1 = wt `(m) = log(1 + e m ) ⌘rL(wt ) ; ⌘>0 <latexit sha1_base64="VJVFbPAq1jx6rJvWLoecIV5GI7c=">AAACUHicfZBRSxtBEMfnola92qr10ZfDUBAp4U5EfRJBH3wpVTBR8ILMbSZxze7esTsnDUe+g6/2Y/XNb+Jbu4kR2igOLPvjP/9lZv9ZoaTjOH4MajOzcx/mFxbDj0ufPi+vrH5puby0gpoiV7m9zNCRkoaaLFnRZWEJdaboIusfjfoXd2SdzM05Dwpqa+wZ2ZUC2UutlBgP4uuVetyIxxW9hmQCdZjU6fVqsJN2clFqMiwUOneVxAW3K7QshaJhmJaOChR97NGVR4OaXLsarzuMvnqlE3Vz64/haKz++6JC7dxAZ96pkW/cdG8kvtnL9NRk7u63K2mKksmI58HdUkWcR6Msoo60JFgNPKCw0u8eiRu0KNgnFobpMfnPWfruB/0oyCLndqtK0fa0NEP/2V76bUTvGfHni9GTzzmZTvU1tLYbyW5j92ynfng0SXwB1mEDNiGBPTiEEziFJgi4hXt4gF/B7+Ap+FMLnq0vN6zBf1UL/wJke7Qx</latexit> : EoS Stage <latexit sha1_base64="uVKnhPBYagm5kR5Bq1HUTeAdM2o=">AAACg3icfVDRahNBFJ2s1ta02lQffRkahFQl7IbQ9kUo1AcflFZI2kInhtnZu5uhM7PLzF1rWPZT+jV91Q/wb5xNU9BUvDBw7rnncuecuFDSYRj+agWPHq89Wd942t7cevZ8u7Pz4szlpRUwFrnK7UXMHShpYIwSFVwUFriOFZzHV8fN/PwbWCdzM8J5ARPNMyNTKTh6ato5+NRjmuMsTqvrejraYwoou5YJoFQJ0BPfp9hjqeUiqkZfBzWzMpvh3rTTDfvhouhDEC1BlyzrdLrTGrIkF6UGg0Jx5y6jsMBJxS1KoaBus9JBwcUVz+DSQ8M1uEm1cFjT155JaJpb/wzSBfvnRsW1c3Mde2Xjxq3OGvKfs1ivXMb0cFJJU5QIRtwdTktFMadNfDSRFgSquQdcWOn/TsWM+3TQh9xusw/gzVn47A+dFGA55vZNxbjNtDS1N5uxdw36n5B/vxd65HOOVlN9CM4G/Wi/v/9l2D06Xia+QV6RXdIjETkgR+QjOSVjIsgNuSU/yM9gLXgbDILhnTRoLXdekr8qeP8bawPFgg==</latexit> e L(wT ) O ✓ 1 T2 ◆ 見 大 二 4 2 0 2 2 6 Wu, J., Bartlett, P. L., Telgarsky, M., & Yu, B. (2024). Large Stepsize Gradient Descent for Logistic Loss: Non-Monotonicity of the Loss Improves Optimization Efficiency. 2 1 📄 [Wu+ ]
<latexit sha1_base64="syXruHIyJZx4DWoGWWlcpIJU6N0=">AAACXHicfZBNSwMxEIbT9avWr6rgxUuwCCpadkXUiyDowYtYwVbBrWU2nbbBJLsk2WJZ+k+86n/y4m8xrRW0igOBh3feMDNvlAhurO+/5byJyanpmfxsYW5+YXGpuLxSM3GqGVZZLGJ9F4FBwRVWLbcC7xKNICOBt9Hj2aB/20VteKxubC/BuoS24i3OwDqpUSyGKMSW3KYnFB+yPdlvFEt+2R8W/Q3BCEpkVJXGcu4gbMYslagsE2DMfeAntp6BtpwJ7BfC1GAC7BHaeO9QgURTz4ar9+mmU5q0FWv3lKVD9fuPDKQxPRk5pwTbMeO9gfhnL5Jjk23ruJ5xlaQWFfsc3EoFtTEd5EKbXCOzoucAmOZud8o6oIFZl16hEJ6jO07jpRt0laAGG+udLATdllz13bHtcHdA/xnh6cvoyOUcjKf6G2r75eCwfHh9UDo9GyWeJ+tkg2yRgByRU3JBKqRKGOmSZ/JCXnPv3qQ35y18Wr3c6M8q+VHe2gcyLrXt</latexit> `(m) = e む hw⇤ , yi xi i <latexit sha1_base64="Ogz5WUyidBxj6A6C8p6HFpWIfhI=">AAACgXicfZDNSuRAEMd74ve46uge99I4CK7IkIhf4EXQw15kFRwVzBAqPZXY2N0J3R11CHkSn8br+gS+jZ1xhHUUCwp+XfUvqusf54Ib6/svDW9icmp6ZnauOf9jYXGptbxyYbJCM+yyTGT6KgaDgivsWm4FXuUaQcYCL+Pbo7p/eYfa8Eyd20GOPQmp4glnYF0pau2EAlQqMJRgb+KkvK+ijc1BxN/fD5VjPZTQMK0TpISo1fY7/jDoZwhG0CajOI2WG9thP2OFRGWZAGOuAz+3vRK05Uxg1QwLgzmwW0jx2qECiaZXDu+r6Jqr9GmSaZfK0mH1/4kSpDEDGTtl/W0z3quLX/ZiObbZJvu9kqu8sKjY2+KkENRmtDaP9rlGZsXAATDN3d8puwENzDqLm83wGN1xGk/cor85arCZ3ihD0KnkqnLHpuFmTd8J4eFd6Mj5HIy7+hkutjrBbmf3bLt9eDRyfJb8IqtknQRkjxySP+SUdAkjj+SJ/CPP3oT32/O9rTep1xjN/CQfwjt4Be1Gxds=</latexit> <latexit sha1_base64="U94f3z6l97Xyl6EofN8Tlv7UmvY=">AAACVHicfZBNSwMxEIaz61et33r0EiyCiJRdEfUkBT14ERWsLbpFZtPpGppklyQrlqX/wqv+LMH/4sG0VtAqDgQe3nnDzLxxJrixQfDm+ROTU9Mzpdny3PzC4tLyyuq1SXPNsM5SkepmDAYFV1i33ApsZhpBxgIbcfd40G88oDY8VVe2l2FLQqJ4hzOwTrqJEpAS6BEN7pYrQTUYFv0N4QgqZFQXdyveXtROWS5RWSbAmNswyGyrAG05E9gvR7nBDFgXErx1qECiaRXDlft00ylt2km1e8rSofr9RwHSmJ6MnVOCvTfjvYH4Zy+WY5Nt57BVcJXlFhX7HNzJBbUpHeRB21wjs6LnAJjmbnfK7kEDsy61cjk6QXecxjM36DxDDTbV20UEOpFc9d2xSbQzoP+M8PhldORyDsdT/Q3Xu9Vwv7p/uVepHY8SL5F1skG2SEgOSI2ckgtSJ4wo8kSeyYv36r37E/7Up9X3Rn/WyI/yFz8A2bq0Yw==</latexit> >0 `(hw, yi xi i) m <latexit sha1_base64="HVN1/W6stNVI4xeh2RL3r2DciNM=">AAACY3icfZDfShtBFMYnq60aa41/wAsRhgYhVg27IuqNIOiFN6JCo4KbhrOTk3Vw/iwzs2JY9mm81QfyAfoencQIbSw9MPDjO9/hzPmSTHDrwvC1EkxMfvo8NT1Tnf0y93W+trB4ZXVuGLaYFtrcJGBRcIUtx53Am8wgyETgdXJ/POhfP6CxXKsfrp9hW0KqeI8zcF7q1FZojEI05MZhLHTaiDbxZ7Ety41OrR42w2HRjxCNoE5GddFZqOzGXc1yicoxAdbeRmHm2gUYx5nAshrnFjNg95DirUcFEm27GF5Q0nWvdGlPG/+Uo0P1z4kCpLV9mXinBHdnx3sD8Z+9RI5tdr2DdsFVljtU7G1xLxfUaTqIh3a5QeZE3wMww/3fKbsDA8z5EKvV+AT9cQbP/KLzDA04bb4XMZhUclX6Y9N4a0D/M8Lju9GTzzkaT/UjXO00o73m3uVu/eh4lPg0WSXfSINEZJ8ckVNyQVqEkZI8kWfyUvkVzAaLwfKbNaiMZpbIXxWs/QaCurhe</latexit> <latexit sha1_base64="UqUu31KDfKZfPPqHtylbqfTZtOY=">AAACiHicfVBNaxRBEO0djSajSTbJ0UvjIsR8LDMSkggGgvHgQTGCmwQyy1LTW7Np0t0zdNeoyzB/xl+Tq978N/ZsVthsxIKCx3uvqKqXFko6iqLfreDBw4VHjxeXwidPl1dW22vrZy4vrcCeyFVuL1JwqKTBHklSeFFYBJ0qPE+vTxr9/CtaJ3PzhcYF9jWMjMykAPLUoP0m0UBXaVZ9qwcVbcc1P+IzFPFdniBBYiBVwD9szmovB+1O1I0mxe+DeAo6bFqng7XWXjLMRanRkFDg3GUcFdSvwJIUCuswKR0WIK5hhJceGtDo+tXkzZq/8MyQZ7n1bYhP2NmJCrRzY516Z3Olm9ca8p9aquc2U3bYr6QpSkIjbhdnpeKU8yZDPpQWBamxByCs9LdzcQUWBPmkwzB5h/45ix/9ok8FWqDcblUJ2JGWpvbPjpKdBv3PCN//Gj3yOcfzqd4HZ6+68X53//Ne5/hkmvgie8aes00WswN2zN6zU9Zjgv1gN+wn+xWEQRQcBK9vrUFrOrPB7lTw9g8vaMeY</latexit> wt+1 = wt `(m) = log(1 + e m ) ⌘rL(wt ) ; <latexit sha1_base64="JW+yb2ql4nYT5YCxGzjH/2b8ytA=">AAACk3icfVBdixMxFE3Hr7V+bFfxyZdgEbqylBlZVsGXxS7og4sr2N2FTSmZ9M40bJIZkjtqCflJ/hqfBP0vZtoK2hUvhJycey435+S1kg7T9HsnuXb9xs1bW7e7d+7eu7/d23lw6qrGChiLSlX2POcOlDQwRokKzmsLXOcKzvLLUds/+wTWycp8xEUNE81LIwspOEZq2nvzbsA0x3le+M9h6naZAsq0NPEukHlWWC58FvyIAfKwt3ozUGqQ7gZvArOynCML014/HabLoldBtgZ9sq6T6U5nn80q0WgwKBR37iJLa5x4blEKBaHLGgc1F5e8hIsIDdfgJn7pONCnkZnRorLxGKRL9s8Jz7VzC51HZevObfZa8p+9XG9sxuLlxEtTNwhGrBYXjaJY0TZOOpMWBKpFBFxYGf9OxZzHkDCG3u2yI4jmLBzHRe9rsBwr+8wzbsuYcYhmS7bXov8J+ZffwohiztlmqlfB6fNhdjA8+LDfPxytE98ij8kTMiAZeUEOyVtyQsZEkK/kG/lBfiaPklfJ6+RoJU0665mH5K9Kjn8B6UDNQw==</latexit> L(ws ) min ⌘>0 <latexit sha1_base64="VJVFbPAq1jx6rJvWLoecIV5GI7c=">AAACUHicfZBRSxtBEMfnola92qr10ZfDUBAp4U5EfRJBH3wpVTBR8ILMbSZxze7esTsnDUe+g6/2Y/XNb+Jbu4kR2igOLPvjP/9lZv9ZoaTjOH4MajOzcx/mFxbDj0ufPi+vrH5puby0gpoiV7m9zNCRkoaaLFnRZWEJdaboIusfjfoXd2SdzM05Dwpqa+wZ2ZUC2UutlBgP4uuVetyIxxW9hmQCdZjU6fVqsJN2clFqMiwUOneVxAW3K7QshaJhmJaOChR97NGVR4OaXLsarzuMvnqlE3Vz64/haKz++6JC7dxAZ96pkW/cdG8kvtnL9NRk7u63K2mKksmI58HdUkWcR6Msoo60JFgNPKCw0u8eiRu0KNgnFobpMfnPWfruB/0oyCLndqtK0fa0NEP/2V76bUTvGfHni9GTzzmZTvU1tLYbyW5j92ynfng0SXwB1mEDNiGBPTiEEziFJgi4hXt4gF/B7+Ap+FMLnq0vN6zBf1UL/wJke7Qx</latexit> ⇢ 1 `(0) , C⌘ n 大 二 4 立 2 0 2 7 Wu, J., Bartlett, P. L., Telgarsky, M., & Yu, B. (2024). Large Stepsize Gradient Descent for Logistic Loss: Non-Monotonicity of the Loss Improves Optimization Efficiency. 2 1 📄 [Wu+ ]
w⇤ <latexit sha1_base64="JNc9LqsQcokMM6+B9jobIILE/zk=">AAACaXicfVBdSxtBFJ1stWrUNmlfiiIMDYKIhN0i6qNgH/oiKhgV3BDuTu7GwfliZlYNS578Nb62v6a/oX/C2RjBRvHChcO553LvOZkR3Pk4/luLPszMfpybX6gvLi1/+txofjlzurAMO0wLbS8ycCi4wo7nXuCFsQgyE3ieXR9U8/MbtI5rdeqHBrsSBornnIEPVK+xlkrwV1lOC5oaq43X9Jm57W32Gq24HY+LvgbJBLTIpI57zdp22teskKg8E+DcZRIb3y3Bes4Ejupp4dAAu4YBXgaoQKLrlmMfI7oemD7NtQ2tPB2zLzdKkM4NZRaU1Y9uelaRb84yOXXZ53vdkitTeFTs6XBeCBrcVyHRPrfIvBgGAMzy8DtlV2CB+RBlvZ7+xGDO4mE4dGTQgtd2s0zBDiRXo2B2kG5V6D0h3D0LAwo5J9OpvgZnP9rJTnvnZLu1fzBJfJ6sku9kgyRkl+yTX+SYdAgj9+SB/CZ/av+iZvQtWnmSRrXJzlfyX0WtR9Wpu9c=</latexit> <latexit sha1_base64="UqUu31KDfKZfPPqHtylbqfTZtOY=">AAACiHicfVBNaxRBEO0djSajSTbJ0UvjIsR8LDMSkggGgvHgQTGCmwQyy1LTW7Np0t0zdNeoyzB/xl+Tq978N/ZsVthsxIKCx3uvqKqXFko6iqLfreDBw4VHjxeXwidPl1dW22vrZy4vrcCeyFVuL1JwqKTBHklSeFFYBJ0qPE+vTxr9/CtaJ3PzhcYF9jWMjMykAPLUoP0m0UBXaVZ9qwcVbcc1P+IzFPFdniBBYiBVwD9szmovB+1O1I0mxe+DeAo6bFqng7XWXjLMRanRkFDg3GUcFdSvwJIUCuswKR0WIK5hhJceGtDo+tXkzZq/8MyQZ7n1bYhP2NmJCrRzY516Z3Olm9ca8p9aquc2U3bYr6QpSkIjbhdnpeKU8yZDPpQWBamxByCs9LdzcQUWBPmkwzB5h/45ix/9ok8FWqDcblUJ2JGWpvbPjpKdBv3PCN//Gj3yOcfzqd4HZ6+68X53//Ne5/hkmvgie8aes00WswN2zN6zU9Zjgv1gN+wn+xWEQRQcBK9vrUFrOrPB7lTw9g8vaMeY</latexit> wt+1 = wt <latexit sha1_base64="bwrIt4mhQRPcOefPV68YjElvXw0=">AAAC9XicfVFdTxQxFO0MKjh+sOCjL40bExTYzGwQfRFJ8MEHjZi4QELXzZ3unaGh05m0HWEz7E/xjZD45O/x39jZXcy4EG/S5PSec3ra27iQwtgw/O35C3fu3ltcuh88ePjo8XJrZfXA5KXm2OO5zPVRDAalUNizwko8KjRCFks8jE/3av7wO2ojcvXVjgrsZ5AqkQgO1rUGrZ/sgmVgT+KEng0qux6NN6/3Jbv41g02aVNhqds3efqWdhlaoEyCSiVSpiCWQD+uNUwvNv56Gn5HMD01rdP6DHeay7rNXycNWu2wE06K3gTRDLTJrPYHK94WG+a8zFBZLsGY4ygsbL8CbQWXOA5YabAAfgopHjuoIEPTryYzHdPnrjOkSa7dUpZOuk1HBZkxoyx2yvqiZp6rm7dycTaXbJM3/UqoorSo+DQ4KSW1Oa0/jA6FRm7lyAHgWri7U34CGrh13xoE7D26x2n85II+F6jB5vplxUCnmVBj99iUbdTof0I4vxY65OYczU/1JjjodqLtzvaXrfbu3mziS+QpeUbWSERek13ygeyTHuHesvfK2/He+Wf+D//Sv5pKfW/meUL+Kf/XH3Hd66o=</latexit> kwt+1 uk 2 ⌘rL(wt ) 2 kwt uk = 2⌘hrL(wt ), u 2 wt i + ⌘ krL(wt )k 2 2 <latexit sha1_base64="U56KgnYgjGsLibTfpZvw12g0ojk=">AAAC73ichVHLbtQwFHXCqx1eU1iysRghFShDElWlLJAqlQULEEVi2kr1NLrx3KRWbSfYDjBK5zvYISRWfBJ/gzMzFe20gitZOj73nvvMKimsi6LfQXjl6rXrN5aWOzdv3b5zt7tyb9eWteE44KUszX4GFqXQOHDCSdyvDILKJO5lx9utf+8zGitK/dGNKxwqKLTIBQfnqbT78xVNGDqgTIIuJFKmIZNA364yBe4oy+mX1D1eO/3UaUyf0TMuZmayp+fT/C9L8lfXyg4Tyk4u1bCTwyTt9qJ+NDV6EcRz0CNz20lXgnU2KnmtUDsuwdqDOKrcsAHjBJc46bDaYgX8GAo88FCDQjtsptuc0EeeGdG8NP5pR6fsWUUDytqxynxk26hd9LXkpb5MLVR2+eawEbqqHWo+K5zXkrqStqeiI2GQOzn2ALgRvnfKj8AAd/6gnQ57jX44g+98ofcVGnCledIwMIUSeuKHLdhai/4VCF9PAz3ye44Xt3oR7Cb9eKO/8WG9t7U93/gSeUAeklUSkxdki7whO2RAeLAcPA82g5fhp/Bb+D38MQsNg7nmPjln4a8/6X7pSQ==</latexit> = 2⌘hrL(wt ), u1 wt i + 2⌘hrL(wt ), u2 i + ⌘ krL 2 2 n X 1 = 2⌘hrL(w ), u w i + 2⌘hrL(w ), u i + ⌘ krL(w )k 0 t 1 t t 2 t = ` (hwt , zi i)hzi , u wt i <latexit sha1_base64="U56KgnYgjGsLibTfpZvw12g0ojk=">AAAC73ichVHLbtQwFHXCqx1eU1iysRghFShDElWlLJAqlQULEEVi2kr1NLrx3KRWbSfYDjBK5zvYISRWfBJ/gzMzFe20gitZOj73nvvMKimsi6LfQXjl6rXrN5aWOzdv3b5zt7tyb9eWteE44KUszX4GFqXQOHDCSdyvDILKJO5lx9utf+8zGitK/dGNKxwqKLTIBQfnqbT78xVNGDqgTIIuJFKmIZNA364yBe4oy+mX1D1eO/3UaUyf0TMuZmayp+fT/C9L8lfXyg4Tyk4u1bCTwyTt9qJ+NDV6EcRz0CNz20lXgnU2KnmtUDsuwdqDOKrcsAHjBJc46bDaYgX8GAo88FCDQjtsptuc0EeeGdG8NP5pR6fsWUUDytqxynxk26hd9LXkpb5MLVR2+eawEbqqHWo+K5zXkrqStqeiI2GQOzn2ALgRvnfKj8AAd/6gnQ57jX44g+98ofcVGnCledIwMIUSeuKHLdhai/4VCF9PAz3ye44Xt3oR7Cb9eKO/8WG9t7U93/gSeUAeklUSkxdki7whO2RAeLAcPA82g5fhp/Bb+D38MQsNg7nmPjln4a8/6X7pSQ==</latexit> <latexit sha1_base64="BGgV/BWIxgjg5aT1gN895LBz0QY=">AAADQHichVJNb9QwEHXCVwkf3cKRi8WKUtB2laCqcKlUqRy4IIrEthXrEDneSdaq7US2AyzRSlzhZ/Ev+AfcEBInTjjbFC27Kxgp8su8N3rjGael4MaG4VfPv3Dx0uUra1eDa9dv3FzvbNw6MkWlGQxYIQp9klIDgisYWG4FnJQaqEwFHKenBw1//Ba04YV6ZSclxJLmimecUetSSefn3ibJNGWRIqaSSc33oukbRUCI+1tEUJULwERSO04z/C6xvXP8IeFEz+gHizJH9f78VNtz1W0FJiTYJM25wllAZoeN/5J9tcocb+N58f9a1Twf2zjpdMN+OAu8DKIWdFEbh8mGt0NGBaskKMsENWYYhaWNa6otZwKmAakMlJSd0hyGDioqwcT1bD1TfM9lRjgrtPuUxbPsfEVNpTETmTpl07JZ5JrkSi6VC842exLXXJWVBcXOjLNKYFvgZvd4xDUwKyYOUKa56x2zMXU7sO6FBAF5Cu5yGp47oxclaGoL/bAmVOeSq6m7bE56DfqXkL4/Fzrk5hwtTnUZHD3qR7v93Zc73f2DduJr6A66i7ZQhB6jffQMHaIBYt5r76P3yfvsf/G/+d/9H2dS32trbqO/wv/1G7bNDaI=</latexit> n i=1 n X 1 [`(hu, zi i) n i=1 (convexity) `(hwt , zi i)] u2 L(u1 ) L(wt ) `( ✓) L(wt ) <latexit sha1_base64="9ZWOyn0nyc/PyHfOKCZdfA0cPFs=">AAACb3icfZDfShtBFMYnq7WattY/F14IMhgKptSwK6K9FPSiF4oWGhXcEM5OzsbBmdll5qw2LHkAn6a39VF8DN+gszGCxuKBgR/f+YZzzpfkSjoKw/taMDX9bub97Fz9w8dP858XFpdOXVZYgW2RqcyeJ+BQSYNtkqTwPLcIOlF4llztV/2za7ROZuYXDXLsaOgbmUoB5KXuQiNWyA83Yg10maS86EZNvvlMuOlS07vCVjgq/hqiMTTYuE66i7XtuJeJQqMhocC5iyjMqVOCJSkUDutx4TAHcQV9vPBoQKPrlKNrhvyLV3o8zax/hvhIff6jBO3cQCfeWS3pJnuV+N9eoicmU/q9U0qTF4RGPA5OC8Up41VUvCctClIDDyCs9LtzcQkWBPlA6/X4AP1xFo/8oOMcLVBmv5Yx2L6WZuiP7cffKnrLCL+fjJ58ztFkqq/hdKsV7bR2fm439vbHic+yVbbONljEdtke+8FOWJsJdsv+sL/srvYQrARrAX+0BrXxn2X2ooLmP+rdvKE=</latexit> `(✓ ) <latexit sha1_base64="h1qMM3lLiQ9zHMfh2x2djBbtOw8=">AAACYXicfZDfShtBFMYn29bG+C8qvfJmMQhWJOwWsV4G7IU3xRQaFZwQzk5ONkNmZpeZs8Ww5GF6q0/Ua1/ESYygifTAwI/vfMM550tyJR1F0b9K8OHjp5XP1dXa2vrG5lZ9e+fKZYUV2BGZyuxNAg6VNNghSQpvcougE4XXyeh82r/+g9bJzPymcY5dDamRAymAvNSrf+EKOSp1yGmIBDwFreFrr96ImtGswmWI59Bg82r3tisnvJ+JQqMhocC52zjKqVuCJSkUTmq8cJiDGEGKtx4NaHTdcrb/JDzwSj8cZNY/Q+FMff2jBO3cWCfeqYGGbrE3Fd/tJXphMg3OuqU0eUFoxPPgQaFCysJpOGFfWhSkxh5AWOl3D8UQLAjyEdZq/Af64yz+9IMuc7RAmT0qOdhUSzPxx6b8eEr/M8Ldi9GTzzleTHUZrr4149Pm6a+TRut8nniV7bF9dshi9p212AVrsw4TrGR/2T17qDwGq0E92Hm2BpX5n132poK9J1HzuN4=</latexit> u1 = ✓w⇤ 非 9 <latexit sha1_base64="367e1GafQeTNBLWm/Mbt2TkJF4o=">AAACa3icfZDRShtBFIYna201rTXqnS04NAgiEnaLWG8EQS+8kVowKrhhOTs5mwzOzC4zZ9uGJZc+jbftw/gQvoOzMUIbSw8MfPznH845f1oo6SgM7xvB3Kv5128WFptv3y29X26trF64vLQCuyJXub1KwaGSBrskSeFVYRF0qvAyvTmq+5ff0TqZm3MaFdjTMDAykwLIS0lrI9ZAwzTjZRLxAx7TEAmetR/JdtJqh51wUvwlRFNos2mdJSuN3bifi1KjIaHAuesoLKhXgSUpFI6bcemwAHEDA7z2aECj61WTS8Z80yt9nuXWP0N8ov75owLt3Ein3lnv6GZ7tfjPXqpnJlO236ukKUpCI54GZ6XilPM6Jt6XFgWpkQcQVvrduRiCBUE+zGYzPkZ/nMVTP+hrgRYot9tVDHagpRn7YwfxTk3/M8LPZ6Mnn3M0m+pLuPjcifY6e99224dH08QX2Af2iW2xiH1hh+yEnbEuE+yW3bFf7HfjIVgL1oOPT9agMf2zxv6qYPMRpHW8Kg==</latexit> 2 1 | <latexit sha1_base64="4zh6H38M2fSQj2yfPP5OtPykIK8=">AAACc3icfZDPbhMxEMad5U9LCjSFYy9WU0SKaLRbVYVjpXLooRVFIm2lOopmndmNVdu7smeBaJVH4Gm4woPwINzrTYMEKWIkSz9981kz86WlVp7i+Gcrunf/wcOV1UfttcdPnq53Np6d+6JyEgey0IW7TMGjVhYHpEjjZekQTKrxIr0+avoXn9B5VdiPNC1xaCC3KlMSKEijzkuhkQvUuidyMAYETZBgh+/yk54wQJM0459HtDPqdON+PC9+F5IFdNmizkYbrX0xLmRl0JLU4P1VEpc0rMGRkhpnbVF5LEFeQ45XAS0Y9MN6ftGMvwjKmGeFC88Sn6t//qjBeD81aXA2S/rlXiP+s5eapcmUvR3WypYVoZW3g7NKcyp4ExcfK4eS9DQASKfC7lxOwIGkEGq7Ld5hOM7haRj0vkQHVLhXtQCXG2Vn4dhcvG7of0b48tsYKOScLKd6F873+slB/+DDfvfwaJH4KttkW6zHEvaGHbJjdsYGTLKv7Bv7zn60fkWb0Va0fWuNWos/z9lfFe3eAOZsvvQ=</latexit> (t ) 0
zi i > 0 <latexit sha1_base64="1cEtjBRSN+vWN3nw8IqXksvbm7I=">AAACdXicfZDfahNBFMYna9WY+iepl1IYjIpKibsSYq+k0F54U1qhSQvZEM5Ozm6HzswuM7PVdNln6NP0Vp+jT9JbZ9MEalJ6YODH953hnPNFmeDG+v51zXu09vjJ0/qzxvrzFy9fNVsbA5PmmmGfpSLVJxEYFFxh33Ir8CTTCDISeByd7Vb+8Tlqw1N1ZKcZjiQkisecgXXSuPkpFKASgaEEexrFxa9ya4EX5ZiHeubS79QfN9t+x58VXYVgDm0yr8Nxq9YNJynLJSrLBBgzDPzMjgrQljOBZSPMDWbAziDBoUMFEs2omN1U0vdOmdA41e4pS2fq3R8FSGOmMnKd1b5m2avEe71ILk228fao4CrLLSp2OzjOBbUprQKjE66RWTF1AExztztlp6CBWRdroxHuoTtO474bdJChBpvqz0UIOpFcle7YJNyq6KFG+L1odORyDpZTXYXB107Q6/R+dts7u/PE6+QNeUs+koB8IzvkBzkkfcLIJbkif8jf2o236b3zPty2erX5n9fkv/K+/AMvsMGB</latexit>
zi i > 0 <latexit sha1_base64="1cEtjBRSN+vWN3nw8IqXksvbm7I=">AAACdXicfZDfahNBFMYna9WY+iepl1IYjIpKibsSYq+k0F54U1qhSQvZEM5Ozm6HzswuM7PVdNln6NP0Vp+jT9JbZ9MEalJ6YODH953hnPNFmeDG+v51zXu09vjJ0/qzxvrzFy9fNVsbA5PmmmGfpSLVJxEYFFxh33Ir8CTTCDISeByd7Vb+8Tlqw1N1ZKcZjiQkisecgXXSuPkpFKASgaEEexrFxa9ya4EX5ZiHeubS79QfN9t+x58VXYVgDm0yr8Nxq9YNJynLJSrLBBgzDPzMjgrQljOBZSPMDWbAziDBoUMFEs2omN1U0vdOmdA41e4pS2fq3R8FSGOmMnKd1b5m2avEe71ILk228fao4CrLLSp2OzjOBbUprQKjE66RWTF1AExztztlp6CBWRdroxHuoTtO474bdJChBpvqz0UIOpFcle7YJNyq6KFG+L1odORyDpZTXYXB107Q6/R+dts7u/PE6+QNeUs+koB8IzvkBzkkfcLIJbkif8jf2o236b3zPty2erX5n9fkv/K+/AMvsMGB</latexit>