$30 off During Our Annual Pro Sale. View Details »

Scala x AWS Server Side Application Recipe

Yusuke Wada
February 11, 2017

Scala x AWS Server Side Application Recipe

Scala x AWS Server Side Application Recipe

Yusuke Wada

February 11, 2017
Tweet

More Decks by Yusuke Wada

Other Decks in Programming

Transcript

  1. ΋͘͡ ֎෦αʔϏεͱͷ͓෇߹͍ 
 - Web API x ElastiCache ϚωʔδϟɺϫʔΧʔͷεϧʔϓοτΛ͍͋͛ͨ
 -

    Akka Worker x Amazon ECS ΤϯδχΞɺDDoSʹک͑ͳ͍҆Β͔ͳ຾ΓΛ
 - AWS Lambda x AWS WAF
  2. Unit Test Code @ Specs2 "locked" should { trait Before

    { self: CommonBefore => mockLockComponent.lock(anyString) returns Left(new IllegalStateException()) } class Context extends CommonContext with Before "throw DailyPresentConflictError" in new Context { val result = service.present(mockUserName) await(result) must throwA[DailyPresentConflictError] } "no update process" in new Context { val result = service.present(mockUserName) await(result) must throwA[Throwable] there was one(mockUserRepository).findByName(anyString) andThen one(mockLockComponent).lock(anyString) andThen no(mockPointRepository).append(anyString, anyInt) andThen no(mockLockComponent).unlock(anyString) } } DailyPresentServiceSpec.scala
  3. Unit Test Code @ Specs2 "locked" should { trait Before

    { self: CommonBefore => mockLockComponent.lock(anyString) returns Left(new IllegalStateException()) } class Context extends CommonContext with Before "throw DailyPresentConflictError" in new Context { val result = service.present(mockUserName) await(result) must throwA[DailyPresentConflictError] } "no update process" in new Context { val result = service.present(mockUserName) await(result) must throwA[Throwable] there was one(mockUserRepository).findByName(anyString) andThen one(mockLockComponent).lock(anyString) andThen no(mockPointRepository).append(anyString, anyInt) andThen no(mockLockComponent).unlock(anyString) } } DailyPresentServiceSpec.scala ϩοΫ࣌Ͱ΋ ࣮ߦ͞ΕΔ͸ͣ
  4. Unit Test Code @ Specs2 "locked" should { trait Before

    { self: CommonBefore => mockLockComponent.lock(anyString) returns Left(new IllegalStateException()) } class Context extends CommonContext with Before "throw DailyPresentConflictError" in new Context { val result = service.present(mockUserName) await(result) must throwA[DailyPresentConflictError] } "no update process" in new Context { val result = service.present(mockUserName) await(result) must throwA[Throwable] there was one(mockUserRepository).findByName(anyString) andThen one(mockLockComponent).lock(anyString) andThen no(mockPointRepository).append(anyString, anyInt) andThen no(mockLockComponent).unlock(anyString) } } DailyPresentServiceSpec.scala ϩοΫ࣌ ࣮ߦ͞Εͳ͍͸ͣ
  5. Akka Worker ͷ
 Docker Image ࡞੒ lazy val news =

    project.in(file("./news")) .enablePlugins(PlayScala) .enablePlugins(DockerPlugin) .settings(commonSettings: _*) .settings( name := "ecn-worker-news", version := "0.1.0" ) .settings(libraryDependencies ++= Seq( )) .settings( maintainer in Docker := "Yusuke Wada <[email protected]>", dockerExposedPorts in Docker := Seq(9000, 9443), dockerRepository := Some("cmwadayusuke") ) .dependsOn(library % “test->test;compile->compile") $ ./activator docker:publish build.sbt shell
  6. Akka Worker ͷ Docker Image ࡞੒ lazy val news =

    project.in(file("./news")) .enablePlugins(PlayScala) .enablePlugins(DockerPlugin) .settings(commonSettings: _*) .settings( name := "ecn-worker-news", version := "0.1.0" ) .settings(libraryDependencies ++= Seq( )) .settings( maintainer in Docker := "Yusuke Wada <[email protected]>", dockerExposedPorts in Docker := Seq(9000, 9443), dockerRepository := Some("cmwadayusuke") ) .dependsOn(library % “test->test;compile->compile") $ ./activator docker:publish build.sbt shell Docker༻ͷ ৘ใΛఆٛ
  7. Akka Worker ͷ Docker Image ࡞੒ lazy val news =

    project.in(file("./news")) .enablePlugins(PlayScala) .enablePlugins(DockerPlugin) .settings(commonSettings: _*) .settings( name := "ecn-worker-news", version := "0.1.0" ) .settings(libraryDependencies ++= Seq( )) .settings( maintainer in Docker := "Yusuke Wada <[email protected]>", dockerExposedPorts in Docker := Seq(9000, 9443), dockerRepository := Some("cmwadayusuke") ) .dependsOn(library % “test->test;compile->compile") $ ./activator docker:publish build.sbt shell activator (sbt ϥούʔ)ͷ ίϚϯυΛ࣮ߦ
  8. ᶃ AWS WAF ͷಋೖ AWS WAF ΛηοτΞοϓɻःஅର৅ͷ
 IP Addresses ͸ͻͱ·ۭͣʹ


    →͜͜Λ AWS Lambda Ͱࣗಈઃఆ AWS WAF Λ CloudFront ʹؔ࿈͚ͮΔ
  9. tg-agent.conf.j2ʢҰ෦ ʣ <match log.webapi.access> type s3 s3_bucket "webapi-access-log" path "webapi/access/"

    s3_object_key_format %{path}%{time_slice}/access_%{index}.% {file_extension} buffer_path /var/log/td-agent/s3/webapi_access time_slice_format %Y/%m/%d/%H time_slice_wait 1m utc buffer_chunk_limit 512k format json include_time_key true time_key log_time </match> tg-agent.conf.j2
  10. tg-agent.conf.j2ʢҰ෦ ʣ <match log.webapi.access> type s3 s3_bucket "webapi-access-log" path "webapi/access/"

    s3_object_key_format %{path}%{time_slice}/access_%{index}.% {file_extension} buffer_path /var/log/td-agent/s3/webapi_access time_slice_format %Y/%m/%d/%H time_slice_wait 1m utc buffer_chunk_limit 512k format json include_time_key true time_key log_time </match> tg-agent.conf.j2 1࣌ؒ΋͘͠͸ 512KB͝ͱʹϩάΛ S3΁
  11. Lambda Function ຊମΛॻ͘ def handle(event, context): # Get the object

    from the event and show its content type bucket = event['Records'][0]['s3']['bucket']['name'] key = urllib.unquote_plus(event['Records'][0]['s3']['object'] ['key'].encode('utf8')) try: # open file, unzip s3.download_file(bucket, key, '/tmp/file.gz') f = gzip.open('/tmp/file.gz', 'rb') content = f.read() f.close # create black list ip_address_list = to_ip_address_list(content.decode('utf-8')) black_list = to_ip_black_list(ip_address_list) main.py
  12. Lambda Function ຊମΛॻ͘ def handle(event, context): # Get the object

    from the event and show its content type bucket = event['Records'][0]['s3']['bucket']['name'] key = urllib.unquote_plus(event['Records'][0]['s3']['object'] ['key'].encode('utf8')) try: # open file, unzip s3.download_file(bucket, key, '/tmp/file.gz') f = gzip.open('/tmp/file.gz', 'rb') content = f.read() f.close # create black list ip_address_list = to_ip_address_list(content.decode('utf-8')) black_list = to_ip_black_list(ip_address_list) main.py IPΞυϨε͚ͩ நग़
  13. Lambda Function ຊମΛॻ͘ def handle(event, context): # Get the object

    from the event and show its content type bucket = event['Records'][0]['s3']['bucket']['name'] key = urllib.unquote_plus(event['Records'][0]['s3']['object'] ['key'].encode('utf8')) try: # open file, unzip s3.download_file(bucket, key, '/tmp/file.gz') f = gzip.open('/tmp/file.gz', 'rb') content = f.read() f.close # create black list ip_address_list = to_ip_address_list(content.decode('utf-8')) black_list = to_ip_black_list(ip_address_list) main.py ूܭɾ͖͍͠஋ ൑ఆ
  14. if len(black_list): # create ip set for waf update_ip_set=map(to_ip_set_for_waf, black_list)

    # create sns settings sns_settings=to_sns_notification_settings(black_list) # waf setting (side-effect function) waf_block(update_ip_set) # notification (side-effect functions) sns_notification(sns_settings) if slack_notification_enabled: slack_notification(black_list) else: pass else: print(u'nothing to do.') pass return black_list except Exception as e: print(e) raise e
  15. if len(black_list): # create ip set for waf update_ip_set=map(to_ip_set_for_waf, black_list)

    # create sns settings sns_settings=to_sns_notification_settings(black_list) # waf setting (side-effect function) waf_block(update_ip_set) # notification (side-effect functions) sns_notification(sns_settings) if slack_notification_enabled: slack_notification(black_list) else: pass else: print(u'nothing to do.') pass return black_list except Exception as e: print(e) raise e AWS WAF API ίʔϧ
  16. if len(black_list): # create ip set for waf update_ip_set=map(to_ip_set_for_waf, black_list)

    # create sns settings sns_settings=to_sns_notification_settings(black_list) # waf setting (side-effect function) waf_block(update_ip_set) # notification (side-effect functions) sns_notification(sns_settings) if slack_notification_enabled: slack_notification(black_list) else: pass else: print(u'nothing to do.') pass return black_list except Exception as e: print(e) raise e ௨஌
  17. λΠϜϥΠϯ 08:00 
 ىচɺΞϥʔτ·ΈΕͷSlack 10:00 
 ग़ࣾɺϚωʔδϟʔ૵ന 12:00 
 ΞʔΩςΫνϟܾఆɺ࣮૷։࢝

    15:00 
 ࣮૷׬ྃɺಈ࡞ݕূ։࢝ 17:00 
 ಈ࡞ݕূ׬ྃɺ
 εςʔδϯά؀ڥʹσϓϩΠ ཌ11:00 ຊ൪ϦϦʔε ཌ18:00 υϠؼ୐
  18. ͜Μͳײ͡Ͱ ָ͘͠΍ͬͯ·͢ ࠷৽ͷٕज़Λ࣮Ҋ݅΁ੵۃతʹ౤ೖ͍ͨ͠ʂ AWSɺαʔόʔαΠυʹؔ͢Δ༇շͳΤϯδχΞͱҰॹʹಇ͖ ͍ͨʂ Developers.IOͰϒϩάॻ͖͍ͨʂ εΩϧ σβΠϯ / iOS

    / Android Java / Scala / Ruby / node.js / Go && AWS ϞόΠϧόοΫΤϯυΤϯδχΞ ― https://classmethod.jp/recruit/jobs/aws-web-app-mobile-engineer/