Upgrade to Pro — share decks privately, control downloads, hide ads and more …

Hacking Drone with Golang

Hacking Drone with Golang

Talk about OpenCV (GoCV) and Gobot framework.

Ladislav Prskavec

October 06, 2018
Tweet

More Decks by Ladislav Prskavec

Other Decks in Technology

Transcript

  1. Who am I? 4 Ladislav Prskavec (@abtris) 4 Developer &

    SRE in Oracle Apiary 4 Ruby & Node.js & Go 4 Follow Golang meetup on twitter @GoMeetupPrague
  2. Drone 4 Ryze Tello 4 Intel Myriad VPU 4 5MPx

    camera (720p30, 2592x1936), 80g 4 Electronic stabilisation 4 no GPS
  3. OpenCV 4 Open Source Computer Vision Library 4 OpenCV was

    built to provide a common infrastructure for computer vision applications and to accelerate the use of machine perception in the commercial products. 4 GoCV
  4. package main import ( "fmt" "time" "gobot.io/x/gobot" "gobot.io/x/gobot/platforms/dji/tello" ) func

    main() { drone := tello.NewDriver("8888") var flightData *tello.FlightData var battery int8 work := func() { drone.TakeOff() drone.On(tello.FlightDataEvent, func(data interface{}) { flightData = data.(*tello.FlightData) battery = flightData.BatteryPercentage fmt.Println("Height:", flightData.Height) }) gobot.After(5*time.Second, func() { fmt.Println("Battery:", battery) drone.Land() }) } robot := gobot.NewRobot("tello", []gobot.Connection{}, []gobot.Device{drone}, work, ) robot.Start() }
  5. func main() { drone := tello.NewDriver("8888") work := func() {

    ... } robot := gobot.NewRobot("tello", []gobot.Connection{}, []gobot.Device{drone}, work, ) robot.Start() }
  6. work := func() { drone.TakeOff() drone.On(tello.FlightDataEvent, func(data interface{}) { flightData

    = data.(*tello.FlightData) battery = flightData.BatteryPercentage fmt.Println("Height:", flightData.Height) }) gobot.After(5*time.Second, func() { fmt.Println("Battery:", battery) drone.Land() }) }
  7. Others GoCV examples 4 Caffe Classifier 4 Motion-detect 4 Pose

    4 SSD Facedetect (Advanced Deep Neural Network example that uses SSD classifier) 4 TF Classifier (Tensorflow)
  8. package main import ( "fmt" "image" "image/color" "io" "os/exec" "strconv"

    "time" "gobot.io/x/gobot" "gobot.io/x/gobot/platforms/dji/tello" "gocv.io/x/gocv" )
  9. const ( frameX = 400 frameY = 300 frameSize =

    frameX * frameY * 3 ) func main() { drone := tello.NewDriver("8890") window := gocv.NewWindow("Tello") xmlFile := "haarcascade_frontalface_default.xml" ffmpeg := exec.Command("ffmpeg", "-hwaccel", "auto", "-hwaccel_device", "opencl", "-i", "pipe:0", "-pix_fmt", "bgr24", "-s", strconv.Itoa(frameX)+"x"+strconv.Itoa(frameY), "-f", "rawvideo", "pipe:1") ffmpegIn, _ := ffmpeg.StdinPipe() ffmpegOut, _ := ffmpeg.StdoutPipe()
  10. work := func() { if err := ffmpeg.Start(); err !=

    nil { fmt.Println(err) return } drone.On(tello.ConnectedEvent, func(data interface{}) { fmt.Println("Connected") drone.StartVideo() drone.SetVideoEncoderRate(tello.VideoBitRateAuto) drone.SetExposure(0) gobot.Every(100*time.Millisecond, func() { drone.StartVideo() }) }) drone.On(tello.VideoFrameEvent, func(data interface{}) { pkt := data.([]byte) if _, err := ffmpegIn.Write(pkt); err != nil { fmt.Println(err) } }) } robot := gobot.NewRobot("tello", []gobot.Connection{}, []gobot.Device{drone}, work, ) // calling Start(false) lets the Start routine return immediately without an additional blocking goroutine robot.Start(false)
  11. // now handle video frames from ffmpeg stream in main

    thread, to be macOS/Windows friendly for { buf := make([]byte, frameSize) if _, err := io.ReadFull(ffmpegOut, buf); err != nil { fmt.Println(err) continue } img, _ := gocv.NewMatFromBytes(frameY, frameX, gocv.MatTypeCV8UC3, buf) if img.Empty() { continue } // detect faces // color for the rect when faces detected blue := color.RGBA{0, 0, 255, 0} // load classifier to recognize faces classifier := gocv.NewCascadeClassifier() defer classifier.Close() if !classifier.Load(xmlFile) { fmt.Printf("Error reading cascade file: %v\n", xmlFile) return } rects := classifier.DetectMultiScale(img) fmt.Printf("found %d faces\n", len(rects)) // draw a rectangle around each face on the original image, // along with text identifying as "Human" for _, r := range rects { gocv.Rectangle(&img, r, blue, 3) size := gocv.GetTextSize("Human", gocv.FontHersheyPlain, 1.2, 2) pt := image.Pt(r.Min.X+(r.Min.X/2)-(size.X/2), r.Min.Y-2) gocv.PutText(&img, "Human", pt, gocv.FontHersheyPlain, 1.2, blue, 2) } window.IMShow(img) if window.WaitKey(1) >= 0 { break } } }