Upgrade to Pro — share decks privately, control downloads, hide ads and more …

Getting Started with Azure Kinect DK

TakashiYoshinaga
September 08, 2019

Getting Started with Azure Kinect DK

Slice deck of tutorial of contents creation with Azure Kinect and Unity.

TakashiYoshinaga

September 08, 2019
Tweet

More Decks by TakashiYoshinaga

Other Decks in Technology

Transcript

  1. Installation of Azure Kinect SDK ①Open following directory by explorer

    Sample¥Plugins ②Drag & Drop all Files into Plugins of this project
  2. Installation of Azure Kinect SDK ①Open the directory of which

    Azure Kinect SDK is installed C:¥Program Files¥Azure Kinect SDK v1.2.0¥sdk¥windows-desktop¥amd64¥release¥bin ②Drag and drop depthengine_2_0.dll into Plugins of your project
  3. Adding Object to Visualize Point Cloud Next step, material for

    drawing point cloud will be set to Element0
  4. Adding a Script to Draw Point Cloud ①Remove search word

    ②New Script ③KinectScript ④Create and Add
  5. Start and Stop Azure Kinect using UnityEngine; using Microsoft.Azure.Kinect.Sensor; public

    class KinectScript : MonoBehaviour { Device device; //Variable to access to Kinect void Start() { InitKinect(); //Initialization of Kinect } void InitKinect() { //See Next Page. } /*Omitted */ }
  6. Adding a Script to Draw Point Cloud void InitKinect() {

    device = Device.Open(0); //OpenKinect device.StartCameras(new DeviceConfiguration { ColorFormat = ImageFormat.ColorBGRA32, ColorResolution = ColorResolution.R720p, DepthMode = DepthMode.NFOV_Unbinned, SynchronizedImagesOnly = true, CameraFPS = FPS.FPS30, }); } void OnDestroy() //Stop Kinect { device.StopCameras(); }
  7. Preparation of Drawing Point Cloud Device device; int width, height,

    num; //width, height and total Pixels void Start(){ /*Omitted*/ } void InitKinect() { device = Device.Open(0); device.StartCameras ( /*Omitted*/ ); //Calculation of total number of pixel. width = device.GetCalibration(). depth_camera_calibration.resolution_width; height = device.GetCalibration(). depth_camera_calibration.resolution_height; num = height * width; }
  8. Preparation of Drawing Point Cloud int width, height, num; Mesh

    mesh; //Used for Drawing Shape Vector3[] vertices; //Position of each point. Color32[] colors; //Color of each point. int[] indices; //Index list of drawing point void Start() { InitKinect(); InitMesh(); //Initialization of mesh } void InitMesh() { //See the next page. }
  9. Preparation of Drawing Point Cloud void InitMesh() { mesh =

    new Mesh(); //Enable drawing more than 65535 points mesh.indexFormat = UnityEngine.Rendering.IndexFormat.UInt32; //allocation of memory for treating information to //draw point cloud. vertices = new Vector3[num]; colors = new Color32[num]; indices = new int[num]; //See next page }
  10. Drawing Points /*Continued from previous step (in InitMesh function)*/ int

    index = 0; //index of vertex array. for (int y = 0; y < height; y++) { for (int x = 0; x < width; x++) { vertices[index].x = x ; vertices[index].y = y ; vertices[index].z = 2; colors[index].r = 0; colors [index].g = 0; colors [index].b = 255; colors [index].a = 255; indices[index] = index; index++; } } Temporally setting the coordinates of each vertex Temporally setting the color of each vertex Setting drawing list.
  11. Drawing Points /*Continued from the previous page.*/ int index =

    0; for (int y = 0; y < height; y++) { for (int x = 0; x < width; x++) { /*Omitted*/ } } //Assignment vertices and colors to mesh. mesh.vertices = vertices; mesh.colors32 = colors; //Setting drawing list and mesh topology. mesh.SetIndices(indices, MeshTopology.Points, 0); //Assingnment of mesh to MeshFilter of the gameObject. gameObject.GetComponent<MeshFilter>().mesh = mesh;
  12. Getting DataStream from Kinect Int[] indices; Transformation trans; //Coordinate tranlation

    void Start() { /*omitted*/ } void InitKinect() { /*Omitted*/ width = device.GetCalibration(). depth_camera_calibration.resolution_width; height = device.GetCalibration(). depth_camera_calibration.resolution_height; num = height * width; /*It’ll be used for transform coordinate system*/ trans = device.GetCalibration().CreateTransformation(); }
  13. Getting DataStream from Kinect using UnityEngine; using Microsoft.Azure.Kinect.Sensor; //Enable to

    use async function. using System.Threading.Tasks; public class KinectScript : MonoBehaviour { Device device; int width, height, num; Mesh mesh; Vector3[] vertices; Color32[] colors; int[] indices; /*Omitted*/
  14. Getting DataStream from Kinect void Start() { InitKinect(); InitMesh(); Task

    t = KinectLoop( ); } private async Task KinectLoop( ) { while (true) { //Continue to get DataStream from Kinect. } }
  15. Getting DataStream from Kinect While (true) { //Getting synchlonyzed frame

    of Kinect using (Capture capture = await Task.Run(() => device.GetCapture()).ConfigureAwait(true)) { //Getting Depth image with capture.Depth //And transform it into xyz by DeptuImageToPointCloud Image pImage = trans.DepthImageToPointCloud(capture.Depth); //Getting vertices data. Short3[] pointCloud = pImage.GetPixels<Short3>().ToArray(); /*See the next page*/ } }
  16. Getting DataStream from Kinect using (Capture capture = await Task.Run(()

    => device.GetCapture()).ConfigureAwait(true)) { Image pImage = trans.DepthImageToPointCloud(capture.Depth); Short3[] pointCloud = pImage.GetPixels<Short3>().ToArray(); //Assigning coordinate of each vertex to vertices. for (int i = 0; i < num; i++) { vertices[i].x = pointCloud[i].X * 0.001f; vertices[i].y = pointCloud[i].Y * 0.001f; vertices[i].z = pointCloud[i].Z * 0.001f; } //set vertices to mesh. mesh.vertices = vertices; mesh.RecalculateBounds(); }
  17. Run Click Scene Tab if you want to see point

    cloud from any view point Upside Down
  18. Modification of Y Coordinate using (Capture capture = await Task.Run(()

    => device.GetCapture()).ConfigureAwait(true)) { Image pImage = trans.DepthImageToPointCloud(capture.Depth); Short3[] pointCloud = pImage.GetPixels<Short3>().ToArray(); // Assigning coordinate of each vertex to vertices. for (int i = 0; i < num; i++) { vertices[i].x = pointCloud[i].X * 0.001f; vertices[i].y = -pointCloud[i].Y * 0.001f; vertices[i].z = pointCloud[i].Z * 0.001f; } // set vertices to mesh. mesh.vertices = vertices; mesh.RecalculateBounds(); }
  19. Assigning Color to Point Cloud while (true) { using (Capture

    capture = await Task.Run(() => device.GetCapture()).ConfigureAwait(true)) { //Getting color image which is matched to Depth resolution Image modifiedColor = trans.ColorImageToDepthCamera(capture); //Getting pixel array. BGRA[] colorArray = modifiedColor.GetPixels<BGRA>().ToArray(); Image pImage = trans.DepthImageToPointCloud(capture.Depth); Short3[] pointCloud = pImage.GetPixels<Short3>().ToArray(); /*See the next page*/ KinectLoop Function
  20. Assigning Color to Point Cloud /*Continued from the previous page*/

    for (int i = 0; i < num; i++) { vertices[i].x = pointCloud[i].X * 0.001f; vertices[i].y = -pointCloud[i].Y * 0.001f; vertices[i].z = pointCloud[i].Z * 0.001f; colors[i].a = 255; colors [i].b = colorArray[i].B; colors [i].g = colorArray[i].G; colors [i].r = colorArray[i].R; } mesh.vertices = vertices; //Setting color to mesh. mesh.colors32 = colors; mesh.RecalculateBounds();
  21. Run

  22. Modification of Whole Size of Point Cloud ①PointCloud ②Set Position

    and Scale as below Position 0 0.5 -1 Scale 1 1 1