none
Need help with combining and overlaying the skeleton and color stream/image RRS feed

  • Question

  • Hey Guys,

    i'm trying to code a programm that shows me the current colorstream and overlays/combines it with the skeleton stream.

    i took the microsoft skeletonviewer example and tried to implement the colorstream. Well that worked, the colorstream is running, but the skeleton disappeared.

    So now my question for you: How can i enable the skeleton so that if person stands right in front of the kinect, the colorstream and the skeleton appears?

    here's my code:

    //------------------------------------------------------------------------------
    // <copyright file="MainWindow.xaml.cs" company="Microsoft">
    //     Copyright (c) Microsoft Corporation.  All rights reserved.
    // </copyright>
    //------------------------------------------------------------------------------
    
    namespace Microsoft.Samples.Kinect.SkeletonBasics
    {
        using System.IO;
        using System.Windows;
        using System.Windows.Media;
        using Microsoft.Kinect;
        using System;
        using System.Windows.Media.Imaging;
    
        /// <summary>
        /// Interaction logic for MainWindow.xaml
        /// </summary>
        public partial class MainWindow : Window
        {
            /// <summary>
            /// Width of output drawing
            /// </summary>
            private const float RenderWidth = 640.0f;
    
            /// <summary>
            /// Height of our output drawing
            /// </summary>
            private const float RenderHeight = 480.0f;
    
            /// <summary>
            /// Thickness of drawn joint lines
            /// </summary>
            private const double JointThickness = 3;
    
            /// <summary>
            /// Thickness of body center ellipse
            /// </summary>
            private const double BodyCenterThickness = 10;
    
            /// <summary>
            /// Thickness of clip edge rectangles
            /// </summary>
            private const double ClipBoundsThickness = 10;
    
            /// <summary>
            /// Brush used to draw skeleton center point
            /// </summary>
            private readonly Brush centerPointBrush = Brushes.Blue;
    
            /// <summary>
            /// Brush used for drawing joints that are currently tracked
            /// </summary>
            private readonly Brush trackedJointBrush = new SolidColorBrush(Color.FromArgb(255, 68, 192, 68));
    
            /// <summary>
            /// Brush used for drawing joints that are currently inferred
            /// </summary>        
            private readonly Brush inferredJointBrush = Brushes.Yellow;
    
            /// <summary>
            /// Pen used for drawing bones that are currently tracked
            /// </summary>
            private readonly Pen trackedBonePen = new Pen(Brushes.Green, 6);
    
            /// <summary>
            /// Pen used for drawing bones that are currently inferred
            /// </summary>        
            private readonly Pen inferredBonePen = new Pen(Brushes.Gray, 1);
    
            /// <summary>
            /// Active Kinect sensor
            /// </summary>
            //private KinectSensor sensor;
            KinectSensor sensor = KinectSensor.KinectSensors[0];
            private byte[] colorPixelData;
            private WriteableBitmap outputImage;
    
            
            /// <summary>
            /// Drawing group for skeleton rendering output
            /// </summary>
            private DrawingGroup drawingGroup;
    
            /// <summary>
            /// Drawing image that we will display
            /// </summary>
            private DrawingImage imageSource;
    
            /// <summary>
            /// Initializes a new instance of the MainWindow class.
            /// </summary>
            public MainWindow()
            {
                InitializeComponent();
            }
    
            
            /// <summary>
            /// Draws indicators to show which edges are clipping skeleton data
            /// </summary>
            /// <param name="skeleton">skeleton to draw clipping information for</param>
            /// <param name="drawingContext">drawing context to draw to</param>
            private static void RenderClippedEdges(Skeleton skeleton, DrawingContext drawingContext)
            {
                if (skeleton.ClippedEdges.HasFlag(FrameEdges.Bottom))
                {
                    drawingContext.DrawRectangle(
                        Brushes.Red,
                        null,
                        new Rect(0, RenderHeight - ClipBoundsThickness, RenderWidth, ClipBoundsThickness));
                }
    
                if (skeleton.ClippedEdges.HasFlag(FrameEdges.Top))
                {
                    drawingContext.DrawRectangle(
                        Brushes.Red,
                        null,
                        new Rect(0, 0, RenderWidth, ClipBoundsThickness));
                }
    
                if (skeleton.ClippedEdges.HasFlag(FrameEdges.Left))
                {
                    drawingContext.DrawRectangle(
                        Brushes.Red,
                        null,
                        new Rect(0, 0, ClipBoundsThickness, RenderHeight));
                }
    
                if (skeleton.ClippedEdges.HasFlag(FrameEdges.Right))
                {
                    drawingContext.DrawRectangle(
                        Brushes.Red,
                        null,
                        new Rect(RenderWidth - ClipBoundsThickness, 0, ClipBoundsThickness, RenderHeight));
                }
            }
    
            /// <summary>
            /// Execute startup tasks
            /// </summary>
            /// <param name="sender">object sending the event</param>
            /// <param name="e">event arguments</param>
            private void WindowLoaded(object sender, RoutedEventArgs e)
            {
                // Create the drawing group we'll use for drawing
                this.drawingGroup = new DrawingGroup();
    
                // Create an image source that we can use in our image control
                this.imageSource = new DrawingImage(this.drawingGroup);
    
                // Display the drawing using our image control
                videoimage.Source = this.imageSource;
    
                // Look through all sensors and start the first connected one.
                // This requires that a Kinect is connected at the time of app startup.
                // To make your app robust against plug/unplug, 
                // it is recommended to use KinectSensorChooser provided in Microsoft.Kinect.Toolkit
                foreach (var potentialSensor in KinectSensor.KinectSensors)
                {
                    if (potentialSensor.Status == KinectStatus.Connected)
                    {
                        this.sensor = potentialSensor;
                        break;
                    }
                }
    
                if (null != this.sensor)
                {
                    // Turn on the skeleton stream to receive skeleton frames
                    this.sensor.SkeletonStream.Enable();
                    this.sensor.ColorStream.Enable(ColorImageFormat.RgbResolution640x480Fps30);
                    this.sensor.ColorFrameReady += new EventHandler<ColorImageFrameReadyEventArgs>(this.SensorColorFrameReady);
    
    
                    // Add an event handler to be called whenever there is new color frame data
                    this.sensor.SkeletonFrameReady += this.SensorSkeletonFrameReady;
                    
    
                    // Start the sensor!
                    try
                    {
                        this.sensor.Start();
                    }
                    catch (IOException)
                    {
                        this.sensor = null;
                    }
                }
    
                if (null == this.sensor)
                {
                    this.statusBarText.Text = Properties.Resources.NoKinectReady;
                }
            }
    
            /// <summary>
            /// Execute shutdown tasks
            /// </summary>
            /// <param name="sender">object sending the event</param>
            /// <param name="e">event arguments</param>
            private void WindowClosing(object sender, System.ComponentModel.CancelEventArgs e)
            {
                if (null != this.sensor)
                {
                    this.sensor.Stop();
                }
            }
    
            /// <summary>
            /// Event handler for Kinect sensor's SkeletonFrameReady event
            /// </summary>
            /// <param name="sender">object sending the event</param>
            /// <param name="e">event arguments</param>
            private void SensorSkeletonFrameReady(object sender, SkeletonFrameReadyEventArgs e)
            {
                Skeleton[] skeletons = new Skeleton[0];
    
                using (SkeletonFrame skeletonFrame = e.OpenSkeletonFrame())
                {
                    if (skeletonFrame != null)
                    {
                        skeletons = new Skeleton[skeletonFrame.SkeletonArrayLength];
                        skeletonFrame.CopySkeletonDataTo(skeletons);
                    }
                }
    
                using (DrawingContext dc = this.drawingGroup.Open())
                {
                    // Draw a transparent background to set the render size
                    //dc.DrawRectangle(Brushes.Black, null, new Rect(0.0, 0.0, RenderWidth, RenderHeight));
    
                    if (skeletons.Length != 0)
                    {
                        foreach (Skeleton skel in skeletons)
                        {
                            RenderClippedEdges(skel, dc);
    
                            if (skel.TrackingState == SkeletonTrackingState.Tracked)
                            {
                                this.DrawBonesAndJoints(skel, dc);
                            }
                            else if (skel.TrackingState == SkeletonTrackingState.PositionOnly)
                            {
                                dc.DrawEllipse(
                                this.centerPointBrush,
                                null,
                                this.SkeletonPointToScreen(skel.Position),
                                BodyCenterThickness,
                                BodyCenterThickness);
                            }
                        }
                    }
    
                    // prevent drawing outside of our render area
                    this.drawingGroup.ClipGeometry = new RectangleGeometry(new Rect(0.0, 0.0, RenderWidth, RenderHeight));
                }
            }
    
            private void SensorColorFrameReady(object sender, ColorImageFrameReadyEventArgs e)
            {
                using (ColorImageFrame colorFrame = e.OpenColorImageFrame())
                {
                    if (colorFrame != null)
                    {
                        this.colorPixelData = new byte[colorFrame.PixelDataLength];
    
                        colorFrame.CopyPixelDataTo(this.colorPixelData);
    
                        this.outputImage = new WriteableBitmap(colorFrame.Width, colorFrame.Height, 96, 96, PixelFormats.Bgr32, null);
                        this.outputImage.WritePixels(new Int32Rect(0, 0, colorFrame.Width, colorFrame.Height), this.colorPixelData, colorFrame.Width * 4, 0);
                        this.videoimage.Source = this.outputImage;
          
                    }
                }
    
             }
    
            /// <summary>
            /// Draws a skeleton's bones and joints
            /// </summary>
            /// <param name="skeleton">skeleton to draw</param>
            /// <param name="drawingContext">drawing context to draw to</param>
            private void DrawBonesAndJoints(Skeleton skeleton, DrawingContext drawingContext)
            {
                // Render Torso
                this.DrawBone(skeleton, drawingContext, JointType.Head, JointType.ShoulderCenter);
                this.DrawBone(skeleton, drawingContext, JointType.ShoulderCenter, JointType.ShoulderLeft);
                this.DrawBone(skeleton, drawingContext, JointType.ShoulderCenter, JointType.ShoulderRight);
                this.DrawBone(skeleton, drawingContext, JointType.ShoulderCenter, JointType.Spine);
                this.DrawBone(skeleton, drawingContext, JointType.Spine, JointType.HipCenter);
                this.DrawBone(skeleton, drawingContext, JointType.HipCenter, JointType.HipLeft);
                this.DrawBone(skeleton, drawingContext, JointType.HipCenter, JointType.HipRight);
    
                // Left Arm
                this.DrawBone(skeleton, drawingContext, JointType.ShoulderLeft, JointType.ElbowLeft);
                this.DrawBone(skeleton, drawingContext, JointType.ElbowLeft, JointType.WristLeft);
                this.DrawBone(skeleton, drawingContext, JointType.WristLeft, JointType.HandLeft);
    
                // Right Arm
                this.DrawBone(skeleton, drawingContext, JointType.ShoulderRight, JointType.ElbowRight);
                this.DrawBone(skeleton, drawingContext, JointType.ElbowRight, JointType.WristRight);
                this.DrawBone(skeleton, drawingContext, JointType.WristRight, JointType.HandRight);
    
                // Left Leg
                this.DrawBone(skeleton, drawingContext, JointType.HipLeft, JointType.KneeLeft);
                this.DrawBone(skeleton, drawingContext, JointType.KneeLeft, JointType.AnkleLeft);
                this.DrawBone(skeleton, drawingContext, JointType.AnkleLeft, JointType.FootLeft);
    
                // Right Leg
                this.DrawBone(skeleton, drawingContext, JointType.HipRight, JointType.KneeRight);
                this.DrawBone(skeleton, drawingContext, JointType.KneeRight, JointType.AnkleRight);
                this.DrawBone(skeleton, drawingContext, JointType.AnkleRight, JointType.FootRight);
     
                // Render Joints
                foreach (Joint joint in skeleton.Joints)
                {
                    Brush drawBrush = null;
    
                    if (joint.TrackingState == JointTrackingState.Tracked)
                    {
                        drawBrush = this.trackedJointBrush;                    
                    }
                    else if (joint.TrackingState == JointTrackingState.Inferred)
                    {
                        drawBrush = this.inferredJointBrush;                    
                    }
    
                    if (drawBrush != null)
                    {
                        drawingContext.DrawEllipse(drawBrush, null, this.SkeletonPointToScreen(joint.Position), JointThickness, JointThickness);
                    }
                
            }
    
            
            }
    
            
            /// <summary>
            /// Maps a SkeletonPoint to lie within our render space and converts to Point
            /// </summary>
            /// <param name="skelpoint">point to map</param>
            /// <returns>mapped point</returns>
            private Point SkeletonPointToScreen(SkeletonPoint skelpoint)
            {
                // Convert point to depth space.  
                // We are not using depth directly, but we do want the points in our 640x480 output resolution.
                DepthImagePoint depthPoint = this.sensor.CoordinateMapper.MapSkeletonPointToDepthPoint(skelpoint, DepthImageFormat.Resolution640x480Fps30);
                return new Point(depthPoint.X, depthPoint.Y);
            }
    
            private Point UpdateDraw(Joint joint)
            {
                //foreach (Joint joint in skeleton.Joints)
                //{
                    ColorImagePoint colorPoint = this.sensor.CoordinateMapper.MapSkeletonPointToColorPoint(joint.Position, ColorImageFormat.RgbResolution640x480Fps30);
                    //var point = new Point((int)ColorImagePoint.X / 640.0 * this.ActualWidth, (int)ColorImagePoint.Y / 480.0 * this.ActualHeight);
                    return new Point(colorPoint.X, colorPoint.Y);
                //}
    
            }
    
            /// <summary>
            /// Draws a bone line between two joints
            /// </summary>
            /// <param name="skeleton">skeleton to draw bones from</param>
            /// <param name="drawingContext">drawing context to draw to</param>
            /// <param name="jointType0">joint to start drawing from</param>
            /// <param name="jointType1">joint to end drawing at</param>
            private void DrawBone(Skeleton skeleton, DrawingContext drawingContext, JointType jointType0, JointType jointType1)
            {
                Joint joint0 = skeleton.Joints[jointType0];
                Joint joint1 = skeleton.Joints[jointType1];
    
                // If we can't find either of these joints, exit
                if (joint0.TrackingState == JointTrackingState.NotTracked ||
                    joint1.TrackingState == JointTrackingState.NotTracked)
                {
                    return;
                }
    
                // Don't draw if both points are inferred
                if (joint0.TrackingState == JointTrackingState.Inferred &&
                    joint1.TrackingState == JointTrackingState.Inferred)
                {
                    return;
                }
    
                // We assume all drawn bones are inferred unless BOTH joints are tracked
                Pen drawPen = this.inferredBonePen;
                if (joint0.TrackingState == JointTrackingState.Tracked && joint1.TrackingState == JointTrackingState.Tracked)
                {
                    drawPen = this.trackedBonePen;
                }
    
                drawingContext.DrawLine(drawPen, this.SkeletonPointToScreen(joint0.Position), this.SkeletonPointToScreen(joint1.Position));
            }
    
            /// <summary>
            /// Handles the checking or unchecking of the seated mode combo box
            /// </summary>
            /// <param name="sender">object sending the event</param>
            /// <param name="e">event arguments</param>
            private void CheckBoxSeatedModeChanged(object sender, RoutedEventArgs e)
            {
                if (null != this.sensor)
                {
                    if (this.checkBoxSeatedMode.IsChecked.GetValueOrDefault())
                    {
                        this.sensor.SkeletonStream.TrackingMode = SkeletonTrackingMode.Seated;
                    }
                    else
                    {
                        this.sensor.SkeletonStream.TrackingMode = SkeletonTrackingMode.Default;
                    }
                }
            }

    and the XML

    <Window x:Class="Microsoft.Samples.Kinect.SkeletonBasics.MainWindow"
            xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
            xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
            Title="SkeletonApp - Kinect configuration" Height="720.075" Width="770" Loaded="WindowLoaded" Closing="WindowClosing" WindowStyle="ToolWindow">
    
        <Window.Resources>
            <SolidColorBrush x:Key="MediumGreyBrush" Color="#ff6e6e6e"/>
            <SolidColorBrush x:Key="KinectPurpleBrush" Color="#ff52318f"/>
            <SolidColorBrush x:Key="KinectBlueBrush" Color="#ff00BCF2"/>
            <Style TargetType="{x:Type Image}">
                <Setter Property="SnapsToDevicePixels" Value="True"/>
            </Style>
            <Style TargetType="{x:Type CheckBox}" x:Key="SquareCheckBox" >
                <Setter Property="Template">
                    <Setter.Value>
                        <ControlTemplate TargetType="{x:Type CheckBox}">
                            <Grid>
                                <StackPanel Orientation="Horizontal" Background="Transparent">
                                    <Grid x:Name="SquareCheckBoxChecked">
                                        <Image x:Name="CheckedNormal" Source="Images\CheckedNormal.png" Stretch="None" HorizontalAlignment="Center"/>
                                        <Image x:Name="CheckedHover" Source="Images\CheckedHover.png" Stretch="None" HorizontalAlignment="Center" Visibility="Collapsed"/>
                                    </Grid>
                                    <Grid x:Name="SquareCheckBoxUnchecked" Visibility="Collapsed">
                                        <Image x:Name="UncheckedNormal" Source="Images\UncheckedNormal.png" Stretch="None" HorizontalAlignment="Center"/>
                                        <Image x:Name="UncheckedHover" Source="Images\UncheckedHover.png" Stretch="None" HorizontalAlignment="Center" Visibility="Collapsed"/>
                                    </Grid>
                                    <TextBlock x:Name="SquareCheckBoxText" Text="{TemplateBinding Content}" TextAlignment="Left"  VerticalAlignment="Center" Foreground="{StaticResource KinectPurpleBrush}" FontSize="15"  Margin="9,0,0,0"/>
                                </StackPanel>
                            </Grid>
                            <ControlTemplate.Triggers>
                                <Trigger Property="IsChecked" Value="false">
                                    <Setter Property="Visibility" Value="Collapsed" TargetName="SquareCheckBoxChecked"/>
                                    <Setter Property="Visibility" Value="Visible" TargetName="SquareCheckBoxUnchecked"/>
                                </Trigger>
                                <Trigger Property="IsMouseOver" Value="true">
                                    <Setter Property="Visibility" Value="Collapsed" TargetName="CheckedNormal"/>
                                    <Setter Property="Visibility" Value="Collapsed" TargetName="UncheckedNormal"/>
                                    <Setter Property="Visibility" Value="Visible" TargetName="CheckedHover"/>
                                    <Setter Property="Visibility" Value="Visible" TargetName="UncheckedHover"/>
                                    <Setter Property="Foreground" Value="{StaticResource KinectBlueBrush}" TargetName="SquareCheckBoxText"/>
                                </Trigger>
                            </ControlTemplate.Triggers>
                        </ControlTemplate>
                    </Setter.Value>
                </Setter>
            </Style>
        </Window.Resources>
        <Window.Background>
            <SolidColorBrush Color="#FF272727"/>
        </Window.Background>
    
        <Grid Name="layoutGrid" Margin="10 0 10 0">
            <Grid.ColumnDefinitions>
                <ColumnDefinition Width="289*"/>
                <ColumnDefinition Width="276*"/>
                <ColumnDefinition Width="177*"/>
            </Grid.ColumnDefinitions>
            <Grid.RowDefinitions>
                <RowDefinition Height="Auto"/>
                <RowDefinition Height="*"/>
                <RowDefinition Height="Auto"/>
                <RowDefinition Height="Auto"/>
            </Grid.RowDefinitions>
            <DockPanel Grid.Row="0" Margin="0 0 0 20" Grid.ColumnSpan="3"/>
            <Viewbox Grid.Row="1" Stretch="Uniform" HorizontalAlignment="Center" Grid.ColumnSpan="3">
                <Grid ClipToBounds="True">
                    <Image Name="videoimage" Width="640" Height="480"/>
                    <Canvas Background="Transparent" Name="ColorImagePoint"></Canvas>
                </Grid>
            </Viewbox>
            <CheckBox Grid.Row="2" Style="{StaticResource SquareCheckBox}" Content="Seated Mode" Height="Auto" VerticalAlignment="Center" Margin="0,10,157,10" Name="checkBoxSeatedMode" Checked="CheckBoxSeatedModeChanged" Unchecked="CheckBoxSeatedModeChanged" Foreground="White"/>
            <StatusBar Grid.Row="3" HorizontalAlignment="Stretch" Name="statusBar" VerticalAlignment="Bottom" Background="White" Foreground="{StaticResource MediumGreyBrush}" Grid.ColumnSpan="3">
                <StatusBarItem Padding="0 0 0 10">
                    <StatusBarItem.Background>
                        <SolidColorBrush Color="#FF272727"/>
                    </StatusBarItem.Background>
                    <TextBlock Name="statusBarText" Margin="-1 0 0 0" Foreground="White">Click 'Seated' to change skeletal mode</TextBlock>
                </StatusBarItem>
            </StatusBar>
            <Button Content="UP" HorizontalAlignment="Left" Margin="198,5,0,0" VerticalAlignment="Top" Width="115" Height="35" Click="btnUp" Grid.Row="2" Grid.Column="1" RenderTransformOrigin="-0.122,0.716" Grid.ColumnSpan="2" Foreground="White" BorderBrush="DodgerBlue" Background="DodgerBlue"/>
            <Button Content="DOWN" HorizontalAlignment="Right" Margin="0,5,10,0" VerticalAlignment="Top" Width="115" Height="35" Grid.Column="2" Click="btnDown" Grid.Row="2" Foreground="White" Background="DodgerBlue" BorderBrush="DodgerBlue"/>
            <Label x:Name="Label_Angle" Content="Adjust Kinect Angle:" Grid.Column="1" HorizontalAlignment="Left" Margin="73,10,0,0" Grid.Row="2" VerticalAlignment="Top" RenderTransformOrigin="0.787,-0.551" Background="#00000000" Foreground="White"/>
        </Grid>
    </Window>
    

    Sunday, July 7, 2013 8:03 PM

Answers

  • Are you familiar with how WPF and Image and Device Context work?

    http://msdn.microsoft.com/en-us/library/ms748373.aspx

    Did you intent to use the same Image control for both the color and skeleton? Why not just create a new image control for the color and overlay the skeleton image control on top?

                    
    <Image Name="ColorImage" Width="640" Height="480"/>
               <Grid ClipToBounds="True">
                    <Image Name="ColorImage" Width="640" Height="480"/>
                    <Image Name="videoimage" Width="640" Height="480"/>
                    <Canvas Background="Transparent" Name="ColorImagePoint"></Canvas>
                </Grid>
    // add 2 local variables
    private WriteableBitmap colorBitmap;
    private byte[] colorPixels;
    // in the WindowLoaded code add
    if (null != this.sensor)
    {
        // Turn on the color stream to receive color frames
        this.sensor.ColorStream.Enable(ColorImageFormat.RgbResolution640x480Fps30);
        // Allocate space to put the pixels we'll receive
        this.colorPixels = new byte[this.sensor.ColorStream.FramePixelDataLength];
        // This is the bitmap we'll display on-screen
        this.colorBitmap = new WriteableBitmap(this.sensor.ColorStream.FrameWidth, this.sensor.ColorStream.FrameHeight, 96.0, 96.0, PixelFormats.Bgr32, null);
        // Set the image we display to point to the bitmap where we'll put the image data
        this.ColorImage.Source = this.colorBitmap;
    ...

    Carmine Si - MSFT




    Tuesday, July 9, 2013 9:17 PM
  • Well,there are some keypoints when draw skeleton data on color stream or depth stream:

    1. 1.you‘d better using WriteableBitmap to render color or depth image on the screen;
    2. 2.the image size of the color ,depth ,skeleton and the screen should be the same,for example,using 640x480Fps30 format,and set the Grid size 640*480。
    3. 3.some space transformation needed when draw skeleton on color or depth image,for example if you need draw skeleton on depeth MapSkeletonPointToColorPoint method,and if skeleton to depth,MapSkeletonPointToDepthPoint should be called.

    the code are as followings:

    the main xml:

    <Window x:Class="KinectSkeletonAlignwithColorandDepthImage.MainWindow"
            xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
            xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
            Title="MainWindow" Width="660" Height="520">
        <Grid  Width="640" Height="480">
            <Image x:Name="ColorImage"  />
            <Image x:Name="DepthImage" Visibility="Hidden"/>
            <Grid x:Name="LayoutRoot" Background="Transparent" >
            </Grid>
        </Grid>
    </Window>

    if you want draw skeleton on depth images,just set the  ColorImage's Vsisibility property to Hidden.

     /// <summary>
        /// Interaction logic for MainWindow.xaml
        /// </summary>
        public partial class MainWindow : Window
        {
            private KinectSensor kinectDevice;
            private readonly Brush[] skeletonBrushes;
    
            private WriteableBitmap depthImageBitMap;
            private Int32Rect depthImageBitmapRect;
            private Int32 depthImageStride;
            private DepthImageFrame lastDepthFrame;
    
            private WriteableBitmap colorImageBitmap;
            private Int32Rect colorImageBitmapRect;
            private int colorImageStride;
            private byte[] colorImagePixelData;
    
            private Skeleton[] frameSkeletons;
    
            public MainWindow()
            {
                InitializeComponent();
    
                skeletonBrushes = new Brush[] { Brushes.Red };
    
                KinectSensor.KinectSensors.StatusChanged += KinectSensors_StatusChanged;
                this.KinectDevice = KinectSensor.KinectSensors.FirstOrDefault(x => x.Status == KinectStatus.Connected);
            }
    
            public KinectSensor KinectDevice
            {
                get { return this.kinectDevice; }
                set
                {
                    if (this.kinectDevice != value)
                    {
                        //Uninitialize
                        if (this.kinectDevice != null)
                        {
                            this.kinectDevice.Stop();
                            this.kinectDevice.SkeletonFrameReady -= kinectDevice_SkeletonFrameReady;
                            this.kinectDevice.ColorFrameReady -= kinectDevice_ColorFrameReady;
                            this.kinectDevice.DepthFrameReady -= kinectDevice_DepthFrameReady;
                            this.kinectDevice.SkeletonStream.Disable();
                            this.kinectDevice.DepthStream.Disable();
                            this.kinectDevice.ColorStream.Disable();
                            this.frameSkeletons = null;
                        }
    
                        this.kinectDevice = value;
    
                        //Initialize
                        if (this.kinectDevice != null)
                        {
                            if (this.kinectDevice.Status == KinectStatus.Connected)
                            {
                                this.kinectDevice.SkeletonStream.Enable();
                                this.kinectDevice.ColorStream.Enable(ColorImageFormat.RgbResolution640x480Fps30);
                                this.kinectDevice.DepthStream.Enable(DepthImageFormat.Resolution640x480Fps30);
                                this.frameSkeletons = new Skeleton[this.kinectDevice.SkeletonStream.FrameSkeletonArrayLength];
                                this.kinectDevice.SkeletonFrameReady += kinectDevice_SkeletonFrameReady;
                                this.kinectDevice.ColorFrameReady += kinectDevice_ColorFrameReady;
                                this.kinectDevice.DepthFrameReady += kinectDevice_DepthFrameReady;
                                this.kinectDevice.Start();
    
                                DepthImageStream depthStream = kinectDevice.DepthStream;
                                depthStream.Enable();
    
                                depthImageBitMap = new WriteableBitmap(depthStream.FrameWidth, depthStream.FrameHeight, 96, 96, PixelFormats.Gray16, null);
                                depthImageBitmapRect = new Int32Rect(0, 0, depthStream.FrameWidth, depthStream.FrameHeight);
                                depthImageStride = depthStream.FrameWidth * depthStream.FrameBytesPerPixel;
    
                                ColorImageStream colorStream = kinectDevice.ColorStream;
                                colorStream.Enable();
                                colorImageBitmap = new WriteableBitmap(colorStream.FrameWidth, colorStream.FrameHeight,
                                                                                                96, 96, PixelFormats.Bgr32, null);
                                this.colorImageBitmapRect = new Int32Rect(0, 0, colorStream.FrameWidth, colorStream.FrameHeight);
                                this.colorImageStride = colorStream.FrameWidth * colorStream.FrameBytesPerPixel;
                                ColorImage.Source = this.colorImageBitmap;
    
                                DepthImage.Source = depthImageBitMap;
                            }
                        }
                    }
                }
            }
    
            void kinectDevice_DepthFrameReady(object sender, DepthImageFrameReadyEventArgs e)
            {
                using (DepthImageFrame depthFrame = e.OpenDepthImageFrame())
                {
                    if (depthFrame != null)
                    {
                        short[] depthPixelDate = new short[depthFrame.PixelDataLength];
                        depthFrame.CopyPixelDataTo(depthPixelDate);
                        depthImageBitMap.WritePixels(depthImageBitmapRect, depthPixelDate, depthImageStride, 0);
                    }
                }
            }
    
            void kinectDevice_ColorFrameReady(object sender, ColorImageFrameReadyEventArgs e)
            {
                using (ColorImageFrame frame = e.OpenColorImageFrame())
                {
                    if (frame != null)
                    {
                        byte[] pixelData = new byte[frame.PixelDataLength];
                        frame.CopyPixelDataTo(pixelData);
                        this.colorImageBitmap.WritePixels(this.colorImageBitmapRect, pixelData, this.colorImageStride, 0);
                    }
                }
            }
    
            void kinectDevice_SkeletonFrameReady(object sender, SkeletonFrameReadyEventArgs e)
            {
                using (SkeletonFrame frame = e.OpenSkeletonFrame())
                {
                    if (frame != null)
                    {
                        Polyline figure;
                        Brush userBrush;
                        Skeleton skeleton;
    
                        LayoutRoot.Children.Clear();
                        frame.CopySkeletonDataTo(this.frameSkeletons);
    
    
                        for (int i = 0; i < this.frameSkeletons.Length; i++)
                        {
                            skeleton = this.frameSkeletons[i];
    
                            if (skeleton.TrackingState == SkeletonTrackingState.Tracked)
                            {
                                userBrush = this.skeletonBrushes[i % this.skeletonBrushes.Length];
    
                                //draw head and body
                                figure = CreateFigure(skeleton, userBrush, new[] { JointType.Head, JointType.ShoulderCenter, JointType.ShoulderLeft, JointType.Spine,
                                                                    JointType.ShoulderRight, JointType.ShoulderCenter, JointType.HipCenter
                                                                    });
                                LayoutRoot.Children.Add(figure);
    
                                figure = CreateFigure(skeleton, userBrush, new[] { JointType.HipLeft, JointType.HipRight });
                                LayoutRoot.Children.Add(figure);
    
                                //draw left leg
                                figure = CreateFigure(skeleton, userBrush, new[] { JointType.HipCenter, JointType.HipLeft, JointType.KneeLeft, JointType.AnkleLeft, JointType.FootLeft });
                                LayoutRoot.Children.Add(figure);
    
                                //draw right leg
                                figure = CreateFigure(skeleton, userBrush, new[] { JointType.HipCenter, JointType.HipRight, JointType.KneeRight, JointType.AnkleRight, JointType.FootRight });
                                LayoutRoot.Children.Add(figure);
    
                                //draw left arm
                                figure = CreateFigure(skeleton, userBrush, new[] { JointType.ShoulderLeft, JointType.ElbowLeft, JointType.WristLeft, JointType.HandLeft });
                                LayoutRoot.Children.Add(figure);
    
                                //draw right arm
                                figure = CreateFigure(skeleton, userBrush, new[] { JointType.ShoulderRight, JointType.ElbowRight, JointType.WristRight, JointType.HandRight });
                                LayoutRoot.Children.Add(figure);
                            }
                        }
                    }
                }
            }
    
            private Polyline CreateFigure(Skeleton skeleton, Brush brush, JointType[] joints)
            {
                Polyline figure = new Polyline();
    
                figure.StrokeThickness = 8;
                figure.Stroke = brush;
    
                for (int i = 0; i < joints.Length; i++)
                {
                    figure.Points.Add(GetJointPoint(skeleton.Joints[joints[i]]));
                }
    
                return figure;
            }
    
            private Point GetJointPoint(Joint joint)
            {
                CoordinateMapper cm = new CoordinateMapper(kinectDevice);
    
                DepthImagePoint point = cm.MapSkeletonPointToDepthPoint(joint.Position, this.KinectDevice.DepthStream.Format);
                //ColorImagePoint point = cm.MapSkeletonPointToColorPoint(joint.Position, this.KinectDevice.ColorStream.Format);
                point.X *= (int)this.LayoutRoot.ActualWidth / KinectDevice.DepthStream.FrameWidth;
                point.Y *= (int)this.LayoutRoot.ActualHeight / KinectDevice.DepthStream.FrameHeight;
    
                return new Point(point.X, point.Y);
            } 
    
            private void KinectSensors_StatusChanged(object sender, StatusChangedEventArgs e)
            {
                switch (e.Status)
                {
                    case KinectStatus.Initializing:
                    case KinectStatus.Connected:
                    case KinectStatus.NotPowered:
                    case KinectStatus.NotReady:
                    case KinectStatus.DeviceNotGenuine:
                        this.KinectDevice = e.Sensor;
                        break;
                    case KinectStatus.Disconnected:
                        //TODO: Give the user feedback to plug-in a Kinect device.                    
                        this.KinectDevice = null;
                        break;
                    default:
                        //TODO: Show an error state
                        break;
                }
            }
    
        }



    Monday, July 15, 2013 1:33 AM

All replies

  • Are you familiar with how WPF and Image and Device Context work?

    http://msdn.microsoft.com/en-us/library/ms748373.aspx

    Did you intent to use the same Image control for both the color and skeleton? Why not just create a new image control for the color and overlay the skeleton image control on top?

                    
    <Image Name="ColorImage" Width="640" Height="480"/>
               <Grid ClipToBounds="True">
                    <Image Name="ColorImage" Width="640" Height="480"/>
                    <Image Name="videoimage" Width="640" Height="480"/>
                    <Canvas Background="Transparent" Name="ColorImagePoint"></Canvas>
                </Grid>
    // add 2 local variables
    private WriteableBitmap colorBitmap;
    private byte[] colorPixels;
    // in the WindowLoaded code add
    if (null != this.sensor)
    {
        // Turn on the color stream to receive color frames
        this.sensor.ColorStream.Enable(ColorImageFormat.RgbResolution640x480Fps30);
        // Allocate space to put the pixels we'll receive
        this.colorPixels = new byte[this.sensor.ColorStream.FramePixelDataLength];
        // This is the bitmap we'll display on-screen
        this.colorBitmap = new WriteableBitmap(this.sensor.ColorStream.FrameWidth, this.sensor.ColorStream.FrameHeight, 96.0, 96.0, PixelFormats.Bgr32, null);
        // Set the image we display to point to the bitmap where we'll put the image data
        this.ColorImage.Source = this.colorBitmap;
    ...

    Carmine Si - MSFT




    Tuesday, July 9, 2013 9:17 PM
  • Are you familiar with how WPF and Image and Device Context work?

    http://msdn.microsoft.com/en-us/library/ms748373.aspx

    Did you intent to use the same Image control for both the color and skeleton? Why not just create a new image control for the color and overlay the skeleton image control on top?

                    
    <Image Name="ColorImage" Width="640" Height="480"/>
               <Grid ClipToBounds="True">
                    <Image Name="ColorImage" Width="640" Height="480"/>
                    <Image Name="videoimage" Width="640" Height="480"/>
                    <Canvas Background="Transparent" Name="ColorImagePoint"></Canvas>
                </Grid>
    // add 2 local variables
    private WriteableBitmap colorBitmap;
    private byte[] colorPixels;
    // in the WindowLoaded code add
    if (null != this.sensor)
    {
        // Turn on the color stream to receive color frames
        this.sensor.ColorStream.Enable(ColorImageFormat.RgbResolution640x480Fps30);
        // Allocate space to put the pixels we'll receive
        this.colorPixels = new byte[this.sensor.ColorStream.FramePixelDataLength];
        // This is the bitmap we'll display on-screen
        this.colorBitmap = new WriteableBitmap(this.sensor.ColorStream.FrameWidth, this.sensor.ColorStream.FrameHeight, 96.0, 96.0, PixelFormats.Bgr32, null);
        // Set the image we display to point to the bitmap where we'll put the image data
        this.ColorImage.Source = this.colorBitmap;
    ...

    Carmine Si - MSFT




    ok, iwill try this out, thanks!
    Sunday, July 14, 2013 8:00 PM
  • Well,there are some keypoints when draw skeleton data on color stream or depth stream:

    1. 1.you‘d better using WriteableBitmap to render color or depth image on the screen;
    2. 2.the image size of the color ,depth ,skeleton and the screen should be the same,for example,using 640x480Fps30 format,and set the Grid size 640*480。
    3. 3.some space transformation needed when draw skeleton on color or depth image,for example if you need draw skeleton on depeth MapSkeletonPointToColorPoint method,and if skeleton to depth,MapSkeletonPointToDepthPoint should be called.

    the code are as followings:

    the main xml:

    <Window x:Class="KinectSkeletonAlignwithColorandDepthImage.MainWindow"
            xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
            xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
            Title="MainWindow" Width="660" Height="520">
        <Grid  Width="640" Height="480">
            <Image x:Name="ColorImage"  />
            <Image x:Name="DepthImage" Visibility="Hidden"/>
            <Grid x:Name="LayoutRoot" Background="Transparent" >
            </Grid>
        </Grid>
    </Window>

    if you want draw skeleton on depth images,just set the  ColorImage's Vsisibility property to Hidden.

     /// <summary>
        /// Interaction logic for MainWindow.xaml
        /// </summary>
        public partial class MainWindow : Window
        {
            private KinectSensor kinectDevice;
            private readonly Brush[] skeletonBrushes;
    
            private WriteableBitmap depthImageBitMap;
            private Int32Rect depthImageBitmapRect;
            private Int32 depthImageStride;
            private DepthImageFrame lastDepthFrame;
    
            private WriteableBitmap colorImageBitmap;
            private Int32Rect colorImageBitmapRect;
            private int colorImageStride;
            private byte[] colorImagePixelData;
    
            private Skeleton[] frameSkeletons;
    
            public MainWindow()
            {
                InitializeComponent();
    
                skeletonBrushes = new Brush[] { Brushes.Red };
    
                KinectSensor.KinectSensors.StatusChanged += KinectSensors_StatusChanged;
                this.KinectDevice = KinectSensor.KinectSensors.FirstOrDefault(x => x.Status == KinectStatus.Connected);
            }
    
            public KinectSensor KinectDevice
            {
                get { return this.kinectDevice; }
                set
                {
                    if (this.kinectDevice != value)
                    {
                        //Uninitialize
                        if (this.kinectDevice != null)
                        {
                            this.kinectDevice.Stop();
                            this.kinectDevice.SkeletonFrameReady -= kinectDevice_SkeletonFrameReady;
                            this.kinectDevice.ColorFrameReady -= kinectDevice_ColorFrameReady;
                            this.kinectDevice.DepthFrameReady -= kinectDevice_DepthFrameReady;
                            this.kinectDevice.SkeletonStream.Disable();
                            this.kinectDevice.DepthStream.Disable();
                            this.kinectDevice.ColorStream.Disable();
                            this.frameSkeletons = null;
                        }
    
                        this.kinectDevice = value;
    
                        //Initialize
                        if (this.kinectDevice != null)
                        {
                            if (this.kinectDevice.Status == KinectStatus.Connected)
                            {
                                this.kinectDevice.SkeletonStream.Enable();
                                this.kinectDevice.ColorStream.Enable(ColorImageFormat.RgbResolution640x480Fps30);
                                this.kinectDevice.DepthStream.Enable(DepthImageFormat.Resolution640x480Fps30);
                                this.frameSkeletons = new Skeleton[this.kinectDevice.SkeletonStream.FrameSkeletonArrayLength];
                                this.kinectDevice.SkeletonFrameReady += kinectDevice_SkeletonFrameReady;
                                this.kinectDevice.ColorFrameReady += kinectDevice_ColorFrameReady;
                                this.kinectDevice.DepthFrameReady += kinectDevice_DepthFrameReady;
                                this.kinectDevice.Start();
    
                                DepthImageStream depthStream = kinectDevice.DepthStream;
                                depthStream.Enable();
    
                                depthImageBitMap = new WriteableBitmap(depthStream.FrameWidth, depthStream.FrameHeight, 96, 96, PixelFormats.Gray16, null);
                                depthImageBitmapRect = new Int32Rect(0, 0, depthStream.FrameWidth, depthStream.FrameHeight);
                                depthImageStride = depthStream.FrameWidth * depthStream.FrameBytesPerPixel;
    
                                ColorImageStream colorStream = kinectDevice.ColorStream;
                                colorStream.Enable();
                                colorImageBitmap = new WriteableBitmap(colorStream.FrameWidth, colorStream.FrameHeight,
                                                                                                96, 96, PixelFormats.Bgr32, null);
                                this.colorImageBitmapRect = new Int32Rect(0, 0, colorStream.FrameWidth, colorStream.FrameHeight);
                                this.colorImageStride = colorStream.FrameWidth * colorStream.FrameBytesPerPixel;
                                ColorImage.Source = this.colorImageBitmap;
    
                                DepthImage.Source = depthImageBitMap;
                            }
                        }
                    }
                }
            }
    
            void kinectDevice_DepthFrameReady(object sender, DepthImageFrameReadyEventArgs e)
            {
                using (DepthImageFrame depthFrame = e.OpenDepthImageFrame())
                {
                    if (depthFrame != null)
                    {
                        short[] depthPixelDate = new short[depthFrame.PixelDataLength];
                        depthFrame.CopyPixelDataTo(depthPixelDate);
                        depthImageBitMap.WritePixels(depthImageBitmapRect, depthPixelDate, depthImageStride, 0);
                    }
                }
            }
    
            void kinectDevice_ColorFrameReady(object sender, ColorImageFrameReadyEventArgs e)
            {
                using (ColorImageFrame frame = e.OpenColorImageFrame())
                {
                    if (frame != null)
                    {
                        byte[] pixelData = new byte[frame.PixelDataLength];
                        frame.CopyPixelDataTo(pixelData);
                        this.colorImageBitmap.WritePixels(this.colorImageBitmapRect, pixelData, this.colorImageStride, 0);
                    }
                }
            }
    
            void kinectDevice_SkeletonFrameReady(object sender, SkeletonFrameReadyEventArgs e)
            {
                using (SkeletonFrame frame = e.OpenSkeletonFrame())
                {
                    if (frame != null)
                    {
                        Polyline figure;
                        Brush userBrush;
                        Skeleton skeleton;
    
                        LayoutRoot.Children.Clear();
                        frame.CopySkeletonDataTo(this.frameSkeletons);
    
    
                        for (int i = 0; i < this.frameSkeletons.Length; i++)
                        {
                            skeleton = this.frameSkeletons[i];
    
                            if (skeleton.TrackingState == SkeletonTrackingState.Tracked)
                            {
                                userBrush = this.skeletonBrushes[i % this.skeletonBrushes.Length];
    
                                //draw head and body
                                figure = CreateFigure(skeleton, userBrush, new[] { JointType.Head, JointType.ShoulderCenter, JointType.ShoulderLeft, JointType.Spine,
                                                                    JointType.ShoulderRight, JointType.ShoulderCenter, JointType.HipCenter
                                                                    });
                                LayoutRoot.Children.Add(figure);
    
                                figure = CreateFigure(skeleton, userBrush, new[] { JointType.HipLeft, JointType.HipRight });
                                LayoutRoot.Children.Add(figure);
    
                                //draw left leg
                                figure = CreateFigure(skeleton, userBrush, new[] { JointType.HipCenter, JointType.HipLeft, JointType.KneeLeft, JointType.AnkleLeft, JointType.FootLeft });
                                LayoutRoot.Children.Add(figure);
    
                                //draw right leg
                                figure = CreateFigure(skeleton, userBrush, new[] { JointType.HipCenter, JointType.HipRight, JointType.KneeRight, JointType.AnkleRight, JointType.FootRight });
                                LayoutRoot.Children.Add(figure);
    
                                //draw left arm
                                figure = CreateFigure(skeleton, userBrush, new[] { JointType.ShoulderLeft, JointType.ElbowLeft, JointType.WristLeft, JointType.HandLeft });
                                LayoutRoot.Children.Add(figure);
    
                                //draw right arm
                                figure = CreateFigure(skeleton, userBrush, new[] { JointType.ShoulderRight, JointType.ElbowRight, JointType.WristRight, JointType.HandRight });
                                LayoutRoot.Children.Add(figure);
                            }
                        }
                    }
                }
            }
    
            private Polyline CreateFigure(Skeleton skeleton, Brush brush, JointType[] joints)
            {
                Polyline figure = new Polyline();
    
                figure.StrokeThickness = 8;
                figure.Stroke = brush;
    
                for (int i = 0; i < joints.Length; i++)
                {
                    figure.Points.Add(GetJointPoint(skeleton.Joints[joints[i]]));
                }
    
                return figure;
            }
    
            private Point GetJointPoint(Joint joint)
            {
                CoordinateMapper cm = new CoordinateMapper(kinectDevice);
    
                DepthImagePoint point = cm.MapSkeletonPointToDepthPoint(joint.Position, this.KinectDevice.DepthStream.Format);
                //ColorImagePoint point = cm.MapSkeletonPointToColorPoint(joint.Position, this.KinectDevice.ColorStream.Format);
                point.X *= (int)this.LayoutRoot.ActualWidth / KinectDevice.DepthStream.FrameWidth;
                point.Y *= (int)this.LayoutRoot.ActualHeight / KinectDevice.DepthStream.FrameHeight;
    
                return new Point(point.X, point.Y);
            } 
    
            private void KinectSensors_StatusChanged(object sender, StatusChangedEventArgs e)
            {
                switch (e.Status)
                {
                    case KinectStatus.Initializing:
                    case KinectStatus.Connected:
                    case KinectStatus.NotPowered:
                    case KinectStatus.NotReady:
                    case KinectStatus.DeviceNotGenuine:
                        this.KinectDevice = e.Sensor;
                        break;
                    case KinectStatus.Disconnected:
                        //TODO: Give the user feedback to plug-in a Kinect device.                    
                        this.KinectDevice = null;
                        break;
                    default:
                        //TODO: Show an error state
                        break;
                }
            }
    
        }



    Monday, July 15, 2013 1:33 AM
  • The same question was asked over http://social.msdn.microsoft.com/Forums/en-US/d0c4935f-3b6f-496a-9a58-c9d4fb164b70/overlaying-skeleton-on-top-of-color-image?forum=k4wv2devpreview I put a quick working example up on there which gives you full 1920x1080 colour feed with skeletons placed over the top.
    Wednesday, December 4, 2013 10:59 PM