none
Kinect video image cost high performance problem in XNA RRS feed

  • Question

  • Does anyone can help me figure out how to programm Kinect color stream displaying in XNA?

    I can successfully have displayed the Kinect color stream image XNA but it costs high performance and sometimes the Kinect Video is a bit delayed in showing video and it impacts badly on user gameplay?

    I found out some ways to do good memory management such as threading for WPF and Kinect but I can't find for XNA and Kinect.

    Hope someone can help me.

    Regards

    Tuesday, December 18, 2012 4:33 PM

All replies

  • Can you lower the resolution on the video and it change any at all?  Higher resolutions of video take up more memory. For better help can you show how you handle the video in code? (We just need the section specifically to initializing and  transferring the video to the screen).

    Once you eliminate the impossible, whatever remains, no matter how improbable, must be the truth. - "Sherlock holmes" "speak softly and carry a big stick" - theodore roosevelt. Fear leads to anger, anger leads to hate, hate leads to suffering - Yoda. Blog - http://www.computerprofessions.co.nr

    Wednesday, December 19, 2012 2:28 PM
  • Thanks a lot. Here is my Kinect Manager Code which cost high performance problem

    using System;
    using System.Collections.Generic;
    using System.Linq;
    using System.Text;
    using Microsoft.Xna.Framework;
    using Microsoft.Xna.Framework.Audio;
    using Microsoft.Xna.Framework.Content;
    using Microsoft.Xna.Framework.GamerServices;
    using Microsoft.Xna.Framework.Graphics;
    using Microsoft.Xna.Framework.Input;
    using Microsoft.Xna.Framework.Media;
    using Microsoft.Kinect;

    namespace Kinect_V10
    {
        public class KinectManager
        {
            // raw kinect stuff
            KinectSensor kinectSensor;

            byte[] pixelData = new byte[640 * 480 * 4]; // 4 bytes/pixel

            Skeleton[] skeletons;

            string connectedStatus = "<error>";

            ColorImagePoint[] colourPoints = new ColorImagePoint[320 * 240];    // same as depthData

            // processed kinect stuff

            Color[] videoColour = new Color[640 * 480]; // 640x480 video info

            Color[] texColour = new Color[640 * 480];   // the data for textures

            public Texture2D texture, lefthand, righthand;

            bool newTexture = false;

            Game1 game;

            public KinectManager(Game1 game)
            {
                this.game = game;
                texture = new Texture2D(this.game.GraphicsDevice, 640, 480);
                KinectSensor.KinectSensors.StatusChanged += new EventHandler<StatusChangedEventArgs>(KinectSensors_StatusChanged);
                DiscoverKinectSensor();
            }



            private void KinectSensors_StatusChanged(object sender, StatusChangedEventArgs e)
            {
                Console.WriteLine("Kinect " + e.Sensor.GetHashCode() + " status:" + e.Status);
                if (this.kinectSensor == e.Sensor)
                {
                    //if (e.Status == KinectStatus.Disconnected ||
                    //    e.Status == KinectStatus.NotPowered)
                    if (e.Status == KinectStatus.Disconnected)
                    {
                        // stop
                        this.kinectSensor.Stop();
                        this.kinectSensor = null;
                    }
                    if (e.Status == KinectStatus.Connected)
                    {
                        InitializeKinect();
                    }
                }
                if (this.kinectSensor == null)  // no sensor
                    this.DiscoverKinectSensor();    // have a look
            }

            private void DiscoverKinectSensor()
            {
                foreach (KinectSensor sensor in KinectSensor.KinectSensors)
                {
                    //if (sensor.Status == KinectStatus.Connected)
                    if (sensor.Status != KinectStatus.Disconnected)
                    {
                        // Found one, set our sensor to this
                        kinectSensor = sensor;
                        break;
                    }
                }

                if (this.kinectSensor == null)
                {
                    connectedStatus = "No Kinect Sensors connected to USB";
                    return;
                }

                // You can use the kinectSensor.Status to check for status
                // and give the user some kind of feedback
                switch (kinectSensor.Status)
                {
                    case KinectStatus.Connected:
                        {
                            connectedStatus = "Kinect Connected";
                            break;
                        }
                    case KinectStatus.Disconnected:
                        {
                            connectedStatus = "Kinect Disconnected";
                            break;
                        }
                    case KinectStatus.NotPowered:
                        {
                            connectedStatus = "Kinect not powered";
                            break;
                        }
                    default:
                        {
                            //connectedStatus = "Status: Error";
                            connectedStatus = "Status: " + kinectSensor.Status.ToString();

                            break;
                        }
                }

                // Init the found and connected device
                if (kinectSensor.Status == KinectStatus.Connected)
                {
                    InitializeKinect();
                }
            }

            /// <summary>
            /// Kinect setup
            /// </summary>
            private void InitializeKinect()
            {
                try
                {
                    // Initalise the kinect
                    // lets have video:640x480, depth:320x240 and skeleton
                    kinectSensor.ColorStream.Enable(ColorImageFormat.RgbResolution640x480Fps30);
                    kinectSensor.DepthStream.Enable(DepthImageFormat.Resolution320x240Fps30);
                    kinectSensor.SkeletonStream.Enable(); // just enable without worrying on smoothing params
                    kinectSensor.Start();
                    // we could use polling to access the data, but I'm using event as its simple
                    //kinectSensor.AllFramesReady += KinectAllFramesReady;  // calls this fn when data is ready
                    //UpdateKinect();

                    GetSkeletonJoints();

                    //kinectSensor.SkeletonFrameReady += KinectSkeletonFramesReady;

                    skeletons = new Skeleton[kinectSensor.SkeletonStream.FrameSkeletonArrayLength];   // max array len

                    //System.Diagnostics.Debug.WriteLine(
                    //    string.Format("Kinect Enabled, current angle={0}, max-min={1} {2}",
                    //    kinectSensor.ElevationAngle, kinectSensor.MaxElevationAngle, kinectSensor.MinElevationAngle));
                }
                catch (Exception e)
                {
                    Console.WriteLine("Exception in InitializeKinect() " + e);
                    kinectSensor = null;  // drop it
                }
            }

            //void KinectSkeletonFramesReady(object sender, SkeletonFrameReadyEventArgs e)
            //{
            //    using (SkeletonFrame sf = kinectSensor.SkeletonStream.OpenNextFrame(10))
            //    {
            //        if (sf != null)   // if we have a skeleton
            //        {
            //            sf.CopySkeletonDataTo(skeletons);   // make a copy                   
            //        }
            //    }
            //}
            void GetSkeletonJoints()
            {
                using (SkeletonFrame sf = kinectSensor.SkeletonStream.OpenNextFrame(0))
                {
                    if (sf != null)   // if we have a skeleton
                    {
                        sf.CopySkeletonDataTo(skeletons);   // make a copy                   
                    }
                }
            }

            public void UpdateKinect()
            {
                this.game.GraphicsDevice.Textures[0] = null;

                GetSkeletonJoints();

                using (ColorImageFrame cif = kinectSensor.ColorStream.OpenNextFrame(0))
                // using (ColorImageFrame cif = e.OpenColorImageFrame())
                {
                    if (cif != null)  // if we have a colour image
                    {
                        cif.CopyPixelDataTo(pixelData); // make a copy
                        newTexture = true;
                    }

                    KinectRgbaToXnaColor(pixelData, videoColour);

                    texture.SetData<Color>(videoColour);
                }
            }
            static private void KinectRgbaToXnaColor(byte[] rgba, Color[] xnaCol)
            {
                for (int i = 0, idx = 0; i < 640 * 480; i++, idx += 4)
                {
                    // Either the XNA format is reversed or the Kinect format is
                    // its an RGB/BGR issue
                    // hence the reversed order for the pixels
                    xnaCol[i] = new Color(rgba[idx + 2], rgba[idx + 1], rgba[idx], 255);
                }
            }

            public Skeleton GetNearestSkeleton()
            {
                Skeleton result = null; // no skeleton
                double dist = 1e7;  // far away
                for (int i = 0; i < skeletons.Length; i++)
                {
                    if (skeletons[i] != null &&
                        skeletons[i].TrackingState == SkeletonTrackingState.Tracked &&
                        skeletons[i].Joints[JointType.ShoulderCenter].Position.Z < dist)
                    {
                        result = skeletons[i];
                        dist = result.Joints[JointType.ShoulderCenter].Position.Z;
                    }
                }
                return result;
            }
            /** Finds and returns the nearest tracked skeleton as an array of Vector2's.
             * May return a null if no skeletons are being tracked.
             * The 3D positions are converted into an array of Vector2's for easy use
             */
            public Vector2[] GetNearestSkeleton2D()
            {
                Skeleton skeleton = GetNearestSkeleton();
                if (skeleton == null) return null;  // no data:
                Vector2[] result = new Vector2[20];
                for (int i = 0; i < 20; i++)
                {
                    ColorImagePoint cip = kinectSensor.MapSkeletonPointToColor(skeleton.Joints[(JointType)(i)].Position,
                        ColorImageFormat.RgbResolution640x480Fps30);
                    result[i] = new Vector2(cip.X, cip.Y);
                }
                return result;
            }

            public Vector2 GetJointsPosition(int index)
            {
                Vector2[] _results = GetNearestSkeleton2D();
                Vector2 pos = new Vector2(0, 0);
                if (_results != null)
                {

                    if (_results.Length > 0)
                        pos = _results[index];
                }
                return pos;
            }
        }
    }

    //................................ end of Kinect Manager..............................................//

    Here is Game code

    using System;
    using System.Collections.Generic;
    using System.Linq;
    using Microsoft.Xna.Framework;
    using Microsoft.Xna.Framework.Audio;
    using Microsoft.Xna.Framework.Content;
    using Microsoft.Xna.Framework.GamerServices;
    using Microsoft.Xna.Framework.Graphics;
    using Microsoft.Xna.Framework.Input;
    using Microsoft.Xna.Framework.Media;
    using Microsoft.Kinect;

    namespace Kinect_V10
    {
        /// <summary>
        /// This is the main type for your game
        /// </summary>
        public class Game1 : Microsoft.Xna.Framework.Game
        {
            GraphicsDeviceManager graphics;
            SpriteBatch spriteBatch;
            KinectManager kmgr;

            Vector2 leftHandPos, rightHandPos;

            Texture2D texture, lefthand, righthand;

            public Game1()
            {
                graphics = new GraphicsDeviceManager(this);
                this.graphics.PreferredBackBufferWidth = 640;
                this.graphics.PreferredBackBufferHeight = 480;
                Content.RootDirectory = "Content";
            }

            /// <summary>
            /// Allows the game to perform any initialization it needs to before starting to run.
            /// This is where it can query for any required services and load any non-graphic
            /// related content.  Calling base.Initialize will enumerate through any components
            /// and initialize them as well.
            /// </summary>
            protected override void Initialize()
            {
                // TODO: Add your initialization logic here
                kmgr = new KinectManager(this);
                base.Initialize();
            }

            /// <summary>
            /// LoadContent will be called once per game and is the place to load
            /// all of your content.
            /// </summary>
            protected override void LoadContent()
            {
                // Create a new SpriteBatch, which can be used to draw textures.
                spriteBatch = new SpriteBatch(GraphicsDevice);

                // TODO: use this.Content to load your game content here
                leftHandPos = new Vector2(0, 0);
                rightHandPos = new Vector2(0, 0);

                lefthand = Content.Load<Texture2D>("lefthand");
                righthand = Content.Load<Texture2D>("righthand");
            }

            /// <summary>
            /// UnloadContent will be called once per game and is the place to unload
            /// all content.
            /// </summary>
            protected override void UnloadContent()
            {
                // TODO: Unload any non ContentManager content here
            }

            /// <summary>
            /// Allows the game to run logic such as updating the world,
            /// checking for collisions, gathering input, and playing audio.
            /// </summary>
            /// <param name="gameTime">Provides a snapshot of timing values.</param>
            protected override void Update(GameTime gameTime)
            {
                // Allows the game to exit
                if (GamePad.GetState(PlayerIndex.One).Buttons.Back == ButtonState.Pressed)
                    this.Exit();

                // TODO: Add your update logic here
                kmgr.UpdateKinect();

                leftHandPos = kmgr.GetJointsPosition(7);
                rightHandPos = kmgr.GetJointsPosition(11);
                base.Update(gameTime);
            }

            /// <summary>
            /// This is called when the game should draw itself.
            /// </summary>
            /// <param name="gameTime">Provides a snapshot of timing values.</param>
            protected override void Draw(GameTime gameTime)
            {
                GraphicsDevice.Clear(Color.CornflowerBlue);

                // TODO: Add your drawing code here
                spriteBatch.Begin();
                spriteBatch.Draw(kmgr.texture, new Rectangle(0, 0, 640, 480), Color.White);
                spriteBatch.Draw(lefthand, new Vector2(leftHandPos.X - lefthand.Width / 2, leftHandPos.Y - lefthand.Height / 2), Color.White);
                spriteBatch.Draw(righthand, new Vector2(rightHandPos.X - righthand.Width / 2, rightHandPos.Y - righthand.Height / 2), Color.White);
                spriteBatch.End();
                base.Draw(gameTime);
            }
        }
    }

    Thursday, December 20, 2012 9:37 AM
  • It seems to me somewhere your loading of a texture might be causing problems or possibly your handling of the colors for the hands is not efficient. I think you could save a stream to a temp binary file for better handling and efficiency. Their is also some memory management code someone might suggest.  

    Note: Although I doubt this is not the case check your power usage of the Kinect. This can be done in device manager. If power is being consumed,


    Once you eliminate the impossible, whatever remains, no matter how improbable, must be the truth. - "Sherlock holmes" "speak softly and carry a big stick" - theodore roosevelt. Fear leads to anger, anger leads to hate, hate leads to suffering - Yoda. Blog - http://www.computerprofessions.co.nr


    • Edited by The Thinker Thursday, December 20, 2012 6:43 PM
    Thursday, December 20, 2012 6:42 PM