none
InteractionStream C# and XNA (HRESULT: 0x80070057.) RRS feed

  • Question

  • Hello, 

    I'm trying to develop a game with Kinect 1.8 and Toolkit.Interaction.dll, 

    While use InteractionStream have some error (below image) for reference,

    "This API has returned an exception from an HRESULT: 0x80070057." on 

    interStream.ProcessDepth(depthFrame.GetRawPixelData(), depthFrame.Timestamp);

    How to fix the problem?

    Thank you!

    using System;
    using System.Collections.Generic;
    using System.Linq;
    using Microsoft.Xna.Framework;
    using Microsoft.Xna.Framework.Audio;
    using Microsoft.Xna.Framework.Content;
    using Microsoft.Xna.Framework.GamerServices;
    using Microsoft.Xna.Framework.Graphics;
    using Microsoft.Xna.Framework.Input;
    using Microsoft.Xna.Framework.Media;
    using Microsoft.Kinect;
    using Microsoft.Kinect.Toolkit.Interaction;
    
    namespace WindowsGame12
    {
        /// <summary>
        /// This is the main type for your game
        /// </summary>
        public class Game1 : Microsoft.Xna.Framework.Game
        {
            GraphicsDeviceManager graphics;
            SpriteBatch spriteBatch;
    
            Texture2D mySpriteTexture2D1;
    
            #region Setup Kinect
    
            KinectSensor myKinect;
    
            InteractionStream interStream;
    
            SpriteFont messageFont;
    
            string errorMessage = "";
            string userMessage = "";
            string rHandMessage = "";
            string lHandMessage = "";
            string rHandStatus = "";
            string lHandStatus = "";
    
            protected bool setupKinect()
            {
                // Check to see if a Kinect is available
                if (KinectSensor.KinectSensors.Count == 0)
                {
                    errorMessage = "No Kinects detected";
                    return false;
                }
    
                // Get the first Kinect on the computer
                myKinect = KinectSensor.KinectSensors[0];
    
                // Start the Kinect running and select all the streams
                try
                {
                    myKinect.SkeletonStream.Enable();
                    myKinect.ColorStream.Enable();
                    myKinect.DepthStream.Enable(DepthImageFormat.Resolution320x240Fps30);
    
                    interStream = new InteractionStream(myKinect, new DummyInteractionClient());
                    interStream.InteractionFrameReady += interStream_InteractionFrameReady;
    
                    myKinect.Start();
                }
                catch
                {
                    errorMessage = "Kinect initialise failed";
                    return false;
                }
    
                // connect a handler to the event that fires when new frames are available
    
                myKinect.AllFramesReady += new EventHandler<AllFramesReadyEventArgs>(myKinect_AllFramesReady);
    
                return true;
            }
    
            #endregion Setup Kinect
    
            #region InterStream
    
            public class DummyInteractionClient : IInteractionClient
            {
                public InteractionInfo GetInteractionInfoAtLocation(
                    int skeletonTrackingId, InteractionHandType handType,
                    double x, double y)
                {
                    var result = new InteractionInfo();
                    result.IsGripTarget = true;
                    result.IsPressTarget = true;
                    result.PressAttractionPointX = 0.5;
                    result.PressAttractionPointY = 0.5;
                    result.PressTargetControlId = 1;
                    return result;
                }
            }
    
            void interStream_InteractionFrameReady(object sender, InteractionFrameReadyEventArgs e)
            {
                using (var iaf = e.OpenInteractionFrame())
                {
                    #region ᄐ￐에ᄈBᄇzᆲ[ᄎc
                    if (iaf == null)
                        return;
    
                    UserInfo[] userInfos = new UserInfo[InteractionFrame.UserInfoArrayLength];
                    iaf.CopyInteractionDataTo(userInfos);
                    #endregion
    
                    #region ᅡ^ᄄᄂᆲᄚᅧᆭ↑ᆲyᄡᆪᄄ￑ᆰᄎᄌ↑ᄚT
                    GetIDs(userInfos);
    
                    var activeuser = (from u in userInfos
                                      where u.SkeletonTrackingId > 0
                                      select u).FirstOrDefault();
    
                    if (activeuser != null)
                    {
                        GetHands(activeuser);
                        GetHandEvent(activeuser);
                    }
                    #endregion
                }
            }
    
            private void GetIDs(UserInfo[] userInfos)
            {
                userMessage = "UserIDs: ";
                foreach (var userInfo in userInfos)
                {
                    if (userInfo.SkeletonTrackingId > 0)
                        userMessage += userInfo.SkeletonTrackingId + "  ";
                }
            }
    
            private static string FormatHandData(InteractionHandPointer hand)
            {
                string rawxloc = String.Format("X:{0:0.0}", hand.RawX);
                string rawyloc = String.Format(",Y:{0:0.0}", hand.RawY);
                string rawzloc = String.Format(",Z:{0:0.0}", hand.RawZ);
                //string xloc = String.Format(",ᆴᅰᆬ﾿Xᆴyᄐ￐:{0:0.0}", hand.X);
                //string yloc = String.Format(",ᆴᅰᆬ﾿Yᆴyᄐ￐:{0:0.0}", hand.Y);
    
                return rawxloc + rawyloc + rawzloc;
            }
    
    
            private void GetHands(UserInfo userInfo)
            {
                rHandMessage = "Not Found User Right Hand";
                lHandMessage = "Not Found User Left Hand";
                var hands = userInfo.HandPointers;
                if (hands.Count > 0)
                {
                    foreach (var hand in hands)
                    {
                        if (hand.HandType == InteractionHandType.Right)
                        {
                            rHandMessage = FormatHandData(hand);
                        }
                        else if (hand.HandType == InteractionHandType.Left)
                        {
                            lHandMessage = FormatHandData(hand);
                        }
                    }
                }
            }
    
            private void GetHandEvent(UserInfo userInfo)
            {
                var hands = userInfo.HandPointers;
                if (hands.Count > 0)
                {
                    foreach (var hand in hands)
                    {
                        ParseHandEvent(hand);
                    }
                }
            }
    
            private void ParseHandEvent(InteractionHandPointer hand)
            {
                if (hand.HandEventType == InteractionHandEventType.None)
                    return;
    
                if (hand.HandType == InteractionHandType.Right)
                {
                    if (hand.HandEventType == InteractionHandEventType.Grip)
                        rHandStatus = "Right Grip";
                    else
                        rHandStatus = "Right Release";
                }
                else if (hand.HandType == InteractionHandType.Left)
                {
                    if (hand.HandEventType == InteractionHandEventType.Grip)
                        lHandStatus = "Left Grip";
                    else
                        lHandStatus = "Left Release";
                }
            }
    
            #endregion InterStream
    
    
            #region Image Processing
    
            byte[] colorData = null;
            short[] depthData = null;
    
            //Texture2D gameMaskTexture = null;
            Texture2D kinectVideoTexture;
            Rectangle fullScreenRectangle;
    
            //Texture2D gameImageTexture;
            //Color[] maskImageColors = null;
    
            Skeleton[] skeletons = null;
            Skeleton activeSkeleton = null;
    
            int activeSkeletonNumber;
    
            void myKinect_AllFramesReady(object sender, AllFramesReadyEventArgs e)
            {
    #if SREENSHOT_PAUSE
                if (trackCount == trackLimit) return;
    #endif
    
                #region Video image
    
                // Puts a copy of the video image into the kinect video texture
    
                using (ColorImageFrame colorFrame = e.OpenColorImageFrame())
                {
                    if (colorFrame == null)
                        return;
    
                    if (colorData == null)
                        colorData = new byte[colorFrame.Width * colorFrame.Height * 4];
    
                    colorFrame.CopyPixelDataTo(colorData);
    
                    kinectVideoTexture = new Texture2D(GraphicsDevice, colorFrame.Width, colorFrame.Height);
    
                    Color[] bitmap = new Color[colorFrame.Width * colorFrame.Height];
    
                    int sourceOffset = 0;
    
                    for (int i = 0; i < bitmap.Length; i++)
                    {
                        bitmap[i] = new Color(colorData[sourceOffset + 2],
                            colorData[sourceOffset + 1], colorData[sourceOffset], 255);
                        sourceOffset += 4;
                    }
    
                    kinectVideoTexture.SetData(bitmap);
                }
    
                #endregion Video image
    
                #region Skeleton
    
                // Finds the currently active skeleton
    
                using (SkeletonFrame frame = e.OpenSkeletonFrame())
                {
                    if (frame == null)
                        return;
    
                    skeletons = new Skeleton[frame.SkeletonArrayLength];
                    frame.CopySkeletonDataTo(skeletons);
    
                    var accelerometerReading = myKinect.AccelerometerGetCurrentReading();
                    interStream.ProcessSkeleton(skeletons, accelerometerReading, frame.Timestamp);
                }
    
                activeSkeletonNumber = 0;
    
                for (int i = 0; i < skeletons.Length; i++)
                {
                    if (skeletons[i].TrackingState == SkeletonTrackingState.Tracked)
                    {
                        activeSkeletonNumber = i + 1;
                        activeSkeleton = skeletons[i];
                        break;
                    }
                }
    
                #endregion Skeleton
    
    
    
                #region Depth image
    
                // Creates a game background image with transparent regions 
                // where the player is displayed
    
                #region original
    
                using (DepthImageFrame depthFrame = e.OpenDepthImageFrame())
                {
                    // Get the depth data
    
                    if (depthFrame == null) return;
    
                    if (depthData == null)
                        depthData = new short[depthFrame.Width * depthFrame.Height];
    
                    depthFrame.CopyPixelDataTo(depthData);
    
                    interStream.ProcessDepth(depthFrame.GetRawPixelData(), depthFrame.Timestamp);
    
                #endregion original
    
                #region kinectsample_simplekinectinteraction
    
                //using (DepthImageFrame depthFrame = e.OpenDepthImageFrame())
                //{
                //    if (depthFrame == null)
                //        return;
    
                //    interStream.ProcessDepth(depthFrame.GetRawPixelData(), depthFrame.Timestamp);
    
                #endregion kinectsample_simplekinectinteraction
    
                #region gistfile1.cs
                //using (var dif = this.myKinect.DepthStream.OpenNextFrame(0)) 
                //{
                //    if (dif != null) 
                //    {
                //    DepthImagePixel[] data = new DepthImagePixel[dif.PixelDataLength];
                //    dif.CopyDepthImagePixelDataTo(data);
                
                //    //Feed depth data to interactionStream
                //    interStream.ProcessDepth(data, dif.Timestamp);
                //    }
                //}
                #endregion gistfile1.cs
    
                    #region Create the mask from the background image
                    
                    ////gameImageTexture.GetData(maskImageColors);
    
                    //if (activeSkeletonNumber != 0)
                    //{
                    //    for (int depthPos = 0; depthPos < depthData.Length; depthPos++)
                    //    {
                    //        // find a player to mask - split off bottom bits
                    //        int playerNo = depthData[depthPos] & 0x07;
    
                    //        if (playerNo == activeSkeletonNumber)
                    //        {
                    //            // We have a player to mask
    
                    //            // find the X and Y positions of the depth point
                    //            int x = depthPos % depthFrame.Width;
                    //            int y = depthPos / depthFrame.Width;
    
                    //            // get the X and Y positions in the video feed
                    //            ColorImagePoint playerPoint = myKinect.MapDepthToColorImagePoint(
                    //                DepthImageFormat.Resolution320x240Fps30, x, y, depthData[depthPos], ColorImageFormat.RgbResolution640x480Fps30);
    
                    //            // Map the player coordinates into our lower resolution background
                    //            // Have to do this because the lowest resultion for the color camera is 640x480
    
                    //            playerPoint.X /= 2;
                    //            playerPoint.Y /= 2;
    
                    //            // convert this into an offset into the mask color data
                    //            //int gameImagePos = (playerPoint.X + (playerPoint.Y * depthFrame.Width));
                    //            //if ( gameImagePos < maskImageColors.Length ) 
                    //            // make this point in the mask transparent
                    //            //maskImageColors[gameImagePos] = Color.FromNonPremultiplied(0, 0, 0, 0);
                    //        }
                    //    }
                    //}
                    ////gameMaskTexture = new Texture2D(GraphicsDevice, depthFrame.Width, depthFrame.Height);
                    ////gameMaskTexture.SetData(maskImageColors);
    
                    #endregion Create the mask from the background image
    
                }
                #endregion Depth image
            }
            # endregion Image Processing
    
            public Game1()
            {
                graphics = new GraphicsDeviceManager(this);
                Content.RootDirectory = "Content";
    
                this.Window.Title = "XNA Kinect Drag & Drop";
                this.IsMouseVisible = true;  // ᄃeᄇ{어ᄍᆱᄡ¥ᄐ￐
                this.Window.AllowUserResizing = true; // ᄂᄍᄈ\ᄉ?ᄉᄀ￁Yᄅ?
                this.graphics.PreferredBackBufferWidth = 640;  //ᄉ?ᄉᄀᆰᄎᄍwᄈ]ᄐe
                this.graphics.PreferredBackBufferHeight = 480; //ᄉ?ᄉᄀᆰᄎᄍwᄈ]ᄚᆰ
            }
    
            /// <summary>
            /// Allows the game to perform any initialization it needs to before starting to run.
            /// This is where it can query for any required services and load any non-graphic
            /// related content.  Calling base.Initialize will enumerate through any components
            /// and initialize them as well.
            /// </summary>
            protected override void Initialize()
            {
                // TODO: Add your initialization logic here
    
                base.Initialize();
    
                fullScreenRectangle = new Rectangle(0, 0, GraphicsDevice.Viewport.Width, GraphicsDevice.Viewport.Height);
            }
    
            /// <summary>
            /// LoadContent will be called once per game and is the place to load
            /// all of your content.
            /// </summary>
            protected override void LoadContent()
            {
                // Create a new SpriteBatch, which can be used to draw textures.
                spriteBatch = new SpriteBatch(GraphicsDevice);
    
                // TODO: use this.Content to load your game content her
    
                setupKinect();
    
                mySpriteTexture2D1 = Content.Load<Texture2D>("D1");
                messageFont = Content.Load<SpriteFont>("MessageFont");
            }
    
            /// <summary>
            /// UnloadContent will be called once per game and is the place to unload
            /// all content.
            /// </summary>
            protected override void UnloadContent()
            {
                // TODO: Unload any non ContentManager content here
            }
    
            /// <summary>
            /// Allows the game to run logic such as updating the world,
            /// checking for collisions, gathering input, and playing audio.
            /// </summary>
            /// <param name="gameTime">Provides a snapshot of timing values.</param>
            protected override void Update(GameTime gameTime)
            {
                // Allows the game to exit
                if (GamePad.GetState(PlayerIndex.One).Buttons.Back == ButtonState.Pressed)
                    this.Exit();
    
                // TODO: Add your update logic here
    
                base.Update(gameTime);
            }
    
            /// <summary>
            /// This is called when the game should draw itself.
            /// </summary>
            /// <param name="gameTime">Provides a snapshot of timing values.</param>
            protected override void Draw(GameTime gameTime)
            {
                GraphicsDevice.Clear(Color.CornflowerBlue);
    
                // TODO: Add your drawing code here
    
                spriteBatch.Begin();
    
                if (kinectVideoTexture != null)
                    spriteBatch.Draw(kinectVideoTexture, fullScreenRectangle, Color.White);
    
                if (errorMessage.Length > 0)
                {
                    spriteBatch.DrawString(messageFont, errorMessage, new Vector2(20, 20), Color.White);
                }
    
                spriteBatch.DrawString(messageFont, userMessage, new Vector2(20, 40), Color.White);
                spriteBatch.DrawString(messageFont, rHandMessage, new Vector2(20, 60), Color.White);
                spriteBatch.DrawString(messageFont, lHandMessage, new Vector2(20, 80), Color.White);
                spriteBatch.DrawString(messageFont, rHandStatus, new Vector2(20, 100), Color.White);
                spriteBatch.DrawString(messageFont, lHandStatus, new Vector2(20, 120), Color.White);
    
                Rectangle recDest = new Rectangle(10, 10, mySpriteTexture2D1.Width, mySpriteTexture2D1.Height);
    
                spriteBatch.Draw(mySpriteTexture2D1, recDest, Color.White);
                
                spriteBatch.End();
    
                base.Draw(gameTime);
            }
        }
    }
    

    Wednesday, October 30, 2013 8:46 AM

Answers

  • ah, the frame itself is null. You should be following the code samples and be sure to check if the frame is not null before trying to use it:

    if (depthFrame != null)
    {
    ...
        depthFrame.CopyDepthImagePixelDataTo(this.depthPixels);
    ...
    }

    Carmine Sirignano - MSFT

    Saturday, November 2, 2013 12:24 AM

All replies

  • The error itself is E_INVALIDARG or invalid parameters. It could be by the time the Interaction engine is ready to parse the data from the stream it has already been released or deleted by the runtime. this is a factor of performance typically.
    http://msdn.microsoft.com/en-us/library/microsoft.kinect.toolkit.interaction.interactionstream.processdepth.aspx

    Change your code to use the CopyDepthImagePixelDataTo method instead of GetRawPixelData to create a local copy of the DepthPixels and use that in the method to see if things change.

    DepthImagePixel[] depthBuffer
    
    if ((null == this.depthBuffer) || (this.depthBuffer.Length != frame.PixelDataLength))
    {
        this.depthBuffer = new DepthImagePixel[frame.PixelDataLength];
    }
    
    frame.CopyDepthImagePixelDataTo(this.depthBuffer);
    


    Carmine Sirignano - MSFT

    Wednesday, October 30, 2013 6:10 PM
  • Thanks your Opinion!

    I try to modify the code, but show same error, 

    "This API has returned an exception from an HRESULT: 0x80070057." 

    How to fix the problem?
    Thank you!

    DepthImagePixel[] depthBuffer;
    
    using (DepthImageFrame frame = e.OpenDepthImageFrame())
                {
                    if ((null == depthBuffer) || (depthBuffer.Length != frame.PixelDataLength))
                    {
                        depthBuffer = new DepthImagePixel[frame.PixelDataLength];
                    }
                    frame.CopyDepthImagePixelDataTo(depthBuffer);
    }
    interStream.ProcessDepth(frame.GetRawPixelData(), frame.Timestamp);
    



    Thursday, October 31, 2013 3:38 AM
  • you didn't update the code for the ProcessDepth method, you are still passing frame.GetRawPixelData(). This line of code should be using the copy data:

    interStream.ProcessDepth(depthBuffer, frame.Timestamp);


    Carmine Sirignano - MSFT


    Thursday, October 31, 2013 4:39 PM
  • Thank you for your help!

    I try that but cannot run and show some errors.

    Thank you! 

    DepthImagePixel[] depthBuffer;
    
    using (DepthImageFrame frame = e.OpenDepthImageFrame())
    {
        // Get the depth data
    
        if ((null == depthBuffer) || (depthBuffer.Length != frame.PixelDataLength))
        {
            depthBuffer = new DepthImagePixel[frame.PixelDataLength];
        }
        frame.CopyDepthImagePixelDataTo(depthBuffer);
    
        interStream.ProcessDepth(depthBuffer, frame.Timestamp);
    



    Friday, November 1, 2013 2:02 AM
  • ah, the frame itself is null. You should be following the code samples and be sure to check if the frame is not null before trying to use it:

    if (depthFrame != null)
    {
    ...
        depthFrame.CopyDepthImagePixelDataTo(this.depthPixels);
    ...
    }

    Carmine Sirignano - MSFT

    Saturday, November 2, 2013 12:24 AM
  • Thank yours help!

    but cannot that.

    Monday, November 4, 2013 2:37 AM
  • You are still trying to use the frame.Timestamp outside of the if( frame != null) call.


    Carmine Sirignano - MSFT

    Monday, November 4, 2013 6:45 PM
  • Thank you!

    I don't know how to modify, can you suggest idea?

    Tuesday, November 5, 2013 1:13 AM