KinectV2_背景消除_多拉A夢任意門空間轉換效果
這次我們要來學習如何寫一個Kinect V2 背景去除效果
首先 我們的第一個目標是如下圖的結果
感覺非常好玩可以到任何國度或者雨林中
我們先觀察 Kinect V2 呈現了彩色資訊 和 身體索引 等等
這是我在一開始所觀察到的
或許還有少些資訊
還有很直觀的一個人的身體輪廓(身體資訊)前景
可以在一張圖上做融入現場環境的有趣小互動
我們先來觀察 Kinect V2 官網上雖說 彩色解析度是1920*1080
第1-1階段程式碼
試著印出 彩色資訊的解析度
#include<iostream>
#include<stdio.h>
//opencv Library
#include"opencv\cv.h"
#include"opencv\highgui.h"
#include"opencv2\highgui\highgui.hpp"
#include"opencv2\opencv.hpp"
//Kiinect SDK
#include "Kinect.h"
using namespace std;
using namespace cv;
int main()
{
//open Kinect V2 sensor
IKinectSensor * mySensor = nullptr;
GetDefaultKinectSensor(&mySensor);
mySensor->Open();
//obtain Kinect V2 color source
IColorFrameSource *myColorSource = nullptr;
mySensor->get_ColorFrameSource(&myColorSource);
//obtain Kinect V2 color frame description
IFrameDescription *myColorDescription = nullptr;
//we want to know the width and height of color source
int width, height;
//we use "IColorFrameSource" to help us get frame description
myColorSource->get_FrameDescription(&myColorDescription);
//we use "IFrameDescription" to help us get width and height
myColorDescription->get_Width(&width);
myColorDescription->get_Height(&height);
cout << "Width:" << width << "\n" << "Height:" << height << endl;
system("PAUSE"); //卡住用的
return 0;
}
一個關於Kinect V2觀念 :
不同種數據會搭配兩個項目(源、閱讀器、對應的描述結構)
IBody , IDepth , IColor .....etc
一個xxxFrameSource(源)
一個xxxFrameReader(閱讀器)
若需要得到源的描述
IFrameDescription * 是一個可以用來取得所有源對應描述之結構體!!!!!!
以目前我們是要獲得 彩色數據源的寬、高等描述
處理三步驟
從「傳感器」獲得「源」
mySensor->get_ColorFrameSource(&myColorSource); myColorSource->get_FrameDescription(&myColorDescription);myColorDescription->get_Width(&width);
myColorDescription->get_Height(&height); 我們來統整一下
所以其實啊~~~ 有兩個步驟是可以同時都做得
應該說可以依據你要做捨麼去擇一
你要秀出影像就必須開啟 IxxxFrameReader (IColorFrameReader ,IDepthFrameReader ...etc)
你要獲取影格描述就必需開啟 IFrameDescription
現在我要秀出影像
所以需使用 Reader
從 source 可開啟 Reader
就會顯示了
第1-2階段程式碼
秀彩影
#include<iostream>
#include<stdio.h>
//opencv Library
#include"opencv\cv.h"
#include"opencv\highgui.h"
#include"opencv2\highgui\highgui.hpp"
#include"opencv2\opencv.hpp"
//Kiinect SDK
#include "Kinect.h"
using namespace std;
using namespace cv;
int main()
{
//open Kinect V2 sensor
IKinectSensor * mySensor = nullptr;
GetDefaultKinectSensor(&mySensor);
mySensor->Open();
//obtain Kinect V2 color source
IColorFrameSource *myColorSource = nullptr;
mySensor->get_ColorFrameSource(&myColorSource);
//obtain Kinect V2 color frame description
IFrameDescription *myColorDescription = nullptr;
//we want to know the width and height of color source
int width, height;
//we use "IColorFrameSource" to help us get frame description
myColorSource->get_FrameDescription(&myColorDescription);
//we use "IFrameDescription" to help us get width and height
myColorDescription->get_Width(&width);
myColorDescription->get_Height(&height);
//cout << "Width:" << width << "\n" << "Height:" << height << endl;
IColorFrameReader *myColorReader = nullptr;
myColorSource->OpenReader(&myColorReader);
Mat img(height, width, CV_8UC4); // R->G->B->A
IColorFrame *myColorFrame = nullptr;
for (;;)
{
if (myColorReader->AcquireLatestFrame(&myColorFrame) == S_OK)
{
UINT size = 0;
myColorFrame->CopyConvertedFrameDataToArray(width*height*4,(BYTE*)img.data,ColorImageFormat_Bgra);
namedWindow("Color",0);
imshow("Color",img);
myColorFrame->Release(); // 沒加會停不更新
}
if (waitKey(30) == VK_ESCAPE) // escape
break;
}
myColorReader->Release();
myColorDescription->Release();
mySensor->Close();
mySensor->Release();
system("PAUSE"); //卡住用的
return 0;
}
緊接著我又好奇了
我們的Kinect V2 是如何得到
身體索引資訊的呢????
依樣畫葫蘆
第2-1階段程式碼
試著印出 身體索引資訊的解析度
#include<iostream>
#include<stdio.h>
//opencv Library
#include"opencv\cv.h"
#include"opencv\highgui.h"
#include"opencv2\highgui\highgui.hpp"
#include"opencv2\opencv.hpp"
//Kiinect SDK
#include "Kinect.h"
using namespace std;
using namespace cv;
int main()
{
//open Kinect V2 sensor
IKinectSensor * mySensor = nullptr;
GetDefaultKinectSensor(&mySensor);
mySensor->Open();
//obtain Kinect V2 BodyIndex source
IBodyIndexFrameSource *myBodyIndexSource = nullptr;
mySensor->get_BodyIndexFrameSource(&myBodyIndexSource);
//obtain Kinect V2 BodyIndex frame description
IFrameDescription * myBodyIndexDescription = nullptr;
int height, width;
myBodyIndexSource->get_FrameDescription(&myBodyIndexDescription);
myBodyIndexDescription->get_Height(&height);
myBodyIndexDescription->get_Width(&width);
cout << "Width:" << width << "\n" << "Height:" << height << endl;
system("PAUSE");
return 0;
}
第2-2階段程式碼
秀Body Index
#include<iostream>
#include<stdio.h>
//opencv Library
#include"opencv\cv.h"
#include"opencv\highgui.h"
#include"opencv2\highgui\highgui.hpp"
#include"opencv2\opencv.hpp"
//Kiinect SDK
#include "Kinect.h"
using namespace std;
using namespace cv;
int main()
{
//open Kinect V2 sensor
IKinectSensor *mySensor = nullptr;
GetDefaultKinectSensor(&mySensor);
mySensor->Open();
//obtain Kinect V2 BodyIndex source
IBodyIndexFrameSource *myBodyIndexSource = nullptr;
mySensor->get_BodyIndexFrameSource(&myBodyIndexSource);
//obtain Kinect V2 BodyIndex frame description
IFrameDescription * myBodyIndexDescription = nullptr;
int height, width;
myBodyIndexSource->get_FrameDescription(&myBodyIndexDescription);
myBodyIndexDescription->get_Height(&height);
myBodyIndexDescription->get_Width(&width);
//cout << "Width:" << width << "\n" << "Height:" << height << endl;
IBodyIndexFrameReader *myBodyIndexReader = nullptr;
myBodyIndexSource->OpenReader(&myBodyIndexReader);
Mat img(height, width, CV_8UC3);
IBodyIndexFrame *myBodyIndexFrame = nullptr;
//use opencv to draw different color
Vec3b color[7] = { Vec3b(0, 0, 255), //R
Vec3b(0, 255, 255), //yello
Vec3b(255, 255, 255), //white
Vec3b(0, 255, 0), //G
Vec3b(255, 0, 0), //B
Vec3b(255, 0, 255), //purple
Vec3b(0, 0, 0) }; //black
for (;;)
{
if (myBodyIndexReader->AcquireLatestFrame(&myBodyIndexFrame) == S_OK)
{
UINT size = 0;
BYTE * buffer = nullptr;
myBodyIndexFrame->AccessUnderlyingBuffer(&size, &buffer);
for (int i = 0; i < height; i++)
for (int j = 0; j < width; j++)
{
int index = buffer[i * width + j];
//0-5代表人體，其他值代表背景，藉此用不同顏色區分人體(前景)
if (index <= 5)
img.at<Vec3b>(i, j) = color[index];
else //大於5的值代表的是背景
img.at<Vec3b>(i, j) = color[6];
}
imshow("Body Index", img);
myBodyIndexFrame->Release();
}
if (waitKey(30) == VK_ESCAPE)
break;
}
myBodyIndexReader->Release();
myBodyIndexDescription->Release();
myBodyIndexSource->Release();
mySensor->Close();
mySensor->Release();
system("PAUSE");
return 0;
}
在Kinect V2 SDK中提供了一個叫
ICoordinateMapper的類
主要是用來做坐標系之間的互相轉換，用來解決各種源(Source)的解析度
不同導致點對應不起來的問題。
我們需要將Color Image中的點與 Depth Image中的點一一對應起來
可透過MapColorFrameToDepthSpace()這個函數。
起初需準備好三種源(Source)：
(1)Color
(2)BodyIndex
(3)Depth
其中前兩個是完成功能本來就需要的
第三個是轉換坐標係時需要，
無法直接把Color的坐標系映射到BodyIndex中，只能映射到Depth中。
緊接著我們需要讀取Background，
讀取之後也要轉換成Color image的尺寸，
這樣把Color中的點貼過去時坐標就不用再轉換
直接替換就行。
接下來也要讀取三種Frame，為了程式碼具有清楚架構和好閱讀性，
我們先將開始的所有配置都弄好
利用MapColorFrameToDepthSpace()
官網介紹 link :
https://msdn.microsoft.com/en-us/library/windowspreview.kinect.coordinatemapper.mapcolorframetodepthspace.aspx
把Color Frame 映射到Depth的坐標系，
它需要4個參數，
第1個是深度幀的大小，
第2個是深度數據，
第3個是彩色幀的大小，
第4個是一個DepthSpacePoint的數組:用來儲存彩色空間中的每個點對應到深度空間的坐標。
要注意，這個函數只是完成坐標系的轉換，也就是說它對於彩色坐標系中的每個點，
都給出了一個此點對應到深度坐標系中的坐標，並不涉及到具體的ColorFrame。
最後，遍歷整張Color Image，
對每一點，都取出其對應的深度坐標系坐標，然後把這個坐標放入BodyIndex的數據中，
判斷此點是否屬於人體，
如果屬於，就把這點從彩色圖中取出，跟背景圖中同一坐標的點替換。
【Hint : DepthSpacePoint中的X和Y的值都是float的
用它們來計算在BodyIndex裡的坐標時，需要做casting運算強制轉型成 int ，
不然畫面就會很差，
不屬於人體的地方也會被標記成了人體被替換掉。
最後效果
最後程式碼
#include<iostream>
#include<stdio.h>
//opencv Library
#include"opencv\cv.h"
#include"opencv\highgui.h"
#include"opencv2\highgui\highgui.hpp"
#include"opencv2\opencv.hpp"
//Kiinect SDK
#include "Kinect.h"
using namespace std;
using namespace cv;
int main()
{
//open Kinect V2 sensor
IKinectSensor *mySensor = nullptr;
GetDefaultKinectSensor(&mySensor);
mySensor->Open();
IFrameDescription * myDescription = nullptr;
// 大家一起共用的 BodyIndex , depth , Color 都可以乘載寬、高描述
/*Color---------------------------------------------------------------*/
//obtain Kinect V2 BodyIndex source
IColorFrameSource * myColorSource = nullptr;
mySensor->get_ColorFrameSource(&myColorSource);
//open Color Reader from ColorFrameSource
IColorFrameReader * myColorReader = nullptr;
myColorSource->OpenReader(&myColorReader);
int colorHeight = 0, colorWidth = 0;
myColorSource->get_FrameDescription(&myDescription);
myDescription->get_Height(&colorHeight);
myDescription->get_Width(&colorWidth);
myDescription->Release();
myColorSource->Release();
/*Depth---------------------------------------------------------------*/
//obtain Kinect V2 Depth source
IDepthFrameSource * myDepthSource = nullptr;
mySensor->get_DepthFrameSource(&myDepthSource);
//open Depth Reader from DepthFrameSource
IDepthFrameReader * myDepthReader = nullptr;
myDepthSource->OpenReader(&myDepthReader);
int depthHeight = 0, depthWidth = 0;
myDepthSource->get_FrameDescription(&myDescription);
myDescription->get_Height(&depthHeight);
myDescription->get_Width(&depthWidth);
myDescription->Release();
myDepthSource->Release();
/*BodyIndex---------------------------------------------------------------*/
//obtain Kinect V2 BodyIndex source
IBodyIndexFrameSource * myBodyIndexSource = nullptr;
mySensor->get_BodyIndexFrameSource(&myBodyIndexSource);
//open BodyIndex Reader from DepthFrameSource
IBodyIndexFrameReader * myBodyIndexReader = nullptr;
myBodyIndexSource->OpenReader(&myBodyIndexReader);
int bodyHeight = 0, bodyWidth = 0;
myDepthSource->get_FrameDescription(&myDescription);
myDescription->get_Height(&bodyHeight);
myDescription->get_Width(&bodyWidth);
myDescription->Release();
myBodyIndexSource->Release();
// prepare buffer for each of kinect sources
UINT colorDataSize = colorHeight * colorWidth;
UINT depthDataSize = depthHeight * depthWidth;
UINT bodyDataSize = bodyHeight * bodyWidth;
//load background
Mat temp = imread("C:\\img_res\\outer_space.jpg"), background;
//resize the background img to the Size of Color source
resize(temp, background, Size(colorWidth, colorHeight));
ICoordinateMapper * myMaper = nullptr; //open mapper
mySensor->get_CoordinateMapper(&myMaper);
Mat colorData(colorHeight, colorWidth, CV_8UC4); //prepare buffer
UINT16 * depthData = new UINT16[depthDataSize];
BYTE * bodyData = new BYTE[bodyDataSize];
DepthSpacePoint * output = new DepthSpacePoint[colorDataSize];
//put each of different frame (IColor , IDepth , IBodyIndex )
while (1)
{
IColorFrame * myColorFrame = nullptr;
while (myColorReader->AcquireLatestFrame(&myColorFrame) != S_OK); //read colorFrame
myColorFrame->CopyConvertedFrameDataToArray(colorDataSize * 4, colorData.data, ColorImageFormat_Bgra);
myColorFrame->Release();
IDepthFrame * myDepthframe = nullptr;
while (myDepthReader->AcquireLatestFrame(&myDepthframe) != S_OK); //read depthFrame
myDepthframe->CopyFrameDataToArray(depthDataSize, depthData);
myDepthframe->Release();
IBodyIndexFrame * myBodyIndexFrame = nullptr; //read BodyIndexFrame
while (myBodyIndexReader->AcquireLatestFrame(&myBodyIndexFrame) != S_OK);
myBodyIndexFrame->CopyFrameDataToArray(bodyDataSize, bodyData);
myBodyIndexFrame->Release();
Mat copy = background.clone(); //copy an bacjground image to do some image process
if (myMaper->MapColorFrameToDepthSpace(depthDataSize, depthData, colorDataSize, output) == S_OK)
{
for (int i = 0; i < colorHeight; ++i)
for (int j = 0; j < colorWidth; ++j)
{
//obtain the points from Color image , It contains the coordinates correspond to the depth image
DepthSpacePoint tPoint = output[i * colorWidth + j];
if (tPoint.X >= 0 && tPoint.X < depthWidth && tPoint.Y >= 0 && tPoint.Y < depthHeight)
{
//Get the value that corresponds to the points where in BodyIndex on the color image
int index = (int)tPoint.Y * depthWidth + (int)tPoint.X; //(need to use casting)
//If it is determined at a point on the color map is the body,
//use it to replace the background image corresponding to the point
if (bodyData[index] <= 5)
{
Vec4b color = colorData.at<Vec4b>(i, j);
copy.at<Vec3b>(i, j) = Vec3b(color[0], color[1], color[2]);
}
}
}
imshow("Back ground remove", copy);
}
if (waitKey(30) == VK_ESCAPE)
break;
}
delete[] depthData;
delete[] bodyData;
delete[] output;
myMaper->Release();
myColorReader->Release();
myDepthReader->Release();
myBodyIndexReader->Release();
mySensor->Close();
mySensor->Release();
return 0;
}
如果覺得太大就在imshow 之前做 resize吧
這裡我調整成 600 * 400 做效果顯示
他把椅子也當成人了 = =|||
以上是這次學習分享
留言
張貼留言