VSAG/vision_sjtu_19
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修改了摄像头读取方式

Browse Source
This commit is contained in:
xinyang
2019-04-27 16:16:53 +08:00
parent 9cfd26cc23
commit 4e47b38d7d
15 changed files with 272 additions and 255 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
armor/include/show_images/show_images.h
View File

@@ -13,5 +13,8 @@
void showArmorBoxVector(std::string windows_name, const cv::Mat &src, const std::vector<cv::Rect2d> &armor_box); void showArmorBoxVector(std::string windows_name, const cv::Mat &src, const std::vector<cv::Rect2d> &armor_box);
void showArmorBox(std::string windows_name, const cv::Mat &src, cv::Rect2d armor_box); void showArmorBox(std::string windows_name, const cv::Mat &src, cv::Rect2d armor_box);
void showContours(std::string windows_name, const cv::Mat &src, const std::vector<LightBlob> &light_blobs); void showContours(std::string windows_name, const cv::Mat &src, const std::vector<LightBlob> &light_blobs);
void showArmorBoxClass(std::string window_names, const cv::Mat &src, vector<cv::Rect2d> boxes_one,
vector<cv::Rect2d> boxes_two, vector<cv::Rect2d> boxes_three);
#endif /* _SHOW_IMAGES_H_ */ #endif /* _SHOW_IMAGES_H_ */

15
armor/src/armor_finder/armor_finder.cpp
View File

@@ -22,14 +22,15 @@ ArmorFinder::ArmorFinder(EnemyColor color, Uart &u, string paras_folder) :
void ArmorFinder::run(cv::Mat &src) { void ArmorFinder::run(cv::Mat &src) {
cv::Mat src_use; cv::Mat src_use;
if (src.type() == CV_8UC3) { // if (src.type() == CV_8UC3) {
cv::cvtColor(src, src_use, CV_RGB2GRAY); // cv::cvtColor(src, src_use, CV_RGB2GRAY);
}else{ // }else{
src_use = src.clone(); src_use = src.clone();
} // }
cv::cvtColor(src_use, src_gray, CV_BayerBG2GRAY); cv::cvtColor(src_use, src_gray, CV_RGB2GRAY);
// return stateSearchingTarget(src_use); stateSearchingTarget(src_use);
return;
switch (state){ switch (state){
case SEARCHING_STATE: case SEARCHING_STATE:
@@ -49,7 +50,7 @@ void ArmorFinder::run(cv::Mat &src) {
} }
break; break;
case TRACKING_STATE: case TRACKING_STATE:
if(!stateTrackingTarget(src_use)){ if(!stateTrackingTarget(src_gray)){
state = SEARCHING_STATE; state = SEARCHING_STATE;
//std::cout << "into search!" << std::endl; //std::cout << "into search!" << std::endl;
} }

17
armor/src/armor_finder/classifier/classifier.cpp
View File

@@ -258,6 +258,9 @@ Classifier::Classifier(const string &folder) : state(true){
fc1_b = load_fc_b(folder+"fc1_b"); fc1_b = load_fc_b(folder+"fc1_b");
fc2_w = load_fc_w(folder+"fc2_w"); fc2_w = load_fc_w(folder+"fc2_w");
fc2_b = load_fc_b(folder+"fc2_b"); fc2_b = load_fc_b(folder+"fc2_b");
if(state){
LOGM("Load paras success!");
}
} }
//#define PRINT_MAT(name) (cout << #name":\n" << name << endl) //#define PRINT_MAT(name) (cout << #name":\n" << name << endl)
@@ -285,10 +288,16 @@ Classifier::operator bool() const {
} }
int Classifier::operator()(const cv::Mat &image) { int Classifier::operator()(const cv::Mat &image) {
MatrixXd x; MatrixXd r, g, b;
cv2eigen(image, x); std::vector<cv::Mat> channels;
x /= 255.0; cv::split(image, channels);
vector<MatrixXd> sub = {x}; cv2eigen(channels[0], b);
cv2eigen(channels[1], g);
cv2eigen(channels[2], r);
r /= 255;
g /= 255;
b /= 255;
vector<MatrixXd> sub = {b, g, r};
vector<vector<MatrixXd>> in = {sub}; vector<vector<MatrixXd>> in = {sub};
MatrixXd result = calculate(in); MatrixXd result = calculate(in);
MatrixXd::Index minRow, minCol; MatrixXd::Index minRow, minCol;

6
armor/src/armor_finder/state_machine/searching_state/image_process/image_process.cpp
View File

@@ -36,15 +36,17 @@ void imageColorSplit(cv::Mat &src_input, cv::Mat &split, EnemyColor color) {
resize(channels.at(0), blue, cv::Size(640, 480)); resize(channels.at(0), blue, cv::Size(640, 480));
resize(channels.at(2), red, cv::Size(640, 480)); resize(channels.at(2), red, cv::Size(640, 480));
if(color == ENEMY_RED){ if(color == ENEMY_RED){
split = red; split = red - blue;
}else if(color == ENEMY_BLUE){ }else if(color == ENEMY_BLUE){
split = blue; split = blue - red;
} }
} }
} }
void imagePreProcess(cv::Mat &src) { void imagePreProcess(cv::Mat &src) {
// cv::medianBlur(src, src, 5);
static cv::Mat kernel_erode = getStructuringElement(cv::MORPH_RECT, cv::Size(1, 4)); static cv::Mat kernel_erode = getStructuringElement(cv::MORPH_RECT, cv::Size(1, 4));
erode(src, src, kernel_erode); erode(src, src, kernel_erode);

46
armor/src/armor_finder/state_machine/searching_state/searching_state.cpp
View File

@@ -114,8 +114,8 @@ static bool findArmorBoxes(LightBlobs &light_blobs, std::vector<cv::Rect2d> &arm
cv::Rect2d rect_left = light_blobs.at(static_cast<unsigned long>(i)).rect.boundingRect(); cv::Rect2d rect_left = light_blobs.at(static_cast<unsigned long>(i)).rect.boundingRect();
cv::Rect2d rect_right = light_blobs.at(static_cast<unsigned long>(j)).rect.boundingRect(); cv::Rect2d rect_right = light_blobs.at(static_cast<unsigned long>(j)).rect.boundingRect();
double min_x, min_y, max_x, max_y; double min_x, min_y, max_x, max_y;
min_x = fmin(rect_left.x, rect_right.x) - 5; min_x = fmin(rect_left.x, rect_right.x);
max_x = fmax(rect_left.x + rect_left.width, rect_right.x + rect_right.width) + 5; max_x = fmax(rect_left.x + rect_left.width, rect_right.x + rect_right.width);
min_y = fmin(rect_left.y, rect_right.y) - 5; min_y = fmin(rect_left.y, rect_right.y) - 5;
max_y = fmax(rect_left.y + rect_left.height, rect_right.y + rect_right.height) + 5; max_y = fmax(rect_left.y + rect_left.height, rect_right.y + rect_right.height) + 5;
if (min_x < 0 || max_x > 640 || min_y < 0 || max_y > 480) { if (min_x < 0 || max_x > 640 || min_y < 0 || max_y > 480) {
@@ -151,7 +151,7 @@ bool judge_light_color(std::vector<LightBlob> &light, std::vector<LightBlob> &co
bool ArmorFinder::stateSearchingTarget(cv::Mat &src) { bool ArmorFinder::stateSearchingTarget(cv::Mat &src) {
cv::Mat split, pmsrc=src.clone(); cv::Mat split, pmsrc=src.clone();
LightBlobs light_blobs, pm_light_blobs, light_blobs_real; LightBlobs light_blobs, pm_light_blobs, light_blobs_real;
std::vector<cv::Rect2d> armor_boxes; std::vector<cv::Rect2d> armor_boxes, boxes_one, boxes_two, boxes_three;
// cv::resize(src, pmsrc, cv::Size(320, 240)); // cv::resize(src, pmsrc, cv::Size(320, 240));
imageColorSplit(src, split, enemy_color); imageColorSplit(src, split, enemy_color);
@@ -174,20 +174,45 @@ bool ArmorFinder::stateSearchingTarget(cv::Mat &src) {
if(!findArmorBoxes(light_blobs, armor_boxes)){ if(!findArmorBoxes(light_blobs, armor_boxes)){
return false; return false;
} }
if(show_armor_boxes){
showArmorBoxVector("boxes", split, armor_boxes);
cv::waitKey(1);
}
if(classifier){ if(classifier){
for(const auto &box : armor_boxes){ for(auto box : armor_boxes){
cv::Mat roi = src(box).clone(); cv::Mat roi = src(box).clone();
cv::resize(roi, roi, cv::Size(60, 45)); cv::resize(roi, roi, cv::Size(48, 36));
if(classifier(roi)){ // cv::imshow("roi", roi);
armor_box = box; // cv::waitKey(0);
break; int c = classifier(roi);
// cout << c << endl;
switch(c){
case 1:
boxes_one.emplace_back(box);
break;
case 2:
boxes_two.emplace_back(box);
break;
case 3:
boxes_three.emplace_back(box);
break;
} }
} }
if(!boxes_one.empty()){
armor_box = boxes_one[0];
}else if(!boxes_two.empty()){
armor_box = boxes_two[0];
}else if(!boxes_three.empty()){
armor_box = boxes_three[0];
}
if(show_armor_box){
showArmorBoxClass("class", src, boxes_one, boxes_two, boxes_three);
}
}else{ }else{
armor_box = armor_boxes[0]; armor_box = armor_boxes[0];
} }
if(show_armor_boxes){ if(show_armor_box){
showArmorBoxVector("boxes", split, armor_boxes); showArmorBox("box", src, armor_box);
cv::waitKey(1); cv::waitKey(1);
} }
if(split.size() == cv::Size(320, 240)){ if(split.size() == cv::Size(320, 240)){
@@ -196,6 +221,5 @@ bool ArmorFinder::stateSearchingTarget(cv::Mat &src) {
armor_box.width *= 2; armor_box.width *= 2;
armor_box.height *= 2; armor_box.height *= 2;
} }
return sendBoxPosition(); return sendBoxPosition();
} }

22
armor/src/show_images/show_images.cpp
View File

@@ -18,6 +18,28 @@ void showArmorBoxVector(std::string windows_name, const cv::Mat &src, const std:
imshow(windows_name, image2show); imshow(windows_name, image2show);
} }
void showArmorBoxClass(std::string windows_name, const cv::Mat &src, vector<cv::Rect2d> boxes_one,
vector<cv::Rect2d> boxes_two, vector<cv::Rect2d> boxes_three){
static Mat image2show;
if (src.type() == CV_8UC1) // 黑白图像
{
cvtColor(src, image2show, COLOR_GRAY2RGB);
} else if(src.type() == CV_8UC3) //RGB 彩色
{
image2show = src.clone();
}
for (auto &box:boxes_one) {
rectangle(image2show, box, Scalar(255, 0, 0), 1);
}
for (auto &box:boxes_two) {
rectangle(image2show, box, Scalar(0, 255, 0), 1);
}
for (auto &box:boxes_three) {
rectangle(image2show, box, Scalar(0, 0, 255), 1);
}
imshow(windows_name, image2show);
}
void showArmorBox(std::string windows_name, const cv::Mat &src, cv::Rect2d armor_box) { void showArmorBox(std::string windows_name, const cv::Mat &src, cv::Rect2d armor_box) {
static Mat image2show; static Mat image2show;
if (src.type() == CV_8UC1) // 黑白图像 if (src.type() == CV_8UC1) // 黑白图像

28
main.cpp
View File

@@ -48,25 +48,28 @@ int main(int argc, char *argv[])
cin >> from_camera; cin >> from_camera;
} }
WrapperHead *video; WrapperHead *video_armor;
if(from_camera) WrapperHead *video_enegy;
video = new CameraWrapper; if(from_camera) {
else video_armor = new CameraWrapper("armor");
video = new VideoWrapper("r_l_640.avi", "fan_640.avi"); video_enegy = new CameraWrapper("energy");
}else {
if (video->init()) { video_armor = new VideoWrapper("/home/xinyang/Desktop/Video.mp4");
video_enegy = new VideoWrapper("/home/xinyang/Desktop/Video.mp4");
}
if (video_enegy->init() && video_armor->init()) {
cout << "Video source initialization successfully." << endl; cout << "Video source initialization successfully." << endl;
} }
Mat energy_src, armor_src; Mat energy_src, armor_src;
ArmorFinder armorFinder(ENEMY_BLUE, uart, "../paras/"); ArmorFinder armorFinder(ENEMY_BLUE, uart, "/home/xinyang/Desktop/AutoAim/tools/para/");
Energy energy(uart); Energy energy(uart);
energy.setAllyColor(ally_color); energy.setAllyColor(ally_color);
energy.setRotation(energy_part_rotation); energy.setRotation(energy_part_rotation);
while (video->read(energy_src, armor_src)) while (video_armor->read(energy_src) && video_armor->read(armor_src))
{ {
if(show_origin) { if(show_origin) {
imshow("enery src", energy_src); imshow("enery src", energy_src);
@@ -78,15 +81,16 @@ int main(int argc, char *argv[])
} }
energy.run(energy_src); energy.run(energy_src);
}else{ }else{
armorFinder.run(armor_src); CNT_TIME(WORD_LIGHT_BLUE, "Armor Time", {
armorFinder.run(armor_src);
});
} }
if (waitKey(1) == 'q') { if (waitKey(1) == 'q') {
flag = false; flag = false;
break; break;
} }
} }
delete video; delete video_enegy, video_armor;
cout << "Program fails. Restarting" << endl; cout << "Program fails. Restarting" << endl;
} }

39
others/include/camera/camera_wrapper.h
View File

@@ -18,40 +18,31 @@
class CameraWrapper: public WrapperHead { class CameraWrapper: public WrapperHead {
private: private:
unsigned char* rgb_buffer0; const std::string name;
unsigned char* rgb_buffer1;
unsigned char* rgb_buffer;
int camera_cnts; int camera_cnts;
int camera_status0, camera_status1; int camera_status;
tSdkCameraDevInfo camera_enum_list[2]; tSdkCameraDevInfo camera_enum_list[2];
int h_camera0; int h_camera;
int h_camera1; char camera_name[32];
char camera_name0[32];
char camera_name1[32];
tSdkCameraCapbility tCapability0; tSdkCameraCapbility tCapability;
tSdkCameraCapbility tCapability1; tSdkFrameHead frame_info;
tSdkFrameHead frame_info0; BYTE *pby_buffer;
tSdkFrameHead frame_info1; IplImage* iplImage;
BYTE *pby_buffer0; int channel;
BYTE *pby_buffer1;
IplImage* iplImage0;
IplImage* iplImage1;
int channel0;
int channel1;
void swapCameraHandle();
public: public:
CameraWrapper(); CameraWrapper();
CameraWrapper(const std::string &n);
~CameraWrapper() final; ~CameraWrapper() final;
bool init() final; bool init() final;
bool read(cv::Mat& src0, cv::Mat& src1) final; bool read(cv::Mat& src) final;
bool readRaw(cv::Mat& src0, cv::Mat& src1); bool readRaw(cv::Mat& src);
bool readProcessed(cv::Mat& src0, cv::Mat& src1); bool readProcessed(cv::Mat& src);
}; };

6
others/include/camera/video_wrapper.h
View File

@@ -16,7 +16,7 @@
class VideoWrapper:public WrapperHead { class VideoWrapper:public WrapperHead {
public: public:
VideoWrapper(const std::string& filename0, const std::string& filename1); VideoWrapper(const std::string& filename);
~VideoWrapper(); ~VideoWrapper();
@@ -33,9 +33,9 @@ public:
* @param src_right : output source video of right camera * @param src_right : output source video of right camera
* @return bool value: whether the reading is successful * @return bool value: whether the reading is successful
*/ */
bool read(cv::Mat &src_left, cv::Mat &src_right) final; bool read(cv::Mat &src) final;
private: private:
cv::VideoCapture video0, video1; cv::VideoCapture video;
}; };

3
others/include/camera/wrapper_head.h
View File

@@ -11,11 +11,10 @@
* @brief A virtual class for wrapper of camera and video files * @brief A virtual class for wrapper of camera and video files
*/ */
class WrapperHead { class WrapperHead {
public: public:
virtual ~WrapperHead() = default;; virtual ~WrapperHead() = default;;
virtual bool init() = 0; virtual bool init() = 0;
virtual bool read(cv::Mat &src_left, cv::Mat &src_right) = 0; virtual bool read(cv::Mat &src) = 0;
}; };

212
others/src/camera/camera_wrapper.cpp
View File

@@ -3,20 +3,26 @@
// //
#include <camera/camera_wrapper.h> #include <camera/camera_wrapper.h>
#include <log.h>
using std::cout; using std::cout;
using std::endl; using std::endl;
using namespace cv; using namespace cv;
CameraWrapper::CameraWrapper() CameraWrapper::CameraWrapper(const std::string &n):
{ name(n),
camera_cnts = 2; camera_cnts(2),
camera_status0 = -1; camera_status(-1),
camera_status1 = -1; iplImage(nullptr),
iplImage0 = nullptr; channel(3){
iplImage1 = nullptr; }
channel0 = 3;
channel1 = 3; CameraWrapper::CameraWrapper():
name("NULL"),
camera_cnts(2),
camera_status(-1),
iplImage(nullptr),
channel(3){
} }
@@ -30,69 +36,52 @@ bool CameraWrapper::init() {
//没有连接设备 //没有连接设备
if (camera_cnts == 0) { if (camera_cnts == 0) {
cout<<"No device detected!"<<endl; LOGE("No camera device detected!");
return false; return false;
}else if(camera_cnts >= 1){
LOGM("%d camera device detected!", camera_cnts);
} }
else if(camera_cnts == 1)
{
cout<<"Only one camera device detected"<<endl;
return false;
}
else if(camera_cnts == 2)
{
cout<<"Two camera devices detected."<<endl;
}
else
{
cout<<"More than 2 cameras detected or some other error occurs."<<endl;
return false;
}
//相机初始化。初始化成功后，才能调用任何其他相机相关的操作接口 //相机初始化。初始化成功后，才能调用任何其他相机相关的操作接口
camera_status0 = CameraInit(&camera_enum_list[0], -1, -1, &h_camera0); int i;
//初始化失败 for(i=0; i<camera_cnts; i++){
if (camera_status0 != CAMERA_STATUS_SUCCESS) { camera_status = CameraInit(&camera_enum_list[i], -1, -1, &h_camera);
cout<<"Camera 0 initialization failed with code "<<camera_status0<<". See camera_status.h to find the code meaning."<<endl; if (camera_status != CAMERA_STATUS_SUCCESS) {
return false; LOGE("Camera 0 initialization failed with code %d. See camera_status.h to find the code meaning.", camera_status);
return false;
}
CameraGetFriendlyName(h_camera, camera_name);
if(name=="NULL" || strcmp(name.data(), camera_name)==0){
break;
}
CameraUnInit(h_camera);
} }
if(i >= camera_cnts){
camera_status1 = CameraInit(&camera_enum_list[1], -1, -1, &h_camera1); LOGE("No device name %s!", name.data());
if (camera_status1 != CAMERA_STATUS_SUCCESS) {
cout<<"Camera 1 initialization failed with code "<<camera_status1<<". See camera_status.h to find the code meaning."<<endl;
return false; return false;
} }
CameraGetFriendlyName(h_camera0, camera_name0);
CameraGetFriendlyName(h_camera1, camera_name1);
cout<<"camera names: "<<camera_name0<<" "<<camera_name1<<endl;
// cout<<camera_name0<<endl;
// cout<<camera_name1<<endl;
//如果读取的相机列表不是energy在左，armor在右，则交换相机句柄
if(strcmp(camera_name0, "armor") != 0)
{
swapCameraHandle();
}
//获得相机的特性描述结构体。该结构体中包含了相机可设置的各种参数的范围信息。决定了相关函数的参数 //获得相机的特性描述结构体。该结构体中包含了相机可设置的各种参数的范围信息。决定了相关函数的参数
CameraGetCapability(h_camera0, &tCapability0); CameraGetCapability(h_camera, &tCapability);
CameraGetCapability(h_camera1, &tCapability1);
// set resolution to 320*240 // set resolution to 320*240
// CameraSetImageResolution(hCamera, &(tCapability.pImageSizeDesc[2])); // CameraSetImageResolution(hCamera, &(tCapability.pImageSizeDesc[2]));
rgb_buffer0 = (unsigned char *)malloc(tCapability0.sResolutionRange.iHeightMax * rgb_buffer = (unsigned char *)malloc(tCapability.sResolutionRange.iHeightMax *
tCapability0.sResolutionRange.iWidthMax * 3); tCapability.sResolutionRange.iWidthMax * 3);
rgb_buffer1 = (unsigned char *)malloc(tCapability1.sResolutionRange.iHeightMax *
tCapability1.sResolutionRange.iWidthMax * 3);
CameraSetAeState(h_camera0, true); //设置是否自动曝光 // 不使用自动曝光
CameraSetAeState(h_camera1, true); CameraSetAeState(h_camera, false);
// 曝光时间10ms
CameraSetExposureTime(h_camera, 10000);
// 模拟增益4
CameraSetAnalogGain(h_camera, 63);
// 使用预设LUT表
CameraSetLutMode(h_camera, LUTMODE_PRESET);
/*让SDK进入工作模式，开始接收来自相机发送的图像 /*让SDK进入工作模式，开始接收来自相机发送的图像
数据。如果当前相机是触发模式，则需要接收到 数据。如果当前相机是触发模式，则需要接收到
触发帧以后才会更新图像。 */ 触发帧以后才会更新图像。 */
CameraPlay(h_camera0); CameraPlay(h_camera);
CameraPlay(h_camera1);
/*其他的相机参数设置 /*其他的相机参数设置
例如 CameraSetExposureTime CameraGetExposureTime 设置/读取曝光时间 例如 CameraSetExposureTime CameraGetExposureTime 设置/读取曝光时间
@@ -100,74 +89,45 @@ bool CameraWrapper::init() {
CameraSetGamma、CameraSetConrast、CameraSetGain等设置图像伽马、对比度、RGB数字增益等等。 CameraSetGamma、CameraSetConrast、CameraSetGain等设置图像伽马、对比度、RGB数字增益等等。
CameraGetFriendlyName CameraSetFriendlyName 获取/设置相机名称（该名称可写入相机硬件） CameraGetFriendlyName CameraSetFriendlyName 获取/设置相机名称（该名称可写入相机硬件）
*/ */
// double exposure_time0, exposure_time1;
// CameraGetExposureTime(h_camera0, &exposure_time0);
// CameraGetExposureTime(h_camera1, &exposure_time1);
// cout<<"exposure time "<<exposure_time0<<" "<<exposure_time1<<endl;
// 抗频闪 // 抗频闪
CameraSetAntiFlick(h_camera0, true); // CameraSetAntiFlick(h_camera, true);
CameraSetAntiFlick(h_camera1, true);
if (tCapability.sIspCapacity.bMonoSensor) {
if (tCapability0.sIspCapacity.bMonoSensor) { channel = 1;
channel0 = 1; CameraSetIspOutFormat(h_camera, CAMERA_MEDIA_TYPE_MONO8);
CameraSetIspOutFormat(h_camera0, CAMERA_MEDIA_TYPE_MONO8); LOGM("camera %s mono ", camera_name);
cout<<"camera0 mono "<<endl;
} else { } else {
channel0 = 3; channel = 3;
CameraSetIspOutFormat(h_camera0, CAMERA_MEDIA_TYPE_BGR8); CameraSetIspOutFormat(h_camera, CAMERA_MEDIA_TYPE_BGR8);
cout<<"camera0 color mode"<<endl; LOGM("camera %s color ", camera_name);
} }
if (tCapability1.sIspCapacity.bMonoSensor) {
channel1 = 1;
CameraSetIspOutFormat(h_camera1, CAMERA_MEDIA_TYPE_MONO8);
cout<<"camera1 mono "<<endl;
} else {
channel1 = 3;
CameraSetIspOutFormat(h_camera1, CAMERA_MEDIA_TYPE_BGR8);
cout<<"camera1 color mode"<<endl;
}
return true; return true;
} }
bool CameraWrapper::read(cv::Mat& src0, cv::Mat& src1) { bool CameraWrapper::read(cv::Mat& src) {
return readRaw(src0, src1); //suit for using bayer hacking in armor_finder to replace process, fast and it can filter red and blue. // return readRaw(src); //suit for using bayer hacking in armor_finder to replace process, fast and it can filter red and blue.
//return readProcessed(src0, src1); // processed color image, but this runs slowly, about half fps of previous one. return readProcessed(src); // processed color image, but this runs slowly, about half fps of previous one.
} }
bool CameraWrapper::readRaw(cv::Mat &src0, cv::Mat &src1) { bool CameraWrapper::readRaw(cv::Mat &src) {
if (CameraGetImageBuffer(h_camera, &frame_info, &pby_buffer, 1000) == CAMERA_STATUS_SUCCESS){
if (iplImage) {
cvReleaseImageHeader(&iplImage);
if (CameraGetImageBuffer(h_camera0, &frame_info0, &pby_buffer0, 1000) == CAMERA_STATUS_SUCCESS &&
CameraGetImageBuffer(h_camera1, &frame_info1, &pby_buffer1, 1000) == CAMERA_STATUS_SUCCESS)
{
if (iplImage0) {
cvReleaseImageHeader(&iplImage0);
}
if (iplImage1){
cvReleaseImageHeader(&iplImage1);
} }
iplImage0 = cvCreateImageHeader(cvSize(frame_info0.iWidth, frame_info0.iHeight), IPL_DEPTH_8U, 1); iplImage = cvCreateImageHeader(cvSize(frame_info.iWidth, frame_info.iHeight), IPL_DEPTH_8U, 1);
iplImage1 = cvCreateImageHeader(cvSize(frame_info1.iWidth, frame_info1.iHeight), IPL_DEPTH_8U, 1);
cvSetData(iplImage0, pby_buffer0, frame_info0.iWidth); //此处只是设置指针，无图像块数据拷贝，不需担心转换效率 cvSetData(iplImage, pby_buffer, frame_info.iWidth); //此处只是设置指针，无图像块数据拷贝，不需担心转换效率
cvSetData(iplImage1, pby_buffer1, frame_info1.iWidth);
src0 = cv::cvarrToMat(iplImage0); src = cv::cvarrToMat(iplImage).clone();
src1 = cv::cvarrToMat(iplImage1);
//在成功调用CameraGetImageBuffer后，必须调用CameraReleaseImageBuffer来释放获得的buffer。 //在成功调用CameraGetImageBuffer后，必须调用CameraReleaseImageBuffer来释放获得的buffer。
//否则再次调用CameraGetImageBuffer时，程序将被挂起一直阻塞，直到其他线程中调用CameraReleaseImageBuffer来释放了buffer //否则再次调用CameraGetImageBuffer时，程序将被挂起一直阻塞，直到其他线程中调用CameraReleaseImageBuffer来释放了buffer
CameraReleaseImageBuffer(h_camera0, pby_buffer0); CameraReleaseImageBuffer(h_camera, pby_buffer);
CameraReleaseImageBuffer(h_camera1, pby_buffer1);
return true; return true;
} else { } else {
@@ -175,56 +135,32 @@ bool CameraWrapper::readRaw(cv::Mat &src0, cv::Mat &src1) {
} }
} }
bool CameraWrapper::readProcessed(cv::Mat &src0, cv::Mat &src1) { bool CameraWrapper::readProcessed(cv::Mat &src) {
if (CameraGetImageBuffer(h_camera0, &frame_info0, &pby_buffer0, 1000) == CAMERA_STATUS_SUCCESS && if (CameraGetImageBuffer(h_camera, &frame_info, &pby_buffer, 1000) == CAMERA_STATUS_SUCCESS){
CameraGetImageBuffer(h_camera1, &frame_info1, &pby_buffer1, 1000) == CAMERA_STATUS_SUCCESS) CameraImageProcess(h_camera, pby_buffer, rgb_buffer, &frame_info); // this function is super slow, better not to use it.
{ if (iplImage) {
cvReleaseImageHeader(&iplImage);
CameraImageProcess(h_camera0, pby_buffer0, rgb_buffer0, &frame_info0); // this function is super slow, better not to use it.
CameraImageProcess(h_camera1, pby_buffer1, rgb_buffer1, &frame_info1);
if (iplImage0) {
cvReleaseImageHeader(&iplImage0);
}
if (iplImage1){
cvReleaseImageHeader(&iplImage1);
} }
iplImage0 = cvCreateImageHeader(cvSize(frame_info0.iWidth, frame_info0.iHeight), IPL_DEPTH_8U, channel0); iplImage = cvCreateImageHeader(cvSize(frame_info.iWidth, frame_info.iHeight), IPL_DEPTH_8U, channel);
iplImage1 = cvCreateImageHeader(cvSize(frame_info1.iWidth, frame_info1.iHeight), IPL_DEPTH_8U, channel1);
cvSetData(iplImage0, rgb_buffer0, frame_info0.iWidth * channel0); //此处只是设置指针，无图像块数据拷贝，不需担心转换效率 cvSetData(iplImage, rgb_buffer, frame_info.iWidth * channel); //此处只是设置指针，无图像块数据拷贝，不需担心转换效率
cvSetData(iplImage1, rgb_buffer1, frame_info1.iWidth * channel1);
src0 = cv::cvarrToMat(iplImage0); src = cv::cvarrToMat(iplImage).clone();
src1 = cv::cvarrToMat(iplImage1);
//在成功调用CameraGetImageBuffer后，必须调用CameraReleaseImageBuffer来释放获得的buffer。 //在成功调用CameraGetImageBuffer后，必须调用CameraReleaseImageBuffer来释放获得的buffer。
//否则再次调用CameraGetImageBuffer时，程序将被挂起一直阻塞，直到其他线程中调用CameraReleaseImageBuffer来释放了buffer //否则再次调用CameraGetImageBuffer时，程序将被挂起一直阻塞，直到其他线程中调用CameraReleaseImageBuffer来释放了buffer
CameraReleaseImageBuffer(h_camera0, pby_buffer0); CameraReleaseImageBuffer(h_camera, pby_buffer);
CameraReleaseImageBuffer(h_camera1, pby_buffer1);
return true; return true;
} else { } else {
return false; return false;
} }
} }
CameraWrapper::~CameraWrapper() CameraWrapper::~CameraWrapper()
{ {
CameraUnInit(h_camera0); CameraUnInit(h_camera);
CameraUnInit(h_camera1);
//注意，先反初始化后再free //注意，先反初始化后再free
free(rgb_buffer0); free(rgb_buffer);
free(rgb_buffer1);
} }
void CameraWrapper::swapCameraHandle() {
int tmp_h_camera = h_camera0;
h_camera0 = h_camera1;
h_camera1 = tmp_h_camera;
}

11
others/src/camera/video_wrapper.cpp
View File

@@ -5,19 +5,18 @@
#include "camera/video_wrapper.h" #include "camera/video_wrapper.h"
VideoWrapper::VideoWrapper(const std::string &filename0, const std::string &filename1) { VideoWrapper::VideoWrapper(const std::string &filename) {
video0.open(filename0); video.open(filename);
video1.open(filename1);
} }
VideoWrapper::~VideoWrapper() = default; VideoWrapper::~VideoWrapper() = default;
bool VideoWrapper::init() { bool VideoWrapper::init() {
return video0.isOpened() && video1.isOpened(); return video.isOpened();
} }
bool VideoWrapper::read(cv::Mat &src_left, cv::Mat &src_right) { bool VideoWrapper::read(cv::Mat &src) {
return video0.read(src_left) && video1.read(src_right); return video.read(src);
} }

38
tools/TrainCNN/backward.py
View File

@@ -2,7 +2,8 @@ import tensorflow as tf
from progressive.bar import Bar from progressive.bar import Bar
import generate import generate
import forward import forward
import cv2
import numpy as np
def save_kernal(fp, val): def save_kernal(fp, val):
print(val.shape[2], file=fp) print(val.shape[2], file=fp)
@@ -49,7 +50,7 @@ def save_para(folder, paras):
save_bias(fp, paras[7]) save_bias(fp, paras[7])
STEPS = 30000 STEPS = 20000
BATCH = 10 BATCH = 10
LEARNING_RATE_BASE = 0.01 LEARNING_RATE_BASE = 0.01
LEARNING_RATE_DECAY = 0.99 LEARNING_RATE_DECAY = 0.99
@@ -59,9 +60,9 @@ MOVING_AVERAGE_DECAY = 0.99
def train(dataset, show_bar=False): def train(dataset, show_bar=False):
test_images, test_labels = dataset.all_test_sets() test_images, test_labels = dataset.all_test_sets()
x = tf.placeholder(tf.float32, [None, forward.SRC_ROWS, forward.SRC_COLS, forward.SRC_CHANNELS]) x = tf.placeholder(tf.float32, [None, generate.SRC_ROWS, generate.SRC_COLS, generate.SRC_CHANNELS])
y_= tf.placeholder(tf.float32, [None, forward.OUTPUT_NODES]) y_= tf.placeholder(tf.float32, [None, forward.OUTPUT_NODES])
nodes, vars = forward.forward(0.001) nodes, vars = forward.forward(x, 0.001)
y = nodes[-1] y = nodes[-1]
ce = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.argmax(y_, 1)) ce = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.argmax(y_, 1))
@@ -72,7 +73,7 @@ def train(dataset, show_bar=False):
learning_rate = tf.train.exponential_decay( learning_rate = tf.train.exponential_decay(
LEARNING_RATE_BASE, LEARNING_RATE_BASE,
global_step, global_step,
len(dataset.train_sets) / BATCH, len(dataset.train_samples) / BATCH,
LEARNING_RATE_DECAY, LEARNING_RATE_DECAY,
staircase=False) staircase=False)
train_step = tf.train.AdamOptimizer(learning_rate).minimize(loss, global_step=global_step) train_step = tf.train.AdamOptimizer(learning_rate).minimize(loss, global_step=global_step)
@@ -112,12 +113,31 @@ def train(dataset, show_bar=False):
bar.cursor.restore() bar.cursor.restore()
bar.draw(value=i+1) bar.draw(value=i+1)
# video = cv2.VideoCapture("/home/xinyang/Desktop/Video.mp4")
# _ = True
# while _:
# _, frame = video.read()
# cv2.imshow("Video", frame)
# if cv2.waitKey(10) == 113:
# bbox = cv2.selectROI("frame", frame, False)
# print(bbox)
# roi = frame[bbox[1]:bbox[1]+bbox[3], bbox[0]:bbox[0]+bbox[2]]
# roi = cv2.resize(roi, (48, 36))
# cv2.imshow("roi", roi)
# cv2.waitKey(0)
# roi = roi.astype(np.float32)
# roi /= 255.0
# roi = roi.reshape([1, 36, 48, 3])
# res = sess.run(y, feed_dict={x: roi})
# res = res.reshape([forward.OUTPUT_NODES])
# print(np.argmax(res))
vars_val = sess.run(vars) vars_val = sess.run(vars)
save_para("paras", vars_val) save_para("/home/xinyang/Desktop/AutoAim/tools/para", vars_val)
# nodes_val = sess.run(nodes, feed_dict={x:test}) nodes_val = sess.run(nodes, feed_dict={x:test_images})
# return vars_val, nodes_val return vars_val, nodes_val
if __name__ == "__main__": if __name__ == "__main__":
dataset = generate.DataSet("images") dataset = generate.DataSet("/home/xinyang/Desktop/DataSets")
train(dataset, show_bar=True) train(dataset, show_bar=True)

21
tools/TrainCNN/forward.py
View File

@@ -3,7 +3,7 @@ import tensorflow as tf
def get_weight(shape, regularizer=None): def get_weight(shape, regularizer=None):
w = tf.Variable(tf.truncated_normal(shape, stddev=0.1)) w = tf.Variable(tf.truncated_normal(shape, stddev=0.1))
if regularizer is None: if regularizer is not None:
tf.add_to_collection('losses', tf.contrib.layers.l2_regularizer(regularizer)(w)) tf.add_to_collection('losses', tf.contrib.layers.l2_regularizer(regularizer)(w))
return w return w
@@ -25,32 +25,23 @@ def max_pool_2x2(x):
return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding="VALID") return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding="VALID")
# 原图像行数
SRC_ROWS = 36
# 原图像列数
SRC_COLS = 48
# 原图像通道数
SRC_CHANNELS = 1
# 第一层卷积核大小 # 第一层卷积核大小
CONV1_KERNAL_SIZE = 5 CONV1_KERNAL_SIZE = 5
# 第一层卷积输出通道数 # 第一层卷积输出通道数
CONV1_OUTPUT_CHANNELS = 8 CONV1_OUTPUT_CHANNELS = 4
# 第二层卷积核大小 # 第二层卷积核大小
CONV2_KERNAL_SIZE = 3 CONV2_KERNAL_SIZE = 3
# 第二层卷积输出通道数 # 第二层卷积输出通道数
CONV2_OUTPUT_CHANNELS = 16 CONV2_OUTPUT_CHANNELS = 8
# 第一层全连接宽度 # 第一层全连接宽度
FC1_OUTPUT_NODES = 32 FC1_OUTPUT_NODES = 16
# 第二层全连接宽度（输出标签类型数） # 第二层全连接宽度（输出标签类型数）
FC2_OUTPUT_NODES = 8 FC2_OUTPUT_NODES = 4
# 输出标签类型数 # 输出标签类型数
OUTPUT_NODES = FC2_OUTPUT_NODES OUTPUT_NODES = FC2_OUTPUT_NODES
@@ -61,7 +52,7 @@ def forward(x, regularizer=None):
nodes = [] nodes = []
conv1_w = get_weight( conv1_w = get_weight(
[CONV1_KERNAL_SIZE, CONV1_KERNAL_SIZE, 1, CONV1_OUTPUT_CHANNELS] [CONV1_KERNAL_SIZE, CONV1_KERNAL_SIZE, int(x.shape[3]), CONV1_OUTPUT_CHANNELS]
) )
conv1_b = get_bias([CONV1_OUTPUT_CHANNELS]) conv1_b = get_bias([CONV1_OUTPUT_CHANNELS])
conv1 = tf.nn.relu(tf.nn.bias_add(conv2d(x, conv1_w), conv1_b)) conv1 = tf.nn.relu(tf.nn.bias_add(conv2d(x, conv1_w), conv1_b))

60
tools/TrainCNN/generate.py
View File

@@ -3,47 +3,63 @@ import os
import cv2 import cv2
import random import random
from forward import OUTPUT_NODES from forward import OUTPUT_NODES
# 原图像行数
SRC_ROWS = 36
# 原图像列数
SRC_COLS = 48
# 原图像通道数
SRC_CHANNELS = 3
class DataSet: class DataSet:
def __init__(self, folder): def __init__(self, folder):
self.train_sets = [] self.train_samples = []
self.test_sets = [] self.train_labels = []
self.test_samples = []
self.test_labels = []
self.generate_data_sets(folder) self.generate_data_sets(folder)
def file2nparray(self, name):
image = cv2.imread(name)
image = cv2.resize(image, (SRC_COLS, SRC_ROWS))
image = image.astype(np.float32)
return image / 255.0
def id2label(self, id):
a = np.zeros([OUTPUT_NODES])
a[id] = 1
return a[:]
def generate_data_sets(self, folder): def generate_data_sets(self, folder):
def file2nparray(name):
image = cv2.imread(name)
return image[:, :, 0]
def id2label(id):
a = np.zeros([OUTPUT_NODES, 1])
a[id] = 1
return a[:]
sets = [] sets = []
for i in range(OUTPUT_NODES): for i in range(OUTPUT_NODES):
dir = "%s/%d" % (folder, i) dir = "%s/%d" % (folder, i)
files = os.listdir(dir) files = os.listdir(dir)
for file in files: for file in files:
sets.append([file2nparray("%s/%s" % (dir, file)), id2label(i)]) if random.random() > 0.2:
sets = np.array(sets) self.train_samples.append(self.file2nparray("%s/%s" % (dir, file)))
np.random.shuffle(sets) self.train_labels.append(self.id2label(i))
length = len(sets) else:
self.train_sets = sets[:length*3//4] self.test_samples.append(self.file2nparray("%s/%s" % (dir, file)))
self.test_sets = sets[length*3//4:] self.test_labels.append(self.id2label(i))
self.train_samples = np.array(self.train_samples)
self.train_labels = np.array(self.train_labels)
self.test_samples = np.array(self.test_samples)
self.test_labels = np.array(self.test_labels)
return sets
def sample_train_sets(self, length): def sample_train_sets(self, length):
samples = [] samples = []
labels = [] labels = []
for i in range(length): for i in range(length):
id = random.randint(0, length-1) id = random.randint(0, len(self.train_samples)-1)
samples.append(self.train_sets[id][0]) samples.append(self.train_samples[id])
labels.append(self.train_sets[id][1]) labels.append(self.train_labels[id])
return np.array(samples), np.array(labels) return np.array(samples), np.array(labels)
def all_train_sets(self): def all_train_sets(self):
return self.train_sets[:, 0, :, :], self.train_sets[:, 1, :, :] return self.train_samples[:], self.train_labels[:]
def all_test_sets(self): def all_test_sets(self):
return self.test_sets[:, 0, :, :], self.test_sets[:, 1, :, :] return self.test_samples[:], self.test_labels[:]