FaceTracker2
VisionExampleDoc.cpp
1 // VisionExampleDoc.cpp : implementation of the CVisionExampleDoc class
2 //
3 
4 #include "stdafx.h"
5 #include "VisionExample.h"
6 #include "VisionExampleDoc.h"
7 #include "OpenGLWndFaceTracker.h"
8 #include "MainFrm.h"
9 #include <process.h>
10 #include "Common.h"
11 #include "VisageRendering.h"
12 
13 #include <highgui.h>
14 #include <videoInput.h>
15 #include <sys/timeb.h>
16 #include <sys/types.h>
17 #include <direct.h>
18 #include <map>
19 
20 #include <ctime>
21 #include <random>
22 #include <algorithm>
23 
24 #ifdef _DEBUG
25 #define new DEBUG_NEW
26 #undef THIS_FILE
27 static char THIS_FILE[] = __FILE__;
28 #endif
29 
30 #define VS_WRITE_AU
31 #define VS_WRITE_HEADPOSE
32 
33 static int startFrame = -1;
34 static int endFrame = -1;
35 static int slowdownFrame = -1;
36 static int slowdownTime = -1;
37 
38 //pure track time variables
39 long startTime = 0;
40 long endTime = 0;
41 long elapsedTime = 0;
42 
43 //logo image
44 IplImage* logo = NULL;
45 int logoViewportWidth;
46 int logoViewportHeight;
47 
48 //font texture image
49 static IplImage* font = NULL;
50 
51 std::string previousImageFolder = ".";
52 std::string previousVideoFolder = ".";
53 std::string previousBatchFolder = ".";
54 
55 /////////////////////////////////////////////////////////////////////////////
56 // CVisionExampleDoc
57 
58 IMPLEMENT_DYNCREATE(CVisionExampleDoc, CDocument)
59 
60 BEGIN_MESSAGE_MAP(CVisionExampleDoc, CDocument)
61  //{{AFX_MSG_MAP(CVisionExampleDoc)
62 
63 
64  //}}AFX_MSG_MAP
65  ON_COMMAND(ID_TRACKING_STOP, &CVisionExampleDoc::OnTrackingStop)
66  ON_COMMAND(ID_TRACKING_TRACKFROMVIDEO, &CVisionExampleDoc::OnTrackingTrackfromvideo)
67  ON_COMMAND(ID_TRACKING_TRACKFROMCAM, &CVisionExampleDoc::OnTrackingTrackfromcam)
68  ON_COMMAND(ID_TRACKING_CAMERASETTINGS, &CVisionExampleDoc::OnTrackingCamerasettings)
69  ON_COMMAND(ID_TRACKING_BATCHPROCESSING, &CVisionExampleDoc::OnTrackingBatchprocessing)
70  ON_COMMAND(ID_TRACKING_TRACKINIMAGE, &CVisionExampleDoc::OnTrackingTrackinimage)
71  ON_COMMAND(ID_TRACKING_WRITEDATA, &CVisionExampleDoc::OnTrackingWritedata)
72  ON_COMMAND(ID_LIVENESS_START, &CVisionExampleDoc::OnLivenessStart)
73  ON_CBN_SELENDOK(ID_COMBO_CONFIG_FILE, &CVisionExampleDoc::OnCfgChange)
74  ON_CBN_SELENDOK(ID_COMBO_CAM_DEVICE, &CVisionExampleDoc::OnTrackingTrackfromcam)
75 END_MESSAGE_MAP()
76 
77 /////////////////////////////////////////////////////////////////////////////
78 // CVisionExampleDoc construction/destruction
79 
80 CVisionExampleDoc::CVisionExampleDoc()
81 {
82  m_Tracker = 0;
83  livenessInited = false;
84  m_OpenGLWnd = 0;
85  m_MainWnd = 0;
86  trackerUsed = false;
87  testImage = 0;
88  datafile = 0;
89  headerSet = false;
90  writeDataEnabled = false;
91  trackingThreadHandle = NULL;
92  cam_device = 0;
93  VI = 0;
94 
95 // Common::RedirectIOToConsole();
96 }
97 
98 static long getCurrentTimeMs(bool init = false)
99 {
100  static long initialTime = 0;
101 
102  struct _timeb timebuffer;
103  _ftime(&timebuffer);
104 
105  long clockTime = 1000 * (long)timebuffer.time + timebuffer.millitm;
106 
107  if (init)
108  initialTime = clockTime;
109 
110  clockTime -= initialTime;
111 
112  return clockTime;
113 }
114 
115 class LivenessGUI
116 {
117 private:
118  long currentTime;
119  long startTime = 0;
120  char currentText[200] = {0};
121  char currentImage[200] = {0};
122  // there are three stages of animation - fade in, no transparency, and fade out; each of these lasts ANIMATION_CYCLE_MS
123  const int ANIMATION_CYCLE_MS = 700;
124  bool displayNewContent;
125  std::map<const char*, VsImage*> images;
126 
127 public:
128 
129  bool getAnimationParams(float &effectValue)
130  {
131  currentTime = getCurrentTimeMs();
132  long timePassed = currentTime - startTime;
133  bool imageChanged = false;
134  if (timePassed > 3*ANIMATION_CYCLE_MS || startTime == 0)
135  {
136  startTime = getCurrentTimeMs();
137  timePassed = 0;
138  displayNewContent = true;
139  }
140  else
141  displayNewContent = false;
142  if (timePassed < ANIMATION_CYCLE_MS)
143  effectValue = float(std::min((double)timePassed, (double)ANIMATION_CYCLE_MS)) / float(ANIMATION_CYCLE_MS);
144  else if (timePassed > 2 * ANIMATION_CYCLE_MS)
145  effectValue = 1 - float(std::min((double)(timePassed - 2 * ANIMATION_CYCLE_MS), (double)ANIMATION_CYCLE_MS)) / float(ANIMATION_CYCLE_MS);
146  else
147  effectValue = 1;
148 
149  return displayNewContent;
150  }
151 
152  bool loadImage(const char* fileName)
153  {
154  if (strcmp(fileName, "") != 0)
155  {
156  if (images.find(fileName) == images.end())
157  {
158  IplImage *image = cvLoadImage(fileName, CV_LOAD_IMAGE_UNCHANGED);
159  if (image)
160  images[fileName] = (VsImage*)image;
161  else
162  return false;
163  }
164  return true;
165  }
166  else
167  return false;
168  }
169 
170  void promptUser(const char* displayText, const char* fileName, bool animate)
171  {
172  float effectValue = 1.0f;
173  bool imageChanged;
174  if (animate)
175  displayNewContent = getAnimationParams(effectValue);
176  else
177  displayNewContent = true;
178 
179  if (displayNewContent)
180  {
181  strcpy(currentText, displayText);
182  if (strcmp(currentImage, fileName) != 0)
183  imageChanged = true;
184  strcpy(currentImage, fileName);
185  }
186 
187  if (strcmp(currentText, "") != 0)
188  VisageRendering::DisplayText(currentText, effectValue, 4.0f);
189 
190  if (!loadImage(currentImage))
191  return;
192 
193  VisageRendering::DisplayImage(images[currentImage], effectValue, imageChanged);
194  }
195 };
196 
198 {
199  if(!m_OpenGLWnd || !m_MainWnd)
200  {
201  // get the OpenGLWindow
202  m_MainWnd = (CMainFrame*)AfxGetMainWnd();
203  m_OpenGLWnd = m_MainWnd->GetOpenGLWnd();
204  }
205  char defaultConfigFile[200];
206  FILE *f = fopen("DefaultConfiguration.txt","r");
207  fgets(defaultConfigFile,200,f);
208  fclose(f);
209  defaultConfigFile[strlen(defaultConfigFile)-1] = 0; //remove line feed character
210  if(!m_Tracker)
211  {
212  m_Tracker = new VisageTracker(defaultConfigFile);
213  }
214 
215  //Load list of configuration files from a current working directory (Samples/data)
216  m_MainWnd->ShowAvailConfigFiles(".", defaultConfigFile);
217  m_MainWnd->ShowAvailCamResolutions();
218  m_MainWnd->ShowAvailCamDevices();
219  m_MainWnd->ShowAvailMaxFaces();
220 
221  //load logo image, working directory is Samples/data
222  logo = cvLoadImage("../OpenGL/data/logo.png", CV_LOAD_IMAGE_UNCHANGED);
223 
224  //load and initialize font texture
225  font = cvLoadImage("../OpenGL/data/font.png", CV_LOAD_IMAGE_UNCHANGED);
226  VisageRendering::SetFontTexture((VsImage*)font);
227 }
228 
229 CVisionExampleDoc::~CVisionExampleDoc()
230 {
231  if (m_Tracker)
232  delete m_Tracker;
233 
234  if (logo)
235  cvReleaseImage(&logo);
236 }
237 
238 void CVisionExampleDoc::OnCloseDocument()
239 {
240  if (m_Tracker) {
241  m_OpenGLWnd->trackerActive = false;
242 
243  WaitForSingleObject(trackingThreadHandle, INFINITE);
244 
245  delete m_Tracker;
246  m_Tracker = 0;
247  }
248 
249  ReleaseActions();
250 
251  CDocument::OnCloseDocument();
252 }
253 
254 
255 /////////////////////////////////////////////////////////////////////////////
256 // CVisionExampleDoc serialization
257 
258 void CVisionExampleDoc::Serialize(CArchive& ar)
259 {
260  if (ar.IsStoring())
261  {
262  // TODO: add storing code here
263  }
264  else
265  {
266  // TODO: add loading code here
267  }
268 }
269 
270 /////////////////////////////////////////////////////////////////////////////
271 // CVisionExampleDoc diagnostics
272 
273 #ifdef _DEBUG
274 void CVisionExampleDoc::AssertValid() const
275 {
276  CDocument::AssertValid();
277 }
278 
279 void CVisionExampleDoc::Dump(CDumpContext& dc) const
280 {
281  CDocument::Dump(dc);
282 }
283 #endif //_DEBUG
284 
285 /////////////////////////////////////////////////////////////////////////////
286 // CVisionExampleDoc commands
287 
288 void writeDataHeader(FILE *fp, int auNum) {
289 
290  char header[2000] = {0};
291  char tmp[50];
292 
293  strcat(header, "Frame;Tracking status;");
294 #ifdef VS_WRITE_HEADPOSE
295  strcat(header, "Head x;Head y;Head z;Head pitch;Head yaw;Head roll;");
296 #endif
297 #ifdef VS_WRITE_AU
298  for (int i=0; i<auNum;i++) {
299  sprintf(tmp, "AU%d;", i);
300  strcat(header, tmp);
301  }
302 #endif
303 
304  header[strlen(header)-1] = 0; // earse last ;
305  fprintf(fp, "%s\n", header);
306 }
307 
308 void CVisionExampleDoc::finishWriteResults()
309 {
310  if (datafile) {
311  fclose(datafile);
312  datafile = 0;
313  headerSet = false;
314  }
315 }
316 
317 /**
318 * Stop tracking.
319 * It is called when Tracking->Stop menu item is chosen.
320 *
321 * This function stops the face tracking operation.
322 *
323 */
324 
326 {
327  if(m_Tracker)
328  {
329  m_OpenGLWnd->trackerActive = false;
330 
331  WaitForSingleObject(trackingThreadHandle, INFINITE);
332  }
333 
334  if(m_OpenGLWnd)
335  {
337  }
338 
339  finishWriteResults();
340 }
341 
342 void CVisionExampleDoc::writeResults(int track_stat, FaceData &trackingData)
343 {
344  if (!datafile)
345  return;
346 
347  //write data to file
348  std::string data = "";
349  char tmp[100];
350 
351  if(!headerSet) {
352  writeDataHeader(datafile, trackingData.actionUnitCount);
353  headerSet = true;
354  frameNum = 1; // first frame is used for initialization, thus skipped
355  }
356 
357  // frame index and tracking status
358  sprintf(tmp, "%d;", frameNum++);
359  data.append(tmp);
360  switch (track_stat) {
361  case TRACK_STAT_OK:
362  data.append("OK;");
363  break;
364  case TRACK_STAT_INIT:
365  data.append("INIT;");
366  break;
367  case TRACK_STAT_RECOVERING:
368  data.append("RECOVERING;");
369  break;
370  default:
371  data.append("OFF;");
372  break;
373  }
374 
375 #ifdef VS_WRITE_HEADPOSE
376  //Head position and rotation
377  float hr[3] = {0.0f,0.0f,0.0f};
378  float ht[3] = {0.0f,0.0f,0.0f};
379  if (track_stat == TRACK_STAT_OK) {
380  for(int i=0;i<3;i++) {
381  hr[i] = (180.0f * trackingData.faceRotation[i]) / 3.14f;
382  ht[i] = trackingData.faceTranslation[i];
383  }
384  }
385  sprintf(tmp, "%f;%f;%f;%f;%f;%f;", ht[0],ht[1],ht[2],hr[0],hr[1],hr[2]);
386  data.append(tmp);
387 #endif
388 #ifdef VS_WRITE_AU
389  // AUs
390  for (int i=0; i < trackingData.actionUnitCount; i++) {
391  sprintf(tmp, "%f;", (track_stat == TRACK_STAT_OK) ? trackingData.actionUnits[i] : 0.0f);
392  data.append(tmp);
393  }
394 #endif
395 
396  data.erase(data.end()-1);
397 
398  fprintf(datafile, "%s\n", data.c_str());
399 }
400 
401 static void displayResults(CVisionExampleDoc *td, int* track_stat, FaceData* trackingData, VsImage* frame, VsImage* logo, int n = 1)
402 {
403  EnterCriticalSection(&td->m_OpenGLWnd->RenderingSection);
404 
405  // display the frame rate, position and other info
406  if(td->m_MainWnd)
407  {
408  char tmpmsg[400];
409  float trackingFrameRate = trackingData[0].frameRate;
410 
411  float hasMask = trackingData[0].hasMask;
412 
413  float r[3];
414  float t[3];
415  if(track_stat[0] == TRACK_STAT_OK)
416  {
417 
418  for(int i=0;i<3;i++) {
419  r[i] = (180.0f * trackingData[0].faceRotation[i]) / 3.141592f;
420  t[i] = trackingData[0].faceTranslation[i] * 100.0f; //translation is expressed in meters so this gives approximate values in centimeters if camera focus distance parameter f in fbft is set correctly
421  }
422  }
423  else
424  {
425  for(int i=0;i<3;i++) {
426  r[i] = 0.0f;
427  t[i] = 0.0f;
428  }
429  }
430 
431  const char* status = "";
432 
433  switch (track_stat[0]) {
434  case TRACK_STAT_OK: status = "OK"; break;
435  case TRACK_STAT_INIT: status = "INIT"; break;
436  case TRACK_STAT_RECOVERING: status = "RECOVERING"; break;
437  default: status = "OFF"; break;
438  }
439 
440  sprintf(tmpmsg, "%4.1f ms (track %ld ms) | %4.1f FPS | Head position %+5.1f %+5.1f %+5.1f | Rotation (deg) %+5.1f %+5.1f %+5.1f | %dx%d | %s",
441  1000.0f / trackingFrameRate, elapsedTime, trackingFrameRate, t[0], t[1], t[2], r[0], r[1], r[2], frame->width, frame->height, status);
442 
443  CString* message = new CString(tmpmsg); //this is deleted at the receiving side so there is no memory leak
444  ::SendNotifyMessage(td->m_MainWnd->m_hWnd, WM_MYMESSAGE, (WPARAM) message, 0);
445 
446  float aspect = (float)frame->width / (float)frame->height;
447 
448  int winWidth = floor(td->m_OpenGLWnd->mWindowHeight * aspect);
449  int winHeight = td->m_OpenGLWnd->mWindowHeight;
450 
452 
453  VisageRendering::DisplayResults(&trackingData[0], track_stat[0], winWidth, winHeight, frame, DISPLAY_DEFAULT);
454  for (int i = 1; i < n; i++)
455  VisageRendering::DisplayResults(&trackingData[i], track_stat[i], winWidth, winHeight, frame,
456  DISPLAY_DEFAULT ^ DISPLAY_FRAME ^ DISPLAY_TRACKING_QUALITY );
457 
458  //display logo
459  if (logo != NULL)
460  VisageRendering::DisplayLogo(logo, winWidth, winHeight);
461 
463  }
464 
465  if (td->m_OpenGLWnd) td->m_OpenGLWnd->RenderToScreen();
466 
467  LeaveCriticalSection(&td->m_OpenGLWnd->RenderingSection);
468 }
469 
470 static void displayResults(CVisionExampleDoc *td, int track_stat, FaceData &trackingData, VsImage* frame, VsImage* logo)
471 {
472  displayResults(td, &track_stat, &trackingData, frame, logo, 1);
473 }
474 
475 void CVisionExampleDoc::trackFromVideo(void *thisdoc)
476 {
477  CVisionExampleDoc *td = (CVisionExampleDoc *)thisdoc;
478 
479  int format = VISAGE_FRAMEGRABBER_FMT_BGR;
480 
481  bool video_file_sync = true;
482 
483  printf("Tracking from video %s...\n", td->videoFileName.c_str());
484 
485  CvCapture* capture = cvCaptureFromFile( td->videoFileName.c_str() );
486 
487  getCurrentTimeMs(true);
488 
489  float frameTime = 1000.0/(float)cvGetCaptureProperty( capture, CV_CAP_PROP_FPS );
490  int frameCount = 0;
491 
492  int max_faces = td->m_MainWnd->GetMaxFaces();
493 
494  FaceData trackingData[10];
495 
496  int *track_stat = 0;
497 
498  while (td->m_OpenGLWnd->trackerActive && cvGrabFrame(capture)) {
499 
500  long currentTime = getCurrentTimeMs();
501 
502  while(video_file_sync && currentTime > frameTime*(frameCount+1) ) // skip frames if track() is too slow
503  {
504  if(!td->m_OpenGLWnd->trackerActive || !cvGrabFrame(capture))
505  break;
506 
507  frameCount++;
508  currentTime = getCurrentTimeMs();
509  }
510 
511  while(video_file_sync && currentTime < frameTime*(frameCount-5)) // wait if track() is too fast
512  {
513  Sleep(1);
514  currentTime = getCurrentTimeMs();
515 
516  if(!td->m_OpenGLWnd->trackerActive)
517  break;
518  }
519 
520  const IplImage *framePtr = cvRetrieveFrame(capture); // grab latest frame
521 
522  if (!framePtr) break;
523 
524  frameCount++;
525 
526  startTime = getCurrentTimeMs();
527  track_stat = td->m_Tracker->track(framePtr->width,framePtr->height,framePtr->imageData, trackingData, format, framePtr->origin, framePtr->widthStep, frameTime*frameCount, max_faces);
528  endTime = getCurrentTimeMs();
529  elapsedTime = endTime - startTime;
530 
531  displayResults(td, track_stat, trackingData, (VsImage*)framePtr, (VsImage*)logo, max_faces);
532 
533  if (td->writeDataEnabled) td->writeResults(track_stat[0], trackingData[0]);
534  }
535 
536  td->m_Tracker->stop();
537  td->finishWriteResults();
538 
539  cvReleaseCapture(&capture);
540 
541  printf("Stop tracking from video.\n");
542 
543  _endthread();
544 }
545 
546 
547 /**
548 * The main function for tracking from a video file.
549 * It is called when the Tracking->Track From Video menu item is chosen.
550 *
551 * @see OnTrackingTrackfromcam()
552 */
554 {
555  m_OpenGLWnd->trackerActive = false;
556 
557  WaitForSingleObject(trackingThreadHandle, INFINITE);
558 
559  finishWriteResults();
560 
561  // get the current path
562  visageFolderManager.pushPath();
563  char fullPath[500];
564  _chdir("../OpenGL/data/Videos/");
565  if (previousVideoFolder.compare(".") == 0)
566  previousVideoFolder = std::string(_fullpath(fullPath, ".", 500));
567  //Ask for video file name
568  CFileDialog dlg(true, NULL, NULL, OFN_HIDEREADONLY | OFN_OVERWRITEPROMPT, _T("Video files (*.mp4;*.avi)|*.mp4;*.avi|All files (*.*)|*.*||"));
569  dlg.m_ofn.lpstrInitialDir = previousVideoFolder.c_str();
570  if(dlg.DoModal() == IDCANCEL) {
571  visageFolderManager.popPath();
572  return;
573  }
574  previousVideoFolder = "";
575  // return to current path after file browser dialog is closed, otherwise the necessary files may not be found
576  visageFolderManager.popPath();
577  std::string strVideoFile = (std::string)dlg.GetPathName();
578  //set the resulting .fba file name to the original video file name (just change the extension to .fba)
579  std::string fbaFileName = strVideoFile;
580  fbaFileName.erase(fbaFileName.find_last_of(".")+1);
581  fbaFileName.append("fba");
582 
583  std::string title = "Tracking from video: ";
584  title.append(strVideoFile.substr(strVideoFile.find_last_of("\\")+1));
585  SetTitle(title.c_str());
586 
587  if (writeDataEnabled) {
588  // open data file, named same as video file but with txt extension
589  std::string datafilename = strVideoFile;
590  datafilename.erase(datafilename.find_last_of(".")+1);
591  datafilename.append("txt");
592  datafile = fopen(datafilename.c_str(), "w");
593  }
594 
595  videoFileName = strVideoFile;
596 
597  // set the configuration file
598  CString strName;
599  m_MainWnd->GetConfigFilename(strName);
600  m_Tracker->setTrackerConfiguration(strName.GetBuffer());
601 
602  m_OpenGLWnd->trackerActive = true;
603 
604  trackingThreadHandle = (HANDLE)_beginthread(trackFromVideo,0,(void*)this);
605 }
606 
607 void CVisionExampleDoc::trackFromCAM(void *thisdoc)
608 {
609  CVisionExampleDoc *td = (CVisionExampleDoc *)thisdoc;
610 
611  int cam_device = 0;
612  int cam_width = 800;
613  int cam_height = 600;
614  int cam_fps = 30;
615  int format = VISAGE_FRAMEGRABBER_FMT_BGR;
616 
617  CString strCamRes;
618  td->m_MainWnd->GetCameraResolution(strCamRes);
619  sscanf((LPCTSTR)strCamRes, "%dx%d", &cam_width, &cam_height);
620 
621  cam_device = td->cam_device = td->m_MainWnd->GetCameraDevice();
622 
623 #define USE_VIDEOINPUT
624 
625 #ifdef USE_VIDEOINPUT
626 
627  // Grabbing from camera using VideoInput library
628  videoInput *VI = td->VI = new videoInput();
629 
630  VI->setIdealFramerate(cam_device, cam_fps);
631  bool r = VI->setupDevice(cam_device, cam_width, cam_height);
632  cam_width = VI->getWidth(cam_device);
633  cam_height = VI->getHeight(cam_device);
634 
635  printf("Start tracking from camera %dx%d@%d\n", cam_width, cam_height, cam_fps);
636 
637  IplImage *framePtr = cvCreateImage(cvSize(cam_width, cam_height), IPL_DEPTH_8U, 3);
638 
639  int max_faces = td->m_MainWnd->GetMaxFaces();
640 
641  FaceData trackingData[10];
642 
643  int *track_stat = 0;
644 
645  while (td->m_OpenGLWnd->trackerActive) {
646 
647  while(!VI->isFrameNew(cam_device))
648  Sleep(1);
649 
650  VI->getPixels(cam_device, (unsigned char*)framePtr->imageData, false, true);
651 #else
652  // Grabbing from camera using OpenCV library
653  CvCapture* capture = cvCaptureFromCAM( cam_device );
654 
655  int ret = cvSetCaptureProperty(capture, CV_CAP_PROP_FRAME_WIDTH, cam_width);
656  ret = cvSetCaptureProperty(capture, CV_CAP_PROP_FRAME_HEIGHT, cam_height);
657  ret = cvSetCaptureProperty(capture, CV_CAP_PROP_FPS, cam_fps);
658 
659  cam_width = (int)cvGetCaptureProperty( capture, CV_CAP_PROP_FRAME_WIDTH );
660  cam_height = (int)cvGetCaptureProperty( capture, CV_CAP_PROP_FRAME_HEIGHT );
661 
662  while (td->m_OpenGLWnd->trackerActive && cvGrabFrame(capture)) {
663 
664  const IplImage *framePtr = cvRetrieveFrame(capture); // grab latest frame
665 #endif
666  startTime = getCurrentTimeMs();
667  track_stat = td->m_Tracker->track(
668  framePtr->width, framePtr->height, framePtr->imageData,
669  trackingData, format, framePtr->origin, framePtr->widthStep, -1, max_faces);
670  endTime = getCurrentTimeMs();
671  elapsedTime = endTime - startTime;
672 
673  displayResults(td, track_stat, trackingData, (VsImage*)framePtr, (VsImage*)logo, max_faces);
674 
675  if (td->writeDataEnabled) td->writeResults(track_stat[0], trackingData[0]);
676  }
677 
678  td->m_Tracker->stop();
679 
680 #ifdef USE_VIDEOINPUT
681  VI->stopDevice(cam_device);
682  delete VI; td->VI = VI = 0;
683  cvReleaseImage(&framePtr);
684 #else
685  cvReleaseCapture(&capture);
686 #endif
687 
688  printf("Stop tracking from camera.\n");
689 
690  td->finishWriteResults();
691 
692  _endthread();
693 }
694 
695 /**
696 * The main function for tracking from a video camera connected to the computer.
697 * It is called when the Tracking->Track from Cam menu item is chosen.
698 *
699 *
700 * @see OnTrackingTrackfromvideo()
701 */
703 {
704  // stop thread if active previously
705  m_OpenGLWnd->trackerActive = false;
706 
707  WaitForSingleObject(trackingThreadHandle, INFINITE);
708 
709  if (writeDataEnabled) {
710  // open data file, named cam_data-HH-MM-SS_DD_MM_YY.txt
711  SYSTEMTIME systemTime;
712  GetSystemTime(&systemTime);
713 
714  char datafilename[50];
715  sprintf(datafilename, "cam_data-%02d-%02d-%02d_%02d-%02d-%02d.txt", systemTime.wHour, systemTime.wMinute, systemTime.wSecond, systemTime.wDay, systemTime.wMonth, systemTime.wYear);
716  datafile = fopen(datafilename, "w");
717  }
718 
719  SetTitle("Tracking from camera");
720 
721  // set the configuration file
722  CString strName;
723  m_MainWnd->GetConfigFilename(strName);
724  m_Tracker->setTrackerConfiguration(strName.GetBuffer());
725 
726  m_OpenGLWnd->trackerActive = true;
727 
728  trackingThreadHandle = (HANDLE)_beginthread(trackFromCAM,0,(void*)this);
729 }
730 
731 void CVisionExampleDoc::OnTrackingCamerasettings()
732 {
733  bool ok = false;
734 
735  if(VI) {
736  VI->showSettingsWindow(cam_device);
737  ok = true;
738  }
739 
740  if(!ok)
741  MessageBox(0,"Camera settings can be changed only when camera is active, i.e. while tracking from camera.", "Can not open camera settings",
742  MB_OK | MB_ICONWARNING );
743 }
744 
745 void CVisionExampleDoc::trackFromImage(void *thisdoc)
746 {
747  CVisionExampleDoc *td = (CVisionExampleDoc *)thisdoc;
748 
749  // get the current path
750  visageFolderManager.pushPath();
751  //Ask for image file name
752  CFileDialog dlg(true, NULL, NULL, OFN_HIDEREADONLY | OFN_OVERWRITEPROMPT, _T("Image files (*.jpg;*.png)|*.jpg;*.png|All files (*.*)|*.*||"));
753  dlg.m_ofn.lpstrInitialDir = previousImageFolder.c_str();
754  if(dlg.DoModal() == IDCANCEL) {
755  visageFolderManager.popPath();
756  _endthread();
757  }
758  previousImageFolder = "";
759  // return to current path after file browser dialog is closed, otherwise the necessary files may not be found
760  visageFolderManager.popPath();
761  std::string strImgFile = (std::string)dlg.GetPathName();
762  if(td->testImage) cvReleaseImage(&td->testImage);
763  td->testImage = cvLoadImage(strImgFile.c_str());
764 
765  if(!td->testImage) {
766  MessageBox(0,"Bad image format!","Error",MB_ICONERROR);
767  _endthread();
768  }
769 
770  std::string title = "Tracking from image: ";
771  title.append(strImgFile.substr(strImgFile.find_last_of("\\")+1));
772  //td->SetTitle(title.c_str());
773 
774  if (td->writeDataEnabled) {
775  // open data file, named same as image file but with txt extension
776  std::string datafilename = strImgFile;
777  datafilename.erase(datafilename.find_last_of(".")+1);
778  datafilename.append("txt");
779  td->datafile = fopen(datafilename.c_str(), "w");
780  }
781 
782  // set the configuration file
783  CString strName;
784  td->m_MainWnd->GetConfigFilename(strName);
785  td->m_Tracker->setTrackerConfiguration(strName.GetBuffer());
786 
787  int format;
788  switch(td->testImage->nChannels)
789  {
790  case 3:
791  format = VISAGE_FRAMEGRABBER_FMT_BGR;
792  break;
793  case 1:
794  format = VISAGE_FRAMEGRABBER_FMT_LUMINANCE;
795  break;
796  default:
797  MessageBox(0,"Unsupported image format.","Error",MB_ICONERROR);
798  _endthread();
799  }
800 
801  int max_faces = td->m_MainWnd->GetMaxFaces();
802 
803  FaceData trackingData[10];
804 
805  int *track_stat = 0;
806 
807  printf("Start tracking from image %s\n", strImgFile.c_str());
808 
809  const IplImage *framePtr = td->testImage;
810 
811  while (td->m_OpenGLWnd->trackerActive) {
812 
813  startTime = getCurrentTimeMs();
814  track_stat = td->m_Tracker->track(
815  framePtr->width, framePtr->height, framePtr->imageData,
816  trackingData, format, framePtr->origin, framePtr->widthStep, -1, max_faces);
817  endTime = getCurrentTimeMs();
818  elapsedTime = endTime - startTime;
819 
820  displayResults(td, track_stat, trackingData, (VsImage*)framePtr, (VsImage*)logo, max_faces);
821 
822  if (td->writeDataEnabled) td->writeResults(track_stat[0], trackingData[0]);
823  }
824 
825  td->m_Tracker->stop();
826 
827  cvReleaseImage(&td->testImage);
828 
829  printf("Stop tracking from image.\n");
830 
831  td->finishWriteResults();
832 
833  _endthread();
834 }
835 
836 /**
837 * The main function for tracking in a static image.
838 * The main purpose is to demonstrate the raw image interface for the tracker.
839 *
840 *
841 * @see OnTrackingTrackfromvideo()
842 */
844 {
845  // stop thread if active previously
846  m_OpenGLWnd->trackerActive = false;
847 
848  WaitForSingleObject(trackingThreadHandle, INFINITE);
849 
850  m_OpenGLWnd->trackerActive = true;
851 
852  trackingThreadHandle = (HANDLE)_beginthread(trackFromImage,0,(void*)this);
853 }
854 
856 {
857  BrowRaiseAction = 0;
858  delete BrowRaiseAction;
859 
860  SmileAction = 0;
861  delete SmileAction;
862 
863  BlinkAction = 0;
864  delete BlinkAction;
865 }
866 
868 {
869 
873 
874  actions[0].action = BrowRaiseAction;
875  actions[0].images[VisageLivenessBrowRaise::STATE_ACTION_INITIALIZING] = "";
876  actions[0].messages[VisageLivenessBrowRaise::STATE_ACTION_INITIALIZING] = "";
877  actions[0].images[VisageLivenessBrowRaise::STATE_WAIT_FOR_FRONTAL] = "";
878  actions[0].messages[VisageLivenessBrowRaise::STATE_WAIT_FOR_FRONTAL] = "Keep your head frontal!";
879  actions[0].images[VisageLivenessBrowRaise::STATE_WAIT_FOR_STABLE] = "";
880  actions[0].messages[VisageLivenessBrowRaise::STATE_WAIT_FOR_STABLE] = "Keep your head stable!";
881  actions[0].images[VisageLivenessBrowRaise::STATE_WAIT_FOR_ACTION] = "";
882  actions[0].messages[VisageLivenessBrowRaise::STATE_WAIT_FOR_ACTION] = "Raise your eyebrows!";
883  actions[0].images[VisageLivenessBrowRaise::STATE_ACTION_VERIFIED] = "";
884  actions[0].messages[VisageLivenessBrowRaise::STATE_ACTION_VERIFIED] = "";
885 
886  actions[1].action = SmileAction;
887  actions[1].images[VisageLivenessSmile::STATE_ACTION_INITIALIZING] = "";
888  actions[1].messages[VisageLivenessSmile::STATE_ACTION_INITIALIZING] = "";
889  actions[1].images[VisageLivenessSmile::STATE_WAIT_FOR_FRONTAL] = "";
890  actions[1].messages[VisageLivenessSmile::STATE_WAIT_FOR_FRONTAL] = "Keep your head frontal!";
891  actions[1].images[VisageLivenessSmile::STATE_WAIT_FOR_STABLE] = "";
892  actions[1].messages[VisageLivenessSmile::STATE_WAIT_FOR_STABLE] = "Keep your head stable!";
893  actions[1].images[VisageLivenessSmile::STATE_WAIT_FOR_ACTION] = "";
894  actions[1].messages[VisageLivenessSmile::STATE_WAIT_FOR_ACTION] = "Smile!";
895  actions[1].images[VisageLivenessSmile::STATE_ACTION_VERIFIED] = "";
896  actions[1].messages[VisageLivenessSmile::STATE_ACTION_VERIFIED] = "";
897 
898  actions[2].action = BlinkAction;
899  actions[2].images[VisageLivenessBlink::STATE_ACTION_INITIALIZING] = "";
900  actions[2].messages[VisageLivenessBlink::STATE_ACTION_INITIALIZING] = "";
901  actions[2].images[VisageLivenessBlink::STATE_WAIT_FOR_FRONTAL] = "";
902  actions[2].messages[VisageLivenessBlink::STATE_WAIT_FOR_FRONTAL] = "Keep your head frontal!";
903  actions[2].images[VisageLivenessBlink::STATE_WAIT_FOR_STABLE] = "";
904  actions[2].messages[VisageLivenessBlink::STATE_WAIT_FOR_STABLE] = "Keep your head stable!";
905  actions[2].images[VisageLivenessBlink::STATE_WAIT_FOR_ACTION] = "";
906  actions[2].messages[VisageLivenessBlink::STATE_WAIT_FOR_ACTION] = "Blink!";
907  actions[2].images[VisageLivenessBlink::STATE_ACTION_VERIFIED] = "";
908  actions[2].messages[VisageLivenessBlink::STATE_ACTION_VERIFIED] = "";
909 
910  livenessInited = true;
911 }
912 
914 {
915  std::random_device rd;
916  std::mt19937 g(rd());
917 
918  shuffle(&actions[0], &actions[3], g);
919 }
920 
921 void CVisionExampleDoc::detectLivenessOnCamera(void *thisdoc)
922 {
923  CVisionExampleDoc *td = (CVisionExampleDoc *)thisdoc;
924 
925  VisageRendering::Reset();
926 
927  if (!td->livenessInited)
928  td->SetLivenessActions();
929 
930  int cam_device = 0;
931  int cam_width = 800;
932  int cam_height = 600;
933  int cam_fps = 30;
934  int format = VISAGE_FRAMEGRABBER_FMT_BGR;
935 
936  CString strCamRes;
937  td->m_MainWnd->GetCameraResolution(strCamRes);
938  sscanf((LPCTSTR)strCamRes, "%dx%d", &cam_width, &cam_height);
939 
940  cam_device = td->cam_device = td->m_MainWnd->GetCameraDevice();
941 
942  // Grabbing from camera using VideoInput library
943  videoInput *VI = td->VI = new videoInput();
944 
945  VI->setIdealFramerate(cam_device, cam_fps);
946  bool r = VI->setupDevice(cam_device, cam_width, cam_height);
947  cam_width = VI->getWidth(cam_device);
948  cam_height = VI->getHeight(cam_device);
949 
950 // printf("Start tracking from camera %dx%d@%d\n", cam_width, cam_height, cam_fps);
951 
952  IplImage *framePtr = cvCreateImage(cvSize(cam_width, cam_height), IPL_DEPTH_8U, 3);
953 
954  FaceData trackingData;
955 
956  int *track_stat = 0;
957  for (int i = 0; i < NUM_ACTIONS; i++)
958  td->actions[i].action->reset();
959  int currAction = 0;
960  int actionState = -1;
961 
962  td->RandomizeActions();
963 
964  LivenessGUI *GUI = new LivenessGUI();
965 
966  while (td->m_OpenGLWnd->trackerActive)
967  {
968  while (!VI->isFrameNew(cam_device))
969  Sleep(1);
970 
971  VI->getPixels(cam_device, (unsigned char*)framePtr->imageData, false, true);
972 
973  //keep verifying liveness until all actions are performed
974  if (currAction < NUM_ACTIONS)
975  {
976  track_stat = td->m_Tracker->track(framePtr->width, framePtr->height, framePtr->imageData, &trackingData, format, framePtr->origin, framePtr->widthStep, -1, 1);
977 
978  if (track_stat[0] == TRACK_STAT_OK)
979  {
980  actionState = td->actions[currAction].action->update(&trackingData, reinterpret_cast<VsImage*>(framePtr));
981  float currentTime = getCurrentTimeMs();
982  }
983  else
984  {
985  //reset liveness detection if face is lost
986  for (int i = 0; i < NUM_ACTIONS; i++)
987  td->actions[i].action->reset();
988  currAction = 0;
989  td->RandomizeActions();
990  }
991  }
992 
993  //rendering
994  if (td->m_MainWnd)
995  {
996  float aspect = (float)framePtr->width / (float)framePtr->height;
997  int winWidth = floor(td->m_OpenGLWnd->mWindowHeight * aspect);
998  int winHeight = td->m_OpenGLWnd->mWindowHeight;
1000  if (td->m_OpenGLWnd)
1001  {
1002  //display frame
1003  VisageRendering::DisplayResults(&trackingData, track_stat[0], winWidth, winHeight, (VsImage*)framePtr, DISPLAY_FRAME);
1004  //display logo
1005  if (logo != NULL)
1006  VisageRendering::DisplayLogo((VsImage*)logo, winWidth, winHeight);
1007 
1008  //prompt user if not all actions are verified
1009  if (currAction < NUM_ACTIONS)
1010  {
1011  if (track_stat[0] == TRACK_STAT_OK)
1012  GUI->promptUser(td->actions[currAction].messages[actionState].c_str(), td->actions[currAction].images[actionState].c_str(), true);
1013  else
1014  {
1015  // if tracker was unable to find face, display bounding box as instruction on where face needs to be positioned
1016  GUI->promptUser("", "../OpenGL/data/FaceTracker2/faceBlank.png", true);
1017  }
1018  }
1019  else
1020  {
1021  //keep displaying frame after liveness is confirmed
1022  GUI->promptUser("Liveness confirmed!", "", true);
1023  }
1024 
1025  }
1027  }
1028  if (td->m_OpenGLWnd)
1029  td->m_OpenGLWnd->RenderToScreen();
1030 
1031  //if action is verifed, proceed to next action
1032  if (actionState == 0)
1033  currAction++;
1034 
1035  }
1036  td->m_Tracker->stop();
1037 
1038 #ifdef USE_VIDEOINPUT
1039  delete GUI;
1040  VI->stopDevice(cam_device);
1041  delete VI; td->VI = VI = 0;
1042  cvReleaseImage(&framePtr);
1043 #else
1044  cvReleaseCapture(&capture);
1045 #endif
1046 
1047  printf("Stop tracking from camera.\n");
1048 
1049  _endthread();
1050 }
1051 /**
1052 * The main function for liveness detection.
1053 * It is called when the Liveness->Perform liveness menu item is chosen.
1054 */
1056 {
1057  // stop thread if active previously
1058  m_OpenGLWnd->trackerActive = false;
1059 
1060  WaitForSingleObject(trackingThreadHandle, INFINITE);
1061 
1062  m_OpenGLWnd->trackerActive = true;
1063 
1064  trackingThreadHandle = (HANDLE)_beginthread(detectLivenessOnCamera, 0, (void*)this);
1065 }
1066 
1067 void CVisionExampleDoc::testingThread(void *thisdoc)
1068 {
1069  CVisionExampleDoc *td = (CVisionExampleDoc *)thisdoc;
1070 
1071  // set the configuration file
1072  CString strName;
1073  td->m_MainWnd->GetConfigFilename(strName);
1074  td->m_Tracker->setTrackerConfiguration(strName.GetBuffer());
1075 
1076  td->finishWriteResults();
1077 
1078  visageFolderManager.pushPath();
1079  char fullPath[500];
1080  _chdir("../OpenGL/data/Images");
1081  if (previousBatchFolder.compare(".") == 0)
1082  previousBatchFolder = std::string(_fullpath(fullPath, ".", 500));
1083  CFileDialog dlg(true, NULL, NULL, OFN_HIDEREADONLY | OFN_OVERWRITEPROMPT, _T("txt (*.txt)|*.txt|All files (*.*)|*.*||"));
1084  if(dlg.DoModal() == IDCANCEL) {
1085  visageFolderManager.popPath();
1086  _endthread();
1087  }
1088  previousBatchFolder = "";
1089  FILE *listFile = fopen((const char*)dlg.GetPathName(),"r");
1090  if (!listFile)
1091  _endthread();
1092  char inString[200];
1093  char fileName[200];
1094  char rootFolder[200];
1095  bool ignore_spec_commands = false;
1096  strcpy(rootFolder, (const char*)dlg.GetFolderPath());
1097  //_getcwd(rootFolder,200);
1098  visageFolderManager.popPath();
1099 
1100  while(td->m_OpenGLWnd->trackerActive && fgets(inString,200,listFile))
1101  {
1102  if(inString[strlen(inString)-1]==10) inString[strlen(inString)-1]=0; //clip Line Feed character from the end
1103  if(strlen(inString)==0) continue; //skip empty lines
1104  if(inString[0]=='#') continue; //skip comments
1105 
1106  if(inString[0]=='$') //process special commands
1107  {
1108  //input folder
1109  if(strstr(inString,"$START:")) startFrame = atoi(inString+7);
1110  if(strstr(inString,"$END:")) endFrame = atoi(inString+5);
1111  if(strstr(inString,"$SLOWDOWN_START:")) slowdownFrame = atoi(inString+16);
1112  if(strstr(inString,"$SLOWDOWN_TIME:")) slowdownTime = atoi(inString+15);
1113  if(strstr(inString,"$IGNORE_SPECIAL_COMMANDS:")) ignore_spec_commands = (atoi(inString+25) != 0);
1114  continue;
1115  }
1116 
1117  if(ignore_spec_commands)
1118  {
1119  startFrame = -1;
1120  endFrame = -1;
1121  slowdownFrame = -1;
1122  slowdownTime = -1;
1123  }
1124  sprintf(fileName,"%s\\%s",rootFolder,inString);
1125 
1126  if (td->writeDataEnabled) {
1127  // open data file, named same as video file but with txt extension
1128  std::string datafilename = fileName;
1129  datafilename.erase(datafilename.find_last_of(".")+1);
1130  datafilename.append("txt");
1131  td->datafile = fopen(datafilename.c_str(), "w");
1132  }
1133 
1134  td->videoFileName = fileName;
1135 
1136  td->m_OpenGLWnd->trackerActive = true;
1137 
1138  HANDLE videoThreadHandle = (HANDLE)_beginthread(trackFromVideo,0,(void*)td);
1139 
1140  WaitForSingleObject(videoThreadHandle, INFINITE);
1141 
1142  startFrame = -1;
1143  endFrame = -1;
1144  slowdownFrame = -1;
1145  slowdownTime = -1;
1146 
1147  td->finishWriteResults();
1148  }
1149 
1150  fclose(listFile);
1151 
1152  _endthread();
1153 }
1154 
1155 void CVisionExampleDoc::OnCfgChange()
1156 {
1157  CString strName;
1158  m_MainWnd->GetConfigFilename(strName);
1159 
1160  VisageConfiguration vc(strName);
1161  vc.enableAuModel();
1162  vc.enableMeshModel();
1164 }
1165 
1166 void CVisionExampleDoc::OnTrackingBatchprocessing()
1167 {
1168  // stop thread if active previously
1169  m_OpenGLWnd->trackerActive = false;
1170 
1171  WaitForSingleObject(trackingThreadHandle, INFINITE);
1172 
1173  m_OpenGLWnd->trackerActive = true;
1174 
1175  trackingThreadHandle = (HANDLE)_beginthread(testingThread,0,(void*)this);
1176 }
1177 
1178 
1179 void CVisionExampleDoc::OnTrackingWritedata()
1180 {
1181  // Get the popup menu which contains the "Toggle State" menu item.
1182  CMenu* mmenu = m_MainWnd->GetMenu();
1183  CMenu* submenu = mmenu->GetSubMenu(1);
1184 
1185  // Check the state of the "Toggle State" menu item. Check the menu item
1186  // if it is currently unchecked. Otherwise, uncheck the menu item
1187  // if it is not currently checked.
1188  UINT state = submenu->GetMenuState(ID_TRACKING_WRITEDATA, MF_BYCOMMAND);
1189  ASSERT(state != 0xFFFFFFFF);
1190 
1191  if (state & MF_CHECKED) {
1192  submenu->CheckMenuItem(ID_TRACKING_WRITEDATA, MF_UNCHECKED | MF_BYCOMMAND);
1193  writeDataEnabled = false;
1194  }
1195  else {
1196  submenu->CheckMenuItem(ID_TRACKING_WRITEDATA, MF_CHECKED | MF_BYCOMMAND);
1197  writeDataEnabled = true;
1198  }
1199 }
int height
afx_msg void OnTrackingTrackfromcam()
The main function for tracking from a video camera connected to the computer.
The document.
void setTrackerConfiguration(const char *trackerConfigFile, bool au_fitting_disabled=false, bool mesh_fitting_disabled=false)
afx_msg void OnLivenessStart()
The main function for liveness detection.
VisageTracker * m_Tracker
The Face Tracker object (VisageTracker).
void RenderToScreen()
Render the model to screen.
void pushPath()
Push path.
void SetLivenessActions()
Initializes liveness detection.
VisageLivenessAction * BrowRaiseAction
VisageLivenessBrowRaise object.
void ReleaseActions()
Releases objects alocated for liveness detection.
virtual int update(const FaceData *faceData, VsImage *frame=0)=0
std::string fbaFileName
fbaFileName.
FILE * datafile
Data file.
virtual int * track(int frameWidth, int frameHeight, const char *p_imageData, FaceData *facedata, int format=VISAGE_FRAMEGRABBER_FMT_RGB, int origin=VISAGE_FRAMEGRABBER_ORIGIN_TL, int widthStep=0, long timeStamp=-1, int maxFaces=1)
VisageLivenessAction * BlinkAction
VisageLivenessBlink object.
float faceTranslation[3]
COpenGLWnd * m_OpenGLWnd
The OpenGL window.
void RandomizeActions()
Shuffles array of actions.
afx_msg void OnTrackingTrackfromvideo()
The main function for tracking from a video file.
void popPath()
Pop path.
void init()
Initialisation.
void swapOpenGLBuffers()
This function is called by method displayResults when it needs to swap OpenGL buffers.
Action actions[NUM_ACTIONS]
Array of actions that will need to be verified.
IplImage * testImage
Test image used for demonstrating raw image tracking interface.
afx_msg void OnTrackingStop()
Stop tracking.
void setOpenGLContext()
This function is called by method displayResults when it needs to clear the OpenGL window and set the...
afx_msg void OnTrackingTrackinimage()
The main function for tracking in a static image.
VisageLivenessAction * SmileAction
VisageLivenessSmile object.
static const int NUM_ACTIONS
Number of actions that will need to be verified in order to confirm liveness.
CRITICAL_SECTION RenderingSection
Critical section.