visageSDK
ShowcaseDemo/ShowcaseDemo/TrackerWrapper.mm
1 #import "TrackerWrapper.h"
2 #import "UIDeviceHardware.h"
3 #import "visageVision.h"
4 #import "VideoGrabber.h"
5 #import "DemoFrameGrabber.h"
6 #import "MathMacros.h"
7 #import <mach/mach_time.h>
8 #include <sys/timeb.h>
9 #include <numeric>
10 #include <deque>
11 #include "Stopwatch.h"
12 
13 using namespace VisageSDK;
14 
15 @implementation TrackerWrapper
16 @synthesize glView;
17 @synthesize ageActivated;
18 @synthesize earsActivated;
19 @synthesize genderActivated;
20 @synthesize emotionsActivated;
21 @synthesize activeDisplay;
22 @synthesize tapPositionx;
23 @synthesize tapPositiony;
24 @synthesize instructionsMessage;
25 
26 const bool REALTIME_VIDEO = true; //Forces tracker speed to sync to video FPS
27 static CameraGrabber *cameraGrabber = 0;
28 long fg_pts = -1;
29 static int *trackerStatus = TRACK_STAT_OFF;
30 const int MAX_FACES = 4;
31 bool isTracking=false;
32 VsRect *face;
33 int currentChosenFace = -1;
34 int numTrackedFaces = 0;
35 int currentNumFaces = 0;
36 int *facesInfo;
37 bool numFacesChanged = false;
38 int analyserInitialized = 0;
39 static bool threadRunning = false;
40 static NSCondition *waitHandle;
41 long currentTime;
42 long startTime;
43 int frameCount; // frame count from beginning of tracking
44 double frameTime; // duration of one frame in milliseconds
45 const NSString *preset;
46 int _device = 0;
47 NSString *path = nil;
48 int _width;
49 int _height;
50 int winWidth;
51 int winHeight;
52 float videoAspect;
53 int cam_width;
54 int cam_height;
55 BOOL fileExists = false;
56 int _framerate;
57 int _rotated = 0;
58 int _isMirrored = 1;
59 int _camera = 0;
60 
61 //pure track time variables
62 long startTrackTime = 0;
63 long endTrackTime = 0;
64 long elapsedTrackTime = 0;
65 mach_timebase_info_data_t timebaseInfo;
66 
67 //analyser
68 int gender = -1;
69 float age = -1.0f;
70 const int NUM_EMOTIONS = 7;
71 float emotion_prob_estimates[NUM_EMOTIONS] = {0.0f};
72 int faceIndex = -1;
73 
74 //wireframe animatiom
75 long durationTimeAnim = 1500;
76 long startTimeAnim;
77 long currTimeAnim;
78 Stopwatch timer;
79 bool animationEnabled[MAX_FACES] = {false};
80 float weight;
81 bool reverseAnim = false;
82 
83 - (long) getElapsedTimeInMs:(uint64_t)startTime :(uint64_t)endTime
84 {
85  uint64_t elapsedTime = endTime - startTime;
86  uint64_t elapsedTimeMilli = elapsedTime * timebaseInfo.numer / (timebaseInfo.denom * 1e6);
87 
88  return elapsedTimeMilli;
89 }
90 
91 void YUV_TO_RGB(unsigned char* yuv, unsigned char* buff, int width, int height);
92 void FaceAnalyserFilter(FaceData& faceData, int index);
93 
94 NSString* resourcePath = 0;
95 
96 int clamp(int x)
97 {
98  unsigned y;
99  return !(y=x>>8) ? x : (0xff ^ (y>>24));
100 }
101 
102 //Returns index of the first tracker that has status TRACK_STATUS_OK, otherwise returns -1
103 int firstTrackerStatusOK()
104 {
105  int index = -1;
106  for(int i = 0; i<MAX_FACES; ++i)
107  {
108  if(trackerStatus[i] == TRACK_STAT_OK)
109  {
110  index = i;
111  break;
112  }
113  }
114 
115  return index;
116 }
117 
118 //Method that returns number of trackers with status OK and index of tracked face if there is one tracker, otherwise returns -1
119 int* getCount()
120 {
121  int count = 0;
122  int trackIndex = -1;
123  static int results[2];
124 
125  for(int i = 0; i<MAX_FACES; ++i)
126  {
127  if(trackerStatus[i] == TRACK_STAT_OK)
128  {
129  count++;
130  trackIndex = i;
131  }
132  }
133 
134  results[1] = count;
135 
136  if(count == 1)
137  results[0] = trackIndex;
138  else
139  results[0] = -1;
140 
141  return results;
142 }
143 
144 static bool getFeaturePoint(VsPoint &point, FaceData* trackingData, int group, int index, int width, int height, bool flipY = false)
145 {
146  const FeaturePoint& fp = trackingData->featurePoints2D->getFP(group,index);
147  point.x = vsRound(fp.pos[0] * width);
148  point.y = vsRound((flipY ? 1 - fp.pos[1] : fp.pos[1]) * height);
149  return fp.defined;
150 }
151 
152 static float fpDistance(const VsPoint* fp1, const VsPoint* fp2)
153 {
154  return sqrt( pow((float)(fp1->x - fp2->x), 2) + pow((float)(fp1->y - fp2->y), 2) );
155 }
156 
157 bool calculateBoundingBox(int width, int height, FaceData* trackingData, VsRect* boundingBox, bool flipY)
158 {
159  //load crucial feature points
160  VsPoint leye1, leye2, reye1, reye2, nose1, nose2, mouth;
161 
162  if(!(getFeaturePoint(leye1, trackingData, 3, 11, width, height, flipY) &&
163  getFeaturePoint(leye2, trackingData, 3, 7, width, height, flipY) &&
164  getFeaturePoint(reye1, trackingData, 3, 12, width, height, flipY) &&
165  getFeaturePoint(reye2, trackingData, 3, 8, width, height, flipY) &&
166  getFeaturePoint(nose1, trackingData, 9, 5, width, height, flipY) &&
167  getFeaturePoint(nose2, trackingData, 9, 4, width, height, flipY) &&
168  getFeaturePoint(mouth, trackingData, 8, 1, width, height, flipY)))
169  return false;
170 
171  //calculate mid-points
172  VsPoint leye = vsPoint(vsRound((leye1.x + leye2.x)/2.0f), vsRound((leye1.y + leye2.y)/2.0f));
173  VsPoint reye = vsPoint(vsRound((reye1.x + reye2.x)/2.0f), vsRound((reye1.y + reye2.y)/2.0f));
174  VsPoint nose = vsPoint(vsRound((nose1.x + nose2.x)/2.0f), vsRound((nose1.y + nose2.y)/2.0f));
175  VsPoint centerEyes = vsPoint(vsRound((leye.x + reye.x)/2.0f), vsRound((leye.y + reye.y)/2.0f));
176 
177  if (centerEyes.y >= nose.y)
178  {
179  //printf("Feature points are flipped\n");
180  }
181 
182  //take the longest distance measure
183  float distance = MAX(fpDistance(&leye, &mouth), MAX(fpDistance(&reye, &mouth), fpDistance(&leye, &reye)));
184 
185  //get bb params
186  boundingBox->width = vsRound(distance * 1.8f);
187  boundingBox->height = vsRound(distance * 2.3f);
188  boundingBox->x = vsRound(nose.x * 0.8f + centerEyes.x * 0.2f - boundingBox->width / 2.0f);
189  boundingBox->y = vsRound(nose.y * 0.8f + centerEyes.y * 0.2f - boundingBox->height / 2.0f);
190 
191  return true;
192 }
193 
194 //helper method for converting UIImage to VsImage
195 VsImage *CreateVsImageFromUIImage(UIImage *image, int nChannels);
196 
197 //comment this to use head tracking configuration
198 #define FACE_TRACKER
199 
200 - (void)initTracker:(CustomGLView *)view withInformer:(Informer *)informer andAnalyserInformer:(AnalyserInformer *)analyserInformer
201 {
202  opQ = [[NSOperationQueue alloc] init];
203  mach_timebase_info(&timebaseInfo);
204 
205  waitHandle = [NSCondition new];
206  threadRunning = false;
207 
208  //initialize licensing
209  //example how to initialize license key
210  //initializeLicenseManager("dev.vlc");
211 
212  if(self->m_informer == NULL)
213  self->m_informer = informer;
214 
215  if(self->analyserInformer == NULL)
216  self->analyserInformer = analyserInformer;
217 
218  glView = view;
219 
220  delete[] trackingData;
221  trackingData = new FaceData[MAX_FACES];
222 
223  // choose configuration based on device at run-time
224  NSString* deviceType = [UIDeviceHardware platform];
225  //NSLog(deviceType);
226 
227 #ifdef FACE_TRACKER
228  if ([deviceType hasPrefix:@"iPhone5"]) //iPhone 5
229  tracker = new VisageTracker("Head Tracker.cfg");
230  else
231  tracker = new VisageTracker("Facial Features Tracker.cfg"); // all other devices
232 #else
233  tracker = new VisageTracker("Head Tracker.cfg");
234 #endif
235 
236  face = new VsRect[MAX_FACES];
237 
238  resourcePath = [[[NSBundle mainBundle] resourcePath] stringByAppendingString:@"/vfa"];
239 
240  delete[] analysisData;
241  analysisData = new AnalysisData[MAX_FACES];
242 
243  //get OpenGL context size
244  glWidth = glView.bounds.size.width;
245  glHeight = glView.bounds.size.height;
246 
247  inGetTrackingResults = false;
248 
249  self->instructionsMessage = false;
250 }
251 
252 - (void)doAnalysis:(int)faceIndex
253 {
254  int options = 0;
255  if (ageActivated)
256  options |= VFA_AGE;
257  if (genderActivated)
258  options |= VFA_GENDER;
259  if (emotionsActivated)
260  options |= VFA_EMOTION;
261 
262  analyser->analyseStream(frameRGB, trackingData[faceIndex], options, analysisData[faceIndex]);
263 }
264 
265 - (void)resetAnalyserFilters:(int)index
266 {
267  if (analyser)
268  analyser->resetStreamAnalysis(index);
269 }
270 
271 - (void) trackingCamOperation
272 {
273  int cam_fps = 30;
274 
275  getOrientation();
276  setDimensions(_rotated);
277 
278  // if camera already works, release
279  if(cameraGrabber)
280  {
281  cameraGrabber = 0;
282  }
283 
284  // initialize new camera
285  cameraGrabber = [[CameraGrabber alloc] initWithSessionPreset:preset UseFPS:cam_fps withDevice:_device];
286  [cameraGrabber startCapture: nil withCamera:_camera];
287 
288  int pixelFormat = [cameraGrabber getPixelFormat];
289  int format = VISAGE_FRAMEGRABBER_FMT_LUMINANCE;
290 
291  if (pixelFormat == 1)
292  format = VISAGE_FRAMEGRABBER_FMT_LUMINANCE;
293  else
294  format = VISAGE_FRAMEGRABBER_FMT_BGRA;
295 
296  frameRGB = vsCreateImageHeader(vsSize(cam_width/2, cam_height/2), VS_DEPTH_8U, 3);
297  if (pixelFormat == 1)
298  frameDisplay = vsCreateImageHeader(vsSize(cam_width, cam_height), VS_DEPTH_8U, 3);
299  else
300  frameDisplay = vsCreateImageHeader(vsSize(cam_width, cam_height), VS_DEPTH_8U, 4);
301 
302  //clear all analysis filters
303  for(int fi=0; fi<MAX_FACES; fi++)
304  {
305  [self resetAnalyserFilters:fi];
306  }
307 
308  while(!currentTrackOperation.isCancelled)
309  {
310  @autoreleasepool {
311  uint64_t loopStart = mach_absolute_time();
312 
313  getOrientation();
314  setDimensions(_rotated);
315  CGImageRef displayImage;
316  CGImageRef trackImage;
317  CGImageRef analysisImage;
318 
319  [cameraGrabber getTrackImage:&trackImage withDisplayImage:&displayImage isBGRA:NO withAnalyserImage:&analysisImage isDownscaled:YES];
320 
321  if (trackImage == nil || displayImage == nil || analysisImage == nil)
322  continue;
323 
324  CFDataRef trackImageDataProvider = CGDataProviderCopyData(CGImageGetDataProvider(trackImage));
325  UInt8* pixels = (UInt8*) CFDataGetBytePtr(trackImageDataProvider);
326 
327  int width = (int)CGImageGetWidth(trackImage);
328  int height = (int)CGImageGetHeight(trackImage);
329  int widthStep = (int)CGImageGetBytesPerRow(trackImage);
330 
331  uint64_t trackStart = mach_absolute_time();
332 
333  trackerStatus = tracker->track(width, height, (const char *)pixels, trackingData, format, VISAGE_FRAMEGRABBER_ORIGIN_TL, widthStep, -1, MAX_FACES);
334 
335  uint64_t trackEnd = mach_absolute_time();
336 
337  //NSLog(@"Tracker status: %u | Track time: %ld", trackerStatus[0], [self getElapsedTimeInMs:trackStart :trackEnd]);
338 
339  CFDataRef analysisImageDataProvider = CGDataProviderCopyData(CGImageGetDataProvider(analysisImage));
340  UInt8* rgbPixels = (UInt8*) CFDataGetBytePtr(analysisImageDataProvider);
341  vsSetImageData(frameRGB, rgbPixels, (int)CGImageGetBytesPerRow(analysisImage));
342 
343  videoAspect = frameRGB->width / (float)frameRGB->height;
344 
345  winWidth = glWidth;
346  winHeight = glWidth / videoAspect;
347 
348  if(ageActivated || genderActivated || emotionsActivated)
349  {
350  facesInfo = getCount();
351 
352  //if only one face is tracked, set the index to current
353  currentChosenFace = facesInfo[0];
354  currentNumFaces = facesInfo[1];
355 
356  //if more than one tracked face
357  if(currentNumFaces > 1)
358  {
359  //if there are more faces, set current index to the previous tracking face
360  currentChosenFace = faceIndex;
361 
362  //check if number of tracked faces changed
363  if(currentNumFaces != numTrackedFaces)
364  {
365  numTrackedFaces = currentNumFaces;
366 
367  //if number of faces changed, choose first with tracking status OK
368  currentChosenFace = firstTrackerStatusOK();
369 
370  //instructional message display only first time when more than one face
371  if(!numFacesChanged)
372  {
373  numFacesChanged = true;
374  self->instructionsMessage = true;
375  }
376  }
377  }
378 
379  //check if touch position is inside any face
380  for(int i=0; i<MAX_FACES; i++)
381  {
382  //calculate bounding boxes for all found faces
383  if(trackerStatus[i] == TRACK_STAT_OK)
384  {
385  calculateBoundingBox(winWidth, winHeight, &trackingData[i], &face[i], false);
386 
387  if(self->tapPositionx > (face[i].x) && self->tapPositionx < (face[i].x + face[i].width) &&
388  self->tapPositiony > (face[i].y) && self->tapPositiony < (face[i].y + face[i].height))
389  {
390  //save index of clicked face
391  currentChosenFace = i;
392 
393  //message show until first click on face
394  self->instructionsMessage = false;
395 
396  //reset tap position
397  self->tapPositionx = -1;
398  self->tapPositiony = -1;
399  }
400  }
401  }
402 
403  //check if the face for which face analysis is performed has changed
404  if(currentChosenFace != faceIndex)
405  {
406  faceIndex = currentChosenFace;
407 
408  //reset filters for all remaining faces
409  for(int fi=0; fi<MAX_FACES; fi++)
410  {
411  ResetWireframeAnimation(fi);
412 
413  if(fi != faceIndex)
414  {
415  [self resetAnalyserFilters:fi];
416  }
417  }
418  }
419 
420  //perform face analysis only for the first tracked face
421  if(currentNumFaces >= 1)
422  {
423  [self doAnalysis:faceIndex];
424 
425  //send analysis results to GUI
426  [self->analyserInformer updateAnalyserInformer:analysisData[faceIndex].emotionProbabilities.data() withAge:analysisData[faceIndex].age andGender:analysisData[faceIndex].gender andEmoState:emotionsActivated andAgeState:ageActivated andGenderState:genderActivated andMessage:instructionsMessage];
427  }
428  else
429  {
430  //clear all analysis filters
431  for(int fi=0; fi<MAX_FACES; fi++)
432  {
433  [self resetAnalyserFilters:fi];
434  }
435 
436  [self->analyserInformer updateAnalyserInformer:NULL withAge:-1.0f andGender:-1 andEmoState:false andAgeState:false andGenderState:false andMessage:instructionsMessage];
437  }
438  }
439 
440  CFDataRef displayImageDataProvider = CGDataProviderCopyData(CGImageGetDataProvider(displayImage));
441  UInt8* displayPixels = (UInt8*) CFDataGetBytePtr(displayImageDataProvider);
442  vsSetImageData(frameDisplay, displayPixels, (int)CGImageGetBytesPerRow(displayImage));
443  [self displayTrackingResults:trackerStatus Frame:frameDisplay Face:face];
444 
445  CFRelease(displayImageDataProvider);
446  CGImageRelease(displayImage);
447  CFRelease(trackImageDataProvider);
448  CGImageRelease(trackImage);
449  CFRelease(analysisImageDataProvider);
450  CGImageRelease(analysisImage);
451 
452  uint64_t loopEnd = mach_absolute_time();
453  //NSLog(@"Loop time: %ld", [self getElapsedTimeInMs:loopStart :loopEnd]);
454 
455  //check if tracker has been turned off
456  if(trackerStatus[0] == TRACK_STAT_OFF)
457  {
458  break;
459  }
460  }
461  }
462 
463  vsReleaseImageHeader(&frameRGB);
464  vsReleaseImageHeader(&frameDisplay);
465 
466  [waitHandle lock];
467  threadRunning = false;
468  [waitHandle broadcast];
469  [waitHandle unlock];
470 }
471 
472 - (void)startTrackingFromCam:(int)camera
473 {
474 #if TARGET_IPHONE_SIMULATOR
475  UIAlertView *alert = [[UIAlertView alloc] initWithTitle:@"Error"
476  message:@"No camera available on simulator."
477  delegate:nil
478  cancelButtonTitle:@"OK"
479  otherButtonTitles:nil];
480  [alert show];
481 #else
482  [opQ cancelAllOperations];
483  [opQ waitUntilAllOperationsAreFinished];
484 
485  _camera = camera;
486  NSBlockOperation* trackOperation = [[NSBlockOperation alloc] init];
487  currentTrackOperation = trackOperation;
488 
489  [trackOperation addExecutionBlock:^{
490  [self trackingCamOperation];
491  }];
492 
493  [opQ addOperation:trackOperation];
494 #endif
495 }
496 
497 - (long) getCurrentTimeMs
498 {
499  return ([[NSDate date] timeIntervalSince1970] * 1000);;
500 }
501 
502 - (void)toggleEarsTracking:(bool)enableEars
503 {
504  dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
505  [waitHandle lock];
506  const std::string newCfg = (enableEars) ? "vft/fm/jk_300_wEars.cfg" : "vft/fm/jk_300.cfg";
507  const unsigned int newValue = (enableEars) ? 1 : 0;
508 
509 
512  sf[7] = (enableEars) ? 1.5f : -1.0f;
513 
516  temp.setAuFittingModelConfiguration(newCfg);
517  temp.setRefineEars(newValue);
518  temp.setSmoothingFactors(sf);
519  tracker->setTrackerConfiguration(temp);
520 
521  [waitHandle unlock];
522  });
523 }
524 
525 - (void)toggleRefineLandmarks:(bool)enableRefineLandmarks
526 {
527  dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
528  [waitHandle lock];
529 
531  (enableRefineLandmarks) ? temp.setRefineLandmarks(1) : temp.setRefineLandmarks(0);
532  tracker->setTrackerConfiguration(temp);
533 
534  [waitHandle unlock];
535  });
536 }
537 
538 - (BOOL)initAnalyser
539 {
540  if (!analyser) analyser = new VisageFaceAnalyser();
541  if (!analyserInitialized)
542  analyserInitialized = analyser->init([resourcePath UTF8String]);
543 
544  if (!analyserInitialized)
545  {
546  NSLog(@"Analyser not initialized");
547  return FALSE;
548  }
549  else
550  {
551  return TRUE;
552  }
553 }
554 
555 - (void)toggleIrisTracking:(bool)enableIris
556 {
557  dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
558  [waitHandle lock];
559  const unsigned int newValue = (enableIris) ? 3 : 0;
560 
562  temp.setProcessEyes(newValue);
563  tracker->setTrackerConfiguration(temp);
564 
565  [waitHandle unlock];
566  });
567 }
568 
569 - (void) trackingImageOperation:(UIImage *)image
570 {
571  demoFrameGrabber = new DemoFrameGrabber(image);
572  unsigned char* pixels = demoFrameGrabber->GrabFrame(fg_pts);
573  if (pixels == NULL)
574  {
575  return;
576  }
577 
578  frameBGRA = vsCreateImage(vsSize(demoFrameGrabber->width, demoFrameGrabber->height), VS_DEPTH_8U, 4);
579  frameRGB = vsCreateImage(vsSize(demoFrameGrabber->width, demoFrameGrabber->height), VS_DEPTH_8U, 3);
580 
581  //clear all analysis filters
582  for(int fi=0; fi<MAX_FACES; fi++)
583  {
584  [self resetAnalyserFilters:fi];
585  }
586 
587  while(!currentTrackOperation.isCancelled)
588  {
589  startTrackTime = [self getCurrentTimeMs];
590  trackerStatus = tracker->track(demoFrameGrabber->width, demoFrameGrabber->height, (const char*)pixels, trackingData, VISAGE_FRAMEGRABBER_FMT_RGBA, VISAGE_FRAMEGRABBER_ORIGIN_TL, 0, -1, MAX_FACES);
591  endTrackTime = [self getCurrentTimeMs];
592  elapsedTrackTime = endTrackTime - startTrackTime;
593 
594  memcpy(frameBGRA->imageData, (char*) pixels, frameBGRA->imageSize);
595  vsCvtColor(frameBGRA, frameRGB, VS_BGRA2RGB);
596 
597  if(ageActivated || genderActivated || emotionsActivated)
598  {
599  facesInfo = getCount();
600 
601  //if only one face is tracked, set the index to current
602  currentChosenFace = facesInfo[0];
603  currentNumFaces = facesInfo[1];
604 
605  //if more than one tracked face
606  if(currentNumFaces > 1)
607  {
608  //if there are more faces, set current index to the previous tracking face
609  currentChosenFace = faceIndex;
610 
611  //check if number of tracked faces changed
612  if(currentNumFaces != numTrackedFaces)
613  {
614  numTrackedFaces = currentNumFaces;
615 
616  //if number of faces changed, choose first with tracking status OK
617  currentChosenFace = firstTrackerStatusOK();
618 
619  //instructional message display only first time when more than one face
620  if(!numFacesChanged)
621  {
622  numFacesChanged = true;
623  self->instructionsMessage = true;
624  }
625  }
626  }
627 
628  //check if touch position is inside any face
629  for(int i=0; i<MAX_FACES; i++)
630  {
631  //calculate bounding boxes for all found faces
632  if(trackerStatus[i] == TRACK_STAT_OK)
633  {
634  calculateBoundingBox(winWidth, winHeight, &trackingData[i], &face[i], false);
635 
636  if(self->tapPositionx > (face[i].x) && self->tapPositionx < (face[i].x + face[i].width) &&
637  self->tapPositiony > (face[i].y) && self->tapPositiony < (face[i].y + face[i].height))
638  {
639  //save index of clicked face
640  currentChosenFace = i;
641 
642  //message show until first click on face
643  self->instructionsMessage = false;
644 
645  //reset tap position
646  self->tapPositionx = -1;
647  self->tapPositiony = -1;
648  }
649  }
650  }
651 
652  //check if the face for which face analysis is performed has changed
653  if(currentChosenFace != faceIndex)
654  {
655  faceIndex = currentChosenFace;
656 
657  //reset filters for all remaining faces
658  for(int fi=0; fi<MAX_FACES; fi++)
659  {
660  ResetWireframeAnimation(fi);
661 
662  if(fi != faceIndex)
663  {
664  [self resetAnalyserFilters:fi];
665  }
666  }
667  }
668 
669  //perform face analysis only for the first tracked face
670  if(currentNumFaces >= 1)
671  {
672  [self doAnalysis:faceIndex];
673 
674  //send analysis results to GUI
675  [self->analyserInformer updateAnalyserInformer:analysisData[faceIndex].emotionProbabilities.data() withAge:analysisData[faceIndex].age andGender:analysisData[faceIndex].gender andEmoState:emotionsActivated andAgeState:ageActivated andGenderState:genderActivated andMessage:instructionsMessage];
676  }
677 
678  else
679  {
680  //clear all analysis filters
681  for(int fi=0; fi<MAX_FACES; fi++)
682  {
683  [self resetAnalyserFilters:fi];
684  }
685 
686  [self->analyserInformer updateAnalyserInformer:NULL withAge:-1.0f andGender:-1 andEmoState:false andAgeState:false andGenderState:false andMessage:instructionsMessage];
687  }
688  }
689 
690  [self displayTrackingResults:trackerStatus Frame:frameRGB Face:face];
691 
692  //check if tracker has been turned off
693  if(trackerStatus[0] == TRACK_STAT_OFF)
694  {
695  break;
696  }
697  }
698 
699  delete demoFrameGrabber;
700  demoFrameGrabber = 0;
701  if (frameBGRA) vsReleaseImage(&frameBGRA);
702  if (frameRGB) vsReleaseImage(&frameRGB);
703 
704  [waitHandle lock];
705  threadRunning = false;
706  [waitHandle broadcast];
707  [waitHandle unlock];
708 }
709 
710 - (void) trackingVideoOperation:(NSString*) videoURL
711 {
712  if ([videoURL isEqual:nil] || [videoURL length] == 0)
713  {
714  NSLog(@"Empty URL");
715  return;
716  }
717 
718  videoGrabber = [[VideoGrabber alloc] init];
719  if (videoGrabber == NULL)
720  {
721  NSLog(@"Could not initialize videoGrabber");
722  return;
723  }
724 
725  [videoGrabber startGrabbing:videoURL];
726 
727  int frameCount = 0;
728  float fps = [videoGrabber framerate];
729  frameTime = 1000/fps;
730 
731  //clear all analysis filters
732  for(int fi=0; fi<MAX_FACES; fi++)
733  {
734  [self resetAnalyserFilters:fi];
735  }
736 
737  Stopwatch sw;
738  sw.start();
739  double currentTime = sw.getTime();
740 
741  while(!currentTrackOperation.isCancelled)
742  {
743  frameBGRA = [videoGrabber getNextMovieFrame:NO];
744  if (frameBGRA == NULL)
745  {
746  break;
747  }
748 
749  frameRGB = vsCreateImage(vsSize(frameBGRA->width, frameBGRA->height), VS_DEPTH_8U, 3);
750  vsCvtColor(frameBGRA, frameRGB, VS_BGRA2RGB);
751 
752  startTrackTime = [self getCurrentTimeMs];
753  trackerStatus = tracker->track(frameBGRA->width, frameBGRA->height, frameBGRA->imageData, trackingData, VISAGE_FRAMEGRABBER_FMT_BGRA, VISAGE_FRAMEGRABBER_ORIGIN_TL, frameBGRA->widthStep, -1, MAX_FACES);
754  endTrackTime = [self getCurrentTimeMs];
755  elapsedTrackTime = endTrackTime - startTrackTime;
756 
758  {
759  facesInfo = getCount();
760 
761  //if only one face is tracked, set the index to current
762  currentChosenFace = facesInfo[0];
763  currentNumFaces = facesInfo[1];
764 
765  //if more than one tracked face
766  if(currentNumFaces > 1)
767  {
768  //if there are more faces, set current index to the previous tracking face
769  currentChosenFace = faceIndex;
770 
771  //check if number of tracked faces changed
772  if(currentNumFaces != numTrackedFaces)
773  {
774  numTrackedFaces = currentNumFaces;
775 
776  //if number of faces changed, choose first with tracking status OK
777  currentChosenFace = firstTrackerStatusOK();
778 
779  //instructional message display only first time when more than one face
780  if(!numFacesChanged)
781  {
782  numFacesChanged = true;
783  self->instructionsMessage = true;
784  }
785  }
786  }
787 
788  //check if touch position is inside any face
789  for(int i=0; i<MAX_FACES; i++)
790  {
791  //calculate bounding boxes for all found faces
792  if(trackerStatus[i] == TRACK_STAT_OK)
793  {
794  calculateBoundingBox(winWidth, winHeight, &trackingData[i], &face[i], false);
795 
796  if(self->tapPositionx > (face[i].x) && self->tapPositionx < (face[i].x + face[i].width) &&
797  self->tapPositiony > (face[i].y) && self->tapPositiony < (face[i].y + face[i].height))
798  {
799  //save index of clicked face
800  currentChosenFace = i;
801 
802  //message show until first click on face
803  self->instructionsMessage = false;
804 
805  //reset tap position
806  self->tapPositionx = -1;
807  self->tapPositiony = -1;
808  }
809  }
810  }
811 
812  //check if the face for which face analysis is performed has changed
813  if(currentChosenFace != faceIndex)
814  {
815  faceIndex = currentChosenFace;
816 
817  //reset filters for all remaining faces
818  for(int fi=0; fi<MAX_FACES; fi++)
819  {
820  ResetWireframeAnimation(fi);
821 
822  if(fi != faceIndex)
823  {
824  [self resetAnalyserFilters:fi];
825  }
826  }
827  }
828 
829  //perform face analysis only for the first tracked face
830  if(currentNumFaces >= 1)
831  {
832  [self doAnalysis:faceIndex];
833 
834  //send analysis results to GUI
835  [self->analyserInformer updateAnalyserInformer:analysisData[faceIndex].emotionProbabilities.data() withAge:analysisData[faceIndex].age andGender:analysisData[faceIndex].gender andEmoState:emotionsActivated andAgeState:ageActivated andGenderState:genderActivated andMessage:instructionsMessage];
836  }
837 
838  else
839  {
840  //clear all analysis filters
841  for(int fi=0; fi<MAX_FACES; fi++)
842  {
843  [self resetAnalyserFilters:fi];
844  }
845 
846  [self->analyserInformer updateAnalyserInformer:NULL withAge:-1.0f andGender:-1 andEmoState:false andAgeState:false andGenderState:false andMessage:instructionsMessage];
847  }
848  }
849 
850  [self displayTrackingResults:trackerStatus Frame:frameRGB Face:face];
851 
852  if (frameRGB) vsReleaseImage(&frameRGB);
853 
854  //check if tracker has been turned off
855  if(trackerStatus[0] == TRACK_STAT_OFF)
856  {
857  break;
858  }
859 
860  while (REALTIME_VIDEO && currentTime > frameTime * (frameCount + 1)) // skip frames if track() is too slow
861  {
862  frameCount++;
863  currentTime = sw.getTime();
864  [videoGrabber getNextMovieFrame:YES];
865  }
866 
867  while (REALTIME_VIDEO && currentTime < frameTime * (frameCount - 5)) // wait if track() is too fast
868  {
869  NSTimeInterval waitTime = (double)1/1000;
870  [NSThread sleepForTimeInterval:waitTime];
871  currentTime = sw.getTime();
872  }
873  frameCount++;
874  }
875 
876  if (frameRGB) vsReleaseImage(&frameRGB);
877  if (demoFrameGrabber) delete demoFrameGrabber;
878  demoFrameGrabber = 0;
879 
880  [waitHandle lock];
881  threadRunning = false;
882  [waitHandle broadcast];
883  [waitHandle unlock];
884 }
885 
886 - (void)startTrackingFromImage:(UIImage *) image
887 {
888  [opQ cancelAllOperations];
889  [opQ waitUntilAllOperationsAreFinished];
890 
891  NSBlockOperation* trackOperation = [[NSBlockOperation alloc] init];
892  currentTrackOperation = trackOperation;
893 
894  [trackOperation addExecutionBlock:^{
895  [self trackingImageOperation:image];
896  }];
897 
898  [opQ addOperation:trackOperation];
899 }
900 
901 - (void)startTrackingFromVideo:(NSString *) video
902 {
903  [opQ cancelAllOperations];
904  [opQ waitUntilAllOperationsAreFinished];
905 
906  NSBlockOperation* trackOperation = [[NSBlockOperation alloc] init];
907  currentTrackOperation = trackOperation;
908 
909  [trackOperation addExecutionBlock:^{
910  [self trackingVideoOperation:video];
911  }];
912 
913  [opQ addOperation:trackOperation];
914 }
915 
916 - (void)stopTracker
917 {
918  [currentTrackOperation cancel];
919 }
920 
921 - (void)dummyThread:(id)object
922 {
923  int n = 0;
924  n++;
925 }
926 
927 - (BOOL)startMultithread
928 {
929  [NSThread detachNewThreadSelector:@selector(dummyThread:) toTarget:self withObject: nil];
930  return [NSThread isMultiThreaded];
931 }
932 
933 #include <mach/mach_time.h>
934 
935 #define MEASURE_FRAMES 10
936 uint64_t last_times[MEASURE_FRAMES];
937 int framecount = -1;
938 int last_pts = 0;
939 
940 
941 - (void)displayTrackingResults:(int*)trackerStatus Frame:(VsImage*)frame Face:(VsRect*)rect
942 {
943  inGetTrackingResults = true;
944 
945  int pts = (int)trackingData[0].timeStamp;
946 
947  if (last_pts == pts) {
948  inGetTrackingResults = false;
949  return;
950  }
951 
952  last_pts = pts;
953 
954  mach_timebase_info_data_t timeBaseInfo;
955  mach_timebase_info(&timeBaseInfo);
956 
957  //measure the frame rate
958  uint64_t currentTime = mach_absolute_time() * timeBaseInfo.numer / (timeBaseInfo.denom * 1e6);
959  if(framecount == -1)
960  {
961  framecount = 0;
962  for(int i=0;i<10;i++)
963  last_times[i]=0;
964  }
965  framecount++;
966  if(framecount == MEASURE_FRAMES) framecount = 0;
967  last_times[framecount] = currentTime;
968 
969  //drawing results
971 
972  videoAspect = frame->width / (float)frame->height;
973 
974  winWidth = glWidth;
975  winHeight = glWidth / videoAspect;
976 
977  //draw frame and tracking results
978  VisageRendering::DisplayResults(&trackingData[0], trackerStatus[0], winWidth, winHeight, frame , activeDisplay + DISPLAY_FRAME + DISPLAY_POINT_QUALITY);
979 
980  for (int i = 1; i < MAX_FACES; i++)
981  VisageRendering::DisplayResults(&trackingData[i], trackerStatus[i], winWidth, winHeight, frame, activeDisplay + DISPLAY_POINT_QUALITY);
982 
983  //draw wireframe animation only if analyser activated
984  if((ageActivated || genderActivated || emotionsActivated) && faceIndex != -1)
985  AnimateWireframe(trackingData, faceIndex, 0.2f, 0.6f);
986 
988 
989  const char *trackingStatusString;
990  switch (trackerStatus [0]) {
991  case TRACK_STAT_OFF:
992  trackingStatusString = "OFF";
993  break;
994  case TRACK_STAT_OK:
995  trackingStatusString = "OK";
996  break;
997  case TRACK_STAT_RECOVERING:
998  trackingStatusString = "RECOVERING";
999  break;
1000  case TRACK_STAT_INIT:
1001  trackingStatusString = "INITIALIZING";
1002  break;
1003  default:
1004  trackingStatusString = "N/A";
1005  break;
1006  }
1007 
1008  // display the frame rate, position and other info
1009 
1010  float r[3] = {0};
1011  float t[3] = {0};
1012  if (trackerStatus[0] == TRACK_STAT_OK)
1013  {
1014  for(int i=0;i<3;i++) {
1015  r[i] = trackingData[0].faceRotation[i] * 180.0f / 3.14159f; //rads to degs
1016  t[i] = trackingData[0].faceTranslation[i] * 100.0f; //translation is expressed in meters so this gives approximate values in centimeters if camera focus distance parameter is set correctly in configuration file
1017  }
1018  }
1019 
1020  NSString* fps = [NSString stringWithFormat:@"%4.1f FPS (track %ld ms) \n", trackingData[0].frameRate, elapsedTrackTime];
1021  NSString* info = [NSString stringWithFormat:@"Head position %+5.1f %+5.1f %+5.1f | Rotation (deg) %+5.1f %+5.1f %+5.1f",t[0],t[1],t[2],r[0],r[1],r[2]];
1022  NSString* status = [NSString stringWithFormat:@"Status: %s", trackingStatusString];
1023 
1024  [self->m_informer showTrackingFps:fps Info:info andStatus:status];
1025 
1026  inGetTrackingResults = false;
1027 }
1028 
1029 void YUV_TO_RGB(unsigned char* yuv, unsigned char* buff, int width, int height)
1030 {
1031  const int frameSize = width * height;
1032 
1033  const int ii = 0;
1034  const int ij = 0;
1035  const int di = +1;
1036  const int dj = +1;
1037 
1038  unsigned char* rgb = buff;
1039 
1040  for (int i = 0, ci = ii; i < height; ++i, ci += di)
1041  {
1042  for (int j = 0, cj = ij; j < width; ++j, cj += dj)
1043  {
1044  int y = (0xff & ((int) yuv[ci * width + cj]));
1045  int u = (0xff & ((int) yuv[frameSize + (ci >> 1) * width + (cj & ~1) + 0]));
1046  int v = (0xff & ((int) yuv[frameSize + (ci >> 1) * width + (cj & ~1) + 1]));
1047 
1048  y = y < 16 ? 16 : y;
1049 
1050  int a0 = 1192 * (y - 16);
1051  int a1 = 1634 * (v - 128);
1052  int a2 = 832 * (v - 128);
1053  int a3 = 400 * (u - 128);
1054  int a4 = 2066 * (u - 128);
1055 
1056  int r = (a0 + a1) >> 10;
1057  int g = (a0 - a2 - a3) >> 10;
1058  int b = (a0 + a4) >> 10;
1059 
1060  *rgb++ = clamp(r);
1061  *rgb++ = clamp(g);
1062  *rgb++ = clamp(b);
1063  }
1064  }
1065 }
1066 
1067 void getOrientation()
1068 {
1069  UIInterfaceOrientation orientation = [UIApplication sharedApplication].statusBarOrientation;
1070  if (orientation == UIInterfaceOrientationPortrait)
1071  _rotated = 0;
1072  else if (orientation == UIInterfaceOrientationLandscapeLeft)
1073  _rotated = 1;
1074  else if (orientation == UIInterfaceOrientationLandscapeRight)
1075  _rotated = 3;
1076  else
1077  _rotated = 0;
1078 }
1079 
1080 void setDimensions(int orientation)
1081 {
1082  NSString* deviceType = [UIDeviceHardware platform];
1083 
1084  if (orientation == 0 || orientation == 2)
1085  {
1086  // portrait mode
1087  cam_width = 480;
1088  cam_height = 640;
1089  preset = AVCaptureSessionPreset640x480;
1090  }
1091  else
1092  {
1093  // landscape mode
1094  cam_width = 640;
1095  cam_height = 480;
1096  preset = AVCaptureSessionPreset640x480;
1097  }
1098 
1099  // override for iPhone 4
1100  if ([deviceType hasPrefix:@"iPhone3"])
1101  { // iPhone4
1102  if (orientation == 0 || orientation == 2)
1103  {
1104  // portrait mode
1105  cam_width = 144;
1106  cam_height = 192;
1107  preset = AVCaptureSessionPresetLow;
1108  }
1109  else
1110  {
1111  // landscape mode
1112  cam_width = 192;
1113  cam_height = 144;
1114  preset = AVCaptureSessionPresetLow;
1115  }
1116  }
1117 }
1118 
1119 bool withinConstraints(FaceData *faceData)
1120 {
1121  double faceRotAppX = faceData->faceRotationApparent[0];
1122  double faceRotAppY = faceData->faceRotationApparent[1];
1123  double faceRotAppZ = faceData->faceRotationApparent[2];
1124 
1125  double headPitchCompensatedRad = faceRotAppX;
1126  double headYawCompensatedRad = faceRotAppY;
1127  double headRollRad = faceRotAppZ;
1128 
1129  double headPitchCompensatedDeg = V_RAD2DEG(headPitchCompensatedRad);
1130  double headYawCompensatedDeg = V_RAD2DEG(headYawCompensatedRad);
1131  double headRollDeg = V_RAD2DEG(headRollRad);
1132 
1133  const double CONSTRAINT_ANGLE = 40;
1134 
1135  if (abs(headPitchCompensatedDeg) > CONSTRAINT_ANGLE ||
1136  abs(headYawCompensatedDeg) > CONSTRAINT_ANGLE ||
1137  abs(headRollDeg) > CONSTRAINT_ANGLE ||
1138  faceData->faceScale < 40)
1139  {
1140  return false;
1141  }
1142 
1143  return true;
1144 }
1145 
1146 float DeltaStep(int ageFilterFramesReached)
1147 {
1148  if (ageFilterFramesReached == 0)
1149  {
1150  return 0.05f;
1151  }
1152  else
1153  return 0.005f;
1154 }
1155 
1156 void AnimateWireframe(FaceData *faceData, int index, float alphaMin, float alphaMax)
1157 {
1158  if(!animationEnabled[index])
1159  {
1160  timer.start();
1161  startTimeAnim = timer.getTime();
1162  animationEnabled[index] = true;
1163  }
1164 
1165  if(animationEnabled[index])
1166  {
1167  currTimeAnim = timer.getTime();
1168  weight = (float)(currTimeAnim - startTimeAnim)/ (float)durationTimeAnim;
1169 
1170  float alpha = (alphaMax - alphaMin)/2 * sin(2*M_PI * weight - M_PI/2) + (alphaMax + alphaMin)/2;
1171  VisageRendering::DisplayWireFrame(&faceData[index], winWidth, winHeight, alpha);
1172 
1173  if(weight > 1)
1174  {
1175  animationEnabled[index] = false;
1176  }
1177  }
1178 }
1179 
1180 void ResetWireframeAnimation(int index)
1181 {
1182  animationEnabled[index] = false;
1183 }
1184 
1185 @end
int height
Image height in pixels.
VisageTracker is a face tracker capable of tracking the head pose, facial features and gaze for multi...
VisageFaceAnalyser contains face analysis algorithms capable of estimating age, gender and emotion fr...
int glHeight
Size of the OpenGL view.
void setTrackerConfiguration(const char *trackerConfigFile, bool au_fitting_disabled=false, bool mesh_fitting_disabled=false)
Sets tracking configuration.
NSString * platform()
Returns exact device name.
Feature point.
Definition: FDP.h:51
Displays tracking information.
const FeaturePoint & getFP(int group, int n) const
Get a feature point by its group and index.
bool setSmoothingFactors(const VsCfgArr< float > &smoothing_factors)
Sets the values of the smoothing_factors parameter.
Analysis data structure, used as a container for all face analysis results.
VisageConfiguration is a class used to change, apply, load and save configuration parameters used by ...
void startTrackingFromImage()
Method for starting tracking from an image.
int faceScale
Scale of facial bounding box expressed in pixels.
Definition: FaceData.h:739
void setOpenGLContext()
Method for setting the OpenGL context.
enable age estimation or check whether age data files have been properly initialized
bool setRefineEars(unsigned int refine_ears)
Sets the value of the refine_ears parameter.
Displays analyser information.
char * imageData
Pointer to aligned image data.
Implements simple high-level Objective-C interface around VisageSDK::VisageTracker functionallity.
void startTrackingFromCam()
Method for starting tracking from a camera.
bool setRefineLandmarks(unsigned int refine_landmarks)
Sets the value of the refine_landmarks parameter.
void swapOpenGLBuffers()
Method for swapping OpenGL buffers.
void trackingImageOperation()
Method for loading picture and performing face tracking on the given frame.
enable emotion estimation or check whether emotion data files have been properly initialized
void stopTracker()
Method for stoping the tracker.
Controls OpenGL view used for tracking output.
This is a templated structure that can store an array of values of varying types, including:
bool ageActivated
Boolean variable that states if age analyser is turned on or off.
void trackingCamOperation()
Method for initializing the camera and performing face tracking on the given frame.
VsCfgArr< float > getSmoothingFactors() const
Returns the value of the smoothing_factors parameter.
int width
Image width in pixels.
virtual int * track(int frameWidth, int frameHeight, const char *p_imageData, FaceData *facedata, int format=VISAGE_FRAMEGRABBER_FMT_RGB, int origin=VISAGE_FRAMEGRABBER_ORIGIN_TL, int widthStep=0, long timeStamp=-1, int maxFaces=1)
Performs face tracking (one or more faces) in the given image and returns tracking results and status...
int defined
Set with regards to the 3D model used.
Definition: FDP.h:70
bool setAuFittingModelConfiguration(std::string au_fitting_model_configuration)
Sets the value of the au_fitting_model_configuration parameter.
bool setMeshFittingModelConfiguration(std::string mesh_fitting_model_configuration)
Sets the value of the mesh_fitting_model_configuration parameter.
bool emotionsActivated
Boolean variable that states if emotions analyser is turned on or off.
bool setProcessEyes(unsigned int process_eyes)
Sets the value of the process_eyes parameter.
float faceTranslation[3]
Translation of the head from the camera.
Definition: FaceData.h:320
int init(const char *dataPath)
Initialise VisageFaceAnalyser.
float pos[3]
Position.
Definition: FDP.h:61
Implements methods that are used to determine specific version of device software is running on.
unsigned char * GrabFrame(long &timeStamp)
VisageTracker calls this function periodically to get the new frame to process.
AnalyserInformer * analyserInformer
Informer used for displaying analyser information.
VisageConfiguration getTrackerConfiguration()
Returns tracking configuration.
enable gender estimation or check whether gender data files have been properly initialized
Namespace.
Definition: ios_header.h:1
float faceRotation[3]
Rotation of the head.
Definition: FaceData.h:369
int activeDisplay
Current drawing options variable.
float faceRotationApparent[3]
Rotation of the head from the camera viewpoint This variable is set only while tracker is tracking (T...
Definition: FaceData.h:386
Face data structure, used as container for all face tracking and detection results.
Definition: FaceData.h:259
bool genderActivated
Boolean variable that states if gender analyser is turned on or off.
long getCurrentTimeMs()
Function used for measuring tracker speed.
int widthStep
Size of aligned image row in bytes.
bool setPoseFittingModelConfiguration(std::string pose_fitting_model_configuration)
Sets the value of the pose_fitting_model_configuration parameter.
Image store container.
Implements a simple frame grabber that serves a single image to the tracker in order to demonstrate t...
FDP * featurePoints2D
Facial feature points (2D coordinates).
Definition: FaceData.h:660
int glWidth
Size of the OpenGL view.