visageSDK
VisageTrackerDemo/Classes/TrackerWrapper.mm
1 #import "TrackerWrapper.h"
2 #import "visageVision.h"
3 #import "DemoFrameGrabber.h"
4 #import "UIDeviceHardware.h"
5 #include <sys/timeb.h>
6 
7 using namespace VisageSDK;
8 
9 @implementation TrackerWrapper
10 @synthesize glView;
11 
12 static CameraGrabber *cameraGrabber = 0;
13 static VideoGrabber *videoGrabber = 0;
14 long fg_pts = -1;
15 static int *trackerStatus = TRACK_STAT_OFF;
16 const int max_faces = 4;
17 bool isTracking=false;
18 static bool threadRunning = false;
19 static NSCondition *waitHandle;
20 long currentTime;
21 long startTime;
22 int frameCount; // frame count from beginning of tracking
23 double frameTime; // duration of one frame in milliseconds
24 const NSString *preset;
25 int _device = 0;
26 NSString *path = nil;
27 int _width;
28 int _height;
29 int cam_width;
30 int cam_height;
31 BOOL fileExists = false;
32 int _framerate;
33 int _rotated = 0;
34 int _isMirrored = 1;
35 
36 //pure track time variables
37 long startTrackTime = 0;
38 long endTrackTime = 0;
39 long elapsedTrackTime = 0;
40 
41 //logo image
42 VsImage* logo = NULL;
43 int logoViewportWidth;
44 int logoViewportHeight;
45 
46 void YUV_TO_RGBA(unsigned char* yuv, unsigned char* buff, int width, int height);
47 
48 int clamp(int x)
49 {
50  unsigned y;
51  return !(y=x>>8) ? x : (0xff ^ (y>>24));
52 }
53 
54 //helper method for converting UIImage to VsImage
55 VsImage *CreateVsImageFromUIImage(UIImage *image, int nChannels);
56 
57 // comment this to use head tracking configuration
58 #define FACE_TRACKER
59 
60 - (void)initTracker:(CustomGLView *)view withInformer:(Informer *)informer
61 {
62  opQ = [[NSOperationQueue alloc] init];
63  waitHandle = [NSCondition new];
64  threadRunning = false;
65 
66  //initialize licensing
67  //example how to initialize license key
68  //initializeLicenseManager("dev.vlc");
69 
70  if (self->m_informer == NULL)
71  self->m_informer = informer;
72 
73  glView = view;
74 
75  // choose configuration based on device at run-time
76  NSString* deviceType = [UIDeviceHardware platform];
77  //NSLog(deviceType);
78 
79 #ifdef FACE_TRACKER
80  if ([deviceType hasPrefix:@"iPhone5"]) //iPhone 5
81  tracker = new VisageTracker("Head Tracker.cfg");
82  else
83  tracker = new VisageTracker("Facial Features Tracker.cfg"); // all other devices
84 #else
85  tracker = new VisageTracker("Head Tracker.cfg");
86 #endif
87  //get OpenGL context size
88  glWidth = glView.bounds.size.width;
89  glHeight = glView.bounds.size.height;
90 
91  inGetTrackingResults = false;
92 
93  //load logo image by the given path
94  NSString *logoPath = [[NSBundle mainBundle] pathForResource:@"logo" ofType:@"png"];
95  UIImage *logoImage = [UIImage imageWithContentsOfFile:logoPath];
96  logo = CreateVsImageFromUIImage(logoImage, 4);
97 }
98 
100 {
101  int cam_fps = 30;
102 
103  //getOrientation();
104  setDimensions(_rotated);
105 
106  // if camera already works, release
107  if(cameraGrabber)
108  {
109  cameraGrabber = 0;
110  }
111 
112  // initialize new camera
113  cameraGrabber = [[CameraGrabber alloc] initWithSessionPreset:preset UseFPS:cam_fps withDevice:_device];
114  [cameraGrabber startCapture:nil withCamera:_device];
115 
116  m_frame = vsCreateImageHeader(vsSize(cam_width, cam_height), VS_DEPTH_8U, 3);
117 
118  int pixelFormat = [cameraGrabber getPixelFormat];
119  int format = VISAGE_FRAMEGRABBER_FMT_LUMINANCE;
120 
121  if (pixelFormat == 1)
122  format = VISAGE_FRAMEGRABBER_FMT_LUMINANCE;
123  else
124  format = VISAGE_FRAMEGRABBER_FMT_BGRA;
125 
126  while(!currentTrackOperation.isCancelled)
127  {
128  //getOrientation();
129  setDimensions(_rotated);
130 
131  CGImageRef displayImage;
132  CGImageRef trackImage;
133 
134  [cameraGrabber getTrackImage:&trackImage withDisplayImage:&displayImage isBGRA:NO withAnalyserImage:nil isDownscaled:NO];
135 
136  if (trackImage == nil || displayImage == nil)
137  continue;
138 
139  CFDataRef trackImageDataProvider = CGDataProviderCopyData(CGImageGetDataProvider(trackImage));
140  UInt8* pixels = (UInt8*) CFDataGetBytePtr(trackImageDataProvider);
141 
142  int width = (int)CGImageGetWidth(trackImage);
143  int height = (int)CGImageGetHeight(trackImage);
144  int widthStep = (int)CGImageGetBytesPerRow(trackImage);
145 
146  startTrackTime = [self getCurrentTimeMs];
147  trackerStatus = tracker->track(width, height, (const char *)pixels, trackingData, format, VISAGE_FRAMEGRABBER_ORIGIN_TL, widthStep, -1, max_faces);
148  endTrackTime = [self getCurrentTimeMs];
149  elapsedTrackTime = endTrackTime - startTrackTime;
150 
151  CFDataRef displayImageDataProvider = CGDataProviderCopyData(CGImageGetDataProvider(displayImage));
152  UInt8* displayPixels = (UInt8*) CFDataGetBytePtr(displayImageDataProvider);
153 
154  if (pixelFormat == 1) {
155  //Convert to RGBA for image to be drawn
156  //YUV_TO_RGBA(pixels, (unsigned char*)m_frame->imageData, cam_width, cam_height);
157  vsSetImageData(m_frame, displayPixels, (int)CGImageGetBytesPerRow(displayImage));
158  }
159  else {
160  //memcpy(m_frame->imageData, pixels, m_frame->imageSize);
161  }
162 
163  m_frame->width = cam_width;
164  m_frame->height = cam_height;
165 
166  [self displayTrackingResults:trackerStatus Frame:m_frame];
167 
168  int trackerCount = 0;
169 
170  for (int i = 0; i < max_faces; i++)
171  {
172  if (trackerStatus[i] == TRACK_STAT_OFF)
173  trackerCount++;
174  }
175 
176  CFRelease(displayImageDataProvider);
177  CGImageRelease(displayImage);
178  CFRelease(trackImageDataProvider);
179  CGImageRelease(trackImage);
180 
181  if (trackerCount == max_faces)
182  break;
183  }
184 
185  vsReleaseImageHeader(&m_frame);
186 
187  [waitHandle lock];
188  threadRunning = false;
189  [waitHandle broadcast];
190  [waitHandle unlock];
191 }
192 
194 {
195 #if TARGET_IPHONE_SIMULATOR
196  UIAlertView *alert = [[UIAlertView alloc] initWithTitle:@"Error"
197  message:@"No camera available on simulator."
198  delegate:nil
199  cancelButtonTitle:@"OK"
200  otherButtonTitles:nil];
201  [alert show];
202 #else
203  [opQ cancelAllOperations];
204  [opQ waitUntilAllOperationsAreFinished];
205 
206  NSBlockOperation* trackOperation = [[NSBlockOperation alloc] init];
207  currentTrackOperation = trackOperation;
208 
209  [trackOperation addExecutionBlock:^{
210  [self trackingCamOperation];
211  }];
212 
213  [opQ addOperation:trackOperation];
214 #endif
215 }
216 
217 - (void) trackingVideoOperation: (NSString *)filename
218 {
219  double fps;
220  bool video_file_sync = true;
221 
222  frameCount = 0;
223 
224  if (filename)
225  [self captureFromFile:filename];
226 
227  else
228  {
229  NSString *bundlefile = [[[NSBundle mainBundle] bundlePath] stringByAppendingPathComponent:@"jam1.mp4"];
230 
231  [self captureFromFile:bundlefile];
232 
233  }
234 
235  if (fileExists) {
236 
237  fps = _framerate;
238  frameTime = 1000.0/fps;
239  startTime = [self getCurrentTimeMs];
240 
241  m_frame = vsCreateImage(vsSize(_width, _height), VS_DEPTH_8U, 4);
242 
243  while(!currentTrackOperation.isCancelled)
244  {
245  if (video_file_sync)
246  {
247  currentTime = [self getCurrentTimeMs];
248 
249  while((currentTime - startTime) > frameTime*(1+frameCount) )
250  {
251  bool ret = [videoGrabber isGrabbing] ? true : false;
252 
253  if (videoGrabber == NULL || !ret)
254  break;
255 
256  [videoGrabber getNextMovieFrame:YES];
257 
258  frameCount++;
259  currentTime = [self getCurrentTimeMs];
260  }
261 
262  while((currentTime - startTime) < frameTime*(frameCount-5))
263  {
264  usleep(1000);
265  currentTime = [self getCurrentTimeMs];
266 
267  }
268 
269  }
270 
271  frameCount++;
272 
273  VsImage *tmp = [videoGrabber getNextMovieFrame:NO];
274  unsigned char *pixels = nil;
275  if (tmp) {
276  pixels = (unsigned char *) tmp->imageData;
277  } else {
278  pixels = nil;
279  }
280 
281  startTrackTime = [self getCurrentTimeMs];
282  trackerStatus = tracker->track(_width, _height, (const char *)pixels, trackingData,VISAGE_FRAMEGRABBER_FMT_RGBA, VISAGE_FRAMEGRABBER_ORIGIN_TL, 0, -1, max_faces);
283  endTrackTime = [self getCurrentTimeMs];
284  elapsedTrackTime = endTrackTime - startTrackTime;
285 
286  if(pixels)
287  memcpy(m_frame->imageData, (char*) pixels, m_frame->imageSize);
288 
289  m_frame->width = _width;
290  m_frame->height = _height;
291 
292  [self displayTrackingResults:trackerStatus Frame:m_frame];
293 
294  int trackerCount = 0;
295 
296  for (int i = 0; i < max_faces; i++)
297  {
298  if (trackerStatus[i] == TRACK_STAT_OFF)
299  trackerCount++;
300  }
301 
302  if (trackerCount == max_faces)
303  break;
304  }
305  }
306 
307  videoGrabber = 0;
308  vsReleaseImage(&m_frame);
309 
310  [waitHandle lock];
311  threadRunning = false;
312  [waitHandle broadcast];
313  [waitHandle unlock];
314 }
315 
316 - (void)startTrackingFromVideo:(NSString *)filename
317 {
318  [opQ cancelAllOperations];
319  [opQ waitUntilAllOperationsAreFinished];
320 
321  NSBlockOperation* trackOperation = [[NSBlockOperation alloc] init];
322  currentTrackOperation = trackOperation;
323 
324  [trackOperation addExecutionBlock:^{
325  [self trackingVideoOperation:filename];
326  }];
327 
328  [opQ addOperation:trackOperation];
329 }
330 
331 -(void) captureFromFile:(NSString *)filename
332 {
333  int _rotated = 0;
334 
335  fileExists = [[NSFileManager defaultManager] fileExistsAtPath:[NSString stringWithUTF8String:[filename UTF8String]]];
336  if (!fileExists) {
337  NSLog(@"File %s does not exist!", [filename UTF8String]);
338  return;
339  }
340 
341  path = [NSString stringWithUTF8String:[filename UTF8String]];
342 
343  // capture from video file
344  videoGrabber = [[VideoGrabber alloc] init];
345 
346  // set output image size
347  videoGrabber.outputWidth = 0;//_width;
348  videoGrabber.outputHeight = 0;//_height;
349 
350  _framerate = 30;
351  _rotated = [videoGrabber startGrabbing:path];
352 
353  _framerate = videoGrabber.framerate;
354  if (_rotated == 0 || _rotated == 1) {
355  _width = videoGrabber.width;
356  _height = videoGrabber.height;
357  } else {
358  _width = videoGrabber.height;
359  _height = videoGrabber.width;
360  }
361 
362 }
363 
365 {
366  struct timeb timebuffer;
367  ftime (&timebuffer);
368 
369  long clockTime = 1000 * (long)timebuffer.time + timebuffer.millitm;
370 
371  return clockTime;
372 
373 }
374 
375 - (void) trackingImageOperation
376 {
377  demoFrameGrabber = new DemoFrameGrabber();
378  unsigned char* pixels = demoFrameGrabber->GrabFrame(fg_pts);
379  m_frame = vsCreateImage(vsSize(demoFrameGrabber->width, demoFrameGrabber->height), VS_DEPTH_8U, 4);
380 
381  while(!currentTrackOperation.isCancelled)
382  {
383  startTrackTime = [self getCurrentTimeMs];
384  trackerStatus = tracker->track(demoFrameGrabber->width, demoFrameGrabber->height, (const char*)pixels, trackingData, VISAGE_FRAMEGRABBER_FMT_RGBA, VISAGE_FRAMEGRABBER_ORIGIN_TL, 0, -1, max_faces);
385  endTrackTime = [self getCurrentTimeMs];
386  elapsedTrackTime = endTrackTime - startTrackTime;
387  memcpy(m_frame->imageData, (char*) pixels, m_frame->imageSize);
388  [self displayTrackingResults:trackerStatus Frame:m_frame];
389  int trackerCount = 0;
390  for (int i = 0; i < max_faces; i++)
391  {
392  if (trackerStatus[i] == TRACK_STAT_OFF)
393  trackerCount++;
394  }
395  if (trackerCount == max_faces)
396  break;
397  }
398 
399  delete demoFrameGrabber;
400  demoFrameGrabber = 0;
401  vsReleaseImage(&m_frame);
402 }
403 
404 - (void)startTrackingFromImage
405 {
406  [opQ cancelAllOperations];
407  [opQ waitUntilAllOperationsAreFinished];
408 
409  NSBlockOperation* trackOperation = [[NSBlockOperation alloc] init];
410  currentTrackOperation = trackOperation;
411 
412  [trackOperation addExecutionBlock:^{
413  [self trackingImageOperation];
414  }];
415 
416  [opQ addOperation:trackOperation];
417 }
418 
419 - (void)stopTracker
420 {
421  [currentTrackOperation cancel];
422 
423  /*[waitHandle lock];
424 
425  isTracking = false;
426 
427  // Keep waiting until either the predicate is true or we timed out
428  while (threadRunning && [waitHandle waitUntilDate:[NSDate dateWithTimeIntervalSinceNow:60]]) {
429  }
430 
431  [waitHandle unlock];
432  */
433 }
434 
435 - (void)dummyThread:(id)object
436 {
437  int n = 0;
438  n++;
439 }
440 
441 - (BOOL)startMultithread
442 {
443  [NSThread detachNewThreadSelector:@selector(dummyThread:) toTarget:self withObject: nil];
444  return [NSThread isMultiThreaded];
445 }
446 
447 #include <mach/mach_time.h>
448 
449 #define MEASURE_FRAMES 10
450 uint64_t last_times[MEASURE_FRAMES];
451 int framecount = -1;
452 int last_pts = 0;
453 
454 
455 - (void)displayTrackingResults:(int*)trackerStatus Frame:(VsImage*)frame
456 {
457  inGetTrackingResults = true;
458 
459  int pts = (int)trackingData[0].timeStamp;
460 
461  if (last_pts == pts) {
462  inGetTrackingResults = false;
463  return;
464  }
465 
466  last_pts = pts;
467 
468  mach_timebase_info_data_t timeBaseInfo;
469  mach_timebase_info(&timeBaseInfo);
470 
471  //measure the frame rate
472  uint64_t currentTime = mach_absolute_time() * timeBaseInfo.numer / (timeBaseInfo.denom * 1e6);
473  if(framecount == -1)
474  {
475  framecount = 0;
476  for(int i=0;i<10;i++)
477  last_times[i]=0;
478  }
479  framecount++;
480  if(framecount == MEASURE_FRAMES) framecount = 0;
481  last_times[framecount] = currentTime;
482 
483  //drawing results
485 
486  float videoAspect = frame->width / (float)frame->height;
487 
488  //
489  int winWidth = glWidth;
490  int winHeight = glWidth / videoAspect;
491 
492  //draw frame and tracking results
493  VisageRendering::DisplayResults(&trackingData[0], trackerStatus[0], winWidth, winHeight, frame);
494 
495  for (int i = 1; i < max_faces; i++)
496  VisageRendering::DisplayResults(&trackingData[i], trackerStatus[i], winWidth, winHeight, frame, DISPLAY_DEFAULT - DISPLAY_FRAME);
497 
498  //set logo viewport dimensions
499  logoViewportWidth = winWidth;
500  logoViewportHeight = winHeight;
501 
502  //draw logo
503  if(logo != NULL)
504  VisageRendering::DisplayLogo(logo, logoViewportWidth, logoViewportHeight);
505 
507 
508  const char *trackingStatusString;
509  switch (trackerStatus [0]) {
510  case TRACK_STAT_OFF:
511  trackingStatusString = "OFF";
512  break;
513  case TRACK_STAT_OK:
514  trackingStatusString = "OK";
515  break;
516  case TRACK_STAT_RECOVERING:
517  trackingStatusString = "RECOVERING";
518  break;
519  case TRACK_STAT_INIT:
520  trackingStatusString = "INITIALIZING";
521  break;
522  default:
523  trackingStatusString = "N/A";
524  break;
525  }
526 
527  // display the frame rate, position and other info
528 
529  float r[3] = {0};
530  float t[3] = {0};
531  if (trackerStatus[0] == TRACK_STAT_OK)
532  {
533  for(int i=0;i<3;i++) {
534  r[i] = trackingData[0].faceRotation[i] * 180.0f / 3.14159f; //rads to degs
535  t[i] = trackingData[0].faceTranslation[i] * 100.0f; //translation is expressed in meters so this gives approximate values in centimeters if camera focus distance parameter is set correctly in configuration file
536  }
537  }
538 
539  NSString* fps = [NSString stringWithFormat:@"%4.1f FPS (track %ld ms) \n", trackingData[0].frameRate, elapsedTrackTime];
540  NSString* info = [NSString stringWithFormat:@"Head position %+5.1f %+5.1f %+5.1f | Rotation (deg) %+5.1f %+5.1f %+5.1f",t[0],t[1],t[2],r[0],r[1],r[2]];
541  NSString* status = [NSString stringWithFormat:@"Status: %s", trackingStatusString];
542 
543  [self->m_informer showTrackingFps:fps Info:info andStatus:status];
544 
545 
546  inGetTrackingResults = false;
547 }
548 
549 
550 void YUV_TO_RGBA(unsigned char* yuv, unsigned char* buff, int width, int height)
551 {
552  const int frameSize = width * height;
553 
554  const int ii = 0;
555  const int ij = 0;
556  const int di = +1;
557  const int dj = +1;
558 
559  unsigned char* rgba = buff;
560 
561  for (int i = 0, ci = ii; i < height; ++i, ci += di)
562  {
563  for (int j = 0, cj = ij; j < width; ++j, cj += dj)
564  {
565  int y = (0xff & ((int) yuv[ci * width + cj]));
566  int v = (0xff & ((int) yuv[frameSize + (ci >> 1) * width + (cj & ~1) + 0]));
567  int u = (0xff & ((int) yuv[frameSize + (ci >> 1) * width + (cj & ~1) + 1]));
568  y = y < 16 ? 16 : y;
569 
570  int a0 = 1192 * (y - 16);
571  int a1 = 1634 * (v - 128);
572  int a2 = 832 * (v - 128);
573  int a3 = 400 * (u - 128);
574  int a4 = 2066 * (u - 128);
575 
576  int r = (a0 + a1) >> 10;
577  int g = (a0 - a2 - a3) >> 10;
578  int b = (a0 + a4) >> 10;
579 
580  *rgba++ = clamp(r);
581  *rgba++ = clamp(g);
582  *rgba++ = clamp(b);
583  *rgba++ = 255;
584  }
585  }
586 }
587 
588 
589 void getOrientation()
590 {
591  UIInterfaceOrientation orientation = [UIApplication sharedApplication].statusBarOrientation;
592 
593  if (orientation == UIInterfaceOrientationPortrait)
594  _rotated = 0;
595  else if (orientation == UIInterfaceOrientationLandscapeLeft)
596  _rotated = 1;
597  else if (orientation == UIInterfaceOrientationLandscapeRight)
598  _rotated = 3;
599  else
600  _rotated = 0;
601 
602 }
603 
604 
605 void setDimensions(int orientation)
606 {
607  NSString* deviceType = [UIDeviceHardware platform];
608 
609  if (orientation == 0 || orientation == 2)
610  {
611  // portrait mode
612  cam_width = 480;
613  cam_height = 640;
614  preset = AVCaptureSessionPreset640x480;
615  }
616 
617  else
618  {
619  // landscape mode
620  cam_width = 640;
621  cam_height = 480;
622  preset = AVCaptureSessionPreset640x480;
623  }
624 
625  // override for iPhone 4
626  if ([deviceType hasPrefix:@"iPhone3"]) { // iPhone4
627  if (orientation == 0 || orientation == 2)
628  {
629  // portrait mode
630  cam_width = 144;
631  cam_height = 192;
632  preset = AVCaptureSessionPresetLow;
633  }
634 
635  else
636  {
637  // landscape mode
638  cam_width = 192;
639  cam_height = 144;
640  preset = AVCaptureSessionPresetLow;
641  }
642 
643  }
644 
645 }
646 
647 
648 @end
int height
Image height in pixels.
static void DisplayResults(FaceData *trackingData, int trackStat, int width, int height, VsImage *frame, int drawingOptions=DISPLAY_DEFAULT)
Method calls other methods for drawing the frame and tracking results.
VisageTracker is a face tracker capable of tracking the head pose, facial features and gaze for multi...
int glHeight
Size of the OpenGL view.
NSString * platform()
Returns exact device name.
Displays tracking information.
void setOpenGLContext()
Method for setting the OpenGL context.
char * imageData
Pointer to aligned image data.
Implements simple high-level Objective-C interface around VisageSDK::VisageTracker functionallity.
void startTrackingFromCam()
Method for starting tracking from a camera.
void swapOpenGLBuffers()
Method for swapping OpenGL buffers.
void trackingImageOperation()
Method for loading picture and performing face tracking on the given frame.
Controls OpenGL view used for tracking output.
void trackingCamOperation()
Method for initializing the camera and performing face tracking on the given frame.
int width
Image width in pixels.
void displayTrackingResults:Frame:(int *trackerStatus, [Frame] VsImage *frame)
Method for displaying tracking results.
Implements methods that are used to determine specific version of device software is running on.
Namespace.
Definition: ios_header.h:1
void trackingVideoOperation:(NSString *filename)
Method for loading video file and performing face tracking on the given frame.
long getCurrentTimeMs()
Function used for measuring tracker speed.
static void DisplayLogo(const VsImage *logo, int width, int height)
Method draws a logo in the upper right corner.
Image store container.
Implements a simple frame grabber that serves a single image to the tracker in order to demonstrate t...
void captureFromFile:(NSString *filename)
Helper function used for capturing from given video file.
int glWidth
Size of the OpenGL view.