7 #include "VisageTracker.h" 8 #include "VisageRendering.h" 9 #include "AndroidImageCapture.h" 10 #include "AndroidCameraCapture.h" 11 #include "VisageLivenessAction.h" 12 #include "VisageLivenessBrowRaise.h" 13 #include "VisageLivenessSmile.h" 14 #include "VisageLivenessBlink.h" 15 #include "LivenessGUI.h" 17 #include "LicenseString.h" 19 #include <android/log.h> 20 #define LOG_TAG "TrackerWrapper" 21 #define LOGI(...) __android_log_print(ANDROID_LOG_INFO,LOG_TAG,__VA_ARGS__) 22 #define LOGE(...) __android_log_print(ANDROID_LOG_ERROR,LOG_TAG,__VA_ARGS__) 29 int initializeLicenseManager(JNIEnv* env, jobject obj,
const char *licenseKeyFileName,
void (*alertFunction)(
const char*) = 0);
34 static VsImage *drawImageBuffer = 0;
35 static VsImage *renderImage = 0;
37 static float m_fps = 0;
39 void Sleep(
int ms) {usleep(ms*1000);}
59 const int MAX_FACES = 4;
61 static FaceData trackingData[MAX_FACES];
62 int *trackingStatus = 0;
63 float displayFramerate = 0;
70 static FaceData trackingDataBuffer[MAX_FACES];
71 int trackingStatusBuffer[MAX_FACES];
120 bool liveness =
false;
123 int actionState = -1;
125 long startTime_liveness;
126 long currentTime_liveness;
135 static FaceData trackingDataRender[MAX_FACES];
136 int trackingStatusRender[MAX_FACES];
141 const char* faceContourImagePath = 0;
146 bool trackingOk =
false;
147 bool isTracking =
false;
148 bool orientationChanged =
false;
149 bool trackerPaused =
false;
157 pthread_mutex_t displayRes_mutex;
158 pthread_mutex_t guardFrame_mutex;
168 GLuint instructionsTexId = 0;
223 jclass dataClass = _env->FindClass(
"com/visagetechnologies/visagetrackerdemo/TrackerActivity");
224 if (_env->ExceptionCheck())
225 _env->ExceptionClear();
226 if (dataClass != NULL)
228 jclass javaClassRef = (jclass) _env->NewGlobalRef(dataClass);
229 jmethodID javaMethodRef = _env->GetMethodID(javaClassRef,
"AlertDialogFunction",
"(Ljava/lang/String;)V");
230 if (_env->ExceptionCheck())
231 _env->ExceptionClear();
232 jstring message = _env->NewStringUTF(warningMessage);
233 if (javaMethodRef != 0)
234 _env->CallVoidMethod(_obj, javaMethodRef, message);
236 _env->DeleteGlobalRef(javaClassRef);
237 _env->DeleteLocalRef(message);
246 clock_gettime(CLOCK_REALTIME, &now);
247 return (
long) ((now.tv_sec*1000000000LL + now.tv_nsec)/1000000LL);
250 void SetLivenessActions()
293 void RandomizeActions()
295 std::random_device rd;
296 std::mt19937 g(rd());
298 std::shuffle(&actions[0], &actions[3], g);
308 static bool endsWith(
const std::string& str,
const std::string& suffix)
310 return str.size() >= suffix.size() && 0 == str.compare(str.size()-suffix.size(), suffix.size(), suffix);
323 jbyteArray logo_, jint width,
326 jbyte *logoBytes = env->GetByteArrayElements(logo_, 0);
328 pthread_mutex_lock(&displayRes_mutex);
332 pthread_mutex_unlock(&displayRes_mutex);
333 env->ReleaseByteArrayElements(logo_, logoBytes, 0);
337 logo = vsCreateImage(vsSize(width, height), VS_DEPTH_8U, 4);
340 pthread_mutex_unlock(&displayRes_mutex);
342 env->ReleaseByteArrayElements(logo_, logoBytes, 0);
353 jstring path, jbyteArray faceContourImage_, jint width,
356 jbyte *faceContourImageBytes = env->GetByteArrayElements(faceContourImage_, 0);
358 pthread_mutex_lock(&displayRes_mutex);
360 if (faceContourImage) {
362 pthread_mutex_unlock(&displayRes_mutex);
363 env->ReleaseByteArrayElements(faceContourImage_, faceContourImageBytes, 0);
367 faceContourImage = vsCreateImage(vsSize(width, height), VS_DEPTH_8U, 4);
368 memcpy(faceContourImage->
imageData, faceContourImageBytes, faceContourImage->
imageSize);
369 faceContourImagePath = env->GetStringUTFChars(path, 0);
371 GUI->setImage(faceContourImagePath, faceContourImage);
373 pthread_mutex_unlock(&displayRes_mutex);
375 env->ReleaseByteArrayElements(faceContourImage_, faceContourImageBytes, 0);
386 jbyteArray fontImage_, jint width,
388 jbyte *fontImageBytes = env->GetByteArrayElements(fontImage_, 0);
390 pthread_mutex_lock(&displayRes_mutex);
394 pthread_mutex_unlock(&displayRes_mutex);
395 env->ReleaseByteArrayElements(fontImage_, fontImageBytes, 0);
399 fontImage = vsCreateImage(vsSize(width, height), VS_DEPTH_8U, 4);
401 pthread_mutex_unlock(&displayRes_mutex);
403 env->ReleaseByteArrayElements(fontImage_, fontImageBytes, 0);
417 const char *_pathToAssets = env->GetStringUTFChars(pathToAssets, 0);
418 trackerStopped =
false;
421 std::string pathToLicense(_pathToAssets + std::string(
"/") + licenseKey);
422 if (endsWith(licenseKey,
".vlc"))
423 initializeLicenseManager(env, obj, pathToLicense.c_str(),
AlertCallback);
425 initializeLicenseManager(env, obj, licenseKey.c_str(),
AlertCallback);
431 pthread_mutex_destroy(&displayRes_mutex);
432 pthread_mutex_init(&displayRes_mutex, NULL);
434 pthread_mutex_destroy(&guardFrame_mutex);
435 pthread_mutex_init(&guardFrame_mutex, NULL);
442 const char *_configFilename = env->GetStringUTFChars(configFilename, 0);
444 std::string pathToConfig = _pathToAssets + std::string(
"/") + _configFilename;
446 m_Tracker =
new VisageTracker((_pathToAssets + std::string(
"/Facial Features Tracker.cfg")).c_str());
449 env->ReleaseStringUTFChars(pathToAssets, _pathToAssets);
452 void Java_com_visagetechnologies_visagetrackerdemo_TrackerActivity_LivenessInit(JNIEnv *env, jobject obj)
454 GUI =
new LivenessGUI();
456 SetLivenessActions();
461 void Java_com_visagetechnologies_visagetrackerdemo_TrackerActivity_RestartLiveness(JNIEnv *env, jobject obj)
464 actions[i].action->reset();
477 trackerPaused =
true;
482 actions[i].action->reset();
498 while(!trackerStopped)
500 if (m_Tracker && a_cap_image && !trackerStopped)
502 pthread_mutex_lock(&guardFrame_mutex);
504 drawImageBuffer = a_cap_image->
GrabFrame(ts);
506 renderImage = vsCloneImage(drawImageBuffer);
507 if (trackerStopped || drawImageBuffer == 0)
509 pthread_mutex_unlock(&guardFrame_mutex);
513 trackingStatus = m_Tracker->
track(width, height, (
const char*)drawImageBuffer->
imageData, trackingData, VISAGE_FRAMEGRABBER_FMT_RGB, VISAGE_FRAMEGRABBER_ORIGIN_TL, 0, -1, MAX_FACES);
515 trackingTime = (int)(endTime - startTime);
516 LOGE(
"Tracking time is %d", trackingTime);
517 pthread_mutex_unlock(&guardFrame_mutex);
518 pthread_mutex_lock(&displayRes_mutex);
519 for (
int i=0; i<MAX_FACES; i++)
521 if (trackingStatus[i] == TRACK_STAT_OFF)
523 std::swap(trackingDataBuffer[i], trackingData[i]);
524 trackingStatusBuffer[i] = trackingStatus[i];
529 pthread_mutex_unlock(&displayRes_mutex);
551 camOrientation = orientation;
557 if (!drawImageBuffer)
559 vsReleaseImage(&drawImageBuffer);
564 if (camOrientation == 90 || camOrientation == 270)
565 drawImageBuffer = vsCreateImage(vsSize(height, width), VS_DEPTH_8U, 3);
567 drawImageBuffer = vsCreateImage(vsSize(width, height), VS_DEPTH_8U, 3);
570 vsReleaseImage(&renderImage);
575 renderImage = vsCloneImage(drawImageBuffer);
577 orientationChanged =
true;
590 while (!trackerStopped)
592 if (m_Tracker && a_cap_camera && !trackerStopped && !trackerPaused)
594 pthread_mutex_lock(&guardFrame_mutex);
597 if (trackerStopped || trackImage == 0)
599 pthread_mutex_unlock(&guardFrame_mutex);
603 if (camOrientation == 90 || camOrientation == 270)
604 trackingStatus = m_Tracker->
track(camHeight, camWidth, trackImage->
imageData, trackingData, VISAGE_FRAMEGRABBER_FMT_RGB, VISAGE_FRAMEGRABBER_ORIGIN_TL, 0, -1, MAX_FACES);
606 trackingStatus = m_Tracker->
track(camWidth, camHeight, trackImage->
imageData, trackingData, VISAGE_FRAMEGRABBER_FMT_RGB, VISAGE_FRAMEGRABBER_ORIGIN_TL, 0, -1, MAX_FACES);
608 trackingTime = (int)endTime - startTime;
610 pthread_mutex_unlock(&guardFrame_mutex);
615 pthread_mutex_lock(&displayRes_mutex);
616 for (
int i=0; i<MAX_FACES; i++)
618 if (trackingStatus[i] == TRACK_STAT_OFF)
620 std::swap(trackingDataBuffer[i], trackingData[i]);
621 trackingStatusBuffer[i] = trackingStatus[i];
627 vsCopy(trackImage, drawImageBuffer);
632 pthread_mutex_unlock(&displayRes_mutex);
641 void Java_com_visagetechnologies_visagetrackerdemo_TrackerActivity_PerformLiveness(JNIEnv *env, jobject obj)
644 actions[i].action->reset();
651 while (!trackerStopped)
653 if (m_Tracker && a_cap_camera && !trackerStopped && !trackerPaused)
655 pthread_mutex_lock(&guardFrame_mutex);
658 if (trackerStopped || trackImage == 0)
660 pthread_mutex_unlock(&guardFrame_mutex);
663 if (currAction >= NUM_ACTIONS)
665 vsCopy(trackImage, drawImageBuffer);
666 trackingStatusBuffer[0] = trackingStatus[0];
667 pthread_mutex_unlock(&guardFrame_mutex);
671 if (camOrientation == 90 || camOrientation == 270)
672 trackingStatus = m_Tracker->
track(camHeight, camWidth, trackImage->
imageData, trackingData, VISAGE_FRAMEGRABBER_FMT_RGB, VISAGE_FRAMEGRABBER_ORIGIN_TL, 0, -1);
674 trackingStatus = m_Tracker->
track(camWidth, camHeight, trackImage->
imageData, trackingData, VISAGE_FRAMEGRABBER_FMT_RGB, VISAGE_FRAMEGRABBER_ORIGIN_TL, 0, -1);
677 pthread_mutex_unlock(&guardFrame_mutex);
679 trackingTime = (int)endTime - startTime;
681 if (trackingStatus[0] == TRACK_STAT_OK)
683 actionState = actions[currAction].action->
update(trackingData, trackImage);
685 timePassed = (int)currentTime_liveness - startTime_liveness;
691 actions[i].action->reset();
696 if (actionState == 0)
705 pthread_mutex_lock(&displayRes_mutex);
706 if (trackingStatus[0] != TRACK_STAT_OFF)
708 vsCopy(trackImage, drawImageBuffer);
709 trackingStatusBuffer[0] = trackingStatus[0];
717 pthread_mutex_unlock(&displayRes_mutex);
727 void ReleaseActions()
747 trackerStopped =
true;
749 pthread_mutex_lock(&guardFrame_mutex);
750 for (
int i=0; i<MAX_FACES; i++)
752 trackingStatusRender[i] = TRACK_STAT_OFF;
753 trackingStatusBuffer[i] = TRACK_STAT_OFF;
754 trackingStatus[i] = TRACK_STAT_OFF;
759 vsReleaseImage(&drawImageBuffer);
761 vsReleaseImage(&renderImage);
763 VisageRendering::Reset();
764 pthread_mutex_unlock(&guardFrame_mutex);
783 jbyte *f = env->GetByteArrayElements(frame, 0);
784 a_cap_image->
WriteFrame((
unsigned char *)f, (
int)width, (
int)height);
785 env->ReleaseByteArrayElements(frame, f, 0);
804 if (!a_cap_camera || orientationChanged)
808 orientationChanged =
false;
809 trackerPaused =
false;
812 jbyte *f = env->GetByteArrayElements(frame, 0);
814 a_cap_camera->WriteFrameYUV((
unsigned char *)f);
815 env->ReleaseByteArrayElements(frame, f, 0);
834 if (!m_Tracker || trackerStopped || !isTracking || !drawImageBuffer || trackerPaused)
839 pthread_mutex_lock(&displayRes_mutex);
841 vsCopy(drawImageBuffer, renderImage);
845 for (
int i=0; i<MAX_FACES; i++)
847 if (trackingStatusBuffer[i] == TRACK_STAT_OFF)
849 trackingDataRender[i] = trackingDataBuffer[i];
850 trackingStatusRender[i] = trackingStatusBuffer[i];
854 trackingStatusRender[0] = trackingStatusBuffer[0];
858 pthread_mutex_unlock(&displayRes_mutex);
861 videoAspect = renderImage->
width / (float) renderImage->
height;
880 glHeight = glHeight*tmp/
glWidth;
892 for (
int i=0; i<MAX_FACES; i++)
894 if(trackingStatusRender[i] == TRACK_STAT_OK)
901 if (currAction < NUM_ACTIONS)
903 if (trackingStatusRender[0] == TRACK_STAT_OK)
904 GUI->promptUser(actions[currAction].messages[actionState].c_str(), actions[currAction].images[actionState].c_str(),
true);
908 GUI->promptUser(
"", faceContourImagePath,
true);
914 GUI->promptUser(
"Liveness confirmed!",
"",
true);
953 for (
int i=0; i<MAX_FACES; i++)
955 if (trackingStatusBuffer[i] == TRACK_STAT_OK)
956 return env->NewStringUTF(
"OK");
959 for (
int i=0; i<MAX_FACES; i++)
961 if (trackingStatusBuffer[i] == TRACK_STAT_RECOVERING)
962 return env->NewStringUTF(
"RECOVERING");
965 for (
int i=0; i<MAX_FACES; i++)
967 if (trackingStatusBuffer[i] == TRACK_STAT_INIT)
968 return env->NewStringUTF(
"INITIALIZING");
971 return env->NewStringUTF(
"OFF");
974 float Java_com_visagetechnologies_visagetrackerdemo_TrackerRenderer_getTrackerFps( JNIEnv* env )
980 float Java_com_visagetechnologies_visagetrackerdemo_TrackerActivity_GetDisplayFps(JNIEnv* env)
982 return displayFramerate;
986 bool Java_com_visagetechnologies_visagetrackerdemo_TrackerView_IsAutoStopped(JNIEnv* env)
992 for (
int i=0; i<MAX_FACES; i++)
994 if (trackingStatus[i] != TRACK_STAT_OK)
int height
Image height in pixels.
static void DisplayResults(FaceData *trackingData, int trackStat, int width, int height, VsImage *frame, int drawingOptions=DISPLAY_DEFAULT)
Method calls other methods for drawing the frame and tracking results.
VisageTracker is a face tracker capable of tracking the head pose, facial features and gaze for multi...
JNIEXPORT jstring JNICALL Java_com_visagetechnologies_visagetrackerdemo_TrackerActivity_GetStatus(JNIEnv *env, jobject obj)
Method for getting the tracking status information from tracker.
static const int STATE_ACTION_INITIALIZING
Describes action's state while the action is initializing, i.e.
void Java_com_visagetechnologies_visagetrackerdemo_TrackerActivity_TrackFromImage(JNIEnv *env, jobject obj, jint width, jint height)
Method for starting tracking in image.
VisageLivenessAction is an abstract class that can be used to implement algorithm which searches for ...
AndroidImageCapture is class that implements a simple frame grabber that serves a single image to the...
VisageLivenessBlink is a class that detects whether the person on the live video stream has blinked...
long getTimeNsec()
Simple timer function.
VisageLivenessAction * SmileAction
VisageLivenessSmile object.
GLuint frameTexId
Texture ID for displaying frames from the tracker.
int glHeight
Size of the OpenGL view.
void Java_com_visagetechnologies_visagetrackerdemo_JavaCamTrackerView_WriteFrameCamera(JNIEnv *env, jobject obj, jbyteArray frame)
Writes raw image data into VisageSDK::AndroidCameraCapture object.
float videoAspect
Aspect of the video.
int yTexSize
Size of the texture for displaying frames from the tracker.
float frameRate
The frame rate of the tracker, in frames per second, measured over last 10 frames.
static const int STATE_ACTION_VERIFIED
Describes action's state where the action was performed and verified.
VisageLivenessAction * BrowRaiseAction
VisageLivenessBrowRaise object.
char * imageData
Pointer to aligned image data.
static const int STATE_WAIT_FOR_ACTION
Describes action's state when eyebrows raise is trying to be verified.
void Java_com_visagetechnologies_visagetrackerdemo_TrackerActivity_TrackerStop(JNIEnv *env, jobject obj)
Stops the tracker and cleans memory.
VisageLivenessSmile is a class that detects whether the person on the live video stream has smiled...
float Java_com_visagetechnologies_visagetrackerdemo_TrackerActivity_GetFps(JNIEnv *env, jobject obj)
Method for getting frame rate information from the tracker.
void Java_com_visagetechnologies_visagetrackerdemo_TrackerActivity_WriteFontImage(JNIEnv *env, jclass type, jbyteArray fontImage_, jint width, jint height)
Creates VsImage object for font image received from TrackerActivity.
static const int MAX_ACTION_STATES
Maximal number of states that the actions will have.
void Java_com_visagetechnologies_visagetrackerdemo_ImageTrackerView_WriteFrameImage(JNIEnv *env, jobject obj, jbyteArray frame, jint width, jint height)
Writes raw image data into VisageSDK::AndroidImageCapture object.
virtual int update(const FaceData *faceData, VsImage *frame=0)=0
Method used to process input data in order to confirm that the action has occured.
static const int STATE_ACTION_INITIALIZING
Describes action's state while the action is initializing, i.e.
void WriteFrame(unsigned char *imageData, int width, int height)
Method for writing imageData to buffer object.
void AlertCallback(const char *warningMessage)
Callback method for license notification.
void Java_com_visagetechnologies_visagetrackerdemo_TrackerActivity_TrackFromCam(JNIEnv *env, jobject obj)
Method for starting tracking from camera.
int xTexSize
Size of the texture for displaying frames from the tracker.
void Java_com_visagetechnologies_visagetrackerdemo_TrackerActivity_WriteFaceContourImage(JNIEnv *env, jclass type, jstring path, jbyteArray faceContourImage_, jint width, jint height)
Creates VsImage object for face countour image received from TrackerActivity.
static const int NUM_ACTIONS
Number of actions that will need to be verified in order to confirm liveness.
VisageLivenessBrowRaise is a class that detects whether the person on the live video stream has raise...
int width
Image width in pixels.
virtual int * track(int frameWidth, int frameHeight, const char *p_imageData, FaceData *facedata, int format=VISAGE_FRAMEGRABBER_FMT_RGB, int origin=VISAGE_FRAMEGRABBER_ORIGIN_TL, int widthStep=0, long timeStamp=-1, int maxFaces=1)
Performs face tracking (one or more faces) in the given image and returns tracking results and status...
VisageLivenessAction * BlinkAction
VisageLivenessBlink object.
bool Java_com_visagetechnologies_visagetrackerdemo_TrackerRenderer_displayTrackingStatus(JNIEnv *env, jobject obj, jint width, jint height)
Method for displaying tracking results.
int Java_com_visagetechnologies_visagetrackerdemo_TrackerActivity_setParameters(JNIEnv *env, jobject obj, jint width, jint height, jint orientation, jint flip)
Method that sets frame parameters.
VsImage * GrabFrame(long &timeStamp)
This function is called to get the frame to process.
void Java_com_visagetechnologies_visagetrackerdemo_TrackerActivity_TrackerInit(JNIEnv *env, jobject obj, jstring pathToAssets, jstring configFilename)
Method for initializing the tracker.
Structure that holds the action and its accompanying messages and images that will be displayed to us...
static const int STATE_ACTION_INITIALIZING
Describes action's state while the action is initializing, i.e.
float yTexCoord
Texture coordinates for displaying frames from the tracker.
static void SetFontTexture(const VsImage *font_tex)
Method sets font texture.
int imageSize
Image data size in bytes (==image->height*image->widthStep in case of interleaved data) ...
void Java_com_visagetechnologies_visagetrackerdemo_TrackerActivity_WriteLogoImage(JNIEnv *env, jclass type, jbyteArray logo_, jint width, jint height)
Creates VsImage object for logo received from TrackerActivity.
int Java_com_visagetechnologies_visagetrackerdemo_TrackerActivity_GetTrackTime(JNIEnv *env, jobject obj)
Method for getting frame rate information from the tracker.
static const int STATE_WAIT_FOR_FRONTAL
Describes action's state where face in the frame is not frontal (head rotations larger than 0...
static const int STATE_WAIT_FOR_STABLE
Describes action's state when user is not keeping his head still.
static const int STATE_WAIT_FOR_ACTION
Describes action's state when smile is trying to be verified.
int glWidth
Size of the OpenGL view.
static const int STATE_WAIT_FOR_STABLE
Describes action's state when user is not keeping his head still.
VsImage * GrabFrame(long &timeStamp)
This function is called periodically to get the new video frame to process.
void Java_com_visagetechnologies_visagetrackerdemo_TrackerActivity_PauseTracker(JNIEnv *env, jobject obj)
Method for "pausing" tracking.
Face data structure, used as container for all face tracking and detection results.
AndroidCameraCapture demonstrates use of raw camera image input to track from Android camera...
float xTexCoord
Texture coordinates for displaying frames from the tracker.
static const int STATE_WAIT_FOR_FRONTAL
Describes action's state where face in the frame is not frontal (head rotations larger than 0...
static const int STATE_WAIT_FOR_STABLE
Describes action's state when user is not keeping his head still.
static const int STATE_WAIT_FOR_FRONTAL
Describes action's state where face in the frame is not frontal (head rotations larger than 0...
static void DisplayLogo(const VsImage *logo, int width, int height)
Method draws a logo in the upper right corner.
Action actions[NUM_ACTIONS]
Array of actions that will need to be verified.
static const int STATE_WAIT_FOR_ACTION
Describes action's state when blink is trying to be verified.