visageSDK
Camera2Fragment.java
1 /*
2  * Copyright 2014 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.visagetechnologies.showcasedemo;
18 
19 import android.Manifest;
20 import android.app.AlertDialog;
21 import android.app.Dialog;
22 import android.content.Context;
23 import android.content.DialogInterface;
24 import android.content.pm.PackageManager;
25 import android.content.res.Configuration;
26 import android.graphics.ImageFormat;
27 import android.graphics.SurfaceTexture;
28 import android.hardware.camera2.CameraAccessException;
29 import android.hardware.camera2.CameraCaptureSession;
30 import android.hardware.camera2.CameraCharacteristics;
31 import android.hardware.camera2.CameraDevice;
32 import android.hardware.camera2.CameraManager;
33 import android.hardware.camera2.CameraMetadata;
34 import android.hardware.camera2.CaptureRequest;
35 import android.hardware.camera2.params.StreamConfigurationMap;
36 import android.media.Image;
37 import android.media.ImageReader;
38 import android.os.Bundle;
39 import android.os.Handler;
40 import android.os.HandlerThread;
41 import androidx.annotation.NonNull;
42 import androidx.legacy.app.FragmentCompat;
43 import androidx.core.app.ActivityCompat;
44 import androidx.fragment.app.DialogFragment;
45 import androidx.fragment.app.Fragment;
46 import androidx.fragment.app.FragmentActivity;
47 import android.util.Log;
48 import android.util.Size;
49 import android.view.LayoutInflater;
50 import android.view.Surface;
51 import android.view.View;
52 import android.view.ViewGroup;
53 import android.widget.Toast;
54 
55 import java.nio.ByteBuffer;
56 import java.util.ArrayList;
57 import java.util.Collections;
58 import java.util.Comparator;
59 import java.util.List;
60 import java.util.concurrent.Semaphore;
61 import java.util.concurrent.TimeUnit;
62 
63 public class Camera2Fragment extends Fragment
64  implements FragmentCompat.OnRequestPermissionsResultCallback {
65 
66  private static Camera2Fragment instance = null;
67 
68  private static final String TAG = "Camera2Fragment";
69  private static final int REQUEST_VIDEO_PERMISSIONS = 1;
70  private static final String FRAGMENT_DIALOG = "dialog";
71 
72  private static final String[] VIDEO_PERMISSIONS = {
73  Manifest.permission.CAMERA,
74  Manifest.permission.RECORD_AUDIO
75  };
76 
77  private VisageWrapper visageWrapper;
78 
82  private CameraDevice mCameraDevice;
83 
84  private boolean frontCameraActive = true;
85 
90  private CameraCaptureSession mPreviewSession;
91 
95  private Size mPreviewSize;
96 
100  private HandlerThread cameraBackgroundThread;
101 
105  private Handler cameraBackgroundHandler;
106 
110  private Semaphore mCameraOpenCloseLock = new Semaphore(1);
111 
115  private CameraDevice.StateCallback mStateCallback = new CameraDevice.StateCallback() {
116 
117  @Override
118  public void onOpened(@NonNull CameraDevice cameraDevice) {
119  mCameraDevice = cameraDevice;
120  startPreview();
121  mCameraOpenCloseLock.release();
122 
123  visageWrapper.SetParameters(mPreviewSize.getWidth(), mPreviewSize.getHeight(), mOrientation, frontCameraActive ? 1 : 0);
124  }
125 
126  @Override
127  public void onDisconnected(@NonNull CameraDevice cameraDevice) {
128  mCameraOpenCloseLock.release();
129  cameraDevice.close();
130  mCameraDevice = null;
131  }
132 
133  @Override
134  public void onError(@NonNull CameraDevice cameraDevice, int error) {
135  mCameraOpenCloseLock.release();
136  cameraDevice.close();
137  mCameraDevice = null;
138  FragmentActivity activity = getActivity();
139  if (null != activity) {
140  activity.finish();
141  }
142  }
143 
144  };
145 
146  private Integer mOrientation;
147  private CaptureRequest.Builder mPreviewBuilder;
148 
149  private ImageReader mImageReader;
150 
151  private static int frameID = 0;
152 
153  private ImageReader.OnImageAvailableListener onImageAvailableListener = new ImageReader.OnImageAvailableListener() {
154  @Override
155  public void onImageAvailable(ImageReader reader) {
156  Image image = null;
157  try {
158  long tsA = System.currentTimeMillis();
159 
160  image = reader.acquireLatestImage();
161  if(image == null)
162  return;
163 
164  Image.Plane[] planes = image.getPlanes();
165 
166  ByteBuffer byteBuffer0 = planes[0].getBuffer();
167  ByteBuffer byteBuffer1 = planes[1].getBuffer();
168  ByteBuffer byteBuffer2 = planes[2].getBuffer();
169 
170  if( byteBuffer0 == null || byteBuffer1 == null || byteBuffer2 == null)
171  return;
172 
173  visageWrapper.WriteFrameStream(byteBuffer0, byteBuffer1, byteBuffer2, tsA, frameID++, planes[1].getPixelStride());
174 
175  } catch(Exception e){
176  e.printStackTrace();
177  throw e;
178  } finally{
179  if(image != null){
180  image.close();
181  }
182  }
183  }
184  };
185 
186  public static Camera2Fragment newInstance() {
187  return new Camera2Fragment();
188  }
189 
190  public static Camera2Fragment get() {
191  if (instance == null) {
192  synchronized (Camera2Fragment.class) {
193  if (instance == null) {
194  instance = Camera2Fragment.newInstance();
195  }
196  }
197  }
198  return instance;
199  }
200 
212  private static Size chooseOptimalSize(Size[] choices, int width, int height, Size aspectRatio) {
213  // Collect the supported resolutions that are at least as big as the preview Surface
214  List<Size> bigEnough = new ArrayList<>();
215  int w = aspectRatio.getWidth();
216  int h = aspectRatio.getHeight();
217 
218  for (Size option : choices) {
219  Log.d(TAG, "size " + option.getWidth() + " " + option.getHeight());
220  if (option.getHeight() == option.getWidth() * h / w
221  && option.getWidth() >= width
222  && option.getHeight() >= height) {
223 
224  bigEnough.add(option);
225  }
226  }
227 
228  // Pick the smallest of those, assuming we found any
229  if (bigEnough.size() > 0) {
230  return Collections.min(bigEnough, new CompareSizesByArea());
231  } else {
232  Log.e(TAG, "Couldn't find any suitable preview size");
233  return choices[choices.length / 2];
234  }
235  }
236 
237  @Override
238  public View onCreateView(LayoutInflater inflater, ViewGroup container,
239  Bundle savedInstanceState) {
240  return inflater.inflate(R.layout.fragment, container, false);
241  }
242 
243  @Override
244  public void onActivityCreated(Bundle savedInstanceState){
245  super.onActivityCreated(savedInstanceState);
246 
247  Context context = getActivity();
248  visageWrapper = VisageWrapper.get(context);
249  visageWrapper.switchToCamera();
250  visageWrapper.onCreate();
251  }
252 
253  @Override
254  public void onResume() {
255  super.onResume();
256  visageWrapper.onResume();
257 
258  startCameraBackgroundThread();
259  openCamera(640, 480, frontCameraActive);
260  }
261 
262  @Override
263  public void onPause() {
264  super.onPause();
265 
266  visageWrapper.onPause();
267  closeCamera();
268  stopCameraBackgroundThread();
269 
270  }
271 
272  @Override
273  public void onDetach() {
274  super.onDetach();
275 // visageWrapper.onStop();
276  }
277 
278  @Override
279  public void onConfigurationChanged(Configuration newConfiguration){
280  super.onConfigurationChanged(newConfiguration);
281 
282  visageWrapper.PauseTracker();
283  CameraManager manager = (CameraManager) getActivity().getSystemService(Context.CAMERA_SERVICE);
284  try {
285  CameraCharacteristics characteristics = manager.getCameraCharacteristics(mCameraDevice.getId());
286  mOrientation = getCorrectCameraOrientation(characteristics);
287 
288  visageWrapper.SetParameters(mPreviewSize.getWidth(), mPreviewSize.getHeight(), mOrientation, frontCameraActive ? 1 : 0);
289  } catch (CameraAccessException e){
290  e.printStackTrace();
291  }
292  }
293 
297  private void startCameraBackgroundThread() {
298  cameraBackgroundThread = new HandlerThread("CameraBackground");
299  cameraBackgroundThread.start();
300  cameraBackgroundHandler = new Handler(cameraBackgroundThread.getLooper());
301  }
302 
306  private void stopCameraBackgroundThread() {
307  cameraBackgroundThread.quitSafely();
308  try {
309  cameraBackgroundThread.join();
310  cameraBackgroundThread = null;
311  cameraBackgroundHandler = null;
312  } catch (InterruptedException e) {
313  e.printStackTrace();
314  }
315  }
316 
323  private boolean shouldShowRequestPermissionRationale(String[] permissions) {
324  for (String permission : permissions) {
325  if (ActivityCompat.shouldShowRequestPermissionRationale(getActivity(), permission)) {
326  return true;
327  }
328  }
329  return false;
330  }
331 
335  private void requestVideoPermissions() {
336  if (shouldShowRequestPermissionRationale(VIDEO_PERMISSIONS)) {
337  new ConfirmationDialog().show(getChildFragmentManager(), FRAGMENT_DIALOG);
338  } else {
339  ActivityCompat.requestPermissions(getActivity(), VIDEO_PERMISSIONS, REQUEST_VIDEO_PERMISSIONS);
340  }
341  }
342 
343  @Override
344  public void onRequestPermissionsResult(int requestCode, @NonNull String[] permissions,
345  @NonNull int[] grantResults) {
346  Log.d(TAG, "onRequestPermissionsResult");
347  if (requestCode == REQUEST_VIDEO_PERMISSIONS) {
348  if (grantResults.length == VIDEO_PERMISSIONS.length) {
349  for (int result : grantResults) {
350  if (result != PackageManager.PERMISSION_GRANTED) {
351  ErrorDialog.newInstance(getString(R.string.permission_request))
352  .show(getChildFragmentManager(), FRAGMENT_DIALOG);
353  break;
354  }
355  }
356  } else {
357  ErrorDialog.newInstance(getString(R.string.permission_request))
358  .show(getChildFragmentManager(), FRAGMENT_DIALOG);
359  }
360  } else {
361  super.onRequestPermissionsResult(requestCode, permissions, grantResults);
362  }
363  }
364 
365  private boolean hasPermissionsGranted(String[] permissions) {
366  for (String permission : permissions) {
367  if (ActivityCompat.checkSelfPermission(getActivity(), permission)
368  != PackageManager.PERMISSION_GRANTED) {
369  return false;
370  }
371  }
372  return true;
373  }
374 
378  @SuppressWarnings("MissingPermission")
379  private void openCamera(int width, int height, boolean frontCamera) {
380  if (!hasPermissionsGranted(VIDEO_PERMISSIONS)) {
381  requestVideoPermissions();
382  return;
383  }
384  final FragmentActivity activity = getActivity();
385  if (null == activity || activity.isFinishing()) {
386  return;
387  }
388 
389  CameraManager manager = (CameraManager) activity.getSystemService(Context.CAMERA_SERVICE);
390  try {
391  Log.d(TAG, "tryAcquire");
392  if (!mCameraOpenCloseLock.tryAcquire(2500, TimeUnit.MILLISECONDS)) {
393  throw new RuntimeException("Time out waiting to lock camera opening.");
394  }
395 
396  String[] cameraList = manager.getCameraIdList();
397  String cameraId = null;
398 
399  for(String cam : cameraList){
400  if(frontCamera){
401  if(CameraCharacteristics.LENS_FACING_FRONT == manager.getCameraCharacteristics(cam).get(CameraCharacteristics.LENS_FACING)){
402  cameraId = cam;
403  break;
404  }
405  } else {
406  if(CameraCharacteristics.LENS_FACING_FRONT != manager.getCameraCharacteristics(cam).get(CameraCharacteristics.LENS_FACING)){
407  cameraId = cam;
408  break;
409  }
410  }
411  }
412  if(cameraId == null && cameraList.length != 0){
413  cameraId = cameraList[0]; //camera facing front not found, use first camera from the list
414  }
415 
416  // Choose the sizes for camera preview
417  CameraCharacteristics characteristics = manager.getCameraCharacteristics(cameraId);
418 
419  StreamConfigurationMap map = characteristics
420  .get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP);
421 
422  mPreviewSize = chooseOptimalSize(map.getOutputSizes(SurfaceTexture.class),
423  width, height, new Size(width, height));
424 
425  Log.i(TAG, "Current preview size is " + mPreviewSize.getWidth() + " " + mPreviewSize.getHeight());
426  mOrientation = getCorrectCameraOrientation(characteristics);
427  manager.openCamera(cameraId, mStateCallback, null);
428 
429  mImageReader = ImageReader.newInstance(mPreviewSize.getWidth(), mPreviewSize.getHeight(), ImageFormat.YUV_420_888, 2);
430  mImageReader.setOnImageAvailableListener(onImageAvailableListener, cameraBackgroundHandler);
431 
432  } catch (CameraAccessException e) {
433  Toast.makeText(activity, "Cannot access the camera.", Toast.LENGTH_SHORT).show();
434  activity.finish();
435  } catch (NullPointerException e) {
436  // Currently an NPE is thrown when the Camera2API is used but not supported on the
437  // device this code runs.
438  ErrorDialog.newInstance(getString(R.string.camera_error))
439  .show(getChildFragmentManager(), FRAGMENT_DIALOG);
440  } catch (InterruptedException e) {
441  throw new RuntimeException("Interrupted while trying to lock camera opening.");
442  }
443  }
444 
445  private void closeCamera() {
446  try {
447  mCameraOpenCloseLock.acquire();
448  closePreviewSession();
449  if (null != mCameraDevice) {
450  mCameraDevice.close();
451  mCameraDevice = null;
452  }
453  if (null != mImageReader) {
454  mImageReader.close();
455  mImageReader = null;
456  }
457  } catch (InterruptedException e) {
458  throw new RuntimeException("Interrupted while trying to lock camera closing.");
459  } finally {
460  mCameraOpenCloseLock.release();
461  }
462  }
463 
467  private void startPreview() {
468  if (null == mCameraDevice || null == mPreviewSize) {
469  return;
470  }
471  try {closePreviewSession();
472 
473  List<Surface> surfaces = new ArrayList<>();
474 
475  mPreviewBuilder = mCameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW); //utjece li nam ova varijabla na sample, probati ostale modove NIJE PRIORITET
476 
477  //Set up Surface for ImageReader
478  Surface imageSurface = mImageReader.getSurface();
479  surfaces.add(imageSurface);
480  mPreviewBuilder.addTarget(imageSurface);
481 
482  mCameraDevice.createCaptureSession(surfaces,
483  new CameraCaptureSession.StateCallback() {
484 
485  @Override
486  public void onConfigured(@NonNull CameraCaptureSession session) {
487  mPreviewSession = session;
488  updatePreview();
489  }
490 
491  @Override
492  public void onConfigureFailed(@NonNull CameraCaptureSession session) {
493  FragmentActivity activity = getActivity();
494  if (null != activity) {
495  Toast.makeText(activity, "Failed", Toast.LENGTH_SHORT).show();
496  }
497  }
498  }, cameraBackgroundHandler);
499  } catch (CameraAccessException e) {
500  e.printStackTrace();
501  }
502  }
503 
507  private void updatePreview() {
508  if (null == mCameraDevice) {
509  return;
510  }
511  try {
512  setUpCaptureRequestBuilder(mPreviewBuilder);
513  mPreviewSession.setRepeatingRequest(mPreviewBuilder.build(), null, cameraBackgroundHandler);
514  } catch (CameraAccessException e) {
515  e.printStackTrace();
516  }
517  }
518 
519  private void setUpCaptureRequestBuilder(CaptureRequest.Builder builder) {
520  builder.set(CaptureRequest.CONTROL_MODE, CameraMetadata.CONTROL_MODE_AUTO);
521  }
522 
523  private void closePreviewSession() {
524  if (mPreviewSession != null) {
525  mPreviewSession.close();
526  mPreviewSession = null;
527  }
528  }
529 
530  public int getCorrectCameraOrientation(CameraCharacteristics info){
531 
532  Integer rotation = getActivity().getWindowManager().getDefaultDisplay().getOrientation();
533  Integer degrees = 0;
534 
535  switch(rotation){
536  case Surface.ROTATION_0:
537  degrees = 0;
538  break;
539 
540  case Surface.ROTATION_90:
541  degrees = 90;
542  break;
543 
544  case Surface.ROTATION_180:
545  degrees = 180;
546  break;
547 
548  case Surface.ROTATION_270:
549  degrees = 270;
550  break;
551 
552  }
553 
554  int result;
555  Integer mSensorOrientation = info.get(CameraCharacteristics.SENSOR_ORIENTATION);
556  Integer lensFacing = info.get(CameraCharacteristics.LENS_FACING);
557  if(lensFacing.intValue() == CameraCharacteristics.LENS_FACING_FRONT){
558  result = (mSensorOrientation.intValue() + degrees) % 360;
559  }else{
560  result = (mSensorOrientation.intValue() - degrees + 360) % 360;
561  }
562 
563  return result;
564  }
565 
566  public void switchCamera() {
567  visageWrapper.PauseTracker();
568  frontCameraActive = !frontCameraActive;
569  closeCamera();
570 
571  openCamera(640, 480, frontCameraActive);
572  }
573 
577  static class CompareSizesByArea implements Comparator<Size> {
578 
579  @Override
580  public int compare(Size lhs, Size rhs) {
581  // We cast here to ensure the multiplications won't overflow
582  return Long.signum((long) lhs.getWidth() * lhs.getHeight() -
583  (long) rhs.getWidth() * rhs.getHeight());
584  }
585 
586  }
587 
588  public static class ErrorDialog extends DialogFragment {
589 
590  private static final String ARG_MESSAGE = "message";
591 
592  public static ErrorDialog newInstance(String message) {
593  ErrorDialog dialog = new ErrorDialog();
594  Bundle args = new Bundle();
595  args.putString(ARG_MESSAGE, message);
596  dialog.setArguments(args);
597  return dialog;
598  }
599 
600  @Override
601  public Dialog onCreateDialog(Bundle savedInstanceState) {
602  final FragmentActivity activity = getActivity();
603  return new AlertDialog.Builder(activity)
604  .setMessage(getArguments().getString(ARG_MESSAGE))
605  .setPositiveButton(android.R.string.ok, new DialogInterface.OnClickListener() {
606  @Override
607  public void onClick(DialogInterface dialogInterface, int i) {
608  activity.finish();
609  }
610  })
611  .create();
612  }
613 
614  }
615 
616  public static class ConfirmationDialog extends DialogFragment {
617 
618  @Override
619  public Dialog onCreateDialog(Bundle savedInstanceState) {
620  return new AlertDialog.Builder(getActivity())
621  .setMessage(R.string.permission_request)
622  .setPositiveButton(android.R.string.ok, new DialogInterface.OnClickListener() {
623  @Override
624  public void onClick(DialogInterface dialog, int which) {
625  ActivityCompat.requestPermissions( getActivity(), VIDEO_PERMISSIONS,
626  REQUEST_VIDEO_PERMISSIONS);
627  }
628  })
629  .setNegativeButton(android.R.string.cancel,
630  new DialogInterface.OnClickListener() {
631  @Override
632  public void onClick(DialogInterface dialog, int which) {
633  getActivity().finish();
634  }
635  })
636  .create();
637  }
638 
639  }
640 
641 }