video recording support in logging

android/app/src/main/java/org/openbot/common/CameraFragment.java

@@ -1,13 +1,18 @@
 package org.openbot.common;
 
 import android.annotation.SuppressLint;
+import android.app.Dialog;
+import android.content.Context;
 import android.content.res.Configuration;
 import android.graphics.Bitmap;
+import android.net.Uri;
 import android.os.Bundle;
+import android.os.Environment;
 import android.util.Size;
 import android.view.LayoutInflater;
 import android.view.View;
 import android.view.ViewGroup;
+import android.view.Window;
 import androidx.activity.result.ActivityResultLauncher;
 import androidx.activity.result.contract.ActivityResultContracts;
 import androidx.annotation.NonNull;
@@ -17,14 +22,19 @@
 import androidx.camera.core.ImageAnalysis;
 import androidx.camera.core.ImageProxy;
 import androidx.camera.core.Preview;
+import androidx.camera.core.VideoCapture;
 import androidx.camera.lifecycle.ProcessCameraProvider;
 import androidx.camera.view.PreviewView;
 import androidx.core.content.ContextCompat;
 import androidx.viewbinding.ViewBinding;
 import com.google.common.util.concurrent.ListenableFuture;
+import java.io.File;
+import java.text.SimpleDateFormat;
+import java.util.Locale;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import org.jetbrains.annotations.NotNull;
 import org.openbot.R;
 import org.openbot.env.ImageUtils;
 import org.openbot.utils.Constants;
@@ -44,6 +54,8 @@ public abstract class CameraFragment extends ControlsFragment {
   private YuvToRgbConverter converter;
   private Bitmap bitmapBuffer;
   private int rotationDegrees;
+  private VideoCapture videoCapture;
+  private Dialog loadingDialog;
 
   protected View inflateFragment(int resId, LayoutInflater inflater, ViewGroup container) {
     return addCamera(inflater.inflate(resId, container, false), inflater, container);
@@ -60,6 +72,7 @@ private View addCamera(View view, LayoutInflater inflater, ViewGroup container)
 
     previewView = cameraView.findViewById(R.id.viewFinder);
     rootView.addView(view);
+    videoCapture = new VideoCapture.Builder().build();
 
     if (!PermissionUtils.hasCameraPermission(requireActivity())) {
       requestPermissionLauncherCamera.launch(Constants.PERMISSION_CAMERA);
@@ -95,9 +108,10 @@ private void setupCamera() {
   }
 
   @SuppressLint({"UnsafeExperimentalUsageError", "UnsafeOptInUsageError"})
-  private void bindCameraUseCases() {
+  protected void bindCameraUseCases() {
     converter = new YuvToRgbConverter(requireContext());
     bitmapBuffer = null;
+    videoCapture = new VideoCapture.Builder().build();
     preview = new Preview.Builder().setTargetAspectRatio(AspectRatio.RATIO_16_9).build();
     final boolean rotated = ImageUtils.getScreenOrientation(requireActivity()) % 180 == 90;
     final PreviewView.ScaleType scaleType =
@@ -113,7 +127,7 @@ private void bindCameraUseCases() {
           new ImageAnalysis.Builder().setTargetAspectRatio(AspectRatio.RATIO_16_9).build();
     else
       imageAnalysis = new ImageAnalysis.Builder().setTargetResolution(analyserResolution).build();
-    // insert your code here.
+
     imageAnalysis.setAnalyzer(
         cameraExecutor,
         image -> {
@@ -130,7 +144,7 @@ private void bindCameraUseCases() {
     try {
       if (cameraProvider != null) {
         cameraProvider.unbindAll();
-        cameraProvider.bindToLifecycle(this, cameraSelector, preview, imageAnalysis);
+        cameraProvider.bindToLifecycle(this, cameraSelector, preview, imageAnalysis, videoCapture);
       }
     } catch (Exception e) {
       Timber.e("Use case binding failed: %s", e.toString());
@@ -188,5 +202,67 @@ public void setAnalyserResolution(Size resolutionSize) {
     bindCameraUseCases();
   }
 
+  @SuppressLint("RestrictedApi")
+  protected void startVideoRecording() {
+    String outputDirectory =
+        Environment.getExternalStorageDirectory().getAbsolutePath()
+            + File.separator
+            + getString(R.string.app_name)
+            + File.separator
+            + "videos";
+    final File myDir = new File(outputDirectory);
+
+    if (!myDir.exists()) {
+      if (!myDir.mkdirs()) {
+        Timber.i("Make dir failed");
+      }
+    }
+
+    File videoFile =
+        new File(
+            outputDirectory,
+            new SimpleDateFormat("yyyyMMdd_HHmmss", Locale.getDefault())
+                    .format(System.currentTimeMillis())
+                + ".mp4");
+    VideoCapture.OutputFileOptions outputOptions =
+        new VideoCapture.OutputFileOptions.Builder(videoFile).build();
+
+    videoCapture.startRecording(
+        outputOptions,
+        ContextCompat.getMainExecutor(requireContext()),
+        new VideoCapture.OnVideoSavedCallback() {
+          @Override
+          public void onVideoSaved(
+              @NonNull @NotNull VideoCapture.OutputFileResults outputFileResults) {
+            Uri savedUri = Uri.fromFile(videoFile);
+            if (loadingDialog != null) loadingDialog.cancel();
+            Timber.d("Video capture succeeded:" + savedUri);
+          }
+
+          @Override
+          public void onError(
+              int videoCaptureError,
+              @NonNull @NotNull String message,
+              @Nullable @org.jetbrains.annotations.Nullable Throwable cause) {
+            Timber.e("Video capture failed: " + message);
+          }
+        });
+  }
+
+  @SuppressLint("RestrictedApi")
+  public void stopVideoRecording() {
+    videoCapture.stopRecording();
+    showLoading(requireContext());
+  }
+
+  public void showLoading(final Context context) {
+    loadingDialog = new Dialog(context);
+    loadingDialog.requestWindowFeature(Window.FEATURE_NO_TITLE);
+    loadingDialog.getWindow().setBackgroundDrawableResource(android.R.color.transparent);
+    loadingDialog.getWindow().setDimAmount(0.1f);
+    loadingDialog.setContentView(R.layout.dialog_loader);
+    loadingDialog.show();
+  }
+
   protected abstract void processFrame(Bitmap image, ImageProxy imageProxy);
 }

android/app/src/main/java/org/openbot/logging/LoggerFragment.java

@@ -308,12 +308,15 @@ private void startLogging() {
             Timber.e(e, "Got interrupted.");
           }
         });
+    if (binding.videoCaptureCheckBox.isChecked()) startVideoRecording();
   }
 
   private void stopLogging() {
     if (sensorConnection != null) requireActivity().unbindService(sensorConnection);
     requireActivity().stopService(intentSensorService);
 
+    if (binding.videoCaptureCheckBox.isChecked()) stopVideoRecording();
+
     // Pack and upload the collected data
     runInBackground(
         () -> {
@@ -357,8 +360,10 @@ protected void setIsLoggingActive(boolean loggingActive) {
           new ActivityResultContracts.RequestMultiplePermissions(),
           result -> {
             result.forEach((permission, granted) -> allGranted = allGranted && granted);
-            if (allGranted) setIsLoggingActive(true);
-            else {
+            if (allGranted) {
+              bindCameraUseCases();
+              setIsLoggingActive(true);
+            } else {
               PermissionUtils.showLoggingPermissionsToast(requireActivity());
             }
           });

android/app/src/main/java/org/openbot/utils/Constants.java

@@ -20,7 +20,7 @@ public class Constants {
   public static final String PERMISSION_AUDIO = Manifest.permission.RECORD_AUDIO;
 
   public static final String[] PERMISSIONS_LOGGING =
-      new String[] {PERMISSION_CAMERA, PERMISSION_STORAGE, PERMISSION_LOCATION};
+      new String[] {PERMISSION_CAMERA, PERMISSION_STORAGE, PERMISSION_AUDIO, PERMISSION_LOCATION};
   public static final String[] PERMISSIONS_CONTROLLER =
       new String[] {PERMISSION_CAMERA, PERMISSION_AUDIO, PERMISSION_LOCATION};
 

android/app/src/main/java/org/openbot/utils/PermissionUtils.java

@@ -56,7 +56,10 @@ public static boolean hasAudioPermission(Activity activity) {
 
   public static boolean hasLoggingPermissions(Activity activity) {
     return hasPermissions(
-        activity, new String[] {PERMISSION_CAMERA, PERMISSION_STORAGE, PERMISSION_LOCATION});
+        activity,
+        new String[] {
+          PERMISSION_CAMERA, PERMISSION_AUDIO, PERMISSION_STORAGE, PERMISSION_LOCATION
+        });
   }
 
   public static boolean hasControllerPermissions(Activity activity) {
@@ -89,7 +92,7 @@ public static void requestAudioPermission(Activity activity) {
   public static void requestLoggingPermissions(Activity activity) {
     requestPermissions(
         activity,
-        new String[] {PERMISSION_CAMERA, PERMISSION_STORAGE, PERMISSION_LOCATION},
+        new String[] {PERMISSION_CAMERA, PERMISSION_AUDIO, PERMISSION_STORAGE, PERMISSION_LOCATION},
         REQUEST_LOGGING_PERMISSIONS);
   }
 
@@ -108,10 +111,11 @@ public static boolean checkControllerPermissions(int[] grantResults) {
   }
 
   public static boolean checkLoggingPermissions(int[] grantResults) {
-    return grantResults.length > 2
+    return grantResults.length > 3
         && grantResults[0] == PackageManager.PERMISSION_GRANTED
         && grantResults[1] == PackageManager.PERMISSION_GRANTED
-        && grantResults[2] == PackageManager.PERMISSION_GRANTED;
+        && grantResults[2] == PackageManager.PERMISSION_GRANTED
+        && grantResults[3] == PackageManager.PERMISSION_GRANTED;
   }
 
   public static void showControllerPermissionsToast(Activity activity) {
@@ -147,6 +151,16 @@ public static void showAudioPermissionControllerToast(Activity activity) {
         .show();
   }
 
+  public static void showAudioPermissionLoggingToast(Activity activity) {
+    Toast.makeText(
+            activity.getApplicationContext(),
+            activity.getResources().getString(R.string.record_audio_permission_denied)
+                + " "
+                + activity.getResources().getString(R.string.permission_reason_save_audio),
+            Toast.LENGTH_LONG)
+        .show();
+  }
+
   public static void showCameraPermissionControllerToast(Activity activity) {
     Toast.makeText(
             activity.getApplicationContext(),
@@ -162,7 +176,7 @@ public static void showCameraPermissionsPreviewToast(Activity activity) {
             activity.getApplicationContext(),
             activity.getResources().getString(R.string.camera_permission_denied)
                 + " "
-                + activity.getResources().getString(R.string.permission_reason_preview),
+                + activity.getResources().getString(R.string.permission_reason_preview_video),
             Toast.LENGTH_LONG)
         .show();
   }
@@ -179,6 +193,10 @@ public static void showLoggingPermissionsToast(Activity activity) {
     if (shouldShowRational(activity, Constants.PERMISSION_STORAGE)) {
       showStoragePermissionLoggingToast(activity);
     }
+
+    if (shouldShowRational(activity, PERMISSION_AUDIO)) {
+      showAudioPermissionLoggingToast(activity);
+    }
   }
 
   public static boolean shouldShowRational(Activity activity, String permission) {

android/app/src/main/res/layout-land/fragment_logger.xml

@@ -156,15 +156,25 @@
                 app:layout_constraintStart_toEndOf="@+id/ipAddress"
                 app:layout_constraintTop_toTopOf="parent" />
 
+
+            <CheckBox
+                android:id="@+id/videoCaptureCheckBox"
+                android:layout_width="wrap_content"
+                android:layout_height="wrap_content"
+                android:checked="true"
+                android:text="Capture Video"
+                app:layout_constraintStart_toStartOf="@id/previewCheckBox"
+                app:layout_constraintTop_toBottomOf="@id/previewCheckBox" />
+
             <CheckBox
                 android:id="@+id/trainingDataCheckBox"
                 android:layout_width="wrap_content"
                 android:layout_height="wrap_content"
                 android:checked="true"
                 android:text="Training Images"
-                app:layout_constraintBottom_toBottomOf="parent"
-                app:layout_constraintStart_toStartOf="@id/previewCheckBox"
-                app:layout_constraintTop_toBottomOf="@+id/previewCheckBox" />
+                app:layout_constraintStart_toStartOf="@id/videoCaptureCheckBox"
+                app:layout_constraintTop_toBottomOf="@+id/videoCaptureCheckBox" />
+
 
             <TextView
                 android:id="@+id/server"

android/app/src/main/res/layout/dialog_loader.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="utf-8"?>
+<ProgressBar style="?android:attr/progressBarStyle"
+    android:id="@+id/progressBar"
+    android:layout_height="wrap_content"
+    android:layout_width="wrap_content"
+    android:indeterminate="true"
+    android:background="@android:color/transparent"
+    xmlns:android="http://schemas.android.com/apk/res/android" />

android/app/src/main/res/layout/fragment_logger.xml

@@ -172,14 +172,24 @@
                 app:layout_constraintStart_toEndOf="@+id/ipAddress"
                 app:layout_constraintTop_toTopOf="parent" />
 
+            <CheckBox
+                android:id="@+id/videoCaptureCheckBox"
+                android:layout_width="wrap_content"
+                android:layout_height="wrap_content"
+                android:checked="true"
+                android:text="Capture Video"
+                app:layout_constraintStart_toStartOf="@id/previewCheckBox"
+                app:layout_constraintTop_toBottomOf="@id/previewCheckBox" />
+
             <CheckBox
                 android:id="@+id/trainingDataCheckBox"
                 android:layout_width="wrap_content"
                 android:layout_height="wrap_content"
                 android:checked="true"
                 android:text="Training Images"
-                app:layout_constraintStart_toStartOf="@id/previewCheckBox"
-                app:layout_constraintTop_toBottomOf="@+id/previewCheckBox" />
+                app:layout_constraintStart_toStartOf="@id/videoCaptureCheckBox"
+                app:layout_constraintTop_toBottomOf="@+id/videoCaptureCheckBox" />
+
 
             <TextView
                 android:id="@+id/server"

android/app/src/main/res/values/strings.xml

@@ -12,7 +12,8 @@
     <string name="permission_reason_logging"> to log datasets.</string>
     <string name="permission_reason_ai"> to run AI models.</string>
     <string name="permission_reason_stream_video"> to stream video to the controller.</string>
-    <string name="permission_reason_preview"> to preview video.</string>
+    <string name="permission_reason_preview_video"> to preview video.</string>
+    <string name="permission_reason_save_audio"> to save video files.</string>
     <string name="permission_reason_stream_audio"> to stream audio to the controller.</string>
     <string name="permission_reason_find_controller"> to find the controller.</string>
     <string name="permission_reason_model_from_phone"> to select a model from phone storage.</string>

android/controller/build.gradle

@@ -78,7 +78,7 @@ dependencies {
     implementation 'org.videolan.android:libvlc-all:3.3.14'
 
     // WebRTC
-    implementation 'org.webrtc:google-webrtc:1.0.+'
+    implementation 'org.webrtc:google-webrtc:1.0.32006'
 
     // For a library module, uncomment the following line and comment the one after
     // apply plugin: 'com.android.library'