From 30e7a2874df954e810c50259772a7c0b5e3fbbd1 Mon Sep 17 00:00:00 2001 From: Hardik Garg <> Date: Tue, 18 Jun 2024 17:45:25 +0530 Subject: [PATCH 1/4] fixed bug in vehicle control on starting auto mode --- .../openbot/autopilot/AutopilotFragment.java | 993 +++++++++--------- .../org/openbot/logging/LoggerFragment.java | 2 - .../openbot/objectNav/ObjectNavFragment.java | 9 +- .../org/openbot/original/CameraActivity.java | 1 - .../org/openbot/robot/FreeRoamFragment.java | 1 - 5 files changed, 510 insertions(+), 496 deletions(-) diff --git a/android/robot/src/main/java/org/openbot/autopilot/AutopilotFragment.java b/android/robot/src/main/java/org/openbot/autopilot/AutopilotFragment.java index a9d7b9060..e05e83e43 100644 --- a/android/robot/src/main/java/org/openbot/autopilot/AutopilotFragment.java +++ b/android/robot/src/main/java/org/openbot/autopilot/AutopilotFragment.java @@ -39,534 +39,547 @@ import org.openbot.utils.Enums; import org.openbot.utils.PermissionUtils; import org.openbot.vehicle.Control; + import timber.log.Timber; public class AutopilotFragment extends CameraFragment { - // options for drop down in object nav? - private FragmentAutopilotBinding binding; - private Handler handler; - private HandlerThread handlerThread; + // options for drop down in object nav? + private FragmentAutopilotBinding binding; + private Handler handler; + private HandlerThread handlerThread; - private long lastProcessingTimeMs; - private boolean computingNetwork = false; + private long lastProcessingTimeMs; + private boolean computingNetwork = false; - private static final float TEXT_SIZE_DIP = 10; + private static final float TEXT_SIZE_DIP = 10; - private Autopilot autopilot; + private Autopilot autopilot; - private Matrix frameToCropTransform; - private Bitmap croppedBitmap; - private int sensorOrientation; + private Matrix frameToCropTransform; + private Bitmap croppedBitmap; + private int sensorOrientation; - private MultiBoxTracker tracker; + private MultiBoxTracker tracker; - private Model model; - private Network.Device device = Network.Device.CPU; - private int numThreads = -1; + private Model model; + private Network.Device device = Network.Device.CPU; + private int numThreads = -1; - @Override - public void onCreate(@Nullable Bundle savedInstanceState) { - super.onCreate(savedInstanceState); - } + @Override + public void onCreate(@Nullable Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + } - @Override - public View onCreateView( - @NotNull LayoutInflater inflater, ViewGroup container, Bundle savedInstanceState) { - // Inflate the layout for this fragment - binding = FragmentAutopilotBinding.inflate(inflater, container, false); + @Override + public View onCreateView( + @NotNull LayoutInflater inflater, ViewGroup container, Bundle savedInstanceState) { + // Inflate the layout for this fragment + binding = FragmentAutopilotBinding.inflate(inflater, container, false); - return inflateFragment(binding, inflater, container); - } + return inflateFragment(binding, inflater, container); + } - @Override - public void onViewCreated(@NonNull View view, @Nullable Bundle savedInstanceState) { - super.onViewCreated(view, savedInstanceState); - binding.controllerContainer.speedInfo.setText(getString(R.string.speedInfo, "---,---")); + @Override + public void onViewCreated(@NonNull View view, @Nullable Bundle savedInstanceState) { + super.onViewCreated(view, savedInstanceState); + binding.controllerContainer.speedInfo.setText(getString(R.string.speedInfo, "---,---")); + + binding.deviceSpinner.setSelection(preferencesManager.getDevice()); + setNumThreads(preferencesManager.getNumThreads()); + binding.threads.setText(String.valueOf(getNumThreads())); + binding.cameraToggle.setOnClickListener(v -> toggleCamera()); + + if (vehicle.getConnectionType().equals("USB")) { + binding.usbToggle.setVisibility(View.VISIBLE); + binding.bleToggle.setVisibility(View.GONE); + } else if (vehicle.getConnectionType().equals("Bluetooth")) { + binding.bleToggle.setVisibility(View.VISIBLE); + binding.usbToggle.setVisibility(View.GONE); + } + List models = + getModelNames(f -> f.type.equals(Model.TYPE.CMDNAV) && f.pathType != Model.PATH_TYPE.URL); + initModelSpinner(binding.modelSpinner, models, preferencesManager.getAutopilotModel()); + initServerSpinner(binding.serverSpinner); + + setAnalyserResolution(Enums.Preview.HD.getValue()); + binding.deviceSpinner.setOnItemSelectedListener( + new AdapterView.OnItemSelectedListener() { + @Override + public void onItemSelected(AdapterView parent, View view, int position, long id) { + String selected = parent.getItemAtPosition(position).toString(); + setDevice(Network.Device.valueOf(selected.toUpperCase())); + } + + @Override + public void onNothingSelected(AdapterView parent) { + } + }); + + binding.plus.setOnClickListener( + v -> { + String threads = binding.threads.getText().toString().trim(); + int numThreads = Integer.parseInt(threads); + if (numThreads >= 9) return; + setNumThreads(++numThreads); + binding.threads.setText(String.valueOf(numThreads)); + }); + binding.minus.setOnClickListener( + v -> { + String threads = binding.threads.getText().toString().trim(); + int numThreads = Integer.parseInt(threads); + if (numThreads == 1) return; + setNumThreads(--numThreads); + binding.threads.setText(String.valueOf(numThreads)); + }); + BottomSheetBehavior.from(binding.aiBottomSheet).setState(BottomSheetBehavior.STATE_EXPANDED); + + mViewModel + .getUsbStatus() + .observe(getViewLifecycleOwner(), status -> binding.usbToggle.setChecked(status)); + + binding.usbToggle.setChecked(vehicle.isUsbConnected()); + binding.bleToggle.setChecked(vehicle.bleConnected()); + + binding.usbToggle.setOnClickListener( + v -> { + binding.usbToggle.setChecked(vehicle.isUsbConnected()); + Navigation.findNavController(requireView()).navigate(R.id.open_usb_fragment); + }); + + binding.bleToggle.setOnClickListener( + v -> { + binding.bleToggle.setChecked(vehicle.bleConnected()); + Navigation.findNavController(requireView()).navigate(R.id.open_bluetooth_fragment); + }); + + setSpeedMode(Enums.SpeedMode.getByID(preferencesManager.getSpeedMode())); + setControlMode(Enums.ControlMode.getByID(preferencesManager.getControlMode())); + System.out.println("drive mode--->" + preferencesManager.getDriveMode()); + setDriveMode(Enums.DriveMode.getByID(preferencesManager.getDriveMode())); + + binding.controllerContainer.controlMode.setOnClickListener( + v -> { + Enums.ControlMode controlMode = + Enums.ControlMode.getByID(preferencesManager.getControlMode()); + if (controlMode != null) setControlMode(Enums.switchControlMode(controlMode)); + }); + binding.controllerContainer.driveMode.setOnClickListener( + v -> setDriveMode(Enums.switchDriveMode(vehicle.getDriveMode()))); + + binding.controllerContainer.speedMode.setOnClickListener( + v -> + setSpeedMode( + Enums.toggleSpeed( + Enums.Direction.CYCLIC.getValue(), + Enums.SpeedMode.getByID(preferencesManager.getSpeedMode())))); + + binding.autoSwitch.setOnClickListener(v -> setNetworkEnabled(binding.autoSwitch.isChecked())); + } - binding.deviceSpinner.setSelection(preferencesManager.getDevice()); - setNumThreads(preferencesManager.getNumThreads()); - binding.threads.setText(String.valueOf(getNumThreads())); - binding.cameraToggle.setOnClickListener(v -> toggleCamera()); + private void updateCropImageInfo() { + // Timber.i("%s x %s",getPreviewSize().getWidth(), getPreviewSize().getHeight()); + // Timber.i("%s x %s",getMaxAnalyseImageSize().getWidth(), + // getMaxAnalyseImageSize().getHeight()); + frameToCropTransform = null; + + sensorOrientation = 90 - ImageUtils.getScreenOrientation(requireActivity()); + + final float textSizePx = + TypedValue.applyDimension( + TypedValue.COMPLEX_UNIT_DIP, TEXT_SIZE_DIP, getResources().getDisplayMetrics()); + BorderedText borderedText = new BorderedText(textSizePx); + borderedText.setTypeface(Typeface.MONOSPACE); + + tracker = new MultiBoxTracker(requireContext()); + + Timber.i("Camera orientation relative to screen canvas: %d", sensorOrientation); + + recreateNetwork(getModel(), getDevice(), getNumThreads()); + if (autopilot == null) { + Timber.e("No network on preview!"); + return; + } + + binding.trackingOverlay.addCallback( + canvas -> { + tracker.draw(canvas); + // tracker.drawDebug(canvas); + }); + tracker.setFrameConfiguration( + getMaxAnalyseImageSize().getWidth(), + getMaxAnalyseImageSize().getHeight(), + sensorOrientation); + } - if (vehicle.getConnectionType().equals("USB")) { - binding.usbToggle.setVisibility(View.VISIBLE); - binding.bleToggle.setVisibility(View.GONE); - } else if (vehicle.getConnectionType().equals("Bluetooth")) { - binding.bleToggle.setVisibility(View.VISIBLE); - binding.usbToggle.setVisibility(View.GONE); + protected void onInferenceConfigurationChanged() { + computingNetwork = false; + if (croppedBitmap == null) { + // Defer creation until we're getting camera frames. + return; + } + final Network.Device device = getDevice(); + final Model model = getModel(); + final int numThreads = getNumThreads(); + runInBackground(() -> recreateNetwork(model, device, numThreads)); } - List models = - getModelNames(f -> f.type.equals(Model.TYPE.CMDNAV) && f.pathType != Model.PATH_TYPE.URL); - initModelSpinner(binding.modelSpinner, models, preferencesManager.getAutopilotModel()); - initServerSpinner(binding.serverSpinner); - - setAnalyserResolution(Enums.Preview.HD.getValue()); - binding.deviceSpinner.setOnItemSelectedListener( - new AdapterView.OnItemSelectedListener() { - @Override - public void onItemSelected(AdapterView parent, View view, int position, long id) { - String selected = parent.getItemAtPosition(position).toString(); - setDevice(Network.Device.valueOf(selected.toUpperCase())); - } - - @Override - public void onNothingSelected(AdapterView parent) {} - }); - - binding.plus.setOnClickListener( - v -> { - String threads = binding.threads.getText().toString().trim(); - int numThreads = Integer.parseInt(threads); - if (numThreads >= 9) return; - setNumThreads(++numThreads); - binding.threads.setText(String.valueOf(numThreads)); - }); - binding.minus.setOnClickListener( - v -> { - String threads = binding.threads.getText().toString().trim(); - int numThreads = Integer.parseInt(threads); - if (numThreads == 1) return; - setNumThreads(--numThreads); - binding.threads.setText(String.valueOf(numThreads)); - }); - BottomSheetBehavior.from(binding.aiBottomSheet).setState(BottomSheetBehavior.STATE_EXPANDED); - - mViewModel - .getUsbStatus() - .observe(getViewLifecycleOwner(), status -> binding.usbToggle.setChecked(status)); - - binding.usbToggle.setChecked(vehicle.isUsbConnected()); - binding.bleToggle.setChecked(vehicle.bleConnected()); - - binding.usbToggle.setOnClickListener( - v -> { - binding.usbToggle.setChecked(vehicle.isUsbConnected()); - Navigation.findNavController(requireView()).navigate(R.id.open_usb_fragment); - }); - - binding.bleToggle.setOnClickListener( - v -> { - binding.bleToggle.setChecked(vehicle.bleConnected()); - Navigation.findNavController(requireView()).navigate(R.id.open_bluetooth_fragment); - }); - - setSpeedMode(Enums.SpeedMode.getByID(preferencesManager.getSpeedMode())); - setControlMode(Enums.ControlMode.getByID(preferencesManager.getControlMode())); - setDriveMode(Enums.DriveMode.getByID(preferencesManager.getDriveMode())); - - binding.controllerContainer.controlMode.setOnClickListener( - v -> { - Enums.ControlMode controlMode = - Enums.ControlMode.getByID(preferencesManager.getControlMode()); - if (controlMode != null) setControlMode(Enums.switchControlMode(controlMode)); - }); - binding.controllerContainer.driveMode.setOnClickListener( - v -> setDriveMode(Enums.switchDriveMode(vehicle.getDriveMode()))); - - binding.controllerContainer.speedMode.setOnClickListener( - v -> - setSpeedMode( - Enums.toggleSpeed( - Enums.Direction.CYCLIC.getValue(), - Enums.SpeedMode.getByID(preferencesManager.getSpeedMode())))); - - binding.autoSwitch.setOnClickListener(v -> setNetworkEnabled(binding.autoSwitch.isChecked())); - } - - private void updateCropImageInfo() { - // Timber.i("%s x %s",getPreviewSize().getWidth(), getPreviewSize().getHeight()); - // Timber.i("%s x %s",getMaxAnalyseImageSize().getWidth(), - // getMaxAnalyseImageSize().getHeight()); - frameToCropTransform = null; - - sensorOrientation = 90 - ImageUtils.getScreenOrientation(requireActivity()); - - final float textSizePx = - TypedValue.applyDimension( - TypedValue.COMPLEX_UNIT_DIP, TEXT_SIZE_DIP, getResources().getDisplayMetrics()); - BorderedText borderedText = new BorderedText(textSizePx); - borderedText.setTypeface(Typeface.MONOSPACE); - - tracker = new MultiBoxTracker(requireContext()); - - Timber.i("Camera orientation relative to screen canvas: %d", sensorOrientation); - - recreateNetwork(getModel(), getDevice(), getNumThreads()); - if (autopilot == null) { - Timber.e("No network on preview!"); - return; + + private void recreateNetwork(Model model, Network.Device device, int numThreads) { + if (model == null) return; + tracker.clearTrackedObjects(); + if (autopilot != null) { + Timber.d("Closing autoPilot."); + autopilot.close(); + autopilot = null; + } + + try { + Timber.d( + "Creating autopilot (model=%s, device=%s, numThreads=%d)", model, device, numThreads); + autopilot = new Autopilot(requireActivity(), model, device, numThreads); + croppedBitmap = + Bitmap.createBitmap( + autopilot.getImageSizeX(), autopilot.getImageSizeY(), Bitmap.Config.ARGB_8888); + frameToCropTransform = + ImageUtils.getTransformationMatrix( + getMaxAnalyseImageSize().getWidth(), + getMaxAnalyseImageSize().getHeight(), + croppedBitmap.getWidth(), + croppedBitmap.getHeight(), + sensorOrientation, + autopilot.getCropRect(), + autopilot.getMaintainAspect()); + requireActivity() + .runOnUiThread( + () -> + binding.inputResolution.setText( + String.format( + Locale.getDefault(), + "%dx%d", + autopilot.getImageSizeX(), + autopilot.getImageSizeY()))); + + Matrix cropToFrameTransform = new Matrix(); + frameToCropTransform.invert(cropToFrameTransform); + + } catch (IllegalArgumentException | IOException e) { + String msg = "Failed to create network."; + Timber.e(e, msg); + requireActivity() + .runOnUiThread( + () -> + Toast.makeText( + requireContext().getApplicationContext(), + e.getMessage(), + Toast.LENGTH_LONG) + .show()); + } } - binding.trackingOverlay.addCallback( - canvas -> { - tracker.draw(canvas); - // tracker.drawDebug(canvas); - }); - tracker.setFrameConfiguration( - getMaxAnalyseImageSize().getWidth(), - getMaxAnalyseImageSize().getHeight(), - sensorOrientation); - } - - protected void onInferenceConfigurationChanged() { - computingNetwork = false; - if (croppedBitmap == null) { - // Defer creation until we're getting camera frames. - return; + @Override + public synchronized void onResume() { + croppedBitmap = null; + tracker = null; + handlerThread = new HandlerThread("inference"); + handlerThread.start(); + handler = new Handler(handlerThread.getLooper()); + binding.bleToggle.setChecked(vehicle.bleConnected()); + super.onResume(); } - final Network.Device device = getDevice(); - final Model model = getModel(); - final int numThreads = getNumThreads(); - runInBackground(() -> recreateNetwork(model, device, numThreads)); - } - - private void recreateNetwork(Model model, Network.Device device, int numThreads) { - if (model == null) return; - tracker.clearTrackedObjects(); - if (autopilot != null) { - Timber.d("Closing autoPilot."); - autopilot.close(); - autopilot = null; + + @Override + public synchronized void onPause() { + handlerThread.quitSafely(); + try { + handlerThread.join(); + handlerThread = null; + handler = null; + } catch (final InterruptedException e) { + e.printStackTrace(); + } + super.onPause(); } - try { - Timber.d( - "Creating autopilot (model=%s, device=%s, numThreads=%d)", model, device, numThreads); - autopilot = new Autopilot(requireActivity(), model, device, numThreads); - croppedBitmap = - Bitmap.createBitmap( - autopilot.getImageSizeX(), autopilot.getImageSizeY(), Bitmap.Config.ARGB_8888); - frameToCropTransform = - ImageUtils.getTransformationMatrix( - getMaxAnalyseImageSize().getWidth(), - getMaxAnalyseImageSize().getHeight(), - croppedBitmap.getWidth(), - croppedBitmap.getHeight(), - sensorOrientation, - autopilot.getCropRect(), - autopilot.getMaintainAspect()); - requireActivity() - .runOnUiThread( - () -> - binding.inputResolution.setText( - String.format( - Locale.getDefault(), - "%dx%d", - autopilot.getImageSizeX(), - autopilot.getImageSizeY()))); - - Matrix cropToFrameTransform = new Matrix(); - frameToCropTransform.invert(cropToFrameTransform); - - } catch (IllegalArgumentException | IOException e) { - String msg = "Failed to create network."; - Timber.e(e, msg); - requireActivity() - .runOnUiThread( - () -> - Toast.makeText( - requireContext().getApplicationContext(), - e.getMessage(), - Toast.LENGTH_LONG) - .show()); + protected synchronized void runInBackground(final Runnable r) { + if (handler != null) { + handler.post(r); + } } - } - - @Override - public synchronized void onResume() { - croppedBitmap = null; - tracker = null; - handlerThread = new HandlerThread("inference"); - handlerThread.start(); - handler = new Handler(handlerThread.getLooper()); - binding.bleToggle.setChecked(vehicle.bleConnected()); - super.onResume(); - } - - @Override - public synchronized void onPause() { - handlerThread.quitSafely(); - try { - handlerThread.join(); - handlerThread = null; - handler = null; - } catch (final InterruptedException e) { - e.printStackTrace(); + + @Override + protected void processUSBData(String data) { + binding.controllerContainer.speedInfo.setText( + getString( + R.string.speedInfo, + String.format( + Locale.US, "%3.0f,%3.0f", vehicle.getLeftWheelRpm(), vehicle.getRightWheelRpm()))); } - super.onPause(); - } - protected synchronized void runInBackground(final Runnable r) { - if (handler != null) { - handler.post(r); + @Override + protected void processKeyEvent(KeyEvent keyCode) { + if (binding.autoSwitch.isChecked() + && (keyCode.getKeyCode() == KeyEvent.KEYCODE_BUTTON_THUMBL + || keyCode.getKeyCode() == KeyEvent.KEYCODE_BUTTON_THUMBR)) { + audioPlayer.playFromString("Autopilot active. Cannot change speed mode."); + } else { + super.processKeyEvent(keyCode); + } } - } - - @Override - protected void processUSBData(String data) { - binding.controllerContainer.speedInfo.setText( - getString( - R.string.speedInfo, - String.format( - Locale.US, "%3.0f,%3.0f", vehicle.getLeftWheelRpm(), vehicle.getRightWheelRpm()))); - } - - @Override - protected void processKeyEvent(KeyEvent keyCode) { - if (binding.autoSwitch.isChecked() - && (keyCode.getKeyCode() == KeyEvent.KEYCODE_BUTTON_THUMBL - || keyCode.getKeyCode() == KeyEvent.KEYCODE_BUTTON_THUMBR)) { - audioPlayer.playFromString("Autopilot active. Cannot change speed mode."); - } else { - super.processKeyEvent(keyCode); + + @Override + protected void processControllerKeyData(String commandType) { + switch (commandType) { + case Constants.CMD_DRIVE: + binding.controllerContainer.controlInfo.setText( + String.format(Locale.US, "%.0f,%.0f", vehicle.getLeftSpeed(), vehicle.getRightSpeed())); + break; + + case Constants.CMD_DRIVE_MODE: + setDriveMode(Enums.switchDriveMode(vehicle.getDriveMode())); + break; + + case Constants.CMD_SPEED_DOWN: + setSpeedMode( + Enums.toggleSpeed( + Enums.Direction.DOWN.getValue(), + Enums.SpeedMode.getByID(preferencesManager.getSpeedMode()))); + break; + + case Constants.CMD_SPEED_UP: + setSpeedMode( + Enums.toggleSpeed( + Enums.Direction.UP.getValue(), + Enums.SpeedMode.getByID(preferencesManager.getSpeedMode()))); + break; + + case Constants.CMD_NETWORK: + setNetworkEnabledWithAudio(!binding.autoSwitch.isChecked()); + break; + } } - } - - @Override - protected void processControllerKeyData(String commandType) { - switch (commandType) { - case Constants.CMD_DRIVE: - binding.controllerContainer.controlInfo.setText( - String.format(Locale.US, "%.0f,%.0f", vehicle.getLeftSpeed(), vehicle.getRightSpeed())); - break; - - case Constants.CMD_DRIVE_MODE: - setDriveMode(Enums.switchDriveMode(vehicle.getDriveMode())); - break; - - case Constants.CMD_SPEED_DOWN: - setSpeedMode( - Enums.toggleSpeed( - Enums.Direction.DOWN.getValue(), - Enums.SpeedMode.getByID(preferencesManager.getSpeedMode()))); - break; - - case Constants.CMD_SPEED_UP: - setSpeedMode( - Enums.toggleSpeed( - Enums.Direction.UP.getValue(), - Enums.SpeedMode.getByID(preferencesManager.getSpeedMode()))); - break; - - case Constants.CMD_NETWORK: - setNetworkEnabledWithAudio(!binding.autoSwitch.isChecked()); - break; + + private void setNetworkEnabledWithAudio(boolean b) { + setNetworkEnabled(b); + + if (b) { + audioPlayer.play(voice, "network_enabled.mp3"); + runInBackground( + () -> { + try { + TimeUnit.MILLISECONDS.sleep(lastProcessingTimeMs); + vehicle.setControl(0, 0); + requireActivity() + .runOnUiThread(() -> binding.inferenceInfo.setText(R.string.time_fps)); + } catch (InterruptedException e) { + Timber.e(e, "Got interrupted."); + } + }); + } else audioPlayer.playDriveMode(voice, vehicle.getDriveMode()); } - } - - private void setNetworkEnabledWithAudio(boolean b) { - setNetworkEnabled(b); - - if (b) { - audioPlayer.play(voice, "network_enabled.mp3"); - runInBackground( - () -> { - try { - TimeUnit.MILLISECONDS.sleep(lastProcessingTimeMs); - vehicle.setControl(0, 0); - requireActivity() - .runOnUiThread(() -> binding.inferenceInfo.setText(R.string.time_fps)); - } catch (InterruptedException e) { - Timber.e(e, "Got interrupted."); + + private void setNetworkEnabled(boolean b) { + binding.autoSwitch.setChecked(b); + System.out.println("drive mode:::" + currentDriveMode); + binding.controllerContainer.controlMode.setEnabled(!b); + binding.controllerContainer.driveMode.setEnabled(!b); + binding.controllerContainer.speedMode.setEnabled(!b); + + System.out.println("drive mode color::" + b); + binding.controllerContainer.controlMode.setAlpha(b ? 0.5f : 1f); + binding.controllerContainer.driveMode.setAlpha(b ? 0.5f : 1f); + binding.controllerContainer.speedMode.setAlpha(b ? 0.5f : 1f); + + if (Enums.DriveMode.getByID(preferencesManager.getDriveMode()) != null) { + if (Enums.DriveMode.getByID(preferencesManager.getDriveMode()) == Enums.DriveMode.DUAL && Enums.ControlMode.getByID(preferencesManager.getControlMode()) == Enums.ControlMode.PHONE) { + binding.controllerContainer.driveMode.setAlpha(0.5f); + binding.controllerContainer.driveMode.setEnabled(false); } - }); - } else audioPlayer.playDriveMode(voice, vehicle.getDriveMode()); - } - - private void setNetworkEnabled(boolean b) { - binding.autoSwitch.setChecked(b); - binding.controllerContainer.controlMode.setEnabled(!b); - binding.controllerContainer.driveMode.setEnabled(!b); - binding.controllerContainer.speedMode.setEnabled(!b); - - binding.controllerContainer.controlMode.setAlpha(b ? 0.5f : 1f); - binding.controllerContainer.driveMode.setAlpha(b ? 0.5f : 1f); - binding.controllerContainer.speedMode.setAlpha(b ? 0.5f : 1f); - - if (!b) { - setSpeedMode(Enums.SpeedMode.getByID(preferencesManager.getSpeedMode())); - handler.postDelayed(() -> vehicle.setControl(0, 0), 500); - } else { - binding.controllerContainer.speedMode.setImageResource(R.drawable.ic_speed_high); - vehicle.setSpeedMultiplier(Enums.SpeedMode.FAST.getValue()); + } + + if (!b) { + setSpeedMode(Enums.SpeedMode.getByID(preferencesManager.getSpeedMode())); + handler.postDelayed(() -> vehicle.setControl(0, 0), 500); + } else { + binding.controllerContainer.speedMode.setImageResource(R.drawable.ic_speed_high); + vehicle.setSpeedMultiplier(Enums.SpeedMode.FAST.getValue()); + } } - } - - private long frameNum = 0; - - @Override - protected void processFrame(Bitmap bitmap, ImageProxy image) { - if (tracker == null) updateCropImageInfo(); - - ++frameNum; - if (binding != null && binding.autoSwitch.isChecked()) { - // If network is busy, return. - if (computingNetwork) { - return; - } - - computingNetwork = true; - Timber.i("Putting image " + frameNum + " for detection in bg thread."); - - runInBackground( - () -> { - final Canvas canvas = new Canvas(croppedBitmap); - canvas.drawBitmap(bitmap, frameToCropTransform, null); - - if (autopilot != null) { - Timber.i("Running autopilot on image %s", frameNum); - final long startTime = SystemClock.elapsedRealtime(); - handleDriveCommand(autopilot.recognizeImage(croppedBitmap, vehicle.getIndicator())); - lastProcessingTimeMs = SystemClock.elapsedRealtime() - startTime; + + private long frameNum = 0; + + @Override + protected void processFrame(Bitmap bitmap, ImageProxy image) { + if (tracker == null) updateCropImageInfo(); + + ++frameNum; + if (binding != null && binding.autoSwitch.isChecked()) { + // If network is busy, return. + if (computingNetwork) { + return; } - computingNetwork = false; - }); - if (lastProcessingTimeMs > 0) + computingNetwork = true; + Timber.i("Putting image " + frameNum + " for detection in bg thread."); + + runInBackground( + () -> { + final Canvas canvas = new Canvas(croppedBitmap); + canvas.drawBitmap(bitmap, frameToCropTransform, null); + + if (autopilot != null) { + Timber.i("Running autopilot on image %s", frameNum); + final long startTime = SystemClock.elapsedRealtime(); + handleDriveCommand(autopilot.recognizeImage(croppedBitmap, vehicle.getIndicator())); + lastProcessingTimeMs = SystemClock.elapsedRealtime() - startTime; + } + + computingNetwork = false; + }); + if (lastProcessingTimeMs > 0) + requireActivity() + .runOnUiThread( + () -> + binding.inferenceInfo.setText( + String.format(Locale.US, "%d fps", 1000 / lastProcessingTimeMs))); + } + } + + protected void handleDriveCommand(Control control) { + vehicle.setControl(control); + float left = vehicle.getLeftSpeed(); + float right = vehicle.getRightSpeed(); requireActivity() - .runOnUiThread( - () -> - binding.inferenceInfo.setText( - String.format(Locale.US, "%d fps", 1000 / lastProcessingTimeMs))); + .runOnUiThread( + () -> + binding.controllerContainer.controlInfo.setText( + String.format(Locale.US, "%.0f,%.0f", left, right))); } - } - - protected void handleDriveCommand(Control control) { - vehicle.setControl(control); - float left = vehicle.getLeftSpeed(); - float right = vehicle.getRightSpeed(); - requireActivity() - .runOnUiThread( - () -> - binding.controllerContainer.controlInfo.setText( - String.format(Locale.US, "%.0f,%.0f", left, right))); - } - - @Override - public void onConnectionEstablished(String ipAddress) { - requireActivity().runOnUiThread(() -> binding.ipAddress.setText(ipAddress)); - } - - protected Model getModel() { - return model; - } - - protected void setModel(Model model) { - if (this.model != model) { - Timber.d("Updating model: %s", model); - this.model = model; - preferencesManager.setAutopilotModel(model.name); - onInferenceConfigurationChanged(); + + @Override + public void onConnectionEstablished(String ipAddress) { + requireActivity().runOnUiThread(() -> binding.ipAddress.setText(ipAddress)); } - } - - protected Network.Device getDevice() { - return device; - } - - private void setDevice(Network.Device device) { - if (this.device != device) { - Timber.d("Updating device: %s", device); - this.device = device; - final boolean threadsEnabled = device == Network.Device.CPU; - binding.plus.setEnabled(threadsEnabled); - binding.minus.setEnabled(threadsEnabled); - binding.threads.setText(threadsEnabled ? String.valueOf(numThreads) : "N/A"); - if (threadsEnabled) binding.threads.setTextColor(Color.BLACK); - else binding.threads.setTextColor(Color.GRAY); - preferencesManager.setDevice(device.ordinal()); - onInferenceConfigurationChanged(); + + protected Model getModel() { + return model; } - } - - protected int getNumThreads() { - return numThreads; - } - - private void setNumThreads(int numThreads) { - if (this.numThreads != numThreads) { - Timber.d("Updating numThreads: %s", numThreads); - this.numThreads = numThreads; - preferencesManager.setNumThreads(numThreads); - onInferenceConfigurationChanged(); + + protected void setModel(Model model) { + if (this.model != model) { + Timber.d("Updating model: %s", model); + this.model = model; + preferencesManager.setAutopilotModel(model.name); + onInferenceConfigurationChanged(); + } } - } - - private void setSpeedMode(Enums.SpeedMode speedMode) { - if (speedMode != null && !binding.autoSwitch.isChecked()) { - switch (speedMode) { - case SLOW: - binding.controllerContainer.speedMode.setImageResource(R.drawable.ic_speed_low); - break; - case NORMAL: - binding.controllerContainer.speedMode.setImageResource(R.drawable.ic_speed_medium); - break; - case FAST: - binding.controllerContainer.speedMode.setImageResource(R.drawable.ic_speed_high); - break; - } - - Timber.d("Updating controlSpeed: %s", speedMode); - preferencesManager.setSpeedMode(speedMode.getValue()); - vehicle.setSpeedMultiplier(speedMode.getValue()); + + protected Network.Device getDevice() { + return device; + } + + private void setDevice(Network.Device device) { + if (this.device != device) { + Timber.d("Updating device: %s", device); + this.device = device; + final boolean threadsEnabled = device == Network.Device.CPU; + binding.plus.setEnabled(threadsEnabled); + binding.minus.setEnabled(threadsEnabled); + binding.threads.setText(threadsEnabled ? String.valueOf(numThreads) : "N/A"); + if (threadsEnabled) binding.threads.setTextColor(Color.BLACK); + else binding.threads.setTextColor(Color.GRAY); + preferencesManager.setDevice(device.ordinal()); + onInferenceConfigurationChanged(); + } + } + + protected int getNumThreads() { + return numThreads; + } + + private void setNumThreads(int numThreads) { + if (this.numThreads != numThreads) { + Timber.d("Updating numThreads: %s", numThreads); + this.numThreads = numThreads; + preferencesManager.setNumThreads(numThreads); + onInferenceConfigurationChanged(); + } + } + + private void setSpeedMode(Enums.SpeedMode speedMode) { + if (speedMode != null && !binding.autoSwitch.isChecked()) { + switch (speedMode) { + case SLOW: + binding.controllerContainer.speedMode.setImageResource(R.drawable.ic_speed_low); + break; + case NORMAL: + binding.controllerContainer.speedMode.setImageResource(R.drawable.ic_speed_medium); + break; + case FAST: + binding.controllerContainer.speedMode.setImageResource(R.drawable.ic_speed_high); + break; + } + + Timber.d("Updating controlSpeed: %s", speedMode); + preferencesManager.setSpeedMode(speedMode.getValue()); + vehicle.setSpeedMultiplier(speedMode.getValue()); + } + } + + private void setControlMode(Enums.ControlMode controlMode) { + if (controlMode != null) { + switch (controlMode) { + case GAMEPAD: + binding.controllerContainer.controlMode.setImageResource(R.drawable.ic_controller); + disconnectPhoneController(); + break; + case PHONE: + binding.controllerContainer.controlMode.setImageResource(R.drawable.ic_phone); + if (!PermissionUtils.hasControllerPermissions(requireActivity())) + requestPermissionLauncher.launch(Constants.PERMISSIONS_CONTROLLER); + else connectPhoneController(); + + break; + } + Timber.d("Updating controlMode: %s", controlMode); + preferencesManager.setControlMode(controlMode.getValue()); + } + } + + protected void setDriveMode(Enums.DriveMode driveMode) { + if (driveMode != null) { + switch (driveMode) { + case DUAL: + binding.controllerContainer.driveMode.setImageResource(R.drawable.ic_dual); + break; + case GAME: + binding.controllerContainer.driveMode.setImageResource(R.drawable.ic_game); + break; + case JOYSTICK: + binding.controllerContainer.driveMode.setImageResource(R.drawable.ic_joystick); + break; + } + + Timber.d("Updating driveMode: %s", driveMode); + vehicle.setDriveMode(driveMode); + System.out.println("set drive mode in setDriveMode-->" + driveMode.getValue()); + preferencesManager.setDriveMode(driveMode.getValue()); + } } - } - - private void setControlMode(Enums.ControlMode controlMode) { - if (controlMode != null) { - switch (controlMode) { - case GAMEPAD: - binding.controllerContainer.controlMode.setImageResource(R.drawable.ic_controller); - disconnectPhoneController(); - break; - case PHONE: - binding.controllerContainer.controlMode.setImageResource(R.drawable.ic_phone); - if (!PermissionUtils.hasControllerPermissions(requireActivity())) - requestPermissionLauncher.launch(Constants.PERMISSIONS_CONTROLLER); - else connectPhoneController(); - - break; - } - Timber.d("Updating controlMode: %s", controlMode); - preferencesManager.setControlMode(controlMode.getValue()); + + private void connectPhoneController() { + phoneController.connect(requireContext()); + Enums.DriveMode oldDriveMode = currentDriveMode; + // Currently only dual drive mode supported + setDriveMode(Enums.DriveMode.DUAL); + binding.controllerContainer.driveMode.setAlpha(0.5f); + binding.controllerContainer.driveMode.setEnabled(false); } - } - - protected void setDriveMode(Enums.DriveMode driveMode) { - if (driveMode != null) { - switch (driveMode) { - case DUAL: - binding.controllerContainer.driveMode.setImageResource(R.drawable.ic_dual); - break; - case GAME: - binding.controllerContainer.driveMode.setImageResource(R.drawable.ic_game); - break; - case JOYSTICK: - binding.controllerContainer.driveMode.setImageResource(R.drawable.ic_joystick); - break; - } - - Timber.d("Updating driveMode: %s", driveMode); - vehicle.setDriveMode(driveMode); - preferencesManager.setDriveMode(driveMode.getValue()); + + private void disconnectPhoneController() { + phoneController.disconnect(); + setDriveMode(Enums.DriveMode.getByID(preferencesManager.getDriveMode())); + binding.controllerContainer.driveMode.setEnabled(true); + System.out.println("drive mode color::" + 1.0f); + binding.controllerContainer.driveMode.setAlpha(1.0f); } - } - - private void connectPhoneController() { - phoneController.connect(requireContext()); - Enums.DriveMode oldDriveMode = currentDriveMode; - // Currently only dual drive mode supported - setDriveMode(Enums.DriveMode.DUAL); - binding.controllerContainer.driveMode.setAlpha(0.5f); - binding.controllerContainer.driveMode.setEnabled(false); - preferencesManager.setDriveMode(oldDriveMode.getValue()); - } - - private void disconnectPhoneController() { - phoneController.disconnect(); - setDriveMode(Enums.DriveMode.getByID(preferencesManager.getDriveMode())); - binding.controllerContainer.driveMode.setEnabled(true); - binding.controllerContainer.driveMode.setAlpha(1.0f); - } } diff --git a/android/robot/src/main/java/org/openbot/logging/LoggerFragment.java b/android/robot/src/main/java/org/openbot/logging/LoggerFragment.java index 9fc8033fe..3ccdcbaf7 100644 --- a/android/robot/src/main/java/org/openbot/logging/LoggerFragment.java +++ b/android/robot/src/main/java/org/openbot/logging/LoggerFragment.java @@ -522,12 +522,10 @@ protected void setDriveMode(Enums.DriveMode driveMode) { private void connectPhoneController() { phoneController.connect(requireContext()); - Enums.DriveMode oldDriveMode = currentDriveMode; // Currently only dual drive mode supported setDriveMode(Enums.DriveMode.DUAL); binding.controllerContainer.driveMode.setAlpha(0.5f); binding.controllerContainer.driveMode.setEnabled(false); - preferencesManager.setDriveMode(oldDriveMode.getValue()); } private void disconnectPhoneController() { diff --git a/android/robot/src/main/java/org/openbot/objectNav/ObjectNavFragment.java b/android/robot/src/main/java/org/openbot/objectNav/ObjectNavFragment.java index 5bbf32cc5..3a043ac8b 100644 --- a/android/robot/src/main/java/org/openbot/objectNav/ObjectNavFragment.java +++ b/android/robot/src/main/java/org/openbot/objectNav/ObjectNavFragment.java @@ -418,6 +418,13 @@ private void setNetworkEnabled(boolean b) { binding.controllerContainer.driveMode.setAlpha(b ? 0.5f : 1f); binding.controllerContainer.speedMode.setAlpha(b ? 0.5f : 1f); + if (Enums.DriveMode.getByID(preferencesManager.getDriveMode()) != null) { + if (Enums.DriveMode.getByID(preferencesManager.getDriveMode()) == Enums.DriveMode.DUAL && Enums.ControlMode.getByID(preferencesManager.getControlMode()) == Enums.ControlMode.PHONE) { + binding.controllerContainer.driveMode.setAlpha(0.5f); + binding.controllerContainer.driveMode.setEnabled(false); + } + } + resetFpsUi(); if (!b) handler.postDelayed(() -> vehicle.setControl(0, 0), Math.max(lastProcessingTimeMs, 50)); } @@ -642,12 +649,10 @@ protected void setDriveMode(Enums.DriveMode driveMode) { private void connectPhoneController() { phoneController.connect(requireContext()); - Enums.DriveMode oldDriveMode = currentDriveMode; // Currently only dual drive mode supported setDriveMode(Enums.DriveMode.DUAL); binding.controllerContainer.driveMode.setAlpha(0.5f); binding.controllerContainer.driveMode.setEnabled(false); - preferencesManager.setDriveMode(oldDriveMode.getValue()); } private void disconnectPhoneController() { diff --git a/android/robot/src/main/java/org/openbot/original/CameraActivity.java b/android/robot/src/main/java/org/openbot/original/CameraActivity.java index 001f8b8dc..6d956c347 100755 --- a/android/robot/src/main/java/org/openbot/original/CameraActivity.java +++ b/android/robot/src/main/java/org/openbot/original/CameraActivity.java @@ -936,7 +936,6 @@ private void connectPhoneController() { // Currently only dual drive mode supported setDriveMode(DriveMode.DUAL); driveModeSpinner.setAlpha(0.5f); - preferencesManager.setDriveMode(oldDriveMode.getValue()); } private void disconnectPhoneController() { diff --git a/android/robot/src/main/java/org/openbot/robot/FreeRoamFragment.java b/android/robot/src/main/java/org/openbot/robot/FreeRoamFragment.java index d83c743b7..8280753fe 100644 --- a/android/robot/src/main/java/org/openbot/robot/FreeRoamFragment.java +++ b/android/robot/src/main/java/org/openbot/robot/FreeRoamFragment.java @@ -249,7 +249,6 @@ private void connectPhoneController() { setDriveMode(DriveMode.DUAL); binding.controllerContainer.driveMode.setAlpha(0.5f); binding.controllerContainer.driveMode.setEnabled(false); - preferencesManager.setDriveMode(oldDriveMode.getValue()); } private void disconnectPhoneController() { From 78c720c66340106694d0cb4993a12cccdf882fc5 Mon Sep 17 00:00:00 2001 From: isha382 Date: Thu, 17 Oct 2024 11:37:10 +0530 Subject: [PATCH 2/4] fixed automode bug in web controller --- .../java/org/openbot/autopilot/AutopilotFragment.java | 8 +++----- .../java/org/openbot/objectNav/ObjectNavFragment.java | 6 ++++-- .../src/main/java/org/openbot/robot/FreeRoamFragment.java | 3 --- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/android/robot/src/main/java/org/openbot/autopilot/AutopilotFragment.java b/android/robot/src/main/java/org/openbot/autopilot/AutopilotFragment.java index 5e050def8..bb4d3c15b 100644 --- a/android/robot/src/main/java/org/openbot/autopilot/AutopilotFragment.java +++ b/android/robot/src/main/java/org/openbot/autopilot/AutopilotFragment.java @@ -155,7 +155,6 @@ public void onNothingSelected(AdapterView parent) { setSpeedMode(Enums.SpeedMode.getByID(preferencesManager.getSpeedMode())); setControlMode(Enums.ControlMode.getByID(preferencesManager.getControlMode())); - System.out.println("drive mode--->" + preferencesManager.getDriveMode()); setDriveMode(Enums.DriveMode.getByID(preferencesManager.getDriveMode())); binding.controllerContainer.controlMode.setOnClickListener( @@ -390,6 +389,9 @@ private void setNetworkEnabled(boolean b) { if (Enums.DriveMode.getByID(preferencesManager.getDriveMode()) == Enums.DriveMode.DUAL && Enums.ControlMode.getByID(preferencesManager.getControlMode()) == Enums.ControlMode.PHONE) { binding.controllerContainer.driveMode.setAlpha(0.5f); binding.controllerContainer.driveMode.setEnabled(false); + } else if (Enums.DriveMode.getByID(preferencesManager.getDriveMode()) == Enums.DriveMode.GAME && Enums.ControlMode.getByID(preferencesManager.getControlMode()) == Enums.ControlMode.WEBSERVER) { + binding.controllerContainer.driveMode.setAlpha(0.5f); + binding.controllerContainer.driveMode.setEnabled(false); } } @@ -463,12 +465,10 @@ protected Model getModel() { private void connectWebController() { phoneController.connectWebServer(); - Enums.DriveMode oldDriveMode = currentDriveMode; // Currently only dual drive mode supported setDriveMode(Enums.DriveMode.GAME); binding.controllerContainer.driveMode.setAlpha(0.5f); binding.controllerContainer.driveMode.setEnabled(false); - preferencesManager.setDriveMode(oldDriveMode.getValue()); } protected void setModel(Model model) { @@ -579,12 +579,10 @@ protected void setDriveMode(Enums.DriveMode driveMode) { private void connectPhoneController() { phoneController.connect(requireContext()); - Enums.DriveMode oldDriveMode = currentDriveMode; // Currently only dual drive mode supported setDriveMode(Enums.DriveMode.DUAL); binding.controllerContainer.driveMode.setAlpha(0.5f); binding.controllerContainer.driveMode.setEnabled(false); - preferencesManager.setDriveMode(oldDriveMode.getValue()); } private void disconnectPhoneController() { diff --git a/android/robot/src/main/java/org/openbot/objectNav/ObjectNavFragment.java b/android/robot/src/main/java/org/openbot/objectNav/ObjectNavFragment.java index 6a533aeaf..e8a5918f4 100644 --- a/android/robot/src/main/java/org/openbot/objectNav/ObjectNavFragment.java +++ b/android/robot/src/main/java/org/openbot/objectNav/ObjectNavFragment.java @@ -428,6 +428,10 @@ private void setNetworkEnabled(boolean b) { binding.controllerContainer.driveMode.setAlpha(0.5f); binding.controllerContainer.driveMode.setEnabled(false); } + else if (Enums.DriveMode.getByID(preferencesManager.getDriveMode()) == Enums.DriveMode.GAME && Enums.ControlMode.getByID(preferencesManager.getControlMode()) == Enums.ControlMode.WEBSERVER) { + binding.controllerContainer.driveMode.setAlpha(0.5f); + binding.controllerContainer.driveMode.setEnabled(false); + } } resetFpsUi(); @@ -667,12 +671,10 @@ private void connectPhoneController() { private void connectWebController() { phoneController.connectWebServer(); - Enums.DriveMode oldDriveMode = currentDriveMode; // Currently only dual drive mode supported setDriveMode(Enums.DriveMode.GAME); binding.controllerContainer.driveMode.setAlpha(0.5f); binding.controllerContainer.driveMode.setEnabled(false); - preferencesManager.setDriveMode(oldDriveMode.getValue()); } private void disconnectPhoneController() { diff --git a/android/robot/src/main/java/org/openbot/robot/FreeRoamFragment.java b/android/robot/src/main/java/org/openbot/robot/FreeRoamFragment.java index 4d413c83e..45b9a8ee9 100644 --- a/android/robot/src/main/java/org/openbot/robot/FreeRoamFragment.java +++ b/android/robot/src/main/java/org/openbot/robot/FreeRoamFragment.java @@ -262,7 +262,6 @@ protected void setDriveMode(DriveMode driveMode) { private void connectPhoneController() { phoneController.connect(requireContext()); - DriveMode oldDriveMode = currentDriveMode; // Currently only dual drive mode supported setDriveMode(DriveMode.DUAL); binding.controllerContainer.driveMode.setAlpha(0.5f); @@ -271,12 +270,10 @@ private void connectPhoneController() { private void connectWebController() { phoneController.connectWebServer(); - Enums.DriveMode oldDriveMode = currentDriveMode; // Currently only dual drive mode supported setDriveMode(Enums.DriveMode.GAME); binding.controllerContainer.driveMode.setAlpha(0.5f); binding.controllerContainer.driveMode.setEnabled(false); - preferencesManager.setDriveMode(oldDriveMode.getValue()); } private void disconnectPhoneController() { From b3e729f487f097dd25573122d527046e613612eb Mon Sep 17 00:00:00 2001 From: isha382 Date: Mon, 11 Nov 2024 15:09:46 +0530 Subject: [PATCH 3/4] added android plugin --- android/robot/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/android/robot/build.gradle b/android/robot/build.gradle index 4d68b5338..c56921162 100755 --- a/android/robot/build.gradle +++ b/android/robot/build.gradle @@ -1,4 +1,5 @@ plugins { + id 'com.android.application' id 'io.github.reactivecircus.app-versioning' version "1.1.2" id 'com.google.gms.google-services' } From a6e5fcc6fcffe78c1c53e32a4e9ec8ea56df50ae Mon Sep 17 00:00:00 2001 From: Harish Thakur Date: Fri, 17 Apr 2026 17:03:52 +0530 Subject: [PATCH 4/4] Remove duplicate plugin declaration --- android/robot/build.gradle | 1 - 1 file changed, 1 deletion(-) diff --git a/android/robot/build.gradle b/android/robot/build.gradle index c21b2671a..1980df485 100755 --- a/android/robot/build.gradle +++ b/android/robot/build.gradle @@ -2,7 +2,6 @@ plugins { id 'com.android.application' id 'io.github.reactivecircus.app-versioning' version "1.1.2" id 'com.google.gms.google-services' - id 'com.android.application' } appVersioning {