Got Camera Feed Working With Republisher (#93)

* Changed compressed img format to jpeg to align with web video server

* Shfited camera feed to compressed image topic

* Added a sample image with food on fork

* Disaggregated MoveFromMouth text

* Formatting

Author: amalnanavati

feeding_web_app_ros2_test/data/resting_carrot_depth.png

@@ -146,26 +146,33 @@ def publish_frames(self):
                     self.num_frames = 0
                     self.video.set(cv2.CAP_PROP_POS_FRAMES, self.num_frames)
 
-            # Publish the RGB Image message
+            # Configure the RGB Image message
             frame_msg = self.bridge.cv2_to_imgmsg(frame, "bgr8")
             frame_msg.header.frame_id = "camera_color_optical_frame"
             frame_msg.header.stamp = self.get_clock().now().to_msg()
-            self.image_publisher.publish(frame_msg)
 
-            # Publish the RGB CompressedImage message
+            # Configure the RGB CompressedImage message
             compressed_frame_msg = self.bridge.cv2_to_compressed_imgmsg(frame)
             compressed_frame_msg.header.frame_id = "camera_color_optical_frame"
             compressed_frame_msg.header.stamp = self.get_clock().now().to_msg()
-            self.compressed_image_publisher.publish(compressed_frame_msg)
+            compressed_frame_msg.format = (
+                "jpeg"  # web_video_server requires "jpeg" not "jpg"
+            )
 
-            # Publish the depth Image message
+            # Configure the depth Image message
             depth_frame_msg = self.bridge.cv2_to_imgmsg(self.depth_frame, "passthrough")
             depth_frame_msg.header.frame_id = "camera_color_optical_frame"
-            depth_frame_msg.header.stamp = self.get_clock().now().to_msg()
-            self.aligned_depth_publisher.publish(depth_frame_msg)
+            depth_frame_msg.header.stamp = (
+                self.get_clock().now() - rclpy.duration.Duration(seconds=0.15)
+            ).to_msg()
+
+            # Configure the Camera Info
+            self.camera_info_msg.header.stamp = depth_frame_msg.header.stamp
 
-            # Publish the Camera Info
-            self.camera_info_msg.header.stamp = compressed_frame_msg.header.stamp
+            # Publish all topics
+            self.image_publisher.publish(frame_msg)
+            self.compressed_image_publisher.publish(compressed_frame_msg)
+            self.aligned_depth_publisher.publish(depth_frame_msg)
             self.camera_info_publisher.publish(self.camera_info_msg)
 
             rate.sleep()

feeding_web_app_ros2_test/data/resting_carrot_rgb.jpg

@@ -42,7 +42,7 @@ NON_MOVING_STATES.add(MEAL_STATE.U_PostMeal)
 export { NON_MOVING_STATES }
 
 // The names of the ROS topic(s)
-export const CAMERA_FEED_TOPIC = '/camera/color/image_raw'
+export const CAMERA_FEED_TOPIC = '/local/camera/color/image_raw'
 export const FACE_DETECTION_TOPIC = '/face_detection'
 export const FACE_DETECTION_TOPIC_MSG = 'ada_feeding_msgs/FaceDetection'
 export const FACE_DETECTION_IMG_TOPIC = '/face_detection_img'

feeding_web_app_ros2_test/feeding_web_app_ros2_test/DummyRealSense.py

@@ -152,7 +152,7 @@ function Home(props) {
          */
         let currentMealState = MEAL_STATE.R_MovingFromMouthToAbovePlate
         let nextMealState = MEAL_STATE.U_BiteSelection
-        let waitingText = 'Waiting to move above the plate...'
+        let waitingText = 'Waiting to move from your mouth to above the plate...'
         return (
           <RobotMotion
             debug={props.debug}
@@ -166,7 +166,7 @@ function Home(props) {
       case MEAL_STATE.R_MovingFromMouthToRestingPosition: {
         let currentMealState = MEAL_STATE.R_MovingFromMouthToRestingPosition
         let nextMealState = MEAL_STATE.U_BiteAcquisitionCheck
-        let waitingText = 'Waiting to move to the resting position...'
+        let waitingText = 'Waiting to move from your mouth to the resting position...'
         return (
           <RobotMotion
             debug={props.debug}

feedingwebapp/src/Pages/Constants.js

@@ -86,15 +86,6 @@ const BiteSelection = (props) => {
   let { actionName, messageType } = ROS_ACTIONS_NAMES[MEAL_STATE.U_BiteSelection]
   let segmentFromPointAction = useRef(createROSActionClient(ros.current, actionName, messageType))
 
-  /**
-   * Callback function for when the user indicates that they want to move the
-   * robot to locate the plate.
-   */
-  const locatePlateClicked = useCallback(() => {
-    console.log('locatePlateClicked')
-    setMealState(MEAL_STATE.U_PlateLocator)
-  }, [setMealState])
-
   /**
    * Callback function for when the user indicates that they are done with their
    * meal.
@@ -163,6 +154,7 @@ const BiteSelection = (props) => {
         setActionStatus({
           actionStatus: ROS_ACTION_STATUS_SUCCEED
         })
+        console.log('Got result', response.values)
         setActionResult(response.values)
       } else {
         if (
@@ -336,7 +328,7 @@ const BiteSelection = (props) => {
           width: Math.round(REALSENSE_WIDTH * maskScaleFactor),
           height: Math.round(REALSENSE_HEIGHT * maskScaleFactor)
         }
-        let imgSrc = `${props.webVideoServerURL}/stream?topic=${CAMERA_FEED_TOPIC}&width=${imgSize.width}&height=${imgSize.height}&quality=20`
+        let imgSrc = `${props.webVideoServerURL}/stream?topic=${CAMERA_FEED_TOPIC}&width=${imgSize.width}&height=${imgSize.height}&type=ros_compressed`
         return (
           <View style={{ flexDirection: 'row', justifyContent: 'center', alignItems: 'center', width: '100%', height: '100%' }}>
             {actionResult.detected_items.map((detected_item, i) => (
@@ -434,40 +426,14 @@ const BiteSelection = (props) => {
           style={{
             flex: 3,
             flexDirection: 'row',
+            justifyContent: 'center',
             alignItems: 'center',
             width: '100%'
           }}
         >
-          <View
-            style={{
-              flex: 1,
-              alignItems: 'center',
-              justifyContent: 'right'
-            }}
-          >
-            <Button
-              className='doneButton'
-              style={{ fontSize: textFontSize, marginTop: '0', marginBottom: '0' }}
-              onClick={locatePlateClicked}
-            >
-              🍽️ Locate Plate
-            </Button>
-          </View>
-          <View
-            style={{
-              flex: 1,
-              alignItems: 'center',
-              justifyContent: 'left'
-            }}
-          >
-            <Button
-              className='doneButton'
-              style={{ fontSize: textFontSize, marginTop: '0', marginBottom: '0' }}
-              onClick={doneEatingClicked}
-            >
-              ✅ Done Eating
-            </Button>
-          </View>
+          <Button className='doneButton' style={{ fontSize: textFontSize, marginTop: '0', marginBottom: '0' }} onClick={doneEatingClicked}>
+            ✅ Done Eating
+          </Button>
         </View>
         {/**
          * Below the buttons, one half of the screen will present the video feed.
@@ -598,7 +564,6 @@ const BiteSelection = (props) => {
       </>
     )
   }, [
-    locatePlateClicked,
     doneEatingClicked,
     dimension,
     margin,

feedingwebapp/src/Pages/Home/Home.jsx

@@ -145,7 +145,7 @@ const VideoFeed = (props) => {
   // Render the component
   return (
     <img
-      src={`${props.webVideoServerURL}/stream?topic=${CAMERA_FEED_TOPIC}&width=${imgWidth}&height=${imgHeight}&quality=20`}
+      src={`${props.webVideoServerURL}/stream?topic=${CAMERA_FEED_TOPIC}&width=${imgWidth}&height=${imgHeight}&type=ros_compressed`}
       alt='Live video feed from the robot'
       style={{
         width: imgWidth,