Improve Acquisition Report (#145)

* Have SkipAcquisition move to staging

* Send Acquisition Report even when SkipAcquisition is clicked'

* Improve rendering of settings on mobile

Author: amalnanavati

feedingwebapp/src/Pages/GlobalState.jsx

@@ -130,8 +130,8 @@ export const useGlobalState = create(
       // message received from the face detection node where a
       // face was detected and within the distance bounds of the camera.
       moveToMouthActionGoal: null,
-      // Last RobotMotion action response
-      lastMotionActionResponse: null,
+      // Last RobotMotion action feedback message
+      lastMotionActionFeedback: null,
       // Whether or not the currently-executing robot motion was paused by the user.
       // NOTE: `paused` may no longer need to be in global state now that we have
       // the `inNonMovingState` flag.
@@ -218,9 +218,9 @@ export const useGlobalState = create(
         set(() => ({
           biteAcquisitionActionGoal: biteAcquisitionActionGoal
         })),
-      setLastMotionActionResponse: (lastMotionActionResponse) =>
+      setLastMotionActionFeedback: (lastMotionActionFeedback) =>
         set(() => ({
-          lastMotionActionResponse: lastMotionActionResponse
+          lastMotionActionFeedback: lastMotionActionFeedback
         })),
       setMoveToMouthActionGoal: (moveToMouthActionGoal) =>
         set(() => {

feedingwebapp/src/Pages/Home/Home.jsx

@@ -1,10 +1,12 @@
 // React imports
-import React, { useCallback, useEffect, useMemo } from 'react'
+import React, { useCallback, useEffect, useMemo, useRef } from 'react'
 import PropTypes from 'prop-types'
+import { toast } from 'react-toastify'
 import { View } from 'react-native'
 
 // Local imports
 import './Home.css'
+import { createROSService, createROSServiceRequest, useROS } from '../../ros/ros_helpers'
 import { useGlobalState, MEAL_STATE } from '../GlobalState'
 import BiteAcquisitionCheck from './MealStates/BiteAcquisitionCheck'
 import BiteDone from './MealStates/BiteDone'
@@ -13,7 +15,13 @@ import DetectingFace from './MealStates/DetectingFace'
 import PostMeal from './MealStates/PostMeal'
 import PreMeal from './MealStates/PreMeal'
 import RobotMotion from './MealStates/RobotMotion'
-import { getRobotMotionText, TIME_TO_RESET_MS } from '../Constants'
+import {
+  ACQUISITION_REPORT_SERVICE_NAME,
+  ACQUISITION_REPORT_SERVICE_TYPE,
+  getRobotMotionText,
+  REGULAR_CONTAINER_ID,
+  TIME_TO_RESET_MS
+} from '../Constants'
 
 /**
  * The Home component displays the state of the meal, solicits user input as
@@ -78,6 +86,47 @@ function Home(props) {
   const moveToMouthActionInput = useMemo(() => moveToMouthActionGoal, [moveToMouthActionGoal])
   const moveToStowPositionActionInput = useMemo(() => ({}), [])
 
+  /**
+   * Create callbacks for acquisition success and failure. This is done here because these
+   * callbacks can be called during BiteAcquisition or the BiteAcquisitionCheck.
+   */
+  const lastMotionActionFeedback = useGlobalState((state) => state.lastMotionActionFeedback)
+  const ros = useRef(useROS().ros)
+  let acquisitionReportService = useRef(createROSService(ros.current, ACQUISITION_REPORT_SERVICE_NAME, ACQUISITION_REPORT_SERVICE_TYPE))
+  let acquisitionResponse = useCallback(
+    (success) => {
+      if (!lastMotionActionFeedback.action_info_populated) {
+        console.info('Cannot report acquisition success or failure without action_info_populated.')
+        return
+      }
+      let msg, loss
+      if (success) {
+        msg = 'Reporting Food Acquisition Success!'
+        loss = 0.0
+      } else {
+        msg = 'Reporting Food Acquisition Failure.'
+        loss = 1.0
+      }
+      // NOTE: This uses the ToastContainer in Header
+      console.log(msg)
+      toast.info(msg, {
+        containerId: REGULAR_CONTAINER_ID,
+        toastId: msg
+      })
+      // Create a service request
+      let request = createROSServiceRequest({
+        loss: loss,
+        action_index: lastMotionActionFeedback.action_index,
+        posthoc: lastMotionActionFeedback.posthoc,
+        id: lastMotionActionFeedback.selection_id
+      })
+      // Call the service
+      let service = acquisitionReportService.current
+      service.callService(request, (response) => console.log('Got acquisition report response', response))
+    },
+    [lastMotionActionFeedback]
+  )
+
   /**
    * Determines what screen to render based on the meal state.
    */
@@ -119,7 +168,8 @@ function Home(props) {
         let nextMealState = MEAL_STATE.U_BiteAcquisitionCheck
         let backMealState = MEAL_STATE.R_MovingAbovePlate
         // TODO: Add an icon for this errorMealState!
-        let errorMealState = MEAL_STATE.R_MovingToRestingPosition
+        let errorMealState = MEAL_STATE.R_MovingToStagingConfiguration
+        let errorCallback = () => acquisitionResponse(true) // Success if the user skips acquisition
         let errorMealStateDescription = 'Skip Acquisition'
         return (
           <RobotMotion
@@ -132,6 +182,7 @@ function Home(props) {
             waitingText={getRobotMotionText(currentMealState)}
             allowRetry={false} // Don't allow retrying bite acquisition
             errorMealState={errorMealState}
+            errorCallback={errorCallback}
             errorMealStateDescription={errorMealStateDescription}
           />
         )
@@ -153,7 +204,7 @@ function Home(props) {
         )
       }
       case MEAL_STATE.U_BiteAcquisitionCheck: {
-        return <BiteAcquisitionCheck debug={props.debug} />
+        return <BiteAcquisitionCheck debug={props.debug} acquisitionResponse={acquisitionResponse} />
       }
       case MEAL_STATE.R_MovingToStagingConfiguration: {
         /**
@@ -257,6 +308,7 @@ function Home(props) {
     props.debug,
     props.webrtcURL,
     biteAcquisitionActionInput,
+    acquisitionResponse,
     mostRecentBiteDoneResponse,
     moveAbovePlateActionInput,
     moveToMouthActionInput,

feedingwebapp/src/Pages/Home/MealStates/BiteAcquisitionCheck.jsx

@@ -2,28 +2,22 @@
 import React, { useCallback, useEffect, useState, useRef } from 'react'
 import Button from 'react-bootstrap/Button'
 import { useMediaQuery } from 'react-responsive'
-import { toast } from 'react-toastify'
 import { View } from 'react-native'
+// PropTypes is used to validate that the used props are in fact passed to this Component
+import PropTypes from 'prop-types'
 
 // Local Imports
 import '../Home.css'
 import { useGlobalState, MEAL_STATE } from '../../GlobalState'
 import { MOVING_STATE_ICON_DICT } from '../../Constants'
 import { useROS, createROSService, createROSServiceRequest, subscribeToROSTopic, unsubscribeFromROSTopic } from '../../../ros/ros_helpers'
-import {
-  ACQUISITION_REPORT_SERVICE_NAME,
-  ACQUISITION_REPORT_SERVICE_TYPE,
-  FOOD_ON_FORK_DETECTION_TOPIC,
-  FOOD_ON_FORK_DETECTION_TOPIC_MSG,
-  REGULAR_CONTAINER_ID,
-  ROS_SERVICE_NAMES
-} from '../../Constants'
+import { FOOD_ON_FORK_DETECTION_TOPIC, FOOD_ON_FORK_DETECTION_TOPIC_MSG, ROS_SERVICE_NAMES } from '../../Constants'
 
 /**
  * The BiteAcquisitionCheck component appears after the robot has attempted to
  * acquire a bite, and asks the user whether it succeeded at acquiring the bite.
  */
-const BiteAcquisitionCheck = () => {
+const BiteAcquisitionCheck = (props) => {
   // Store the remining time before auto-continuing
   const [remainingSeconds, setRemainingSeconds] = useState(null)
   // Get the relevant global variables
@@ -50,17 +44,11 @@ const BiteAcquisitionCheck = () => {
   let iconWidth = isPortrait ? '28vh' : '28vw'
   let iconHeight = isPortrait ? '18vh' : '18vw'
 
-  // Configure AcquisitionReport service
-  const lastMotionActionResponse = useGlobalState((state) => state.lastMotionActionResponse)
   /**
    * Connect to ROS, if not already connected. Put this in useRef to avoid
    * re-connecting upon re-renders.
    */
   const ros = useRef(useROS().ros)
-  /**
-   * Create the ROS Service Client for reporting success/failure
-   */
-  let acquisitionReportService = useRef(createROSService(ros.current, ACQUISITION_REPORT_SERVICE_NAME, ACQUISITION_REPORT_SERVICE_TYPE))
   /**
    * Create the ROS Service. This is created in local state to avoid re-creating
    * it upon every re-render.
@@ -73,48 +61,20 @@ const BiteAcquisitionCheck = () => {
    * succeeded.
    */
   const acquisitionSuccess = useCallback(() => {
-    console.log('acquisitionSuccess')
-    // NOTE: This uses the ToastContainer in Header
-    toast.info('Reporting Food Acquisition Success!', {
-      containerId: REGULAR_CONTAINER_ID,
-      toastId: 'foodAcquisitionSuccess'
-    })
-    // Create a service request
-    let request = createROSServiceRequest({
-      loss: 0.0,
-      action_index: lastMotionActionResponse.action_index,
-      posthoc: lastMotionActionResponse.posthoc,
-      id: lastMotionActionResponse.selection_id
-    })
-    // Call the service
-    let service = acquisitionReportService.current
-    service.callService(request, (response) => console.log('Got acquisition report response', response))
+    let acquisitionResponse = props.acquisitionResponse
+    acquisitionResponse(true)
     setMealState(MEAL_STATE.R_MovingToStagingConfiguration)
-  }, [lastMotionActionResponse, setMealState])
+  }, [props.acquisitionResponse, setMealState])
 
   /**
    * Callback function for when the user indicates that the bite acquisition
    * failed.
    */
   const acquisitionFailure = useCallback(() => {
-    console.log('acquisitionFailure')
-    // NOTE: This uses the ToastContainer in Header
-    toast.info('Reporting Food Acquisition Failure.', {
-      containerId: REGULAR_CONTAINER_ID,
-      toastId: 'foodAcquisitionFailure'
-    })
-    // Create a service request
-    let request = createROSServiceRequest({
-      loss: 1.0,
-      action_index: lastMotionActionResponse.action_index,
-      posthoc: lastMotionActionResponse.posthoc,
-      id: lastMotionActionResponse.selection_id
-    })
-    // Call the service
-    let service = acquisitionReportService.current
-    service.callService(request, (response) => console.log('Got acquisition report response', response))
+    let acquisitionResponse = props.acquisitionResponse
+    acquisitionResponse(false)
     setMealState(MEAL_STATE.R_MovingAbovePlate)
-  }, [lastMotionActionResponse, setMealState])
+  }, [props.acquisitionResponse, setMealState])
 
   /*
    * Create refs to store the interval for the food-on-fork detection timers.
@@ -429,4 +389,11 @@ const BiteAcquisitionCheck = () => {
   return <>{fullPageView()}</>
 }
 
+BiteAcquisitionCheck.propTypes = {
+  debug: PropTypes.bool,
+  // A function that takes a boolean indicating whether the robot succeeded at acquiring the bite,
+  // and processes the response. Note that it does not transition to the next state.
+  acquisitionResponse: PropTypes.func.isRequired
+}
+
 export default BiteAcquisitionCheck

feedingwebapp/src/Pages/Home/MealStates/RobotMotion.jsx

@@ -62,8 +62,8 @@ const RobotMotion = (props) => {
   const paused = useGlobalState((state) => state.paused)
   const setPaused = useGlobalState((state) => state.setPaused)
 
-  // Setter for last motion action response
-  const setLastMotionActionResponse = useGlobalState((state) => state.setLastMotionActionResponse)
+  // Setter for last motion action feedback msg
+  const setLastMotionActionFeedback = useGlobalState((state) => state.setLastMotionActionFeedback)
 
   /**
    * Connect to ROS, if not already connected. Put this in useRef to avoid
@@ -115,23 +115,27 @@ const RobotMotion = (props) => {
   const feedbackCallback = useCallback(
     (feedbackMsg) => {
       console.log('Got feedback message', feedbackMsg)
+      setLastMotionActionFeedback(feedbackMsg.values.feedback)
       setActionStatus({
         actionStatus: ROS_ACTION_STATUS_EXECUTE,
         feedback: feedbackMsg.values.feedback
       })
     },
-    [setActionStatus]
+    [setActionStatus, setLastMotionActionFeedback]
   )
 
   /**
    * Callback function to change the meal state.
    */
   const changeMealState = useCallback(
-    (nextMealState, msg = null) => {
+    (nextMealState, msg = null, callback = null) => {
       if (msg) {
         console.log(msg)
       }
       setPaused(false)
+      if (callback) {
+        callback()
+      }
       let setMealState = props.setMealState
       setMealState(nextMealState)
     },
@@ -164,7 +168,6 @@ const RobotMotion = (props) => {
         setActionStatus({
           actionStatus: ROS_ACTION_STATUS_SUCCEED
         })
-        setLastMotionActionResponse(response.values)
         robotMotionDone()
       } else {
         if (
@@ -185,7 +188,7 @@ const RobotMotion = (props) => {
         }
       }
     },
-    [setLastMotionActionResponse, setActionStatus, setPaused, robotMotionDone]
+    [setActionStatus, setPaused, robotMotionDone]
   )
 
   /**
@@ -327,7 +330,7 @@ const RobotMotion = (props) => {
                       variant='warning'
                       className='mx-2 btn-huge'
                       size='lg'
-                      onClick={() => changeMealState(props.errorMealState, 'errorMealState')}
+                      onClick={() => changeMealState(props.errorMealState, 'errorMealState', props.errorCallback)}
                       style={{
                         width: '90%',
                         height: '20%'
@@ -363,6 +366,7 @@ const RobotMotion = (props) => {
       props.waitingText,
       props.allowRetry,
       props.errorMealState,
+      props.errorCallback,
       props.errorMealStateDescription,
       motionTextFontSize,
       waitingTextFontSize,
@@ -501,6 +505,7 @@ RobotMotion.propTypes = {
   allowRetry: PropTypes.bool,
   // If error, show the user the option to transition to this meal state
   errorMealState: PropTypes.string,
+  errorCallback: PropTypes.func,
   errorMealStateDescription: PropTypes.string
 }