wired up voice and vision, testing motion

2026-08-28 11:26:40 +02:00
parent 8d88495f6f
commit 3982c6fb6b
84 changed files with 7696 additions and 8984 deletions
@@ -9,9 +9,11 @@ set(CMAKE_CXX_EXTENSIONS OFF)
 option(BUILD_VISION "Build vision components" ON)
 option(BUILD_VOICE "Build R1 voice assistant" ON)

-set(UNITREE_SDK2_DIR
+set(
+    UNITREE_SDK2_DIR
    "/home/jvizner/unitree_sdk2"
-    CACHE PATH "Path to unitree_sdk2"
+    CACHE PATH
+    "Path to unitree_sdk2"
 )

 # ============================================================
@@ -21,7 +23,8 @@ set(UNITREE_SDK2_DIR
 if(BUILD_VISION OR BUILD_VOICE)

    if(NOT EXISTS "${UNITREE_SDK2_DIR}/CMakeLists.txt")
-        message(FATAL_ERROR
+        message(
+            FATAL_ERROR
            "Invalid UNITREE_SDK2_DIR: ${UNITREE_SDK2_DIR}"
        )
    endif()
@@ -35,29 +38,41 @@ if(BUILD_VISION OR BUILD_VOICE)
 endif()

 # ============================================================
-# Vision
+# Vision / Motion
 # ============================================================

 if(BUILD_VISION)

    find_package(Eigen3 REQUIRED)

-    find_package(OpenCV REQUIRED COMPONENTS
-        core
-        imgproc
-        highgui
-        imgcodecs
+    find_package(
+        OpenCV REQUIRED
+        COMPONENTS
+            core
+            imgproc
+            highgui
+            imgcodecs
+            calib3d
    )

    find_package(pinocchio REQUIRED)
+    find_package(fmt REQUIRED)

-    add_library(r1_vision_common INTERFACE)
+    # --------------------------------------------------------
+    # Shared vision dependencies
+    # --------------------------------------------------------
+
+    add_library(
+        r1_vision_common
+        INTERFACE
+    )

    target_include_directories(
        r1_vision_common
        INTERFACE
            "${CMAKE_CURRENT_SOURCE_DIR}/include"
            "${UNITREE_SDK2_DIR}/include"
+            ${OpenCV_INCLUDE_DIRS}
    )

    target_link_libraries(
@@ -75,17 +90,47 @@ if(BUILD_VISION)
            UNITREE_SDK2_DIR="${UNITREE_SDK2_DIR}"
    )

+    # --------------------------------------------------------
+    # Camera frame publisher
+    #
+    # Named r1_vision_node because Unitree SDK2 already
+    # defines its own "vision_node" target.
+    # --------------------------------------------------------
+
    add_executable(
-        vision_node
+        r1_vision_node
        src/vision_node.cc
    )

    target_link_libraries(
-        vision_node
+        r1_vision_node
        PRIVATE
            r1_vision_common
    )

+    # --------------------------------------------------------
+    # Motion controller
+    #
+    # Reads YOLO detections and controls R1 head.
+    # fmt is required by Unitree ArmSdk / spdlog.
+    # --------------------------------------------------------
+
+    add_executable(
+        r1_motion_node
+        src/motion_node.cc
+    )
+
+    target_link_libraries(
+        r1_motion_node
+        PRIVATE
+            r1_vision_common
+            fmt::fmt
+    )
+
+    # --------------------------------------------------------
+    # Hand / camera calibration
+    # --------------------------------------------------------
+
    add_executable(
        r1_hand_calibrate
        src/r1_hand_calibrate.cc
@@ -107,27 +152,20 @@ if(BUILD_VOICE)

    find_package(CURL REQUIRED)

+    # --------------------------------------------------------
+    # Voice assistant
+    # --------------------------------------------------------
+
    add_executable(
        r1_voice
-
        src/voice_node.cc
        src/stt.cc
        src/llm.cc
        src/tts.cc
        src/conversation.cc
+        src/vision_context.cc
    )

-    add_executable(
-    r1_mic_test
-    src/r1_mic_test.cc
-)
-
-target_include_directories(
-    r1_mic_test
-    PRIVATE
-        "${UNITREE_SDK2_DIR}/include"
-)
-
    target_include_directories(
        r1_voice
        PRIVATE
@@ -142,6 +180,28 @@ target_include_directories(
            CURL::libcurl
    )

+    # --------------------------------------------------------
+    # Raw microphone test
+    # --------------------------------------------------------
+
+    add_executable(
+        r1_mic_test
+        src/r1_mic_test.cc
+    )
+
+    target_include_directories(
+        r1_mic_test
+        PRIVATE
+            "${CMAKE_CURRENT_SOURCE_DIR}/include"
+            "${UNITREE_SDK2_DIR}/include"
+    )
+
+    target_link_libraries(
+        r1_mic_test
+        PRIVATE
+            unitree_sdk2
+    )
+
 endif()

 # ============================================================
@@ -151,33 +211,54 @@ endif()
 set(INSTALL_TARGETS "")

 if(BUILD_VISION)
-    list(APPEND INSTALL_TARGETS
-        vision_node
-        r1_hand_calibrate
+
+    list(
+        APPEND INSTALL_TARGETS
+            r1_vision_node
+            r1_motion_node
+            r1_hand_calibrate
    )
+
 endif()

 if(BUILD_VOICE)
-    list(APPEND INSTALL_TARGETS
-        r1_voice
+
+    list(
+        APPEND INSTALL_TARGETS
+            r1_voice
+            r1_mic_test
    )
+
 endif()

-install(
-    TARGETS ${INSTALL_TARGETS}
-    RUNTIME DESTINATION bin
-)
+if(INSTALL_TARGETS)
+
+    install(
+        TARGETS
+            ${INSTALL_TARGETS}
+        RUNTIME DESTINATION bin
+    )
+
+endif()
+
+# ============================================================
+# Install configuration / scripts
+# ============================================================

 if(BUILD_VISION)

    install(
-        DIRECTORY config/
-        DESTINATION share/r1_vision/config
+        DIRECTORY
+            config/
+        DESTINATION
+            share/r1_vision/config
    )

    install(
-        PROGRAMS scripts/yolo_detector.py
-        DESTINATION share/r1_vision/scripts
+        PROGRAMS
+            scripts/yolo_detector.py
+        DESTINATION
+            share/r1_vision/scripts
    )

 endif()