wired up voice and vision, testing motion

2026-08-28 11:26:40 +02:00
parent 8d88495f6f
commit 3982c6fb6b
84 changed files with 7696 additions and 8984 deletions
@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+set -e
+
+PROJECT="$HOME/esarobotech/r1"
+INTERFACE="${1:-enp3s0}"
+
+cd "$PROJECT"
+
+PIDS=()
+
+cleanup() {
+    echo
+    echo "[R1] Stopping all services..."
+
+    for pid in "${PIDS[@]}"; do
+        if kill -0 "$pid" 2>/dev/null; then
+            kill "$pid" 2>/dev/null || true
+        fi
+    done
+
+    wait 2>/dev/null || true
+
+    echo "[R1] All services stopped."
+}
+
+trap cleanup EXIT INT TERM
+
+echo "======================================"
+echo " R1 complete stack"
+echo "======================================"
+echo "Interface: $INTERFACE"
+echo
+
+echo "[R1] Starting camera..."
+./scripts/start_camera.sh "$INTERFACE" &
+PIDS+=("$!")
+
+sleep 1
+
+echo "[R1] Starting YOLO..."
+./scripts/start_yolo.sh &
+PIDS+=("$!")
+
+sleep 1
+
+echo "[R1] Starting LLM..."
+./scripts/start_llm.sh &
+PIDS+=("$!")
+
+echo "[R1] Waiting for LLM server..."
+
+for i in {1..60}; do
+    if curl -fsS \
+        http://127.0.0.1:8080/health \
+        >/dev/null 2>&1; then
+
+        echo "[R1] LLM ready."
+        break
+    fi
+
+    sleep 1
+
+    if [[ "$i" -eq 60 ]]; then
+        echo "[ERROR] LLM failed to become ready."
+        exit 1
+    fi
+done
+
+echo "[R1] Waiting for camera frame..."
+
+for i in {1..20}; do
+    if [[ -s /dev/shm/robot_frame.jpg ]]; then
+        echo "[R1] Camera ready."
+        break
+    fi
+
+    sleep 0.5
+
+    if [[ "$i" -eq 20 ]]; then
+        echo "[WARN] Camera frame not available yet."
+    fi
+done
+
+echo "[R1] Starting voice assistant..."
+./scripts/start_voice.sh "$INTERFACE" &
+PIDS+=("$!")
+
+echo "[R1] Starting motion..."
+./scripts/start_motion.sh "$INTERFACE" &
+PIDS+=("$!")
+
+sleep 1
+
+echo
+echo "======================================"
+echo " R1 stack running"
+echo "======================================"
+echo "Camera: /dev/shm/robot_frame.jpg"
+echo "YOLO:   /dev/shm/r1_detections.json"
+echo "LLM:    http://127.0.0.1:8080"
+echo
+echo "Press Ctrl+C to stop everything."
+echo
+
+# Wait until one service exits.
+wait -n "${PIDS[@]}"
+
+echo "[R1] A service exited; shutting down the stack."
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -e
+
+PROJECT="$HOME/esarobotech/r1"
+INTERFACE="${1:-enp3s0}"
+
+cd "$PROJECT"
+
+echo "[R1] Starting camera on $INTERFACE"
+
+exec ./build/r1_vision_node "$INTERFACE"
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -e
+
+LLAMA="$HOME/esarobotech/llama.cpp"
+
+cd "$LLAMA"
+
+echo "[R1] Starting Qwen LLM on port 8080"
+
+exec ./build/bin/llama-server \
+    -hf Mungert/Qwen3-8B-abliterated-GGUF:Q4_K_M \
+    -c 8192 \
+    --host 127.0.0.1 \
+    --port 8080
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -e
+
+PROJECT="$HOME/esarobotech/r1"
+CONDA="$HOME/miniconda3"
+
+INTERFACE="${1:-enp3s0}"
+
+source "$CONDA/etc/profile.d/conda.sh"
+conda activate ai_robot
+
+cd "$PROJECT"
+
+echo "[R1] Starting motion controller"
+echo "[R1] Mode: person head tracking"
+echo "[R1] Interface: $INTERFACE"
+
+exec ./build/r1_motion_node \
+    "$INTERFACE"
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+set -e
+
+PROJECT="$HOME/esarobotech/r1"
+CONDA="$HOME/miniconda3"
+
+INTERFACE="${1:-enp3s0}"
+
+WHISPER="$HOME/esarobotech/whisper.cpp/build/bin/whisper-cli"
+MODEL="$HOME/esarobotech/whisper.cpp/models/ggml-medium.bin"
+LLM_URL="http://127.0.0.1:8080/v1/chat/completions"
+
+source "$CONDA/etc/profile.d/conda.sh"
+conda activate ai_robot
+
+cd "$PROJECT"
+
+echo "[R1] Starting voice assistant"
+echo "[R1] Interface: $INTERFACE"
+echo "[R1] Whisper:   $MODEL"
+echo "[R1] LLM:       $LLM_URL"
+
+exec ./build/r1_voice \
+    "$INTERFACE" \
+    "$WHISPER" \
+    "$MODEL" \
+    "$LLM_URL"
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+set -e
+
+PROJECT="$HOME/esarobotech/r1"
+CONDA="$HOME/miniconda3"
+
+source "$CONDA/etc/profile.d/conda.sh"
+conda activate ai_robot
+
+cd "$PROJECT"
+
+echo "[R1] Starting YOLO detector"
+echo "[R1] Input:  /dev/shm/robot_frame.jpg"
+echo "[R1] Output: /dev/shm/r1_detections.json"
+
+exec python scripts/yolo_detector.py
@@ -0,0 +1,149 @@
+#!/usr/bin/env python3
+
+import json
+import os
+import tempfile
+import time
+from pathlib import Path
+
+from ultralytics import YOLO
+
+
+IMAGE_PATH = Path("/dev/shm/robot_frame.jpg")
+OUTPUT_PATH = Path("/dev/shm/r1_detections.json")
+
+MODEL = "yolo26n.pt"
+
+CONFIDENCE = 0.35
+IMAGE_SIZE = 640
+
+
+def atomic_write_json(path: Path, data):
+    with tempfile.NamedTemporaryFile(
+        mode="w",
+        dir=path.parent,
+        delete=False,
+        suffix=".tmp",
+    ) as f:
+        json.dump(
+            data,
+            f,
+            ensure_ascii=False,
+            indent=2,
+        )
+
+        tmp = f.name
+
+    os.replace(tmp, path)
+
+
+def main():
+    print(f"[YOLO] Loading {MODEL}")
+
+    model = YOLO(MODEL)
+
+    print("[YOLO] Ready")
+    print(f"[YOLO] Camera: {IMAGE_PATH}")
+    print(f"[YOLO] Output: {OUTPUT_PATH}")
+
+    last_mtime = None
+
+    while True:
+        try:
+            if not IMAGE_PATH.exists():
+                time.sleep(0.05)
+                continue
+
+            mtime = IMAGE_PATH.stat().st_mtime_ns
+
+            if mtime == last_mtime:
+                time.sleep(0.01)
+                continue
+
+            last_mtime = mtime
+
+            results = model.predict(
+                source=str(IMAGE_PATH),
+                conf=CONFIDENCE,
+                imgsz=IMAGE_SIZE,
+                verbose=False,
+            )
+
+            if not results:
+                continue
+
+            result = results[0]
+
+            height, width = result.orig_shape
+
+            detections = []
+
+            if result.boxes is not None:
+                for box in result.boxes:
+                    class_id = int(box.cls[0])
+                    confidence = float(box.conf[0])
+
+                    x1, y1, x2, y2 = [
+                        float(v)
+                        for v in box.xyxy[0]
+                    ]
+
+                    cx = (x1 + x2) / 2.0
+                    cy = (y1 + y2) / 2.0
+
+                    detections.append({
+                        "class_id": class_id,
+                        "class": result.names[class_id],
+                        "confidence": confidence,
+
+                        "box": {
+                            "x1": x1,
+                            "y1": y1,
+                            "x2": x2,
+                            "y2": y2,
+                        },
+
+                        "center": {
+                            "x": cx,
+                            "y": cy,
+                        },
+
+                        "normalized_center": {
+                            "x": cx / width,
+                            "y": cy / height,
+                        },
+                    })
+
+            data = {
+                "timestamp": time.time(),
+                "width": width,
+                "height": height,
+                "detections": detections,
+            }
+
+            atomic_write_json(
+                OUTPUT_PATH,
+                data,
+            )
+
+            objects = ", ".join(
+                f"{d['class']} {d['confidence']:.2f}"
+                for d in detections
+            )
+
+            if objects:
+                print(f"[YOLO] {objects}")
+            else:
+                print("[YOLO] no objects")
+
+        except KeyboardInterrupt:
+            print("\n[YOLO] stopped")
+            break
+
+        except Exception as e:
+            print(f"[YOLO] error: {e}")
+            time.sleep(0.25)
+
+
+if __name__ == "__main__":
+    main()