Feature/stackchan display

.github/workflows/ci-platformio.yml

@@ -7,21 +7,21 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v6
       - name: Cache pip
-        uses: actions/cache@v2
+        uses: actions/cache@v6
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
           restore-keys: |
             ${{ runner.os }}-pip-
       - name: Cache PlatformIO
-        uses: actions/cache@v2
+        uses: actions/cache@v6
         with:
           path: ~/.platformio
           key: ${{ runner.os }}-${{ hashFiles('**/lockfiles') }}
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v6
       - name: Install PlatformIO
         run: |
           python -m pip install --upgrade pip

include/BLEStackchanService.hpp

@@ -4,12 +4,17 @@
 #define BLE_STACKCHAN_SERVICE_HPP
 
 #include <ArduinoBLE.h>
-#include <Avatar.h>
 
 #include "Animation.hpp"
 #include "BLEFormat.hpp"
 #include "BLEUnit.hpp"
 
+using stackchan::display::ColorPalette;
+using stackchan::display::Display;
+using stackchan::display::Expression;
+using stackchan::display::ExpressionWeight;
+using stackchan::display::Face;
+
 namespace color {
 u_int16_t to16bitscolor(u_int32_t color) {
   // https://docs.arduino.cc/library-examples/tft-library/TFTColorPicker
@@ -95,22 +100,19 @@ class StackchanService : public BLEService {
   // ~StackchanService();
   void setInitialValues();
 
-  void servoPoll(botamochi::AnimationController &controller);
+  void servoPoll(botamochi::AnimationController& controller);
 
-  void animationPoll(botamochi::AnimationController &controller);
+  void animationPoll(botamochi::AnimationController& controller);
 
-  void facePoll(m5avatar::Avatar &avatar, m5avatar::Face *faces[],
-                uint8_t face_size);
+  void facePoll(Display& display, Face* faces[], uint8_t face_size);
 
-  void facialExpressionPoll(m5avatar::Avatar &avatar,
-                            const m5avatar::Expression expressions[],
+  void facialExpressionPoll(Display& display, const Expression expressions[],
                             uint8_t expression_size);
 
-  void facialColorPoll(m5avatar::Avatar &avatar,
-                       m5avatar::ColorPalette *palettes[],
+  void facialColorPoll(Display& display, ColorPalette* palettes[],
                        uint8_t palette_size);
 
-  void mouseOpenPoll(m5avatar::Avatar &avatar);
+  void mouseOpenPoll(Display& display);
 };
 
 StackchanService::StackchanService()
@@ -210,7 +212,7 @@ void StackchanService::setInitialValues() {
   // this->background_color_chr.writeValueLE(0x000000);
 };
 
-void StackchanService::servoPoll(botamochi::AnimationController &controller) {
+void StackchanService::servoPoll(botamochi::AnimationController& controller) {
   unsigned short pos1 = controller.servo_driver.getCurrentPosition(
       controller.joint_servo_map.get(botamochi::JointName::kHeadPan));
   this->servo_pan_angle_chr.writeValue(pos1);
@@ -220,67 +222,65 @@ void StackchanService::servoPoll(botamochi::AnimationController &controller) {
 }
 
 void StackchanService::animationPoll(
-    botamochi::AnimationController &controller) {
+    botamochi::AnimationController& controller) {
   if (this->animation_clip_chr.written()) {
     auto idx = this->animation_clip_chr.value();
     controller.play(idx);
   }
 }
 
-void StackchanService::facePoll(m5avatar::Avatar &avatar,
-                                m5avatar::Face *faces[], uint8_t face_size) {
+void StackchanService::facePoll(Display& display, Face* faces[],
+                                uint8_t face_size) {
   if (this->face_chr.written()) {
     auto idx = this->face_chr.value();
     if (face_size <= idx) {
       return;
     }
 
-    avatar.setFace(faces[idx]);
+    display.setFace(faces[idx]);
   }
 }
 
-void StackchanService::facialExpressionPoll(
-    m5avatar::Avatar &avatar, const m5avatar::Expression expressions[],
-    uint8_t expression_size) {
-  if (this->facial_expression_chr.written()) {
-    auto idx = this->facial_expression_chr.value();
-    if (expression_size <= idx) {
-      return;
-    }
+void StackchanService::facialExpressionPoll(Display& display,
+                                            const Expression expressions[],
+                                            uint8_t expression_size) {
+  if (!this->facial_expression_chr.written()) {
+    return;
+  }
 
-    avatar.setExpression(expressions[idx]);
+  auto idx = this->facial_expression_chr.value();
+  if (expression_size <= idx) {
+    return;
   }
+
+  display.getExpressionWeight().setEmotionalExpression(
+      static_cast<stackchan::display::Expression>(
+          idx % (static_cast<int>(stackchan::display::Expression::kRelax) + 1)),
+      255);
 }
 
-void StackchanService::facialColorPoll(m5avatar::Avatar &avatar,
-                                       m5avatar::ColorPalette *palettes[],
+void StackchanService::facialColorPoll(Display& display,
+                                       ColorPalette* palettes[],
                                        uint8_t palette_size) {
-  if (this->facial_color_chr.written()) {
-    auto idx = this->facial_color_chr.value();
-    if (palette_size <= idx) {
-      return;  // out of index
-    }
-    avatar.setColorPalette(*palettes[idx]);
+  if (!this->facial_color_chr.written()) {
+    return;
   }
 
-  // if (this->primary_color_chr.written() ||
-  //     this->background_color_chr.written()) {
-  //   auto idx = this->facial_color_chr.value();
-  //   // convert 24 bit color to 16bit color
-  //   palettes[idx]->set(COLOR_PRIMARY,
-  //                      color::to16bitscolor(this->primary_color_chr.valueLE()));
-  //   palettes[idx]->set(
-  //       COLOR_BACKGROUND,
-  //       color::to16bitscolor(this->background_color_chr.valueLE()));
-  //   avatar.setColorPalette(*palettes[idx]);
-  // }
+  // processes on characteristic-written
+  auto idx = this->facial_color_chr.value();
+  if (palette_size <= idx) {
+    return;  // out of index
+  }
+  display.setColorPalette(palettes[idx]);
 }
 
-void StackchanService::mouseOpenPoll(m5avatar::Avatar &avatar) {
-  if (this->mouse_open_ratio_chr.written()) {
-    avatar.setMouthOpenRatio(
-        static_cast<float>(this->mouse_open_ratio_chr.value()) / 255U);
+void StackchanService::mouseOpenPoll(Display& display) {
+  if (!this->mouse_open_ratio_chr.written()) {
+    return;
   }
+
+  display.getExpressionWeight().set(Expression::kAa,
+                                    this->mouse_open_ratio_chr.value());
 }
 
 }  // namespace ble

include/PaletteExamples.hpp

@@ -2,7 +2,7 @@
 #define PALETTE_EXAMPLES_HPP
 #include <ColorPalette.h>
 
-namespace m5avatar {
+namespace stackchan::display {
 
 void assignUiPalette(ColorPalette* palette) {
   palette->set(DrawingLocation::kIris1,
@@ -20,7 +20,7 @@ void assignUiPalette(ColorPalette* palette) {
                M5.Lcd.color24to16(0xfac2a8));  // skin
   palette->set(DrawingLocation::kEyeBrow, M5.Lcd.color24to16(0x321F39));
   palette->set(DrawingLocation::kMouthBackground, TFT_BLACK);
-  palette->set(DrawingLocation::kInnerMouse, TFT_RED);
+  palette->set(DrawingLocation::kInnerMouth, TFT_RED);
   palette->set(DrawingLocation::kCheek1,
                TFT_PINK);  // cheek
 }
@@ -33,7 +33,7 @@ void assignFbkPalette(ColorPalette* palette) {
   palette->set(DrawingLocation::kCheek1,
                M5.Lcd.color24to16(0xea9c60));  // cheek, #ea9c60
   palette->set(DrawingLocation::kMouthBackground, TFT_BLACK);
-  palette->set(DrawingLocation::kInnerMouse, TFT_RED);
+  palette->set(DrawingLocation::kInnerMouth, TFT_RED);
 }
 
 void assignLeonaPalette(ColorPalette* palette) {
@@ -53,7 +53,7 @@ void assignLeonaPalette(ColorPalette* palette) {
   palette->set(DrawingLocation::kEyeBrow,
                M5.Lcd.color24to16(0x63434F));  // #63434F
   palette->set(DrawingLocation::kMouthBackground, M5.Lcd.color24to16(0x63434F));
-  palette->set(DrawingLocation::kInnerMouse,
+  palette->set(DrawingLocation::kInnerMouth,
                M5.Lcd.color24to16(0x8B0035));  // #8B0035
   palette->set(DrawingLocation::kCheek1,
                M5.Lcd.color24to16(0xF3B5CC));  // cheek #F3B5CC
@@ -76,7 +76,7 @@ void assignNaorePalette(ColorPalette* palette) {
   palette->set(DrawingLocation::kEyeBrow,
                M5.Lcd.color24to16(0x63434F));  // #63434F
   palette->set(DrawingLocation::kMouthBackground, M5.Lcd.color24to16(0x63434F));
-  palette->set(DrawingLocation::kInnerMouse,
+  palette->set(DrawingLocation::kInnerMouth,
                M5.Lcd.color24to16(0x8B0035));  // #8B0035
   palette->set(DrawingLocation::kCheek1,
                M5.Lcd.color24to16(0xd77398));  // cheek #d77398
@@ -95,7 +95,7 @@ void assignPinkDemonPalette(ColorPalette* palette) {
   palette->set(DrawingLocation::kCheek1,
                M5.Lcd.color24to16(0xEB6896));  // cheek, #EB6896
   palette->set(DrawingLocation::kMouthBackground, TFT_BLACK);
-  palette->set(DrawingLocation::kInnerMouse, TFT_RED);
+  palette->set(DrawingLocation::kInnerMouth, TFT_RED);
 }
-}  // namespace m5avatar
+}  // namespace stackchan::display
 #endif  // PALETTE_EXAMPLES_HPP

platformio.ini

@@ -14,9 +14,9 @@ default_envs = m5stack-core1
 monitor_speed = 115200
 lib_deps_external = 
 	arduino-libraries/ArduinoBLE@^1.3.5
-  m5stack/M5Unified@^0.1.11
+  m5stack/M5Unified@^0.2.14
   hideakitai/TaskManager@^0.5.2
-	https://github.com/botamochi6277/m5stack-avatar.git#3764520
+	https://github.com/botamochi6277/stackchan-display.git#0.3.0
   https://github.com/botamochi6277/STS_servos.git#esp32s3
 
 [env:m5stack-core1]

policy.md

@@ -0,0 +1,95 @@
+# StackChan policy
+
+Draft for policy models
+
+```mermaid
+---
+config:
+  theme: 'redux-dark'
+title: 'StackChan Policy'
+---
+graph LR
+
+%% sensor inputs
+
+subgraph sensors
+    camera[fas:fa-camera camera]
+    touch[head touch sensor]
+    head_rot[head rotation sensor]
+    mic[microphone]
+    bat[battery sensor]
+end 
+
+subgraph observations
+    cam_image([image])
+    tactile([tactile])
+    joint_pos([pan tilt])
+    sound([sound])
+    bat_lv([battery voltages])
+end
+camera-->cam_image
+touch-->tactile
+head_rot-->joint_pos
+mic-->sound
+bat-->bat_lv
+%% comm inputs
+
+
+subgraph interceptions
+    esp_now[esp now]
+    ble[bluetooth]
+    usb[usb]
+    wifi[WiFi]
+end
+%% extra
+
+llm[large language model]-.->wifi
+interceptions-.->observations
+
+%% outputs
+
+
+subgraph display[display control]
+    face([face])
+    speech([speech bubble])
+    decorators([decorators])
+end
+
+display-->display_lib["display lib<br/>(M5Unified)"]
+
+%%mot-->servo_lib[servo lib]
+
+%% head neopixels on M5StackChan Head
+%%head_led[head LED control]
+%%head_led-->led_lib[LED lib]
+
+
+
+
+
+subgraph actions
+    led_colors([LED colors])
+    head_act_rot([head rotation])
+    facial_expression([facial expression])
+    voice([voice])
+end
+
+subgraph executors
+    leds
+    servos
+    speaker
+    display
+end
+
+led_colors-->leds
+head_act_rot-->servos
+voice-->speaker
+facial_expression-->display
+
+policy((policy))
+observations-->policy-->actions
+
+```
+
+How to train the policy?
+To use imitation learning, reference motions are required. Someone should be stackchan during a few days