From b9f19dfeed4b827db03029b7a685e6c685fdc67f Mon Sep 17 00:00:00 2001 From: KennardWang Date: Tue, 24 Oct 2023 03:08:41 +0800 Subject: [PATCH] rewrite_csharp_and_add_calibration --- UnityAssets/Script/AnimationTest.cs | 10 ++- UnityAssets/Script/Momose.cs | 102 +++++++++++++++------------- UnityAssets/Script/SocketTest.cs | 20 +++--- main.py | 35 +++++----- sock.py | 18 +++-- utils.py | 33 +++++++++ 6 files changed, 130 insertions(+), 88 deletions(-) diff --git a/UnityAssets/Script/AnimationTest.cs b/UnityAssets/Script/AnimationTest.cs index 2a739f6..688f96f 100644 --- a/UnityAssets/Script/AnimationTest.cs +++ b/UnityAssets/Script/AnimationTest.cs @@ -12,7 +12,7 @@ public class AnimationTest : MonoBehaviour { private CubismModel Momose; private CubismParameter parameter; - private float _t; // time controller + private float _t; // time controller void Start() { @@ -23,12 +23,10 @@ void Start() void Update() { _t += (Time.deltaTime * 4f); - float value = Mathf.Sin(_t) * 30f; // amplitude - parameter = Momose.Parameters[2]; // head angle Z + float value = Mathf.Sin(_t) * 30f; // amplitude + parameter = Momose.Parameters[2]; // head angle Z - parameter.Value = value; // set value + parameter.Value = value; // set value Debug.Log(parameter.Value); - } - } diff --git a/UnityAssets/Script/Momose.cs b/UnityAssets/Script/Momose.cs index ca0fe3e..3a4ece3 100644 --- a/UnityAssets/Script/Momose.cs +++ b/UnityAssets/Script/Momose.cs @@ -1,12 +1,12 @@ using System.Collections; using System.Collections.Generic; -using UnityEngine; - -// Socket -using System; -using System.Text; -using System.Threading; -using System.Net; +using UnityEngine; + +// Socket +using System; +using System.Text; +using System.Threading; +using System.Net; using System.Net.Sockets; // Model @@ -59,8 +59,8 @@ public class Momose : MonoBehaviour // Model parameter, reference: https://docs.live2d.com/cubism-sdk-tutorials/about-parameterupdating-of-model/?locale=ja private CubismModel Model; private CubismParameter parameter; - private float t1; // time controller for breath - private float t2; // time controller for hands + private float t1; // time controller for breath + private float t2; // time controller for hands private float angleX; // head angle private float angleY; // head angle private float angleZ; // head angle @@ -91,13 +91,13 @@ void init() mouthWidth = 1.0f; mouthVar = 0.0f; eyeBallX = eyeBallY = 0.0f; -} + } // Start is called before the first frame update void Start() { - Screen.SetResolution(GUIwidth, GUIheight, false); // set resolution - StartCoroutine("displaySetting"); + //Screen.SetResolution(GUIwidth, GUIheight, false); // set resolution + //StartCoroutine("displaySetting"); init(); } @@ -105,7 +105,6 @@ void Start() // Update is called once per frame void Update() { - // breath t1 += (Time.deltaTime * 3f); float value = Mathf.Sin(t1) * 0.5f + 0.5f; @@ -123,7 +122,7 @@ void Update() // right hand float value3 = Mathf.Sin(t2) * 1.0f; parameter = Model.Parameters[33]; - parameter.Value = value2; + parameter.Value = value3; // update yaw parameter = Model.Parameters[0]; @@ -138,17 +137,17 @@ void Update() parameter.Value = angleZ; // update eyes - parameter = Model.Parameters[4]; // left + parameter = Model.Parameters[4]; // left parameter.Value = eyeLeft; - parameter = Model.Parameters[6]; // right + parameter = Model.Parameters[6]; // right parameter.Value = eyeRight; // update eyeballs - parameter = Model.Parameters[8]; // X axis + parameter = Model.Parameters[8]; // X axis parameter.Value = eyeBallX; - parameter = Model.Parameters[9]; // Y axis + parameter = Model.Parameters[9]; // Y axis parameter.Value = eyeBallY; // update mouth @@ -159,17 +158,19 @@ void Update() parameter.Value = mouthVar; } - void SocketConnect() - { - if (clientSocket != null) { clientSocket.Close(); } - - clientSocket = serverSocket.Accept(); - } - + void SocketConnect() + { + if (clientSocket != null) { clientSocket.Close(); } + + clientSocket = serverSocket.Accept(); + } + void paraUpdate() { string buff = ""; - string[] para; // parameters + string[] para; // parameters + int cnt1 = 0, cnt2 = 0; // counters to estimate time + SocketConnect(); while (true) @@ -184,39 +185,44 @@ void paraUpdate() { continue; } - buff = Encoding.ASCII.GetString(recData, 0, len); // store data in buffer as string type - para = buff.Split(' '); // get 9 parameters + buff = Encoding.ASCII.GetString(recData, 0, len); // store data in buffer as string type + para = buff.Split(' '); // get 11 parameters - angleZ = Convert.ToSingle(para[0]); // roll - angleY = Convert.ToSingle(para[1]); // pitch - angleX = Convert.ToSingle(para[2]); // yaw + angleZ = Convert.ToSingle(para[0]); // roll + angleY = Convert.ToSingle(para[1]); // pitch + angleX = Convert.ToSingle(para[2]); // yaw // eyes open - float open = Convert.ToSingle(para[3]); - if (open < 0.17f) { eyeLeft = eyeRight = 0.0f; } - else if (open >= 0.17f && open <= 0.22f) { eyeLeft = eyeRight = 0.6f; } - else if(open > 0.22f && open <= 0.35f) { eyeLeft = eyeRight = 1.0f; } - else if(open > 0.35f) { eyeLeft = eyeRight = 1.2f; } + eyeLeft = Convert.ToSingle(para[3]); + eyeRight = Convert.ToSingle(para[4]); + + // one eye open, one eye close + float diff = Convert.ToSingle(para[5]); + float thres = Convert.ToSingle(para[6]); - // eyes blink - float var = Convert.ToSingle(para[4]); + if (diff < -thres) { if (cnt1 < 20) cnt1++; } else { if (cnt1 > 0) cnt1--; } + if (diff > thres) { if (cnt2 < 20) cnt2++; } else { if (cnt2 > 0) cnt2--; } + + // if time is enough, assume the pose is kept + if (cnt1 >= 10) { eyeLeft = 0.0f; eyeRight = 1.0f; } // left close, right open + if (cnt2 >= 10) { eyeLeft = 1.0f; eyeRight = 0.0f; } // left open, right close // eye balls - eyeBallX = Convert.ToSingle(para[5]); - eyeBallY = Convert.ToSingle(para[6]); + eyeBallX = Convert.ToSingle(para[7]); + eyeBallY = Convert.ToSingle(para[8]); // mouth - mouthWidth = Convert.ToSingle(para[7]); - mouthVar = Convert.ToSingle(para[8]); - + mouthWidth = Convert.ToSingle(para[9]); + mouthVar = Convert.ToSingle(para[10]); } - } - + } + + /* IEnumerator displaySetting() { yield return new WaitForSeconds(0.1f); // wait for 0.1 second - SetWindowLong(GetForegroundWindow(), STYLE, BORDER); // pop up window + SetWindowLong(GetForegroundWindow(), STYLE, BORDER); // pop up window bool result = SetWindowPos(GetForegroundWindow(), TOP, GUIposX, GUIposY, GUIwidth, GUIheight, SHOWWINDOW); // set position and display at top - } - + } + */ } diff --git a/UnityAssets/Script/SocketTest.cs b/UnityAssets/Script/SocketTest.cs index 8ba0ad4..5c3cf29 100644 --- a/UnityAssets/Script/SocketTest.cs +++ b/UnityAssets/Script/SocketTest.cs @@ -2,10 +2,10 @@ using System.Collections.Generic; using UnityEngine; -using System; -using System.Text; -using System.Threading; -using System.Net; +using System; +using System.Text; +using System.Threading; +using System.Net; using System.Net.Sockets; // Reference: https://blog.csdn.net/u012234115/article/details/46481845 @@ -32,7 +32,6 @@ void init() // start a new thread to update parameters Thread connect = new Thread(new ThreadStart(DataReception)); connect.Start(); - } // Start is called before the first frame update @@ -48,16 +47,15 @@ void Update() } - void SocketConnect() - { + void SocketConnect() + { if (clientSocket != null) { clientSocket.Close(); } clientSocket = serverSocket.Accept(); Debug.Log("Connect"); - } - + } + void DataReception() { - SocketConnect(); while (true) @@ -73,7 +71,7 @@ void DataReception() // change data type to string, then to float string str = Encoding.ASCII.GetString(recData, 0, len); - string[] para = str.Split(' '); // parameters group + string[] para = str.Split(' '); // parameters group float v = Convert.ToSingle(para[5]); Debug.Log(v); diff --git a/main.py b/main.py index 23bd9b2..8b8f369 100644 --- a/main.py +++ b/main.py @@ -68,11 +68,9 @@ def run(): t = time.time() # Loop - # 1. Face detection - # 2. Draw face and iris landmarks - # 3. Pose estimation and stabilization (face + iris) - # 4. Calculate and calibrate message data if low error - # 5. Data transmission with socket + # 1. Face detection, draw face and iris landmarks + # 2. Pose estimation and stabilization (face + iris), calculate and calibrate data if low error + # 3. Data transmission with socket # Face detection on every odd frame if frame_count % 2 == 1: @@ -145,30 +143,35 @@ def run(): steady_pose.append(ps_stb.state[0]) if args.connect: - # Calibrate: pitch(15 is camera angle), eyeballX, eyeballY, mouthWidth + # Calibrate: pitch(15 is camera angle) # Head roll = np.clip( -(180 + np.degrees(steady_pose[2])), -50, 50) pitch = np.clip( - -(np.degrees(steady_pose[1]) + 15), -40, 40) + -(np.degrees(steady_pose[1])), -50, 50) yaw = np.clip(-(np.degrees(steady_pose[0])), -50, 50) # Eyes - eye_left = utils.eye_aspect_ratio(marks[36:42]) - eye_right = utils.eye_aspect_ratio(marks[42:48]) - eye_open = (eye_left + eye_right) / 2 - eye_diff = abs(eye_left - eye_right) - eyeballX = (steady_pose[6] - 0.45) * (-4) - eyeballY = (steady_pose[7] - 0.38) * 2 + LeyeVar = utils.eye_aspect_ratio(marks[36:42]) + ReyeVar = utils.eye_aspect_ratio(marks[42:48]) + meyeVar = (LeyeVar + ReyeVar) / 2 + eyediff = LeyeVar - ReyeVar + eyeballX = steady_pose[6] + eyeballY = steady_pose[7] # Mouth mouthWidth = utils.mouth_distance( - marks[60:68]) / (facebox[2] - facebox[0]) + 0.4 + marks[60:68]) / (facebox[2] - facebox[0]) mouthVar = utils.mouth_aspect_ration(marks[60:68]) + # Calibrate + if not args.gpu: + eyeLeft, eyeRight, diffthres, cali_eyeballX, cali_eyeballY, cali_mouthWidth = utils.calibrate_cpu( + meyeVar, eyeballX, eyeballY, mouthWidth) + # Update - sock.update_all(roll, pitch, yaw, eye_open, eye_diff, - eyeballX, eyeballY, mouthWidth, mouthVar) + sock.update_all(roll, pitch, yaw, eyeLeft, eyeRight, eyediff, diffthres, + cali_eyeballX, cali_eyeballY, cali_mouthWidth, mouthVar) # In debug mode, show the marks if args.debug: diff --git a/sock.py b/sock.py index 48e4f3a..ed6e00b 100644 --- a/sock.py +++ b/sock.py @@ -8,8 +8,10 @@ def __init__(self): self.roll = 0.0 self.pitch = 0.0 self.yaw = 0.0 - self.eye_open = 0.0 - self.eye_diff = 0.0 + self.eyeLeft = 0.0 + self.eyeRight = 0.0 + self.eyediff = 0.0 + self.diffthres = 0.0 # diff threshold self.eyeballX = 0.0 self.eyeballY = 0.0 self.mouthWidth = 0.0 @@ -17,13 +19,15 @@ def __init__(self): self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - def update_all(self, roll, pitch, yaw, eye_open, eye_diff, eyeballX, eyeballY, mouthWidth, mouthVar): + def update_all(self, roll, pitch, yaw, eyeLeft, eyeRight, eyediff, diffthres, eyeballX, eyeballY, mouthWidth, mouthVar): """Update all variables""" self.roll = roll self.pitch = pitch self.yaw = yaw - self.eye_open = eye_open - self.eye_diff = eye_diff + self.eyeLeft = eyeLeft + self.eyeRight = eyeRight + self.eyediff = eyediff + self.diffthres = diffthres self.eyeballX = eyeballX self.eyeballY = eyeballY self.mouthWidth = mouthWidth @@ -31,8 +35,8 @@ def update_all(self, roll, pitch, yaw, eye_open, eye_diff, eyeballX, eyeballY, m def conv2msg(self): """Convert all variables to message data""" - self.msg = '%.4f %.4f %.4f %.4f %.4f %.4f %.4f %.4f %.4f' % \ - (self.roll, self.pitch, self.yaw, self.eye_open, self.eye_diff, + self.msg = '%.4f %.4f %.4f %.4f %.4f %.4f %.4f %.4f %.4f %.4f %.4f' % \ + (self.roll, self.pitch, self.yaw, self.eyeLeft, self.eyeRight, self.eyediff, self.diffthres, self.eyeballX, self.eyeballY, self.mouthWidth, self.mouthVar) def connect(self, addr): diff --git a/utils.py b/utils.py index 97f8ea1..2488e54 100644 --- a/utils.py +++ b/utils.py @@ -134,3 +134,36 @@ def draw_FPS(frame, FPS): """Draw FPS""" cv2.putText(frame, "FPS: %d" % FPS, (40, 40), cv2.FONT_HERSHEY_DUPLEX, 1, (0, 255, 0), 1) + + +def calibrate_cpu(meyeVar, eyeballX, eyeballY, mouthWidth): + """Calibrate some variables in CPU env""" + + # Calibrate mean eye variance + if meyeVar <= 0.11: + eyeLeft = eyeRight = 0.0 + elif meyeVar > 0.11 and meyeVar <= 0.13: + eyeLeft = eyeRight = 50.0 * meyeVar - 5.5 + elif meyeVar > 0.13 and meyeVar <= 0.17: + eyeLeft = eyeRight = 1.0 + elif meyeVar > 0.17 and meyeVar <= 0.22: + eyeLeft = eyeRight = 4.0 * meyeVar + 0.32 + else: + eyeLeft = eyeRight = 1.2 + + # Calibrate eye diff threshold + diffthres = 0.04 + + # Calibrate eyeballs + cali_eyeballX = (eyeballX - 0.45) * (-4.0) + cali_eyeballY = (eyeballY - 0.38) * 2.0 + + # Calibrate mouth width + if mouthWidth <= 0.27: + cali_mouthWidth = -0.5 + elif mouthWidth > 0.27 and mouthWidth <= 0.35: + cali_mouthWidth = 18.75 * mouthWidth - 5.5625 + else: + cali_mouthWidth = 1.0 + + return eyeLeft, eyeRight, diffthres, cali_eyeballX, cali_eyeballY, cali_mouthWidth