Skip to content

Commit

Permalink
rewrite_csharp_and_add_calibration
Browse files Browse the repository at this point in the history
  • Loading branch information
KennardWang committed Oct 23, 2023
1 parent 2991fcc commit b9f19df
Show file tree
Hide file tree
Showing 6 changed files with 130 additions and 88 deletions.
10 changes: 4 additions & 6 deletions UnityAssets/Script/AnimationTest.cs
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ public class AnimationTest : MonoBehaviour
{
private CubismModel Momose;
private CubismParameter parameter;
private float _t; // time controller
private float _t; // time controller

void Start()
{
Expand All @@ -23,12 +23,10 @@ void Start()
void Update()
{
_t += (Time.deltaTime * 4f);
float value = Mathf.Sin(_t) * 30f; // amplitude
parameter = Momose.Parameters[2]; // head angle Z
float value = Mathf.Sin(_t) * 30f; // amplitude
parameter = Momose.Parameters[2]; // head angle Z

parameter.Value = value; // set value
parameter.Value = value; // set value
Debug.Log(parameter.Value);

}

}
102 changes: 54 additions & 48 deletions UnityAssets/Script/Momose.cs
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
using System.Collections;
using System.Collections.Generic;
using UnityEngine;

// Socket
using System;
using System.Text;
using System.Threading;
using System.Net;
using UnityEngine;

// Socket
using System;
using System.Text;
using System.Threading;
using System.Net;
using System.Net.Sockets;

// Model
Expand Down Expand Up @@ -59,8 +59,8 @@ public class Momose : MonoBehaviour
// Model parameter, reference: https://docs.live2d.com/cubism-sdk-tutorials/about-parameterupdating-of-model/?locale=ja
private CubismModel Model;
private CubismParameter parameter;
private float t1; // time controller for breath
private float t2; // time controller for hands
private float t1; // time controller for breath
private float t2; // time controller for hands
private float angleX; // head angle
private float angleY; // head angle
private float angleZ; // head angle
Expand Down Expand Up @@ -91,21 +91,20 @@ void init()
mouthWidth = 1.0f;
mouthVar = 0.0f;
eyeBallX = eyeBallY = 0.0f;
}
}

// Start is called before the first frame update
void Start()
{
Screen.SetResolution(GUIwidth, GUIheight, false); // set resolution
StartCoroutine("displaySetting");
//Screen.SetResolution(GUIwidth, GUIheight, false); // set resolution
//StartCoroutine("displaySetting");

init();
}

// Update is called once per frame
void Update()
{

// breath
t1 += (Time.deltaTime * 3f);
float value = Mathf.Sin(t1) * 0.5f + 0.5f;
Expand All @@ -123,7 +122,7 @@ void Update()
// right hand
float value3 = Mathf.Sin(t2) * 1.0f;
parameter = Model.Parameters[33];
parameter.Value = value2;
parameter.Value = value3;

// update yaw
parameter = Model.Parameters[0];
Expand All @@ -138,17 +137,17 @@ void Update()
parameter.Value = angleZ;

// update eyes
parameter = Model.Parameters[4]; // left
parameter = Model.Parameters[4]; // left
parameter.Value = eyeLeft;

parameter = Model.Parameters[6]; // right
parameter = Model.Parameters[6]; // right
parameter.Value = eyeRight;

// update eyeballs
parameter = Model.Parameters[8]; // X axis
parameter = Model.Parameters[8]; // X axis
parameter.Value = eyeBallX;

parameter = Model.Parameters[9]; // Y axis
parameter = Model.Parameters[9]; // Y axis
parameter.Value = eyeBallY;

// update mouth
Expand All @@ -159,17 +158,19 @@ void Update()
parameter.Value = mouthVar;
}

void SocketConnect()
{
if (clientSocket != null) { clientSocket.Close(); }

clientSocket = serverSocket.Accept();
}

void SocketConnect()
{
if (clientSocket != null) { clientSocket.Close(); }

clientSocket = serverSocket.Accept();
}

void paraUpdate() {

string buff = "";
string[] para; // parameters
string[] para; // parameters
int cnt1 = 0, cnt2 = 0; // counters to estimate time

SocketConnect();

while (true)
Expand All @@ -184,39 +185,44 @@ void paraUpdate() {
continue;
}

buff = Encoding.ASCII.GetString(recData, 0, len); // store data in buffer as string type
para = buff.Split(' '); // get 9 parameters
buff = Encoding.ASCII.GetString(recData, 0, len); // store data in buffer as string type
para = buff.Split(' '); // get 11 parameters

angleZ = Convert.ToSingle(para[0]); // roll
angleY = Convert.ToSingle(para[1]); // pitch
angleX = Convert.ToSingle(para[2]); // yaw
angleZ = Convert.ToSingle(para[0]); // roll
angleY = Convert.ToSingle(para[1]); // pitch
angleX = Convert.ToSingle(para[2]); // yaw

// eyes open
float open = Convert.ToSingle(para[3]);
if (open < 0.17f) { eyeLeft = eyeRight = 0.0f; }
else if (open >= 0.17f && open <= 0.22f) { eyeLeft = eyeRight = 0.6f; }
else if(open > 0.22f && open <= 0.35f) { eyeLeft = eyeRight = 1.0f; }
else if(open > 0.35f) { eyeLeft = eyeRight = 1.2f; }
eyeLeft = Convert.ToSingle(para[3]);
eyeRight = Convert.ToSingle(para[4]);

// one eye open, one eye close
float diff = Convert.ToSingle(para[5]);
float thres = Convert.ToSingle(para[6]);

// eyes blink
float var = Convert.ToSingle(para[4]);
if (diff < -thres) { if (cnt1 < 20) cnt1++; } else { if (cnt1 > 0) cnt1--; }
if (diff > thres) { if (cnt2 < 20) cnt2++; } else { if (cnt2 > 0) cnt2--; }

// if time is enough, assume the pose is kept
if (cnt1 >= 10) { eyeLeft = 0.0f; eyeRight = 1.0f; } // left close, right open
if (cnt2 >= 10) { eyeLeft = 1.0f; eyeRight = 0.0f; } // left open, right close

// eye balls
eyeBallX = Convert.ToSingle(para[5]);
eyeBallY = Convert.ToSingle(para[6]);
eyeBallX = Convert.ToSingle(para[7]);
eyeBallY = Convert.ToSingle(para[8]);

// mouth
mouthWidth = Convert.ToSingle(para[7]);
mouthVar = Convert.ToSingle(para[8]);

mouthWidth = Convert.ToSingle(para[9]);
mouthVar = Convert.ToSingle(para[10]);
}
}

}

/*
IEnumerator displaySetting()
{
yield return new WaitForSeconds(0.1f); // wait for 0.1 second
SetWindowLong(GetForegroundWindow(), STYLE, BORDER); // pop up window
SetWindowLong(GetForegroundWindow(), STYLE, BORDER); // pop up window
bool result = SetWindowPos(GetForegroundWindow(), TOP, GUIposX, GUIposY, GUIwidth, GUIheight, SHOWWINDOW); // set position and display at top
}

}
*/
}
20 changes: 9 additions & 11 deletions UnityAssets/Script/SocketTest.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,10 @@
using System.Collections.Generic;
using UnityEngine;

using System;
using System.Text;
using System.Threading;
using System.Net;
using System;
using System.Text;
using System.Threading;
using System.Net;
using System.Net.Sockets;

// Reference: https://blog.csdn.net/u012234115/article/details/46481845
Expand All @@ -32,7 +32,6 @@ void init()
// start a new thread to update parameters
Thread connect = new Thread(new ThreadStart(DataReception));
connect.Start();

}

// Start is called before the first frame update
Expand All @@ -48,16 +47,15 @@ void Update()

}

void SocketConnect()
{
void SocketConnect()
{
if (clientSocket != null) { clientSocket.Close(); }
clientSocket = serverSocket.Accept();
Debug.Log("Connect");
}

}

void DataReception()
{

SocketConnect();

while (true)
Expand All @@ -73,7 +71,7 @@ void DataReception()

// change data type to string, then to float
string str = Encoding.ASCII.GetString(recData, 0, len);
string[] para = str.Split(' '); // parameters group
string[] para = str.Split(' '); // parameters group

float v = Convert.ToSingle(para[5]);
Debug.Log(v);
Expand Down
35 changes: 19 additions & 16 deletions main.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,11 +68,9 @@ def run():
t = time.time()

# Loop
# 1. Face detection
# 2. Draw face and iris landmarks
# 3. Pose estimation and stabilization (face + iris)
# 4. Calculate and calibrate message data if low error
# 5. Data transmission with socket
# 1. Face detection, draw face and iris landmarks
# 2. Pose estimation and stabilization (face + iris), calculate and calibrate data if low error
# 3. Data transmission with socket

# Face detection on every odd frame
if frame_count % 2 == 1:
Expand Down Expand Up @@ -145,30 +143,35 @@ def run():
steady_pose.append(ps_stb.state[0])

if args.connect:
# Calibrate: pitch(15 is camera angle), eyeballX, eyeballY, mouthWidth
# Calibrate: pitch(15 is camera angle)
# Head
roll = np.clip(
-(180 + np.degrees(steady_pose[2])), -50, 50)
pitch = np.clip(
-(np.degrees(steady_pose[1]) + 15), -40, 40)
-(np.degrees(steady_pose[1])), -50, 50)
yaw = np.clip(-(np.degrees(steady_pose[0])), -50, 50)

# Eyes
eye_left = utils.eye_aspect_ratio(marks[36:42])
eye_right = utils.eye_aspect_ratio(marks[42:48])
eye_open = (eye_left + eye_right) / 2
eye_diff = abs(eye_left - eye_right)
eyeballX = (steady_pose[6] - 0.45) * (-4)
eyeballY = (steady_pose[7] - 0.38) * 2
LeyeVar = utils.eye_aspect_ratio(marks[36:42])
ReyeVar = utils.eye_aspect_ratio(marks[42:48])
meyeVar = (LeyeVar + ReyeVar) / 2
eyediff = LeyeVar - ReyeVar
eyeballX = steady_pose[6]
eyeballY = steady_pose[7]

# Mouth
mouthWidth = utils.mouth_distance(
marks[60:68]) / (facebox[2] - facebox[0]) + 0.4
marks[60:68]) / (facebox[2] - facebox[0])
mouthVar = utils.mouth_aspect_ration(marks[60:68])

# Calibrate
if not args.gpu:
eyeLeft, eyeRight, diffthres, cali_eyeballX, cali_eyeballY, cali_mouthWidth = utils.calibrate_cpu(
meyeVar, eyeballX, eyeballY, mouthWidth)

# Update
sock.update_all(roll, pitch, yaw, eye_open, eye_diff,
eyeballX, eyeballY, mouthWidth, mouthVar)
sock.update_all(roll, pitch, yaw, eyeLeft, eyeRight, eyediff, diffthres,
cali_eyeballX, cali_eyeballY, cali_mouthWidth, mouthVar)

# In debug mode, show the marks
if args.debug:
Expand Down
18 changes: 11 additions & 7 deletions sock.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,31 +8,35 @@ def __init__(self):
self.roll = 0.0
self.pitch = 0.0
self.yaw = 0.0
self.eye_open = 0.0
self.eye_diff = 0.0
self.eyeLeft = 0.0
self.eyeRight = 0.0
self.eyediff = 0.0
self.diffthres = 0.0 # diff threshold
self.eyeballX = 0.0
self.eyeballY = 0.0
self.mouthWidth = 0.0
self.mouthVar = 0.0

self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

def update_all(self, roll, pitch, yaw, eye_open, eye_diff, eyeballX, eyeballY, mouthWidth, mouthVar):
def update_all(self, roll, pitch, yaw, eyeLeft, eyeRight, eyediff, diffthres, eyeballX, eyeballY, mouthWidth, mouthVar):
"""Update all variables"""
self.roll = roll
self.pitch = pitch
self.yaw = yaw
self.eye_open = eye_open
self.eye_diff = eye_diff
self.eyeLeft = eyeLeft
self.eyeRight = eyeRight
self.eyediff = eyediff
self.diffthres = diffthres
self.eyeballX = eyeballX
self.eyeballY = eyeballY
self.mouthWidth = mouthWidth
self.mouthVar = mouthVar

def conv2msg(self):
"""Convert all variables to message data"""
self.msg = '%.4f %.4f %.4f %.4f %.4f %.4f %.4f %.4f %.4f' % \
(self.roll, self.pitch, self.yaw, self.eye_open, self.eye_diff,
self.msg = '%.4f %.4f %.4f %.4f %.4f %.4f %.4f %.4f %.4f %.4f %.4f' % \
(self.roll, self.pitch, self.yaw, self.eyeLeft, self.eyeRight, self.eyediff, self.diffthres,
self.eyeballX, self.eyeballY, self.mouthWidth, self.mouthVar)

def connect(self, addr):
Expand Down
Loading

0 comments on commit b9f19df

Please sign in to comment.