• Products
  • Solutions
  • Made with Unity
  • Learning
  • Support & Services
  • Community
  • Asset Store
  • Get Unity

UNITY ACCOUNT

You need a Unity Account to shop in the Online and Asset Stores, participate in the Unity Community and manage your license portfolio. Login Create account
  • Blog
  • Forums
  • Answers
  • Evangelists
  • User Groups
  • Beta Program
  • Advisory Panel

Navigation

  • Home
  • Products
  • Solutions
  • Made with Unity
  • Learning
  • Support & Services
  • Community
    • Blog
    • Forums
    • Answers
    • Evangelists
    • User Groups
    • Beta Program
    • Advisory Panel

Unity account

You need a Unity Account to shop in the Online and Asset Stores, participate in the Unity Community and manage your license portfolio. Login Create account

Language

  • Chinese
  • Spanish
  • Japanese
  • Korean
  • Portuguese
  • Ask a question
  • Spaces
    • Default
    • Help Room
    • META
    • Moderators
    • Topics
    • Questions
    • Users
    • Badges
  • Home /
avatar image
0
Question by unity_bcKPWdrKWohHiw · Oct 12, 2017 at 12:16 PM · unity 5detectautomaticvoicelipsync

Automatic lip syncing detect human voice and set voice frequency ?

I'm new in Unity 3d, And developing one App. In that there is features of "automatic" lip syncing.

I'm following below tutorial

http://answers.unity3d.com/questions/139323/any-way-of-quotautomaticquot-lip-syncing.html

And look at below my code

 using UnityEngine;
 using System.Collections;
 
 public class  lipmovement2: MonoBehaviour
 {
 
     // Use this for initialization
 
     /*Class for implementing Lips Syncronisation*/
 
     public AudioClip source_clip;
     public float[] freqData;
     int nSamples = 256;
     int fMax = 24000;
 
     public Transform upmouth0_M, upmouth01_L, upmouth02_R, downmouth1_M, downmouth11_L, downmouth12_R;
     float volume = 1000;
 //    float freqLow = 200;
 //    float freqHigh = 800;
     //value change
 
     float freqLow = 200;
     float freqHigh = 1600;
 
     int sizeFilter = 5;
     float[] filter;
     float filterSum;
     int posFilter = 0;
     int qSample = 0;
 
     int video_Length, secCounter;
 
     float y0, y1;
 
     void OnEnable ()
     {
         secCounter = 0;
 
 //        y0 = mouth0.localPosition.y;
 //        y1 = mouth1.localPosition.y;
 
         y0 = upmouth0_M.localPosition.y;
         y0 = upmouth01_L.localPosition.y;
         y0 = upmouth02_R.localPosition.y;
         y1 = downmouth1_M.localPosition.y;
         y1 = downmouth11_L.localPosition.y;
         y1 = downmouth12_R.localPosition.y;
 
         freqData = new float[nSamples];
         //source_clip = SetFace.voiceOver;
         GetComponent<AudioSource> ().clip = Rec_voice.instance.voiceFeed.clip;
         GetComponent<AudioSource> ().Play ();
         video_Length = Mathf.CeilToInt (source_clip.length);
 
     }
 

 float BandVol (float fLow, float fHigh)
     {
         fLow = Mathf.Clamp (fLow, 20, fMax);
         fHigh = Mathf.Clamp (fHigh, fLow, fMax);
 
         GetComponent<AudioSource> ().GetSpectrumData (freqData, 0, FFTWindow.BlackmanHarris);
 
         int n1 = Mathf.FloorToInt (fLow * nSamples / fMax);
         int n2 = Mathf.FloorToInt (fHigh * nSamples / fMax);
     
         float sum = 0;
 
         for (int i = n1; i <= n2; i++) {
             sum = freqData [i];
         }
             
         return sum;
     }
 
     float MovingAverage (float sample)
     {
         if (qSample == 0)
             filter = new float[sizeFilter];
 
         filterSum += sample - filter [posFilter];
         filter [posFilter++] = sample;
 
         if (posFilter > qSample) {
             qSample = posFilter;
         }
 
         posFilter = posFilter % sizeFilter;
         return filterSum / qSample;
     }
 
     void Start ()
     {
         /*secCounter = 0;
 
         y0 = mouth0.localPosition.y;
         y1 = mouth1.localPosition.y;
 
         freqData = new float[nSamples];
         //source_clip = SetFace.voiceOver;
         GetComponent<AudioSource> ().clip = Rec_voice.instance.voiceOver;
         GetComponent<AudioSource> ().Play ();
         video_Length = Mathf.CeilToInt (source_clip.length);
 */
         //Debug.Log (y0);
         //    DebugConsole.Log (y0.ToString ());
     
 
         //    Debug.Log (Application.persistentDataPath);
 
         /*StartCoroutine (Timer ());
         StartCoroutine (recordScreen ());
 */
     }
 
     /*    IEnumerator Timer ()
     {
         while (secCounter < video_Length) {
             yield return new WaitForSeconds (1f);
             secCounter += 1;
         }
     }*/
 
 
     float limValue;
     // Update is called once per frame
     void Update ()
     {
         float band_vol = BandVol (freqLow, freqHigh);
         float val = MovingAverage (band_vol) * volume;
         //limValue = val;//Mathf.Clamp (val, 0, 0.1f);
         //limValue = Mathf.Clamp (val, 0, 10f);
         //check new lip movement abd set clamp val
         limValue = Mathf.Clamp (val, 0, 25f);
         //Debug.Log (y0 - limValue);
         if (Input.GetKeyDown (KeyCode.Escape)) {
             Application.Quit ();
         }
         /*    mouth0.position = new Vector3 (mouth0.position.x, y0 - MovingAverage (band_vol) * volume, mouth0.position.z);
         mouth1.position = new Vector3 (mouth1.position.x, y1 + MovingAverage (band_vol) * volume * 0.3f, mouth1.position.z);*/
     }
     void LateUpdate ()
     {
 //        mouth0.localPosition = new Vector3 (mouth0.localPosition.x, y0 - limValue, mouth0.localPosition.z);
 //        mouth1.localPosition = new Vector3 (mouth1.localPosition.x, y1 + limValue, mouth1.localPosition.z);
         upmouth0_M.localPosition = new Vector3 (upmouth0_M.localPosition.x, y0 - limValue, upmouth0_M.localPosition.z);
         upmouth01_L.localPosition = new Vector3 (upmouth01_L.localPosition.x, y0 - limValue, upmouth01_L.localPosition.z);
         upmouth02_R.localPosition = new Vector3 (upmouth02_R.localPosition.x, y0 - limValue, upmouth02_R.localPosition.z);
         downmouth1_M.localPosition = new Vector3 (downmouth1_M.localPosition.x, y1 + limValue, downmouth1_M.localPosition.z);
         downmouth11_L.localPosition = new Vector3 (downmouth11_L.localPosition.x, y1 + limValue, downmouth11_L.localPosition.z);
         downmouth12_R.localPosition = new Vector3 (downmouth12_R.localPosition.x, y1 + limValue, downmouth12_R.localPosition.z);
 
     }
         
 }

Here I'm facing some issue like below

1) How to recognise voice of Human? : Because if another voice like music or etc then it will be detected so how can we stop? I want lips will syncing only for human voice.

2) When I recording if distance is close to device then it working perfectly but if distance is little more away then lips is not syncing.

So suggest me where I'm going to wrong? and how to solve above issues ?

Comment
Add comment
10 |3000 characters needed characters left characters exceeded
▼
  • Viewable by all users
  • Viewable by moderators
  • Viewable by moderators and the original poster
  • Advanced visibility
Viewable by all users

0 Replies

· Add your reply
  • Sort: 

Your answer

Hint: You can notify a user about this post by typing @username

Up to 2 attachments (including images) can be used with a maximum of 524.3 kB each and 1.0 MB total.

Welcome to Unity Answers

The best place to ask and answer questions about development with Unity.

To help users navigate the site we have posted a site navigation guide.

If you are a new user to Unity Answers, check out our FAQ for more information.

Make sure to check out our Knowledge Base for commonly asked Unity questions.

If you are a moderator, see our Moderator Guidelines page.

We are making improvements to UA, see the list of changes.



Follow this Question

Answers Answers and Comments

144 People are following this question.

avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image avatar image

Related Questions

Function to assign gameobject components with parameter strings... 1 Answer

I wanted to know whether is it possible to use the SpeechRecognizer API in Unity Project to make an Android App? If yes, then how? 0 Answers

Detect if UI Image is in a trigger 1 Answer

Text Highlight word by word when background voice is playing. 0 Answers

Bad lighting in automated builds 0 Answers

  • Anonymous
  • Sign in
  • Create
  • Ask a question
  • Spaces
  • Default
  • Help Room
  • META
  • Moderators
  • Explore
  • Topics
  • Questions
  • Users
  • Badges