Align Player to Surface Jitters

On my terrain, I set an empty child to to be a collider of it's mesh like below:

Thanks to some of the community on the Unity forms I was able to get the PlayerController to rotate/hug the slope or terrain smoothly with Racasting as it navigates, but occasionally, certain areas where you rest will jitter like below:

Here is the code for the PlayerController

     using System.Collections;
     using System.Collections.Generic;
     using UnityEngine;

     public class PlayerMovement : MonoBehaviour {

     public float speed     = 10f;
     public float runSpeed  = 20f;
     public float jumpForce = 7f;
     public float gravity   = 15f;
     public bool isRunning  = false;
     private Vector3 moveDir = Vector3.zero;
     public bool grounded;
     private Vector3 turnVector;
     private Vector3 posCur = Vector3.zero;
     private Quaternion rotCur;

     // Use this for initialization
     void Start () {

     }

     // Update is called once per frame
     void Update () {
         // CharacterController reference
         CharacterController controller = gameObject.GetComponent<CharacterController> ();
         //Check if character is on the ground
         if( controller.isGrounded ) {
             //Set new Vector3 with X,Y,Z coordinates
             moveDir = new Vector3 (Input.GetAxis("Horizontal"), 0, Input.GetAxis("Vertical"));

             moveDir = transform.TransformDirection (moveDir);

             // Check for run input
             if (Input.GetButtonDown ("Run")) {
                 isRunning = true;
             } else if (Input.GetButtonUp ("Run")) {
                 isRunning = false;
             }

             moveDir *= speed;

             // Check for jump input
             if (Input.GetButtonDown ("Jump")) {
                 moveDir.y = jumpForce;
             }
             //Allign player to the terrain
             Ray ray = new Ray(transform.position, -transform.up);
             RaycastHit hit;
             if(Physics.Raycast(ray, out hit, 1.5f) == true) {
                Debug.DrawLine(transform.position, hit.point, Color.green);
                rotCur = Quaternion.FromToRotation(transform.up, hit.normal) * transform.rotation;
                posCur = new Vector3(transform.position.x, hit.point.y + 1f, transform.position.z);

                grounded = true;
             }
             else {
                grounded = false;
             }
             if(grounded == true) {
                transform.rotation = Quaternion.Lerp(transform.rotation, rotCur, Time.deltaTime * 5);
             }
             else {
                transform.position = Vector3.Lerp(transform.position, transform.position - Vector3.up * 1f, Time.deltaTime * 5);
                if(transform.eulerAngles.x > 15) {
                    turnVector.x -= Time.deltaTime * 1000;
                }
                else if(transform.eulerAngles.x < 15) {
                    turnVector.x += Time.deltaTime * 1000;
                }
                rotCur.eulerAngles = Vector3.zero;
                transform.rotation = Quaternion.Lerp(transform.rotation, rotCur, Time.deltaTime);
             }
         }
         moveDir.y -= gravity * Time.deltaTime;

         controller.Move (moveDir * Time.deltaTime);

     }

     void FixedUpdate() {
     }

 }

I've tried tweaking speed settings, collider size but I just can't seem to get the jitter to be gone. Any ideas what is going on?