Y position keeps changing, after action is complete?

So I have this script that causes the player to duck on a timer when they hit the down key. I want the player to go back to it's original height when the timer is up, the player goes back to close to it's original height but instead the height is dragged down a little each time the action is complete. For example if the starting Y position was 1 and the player ducked, after the timer is up the Y position becomes 0.8 instead of 1. Here is my script:

 public class CharacterDuck : MonoBehaviour {
     public float countDown = 2;//this count down will be used to determine when the character comes back up
     public bool goDown;
     public float startHeight;
     public Vector3 startPos;
     
     void Start()
     {
         startHeight = transform.position.y;
     }
     
     void Update()
     {
         startPos = new Vector3(transform.position.x, startHeight, transform.position.z);
         if (Input.GetKeyDown ("down"))
         {
             goDown = true;
         }
         
         if (goDown)
         {
             countDown -= Time.deltaTime;//count down the countDown to 0
             if (countDown > 1)
             {
                 transform.position = Vector3.Lerp (transform.position, (startPos + new Vector3(0, -5, 0)), Time.deltaTime*2);//use lerp to move the character from their current position to the "down" position
             }
             if (countDown <= 1)
             {
                 transform.position = Vector3.Lerp (transform.position, startPos, Time.deltaTime*2);//use lerp to move the character back to the idle position
                 if (countDown <= 0)
                 {
                     countDown = 2;//reset the cooldown
                     goDown = false;//turn goDown to false
                 }
             }
         }
     }
     
 }