This is not working as I expected...

So, I am trying to make a 2D open world simulation game where you can locate around the world using mouse click and can also zoom in and out using the mouse scroll wheel. But for some reason, the camera does not stay smooth and moves around when I have stopped moving my mouse pointer...

 using System;
 using System.Collections;
 using System.Collections.Generic;
 using UnityEngine;
 
 public class MouseMove : MonoBehaviour
 {
 
     public bool mouseDown;
     public Vector2 staticMousePos;
     public Vector2 d;
     public Vector2 cmp;
 
     // Start is called before the first frame update
     void Start()
     {
         
     }
 
     // Update is called once per frame
     void Update()
     {
         Vector2 currentMousePos = Camera.main.ScreenToWorldPoint(Input.mousePosition);
         cmp = currentMousePos;
         float mouseScroll = Input.GetAxis("Mouse ScrollWheel") * -2;
 
         if (Input.GetMouseButton(0))
         {
             if (!mouseDown)
             {
                 staticMousePos = Camera.main.ScreenToWorldPoint(Input.mousePosition);
             }
             mouseDown = true;
         }
         else
         {
             mouseDown = false;
         }
 
         if (mouseDown)
         {
             Vector2 diff = (currentMousePos - staticMousePos).normalized / 10;
             d = diff;
             transform.position = transform.position - new Vector3(diff.x, diff.y, 0f);
             
         }
 
         GetComponent<Camera>().orthographicSize += mouseScroll;
 
         GetComponent<Camera>().orthographicSize = Mathf.Clamp(GetComponent<Camera>().orthographicSize, 4f, 8f);
         
     }
 }

I can't find any logical mistake in the code... Please help me out