Rotate camera outside of movement lerp?

I modified a standard mouse orbit script to allow for smooth movement between targets using lerp although I now have the issue in that the mouse drag functionality to rotate the object around the object is also using lerp so the object moves off centre and slowly moves back to centre which is the effect I'm trying to get rid of, as you can see below I've moved the X/Y movement outside of the lerp although it doesn't seem to be doing anything, can anyone point me in the correct direction?

 using UnityEngine;
 using System.Collections;
 
 public class OrbiterCamera : MonoBehaviour {
     // Min|Max distance of the camera from the target, set when the target is set but these are defaults.
     private float minDistance = 5.0f; 
     private float maxDistance = 20.0f;
 
     
     public float MinDistance = 5.0f; 
     public float MaxDistance = 20.0f;
     public float TransformSpeed = 3.0f;
     
     [SerializeField]
     private Transform target;
     public Transform Target {
         get {
             return target;
         }
         set {
             if(value != null && value != target) {
                 cameraSpeed = Mathf.Abs(Vector3.Distance(transform.position, value.position)) / TransformSpeed;
                 float size = Mathf.Max(Mathf.Max(value.renderer.bounds.size.x, value.renderer.bounds.size.y), value.renderer.bounds.size.z);
                 minDistance = size * MinDistance;
                 maxDistance = size * MaxDistance;
                 if(distance == 0f)
                     distance = size * 3;
             }
             target = value;
         }
     }
     private float distance = 0f;
     private float cameraSpeed = 1.0f;
     public float xSpeed = 175.0f;
     public float ySpeed = 75.0f;
     public float yMinLimit = -80.0f; //Lowest vertical angle in respect with the target.
     public float yMaxLimit = 80.0f;
     private float x = 0.0f;
     private float y = 0.0f;
     
     void Start () {
         this.Target = this.Target;
         Vector3 angles = transform.eulerAngles;
         x = angles.y;
         y = angles.x;
         // Make the rigid body not change rotation
         if (rigidbody)
             rigidbody.freezeRotation = true;
     }
     void OnGUI() {
         GUI.Label(new Rect(10,10,100,20), x.ToString());
     }
     
     void Update () {
         //Detect mouse drag;
         if (target) {
             //Zooming with mouse
             distance += Input.GetAxis("Mouse ScrollWheel")*distance;
             distance = Mathf.Clamp(distance, minDistance, maxDistance);
         }
         if(Input.GetMouseButton(0))   {
             x += Input.GetAxis("Mouse X") * xSpeed * Time.deltaTime;
             y -= Input.GetAxis("Mouse Y") * ySpeed * Time.deltaTime;
         }
         y = ClampAngle(y, yMinLimit, yMaxLimit);
         
         //Quaternion rotation = Quaternion.Euler(0, 0, 0);
         Vector3 position = new Vector3(0.0f, 0.0f, -distance) + target.position;
         position = Vector3.Lerp (transform.position, position, cameraSpeed * Time.deltaTime);



         // Allow mouse movement outside of the Lerp. although this doesn't seem to do anything :(
         Quaternion rotation = Quaternion.Euler(y, x,0);
         position = rotation  * new Vector3(0,0,-distance) + position;
         transform.position = position;


         // Get the rotation from the new position
         Vector3 stareAt = (target.position - transform.position);
         transform.rotation = Quaternion.Lerp (transform.rotation, Quaternion.LookRotation(stareAt), cameraSpeed * Time.deltaTime);
     }
     static float ClampAngle (float angle, float min, float max) {
         if (angle < -360)
             angle += 360;
     
         if (angle > 360)
             angle -= 360;
     
         return Mathf.Clamp(angle, min, max);
     }
 }