Spring or configurable joint - fluent moving of connected rigidbody

Setting interpolation for carried rigidbody has completely fixed the problem.

However I just found a better overall solution for carrying items anyway.

Attaching the first draft code for those interested:

 using UnityEngine;
 
 public class ItemCarrying : MonoBehaviour
 {
     [SerializeField] private Transform _grabHolder;
     [SerializeField] private float _grabReach = 4f;
     [SerializeField] private float _followSpeed = 5f;
     [SerializeField] private float _carriedItemAngularDrag = 10f;
     [SerializeField] private Transform _camera;
 
     private bool _carrying;
     private Rigidbody _carriedRigidbody;
     private Vector3 _offset;
     private float _originalAngularDrag;
 
     private void Update()
     {
         //also use layer mask to avoid colliding with character controller or other collider which is part of player
         if (Input.GetMouseButtonDown(0) && Physics.Raycast(_camera.position, _camera.forward, out RaycastHit hit, _grabReach))
         {
             _carriedRigidbody = hit.collider.GetComponent<Rigidbody>();
             if (_carriedRigidbody)
             {
                 _carrying = true;
                 _offset = hit.point - _carriedRigidbody.transform.position;
                 _originalAngularDrag = _carriedRigidbody.angularDrag;
                 _carriedRigidbody.angularDrag = _carriedItemAngularDrag;
             }
         }
         else if (Input.GetMouseButtonUp(0) && _carrying)
         {
             _carrying = false;
             _carriedRigidbody.angularDrag = _originalAngularDrag;
             _carriedRigidbody = null;
         }
 
         if (_carrying)
         {
             _carriedRigidbody.velocity = (_grabHolder.position - _offset - _carriedRigidbody.transform.position) * _followSpeed;
         }
     }
 }