rigidBody on client and server apply different force?

I've been trying to figure this out all day and can't find the answer anywhere. I have a rigidBody that is controlled by the player and environmental objects that are rigidBodies.

When the client hits the environmental objects they barely move, but when the server hits them they move freely like i'd expect.

The all the physics are handled by the server, the players send their position to the server and it updates everything. So far the only thing I've found that kind of works is to do an rpc call to manually apply 10x as much force when there is a collision (but that doesn't seem good in the long run)

I think it has something to do with the server calculating velocity wrong because its just moving based on position updates, but I don't know how to fix that. Any suggestions?

 public class networkPlayer : MonoBehaviour
 {
     public int speed = 10;
 
     void Start()
     {
         if (!networkView.isMine)
         {
             enabled = false;
             rigidbody.isKinematic = true;
             rigidbody.useGravity = false;
         }
         
     }
 
 
     void FixedUpdate()
     {
         rigidbody.AddForce(new Vector3(Input.GetAxis("Horizontal"),0,Input.GetAxis("Vertical"))*Time.deltaTime*speed);
     }
 
     void OnCollisionEnter(Collision hit)
     {
         
         if (hit.gameObject.tag == "networkItem")
         {
             
             NetworkViewID id = hit.gameObject.networkView.viewID;
             Vector3 force = Vector3.Dot(hit.contacts[0].normal, hit.relativeVelocity) * rigidbody.mass * rigidbody.velocity;
             Vector3 forcePoint = hit.contacts[0].point;
             Debug.Log("hit " + hit.gameObject.networkView.viewID.ToString());
             networkView.RPC("applyHit", RPCMode.Server, id, force, forcePoint);
         }
     }
 
     void OnCollisionStay(Collision hit)
     {
         if (hit.gameObject.tag == "networkItem")
         {
 
             NetworkViewID id = hit.gameObject.networkView.viewID;
             Vector3 force = Vector3.Dot(hit.contacts[0].normal, hit.relativeVelocity) * rigidbody.mass * rigidbody.velocity;
             Vector3 forcePoint = hit.contacts[0].point;
             Debug.Log("hit " + hit.gameObject.networkView.viewID.ToString());
             networkView.RPC("applyStay", RPCMode.Server, id, force, forcePoint);
         }
     }
 
     [RPC]
     public void applyHit(NetworkViewID id, Vector3 force, Vector3 forcePoint)
     {
         Rigidbody receiver = NetworkView.Find(id).rigidbody;
         receiver.AddForceAtPosition(force*10, forcePoint);
     }
 
     [RPC]
     public void applyStay(NetworkViewID id, Vector3 force, Vector3 forcePoint)
     {
         Rigidbody receiver = NetworkView.Find(id).rigidbody;
         receiver.AddForceAtPosition(force*5, forcePoint);
     }
 }

 public class EnvObject: MonoBehaviour {
 
     private Vector3 lastPos;
 
     void Start()
     {
         if (Network.isServer)
         {
             rigidbody.isKinematic = false;
             rigidbody.useGravity = true;
             lastPos = transform.position;
         }
     }
     
     void FixedUpdate()
     {
         if (Network.isServer)
         {
             if (lastPos != transform.position)
             {
                 networkView.RPC("updatePos", RPCMode.Others, transform.position,transform.rotation);
             }
             lastPos = transform.position;
         }
     }
 
     [RPC]
     public void updatePos(Vector3 pos, Quaternion rot)
     {
         transform.position = pos;
         transform.rotation = rot;
     }
 }