Improve ray detection?

In the platformer im working on, the player can hang on ledges by casting 2 rays, and stopping the player if one detects a wall and the other doesn't.

This works great, but the issue comes when I am falling on to a ledge, about 10% of the time, it wont get detected.

Is there anything to do to improve the detection? I have lowered the fixed timestep to 0.01, ive tried casting the rays in fixed update(this makes it worse?)

Heres my code if it helps void Update() { #region initialize rays

     float widthOffset = transform.localScale.x / 2;
     float heightOffset = transform.localScale.y / 2;

     dir = transform.localScale.x; //right or left

     rayPos[0] = (Vector2)transform.position + new Vector2(widthOffset, heightOffset + ledgeCheckOffset);
     rayPos[1] = (Vector2)transform.position + new Vector2(widthOffset, heightOffset + ledgeCheckOffset - rayGap);


     #endregion

     #region check for wall

     for (int i = 0; i < rayPos.Length; i++)
     {
         wallHit[i] = Physics2D.Raycast(rayPos[i], Vector2.right * dir, rayDist, pm.groundLayer);//ledge check
     }

     if (pm.isGrounded) return;

     if (wallHit[1] && !wallHit[0])
     {
         if (ledgeDetected) return;

         bool wallAbove = Physics2D.OverlapCircle((Vector2)transform.position + new Vector2(transform.localScale.x / 2, .75f), .2f, pm.groundLayer);
         if (wallAbove) return;

         ledgeDetected = true;

         WallCling(wallHit[1], dir);
     }

     #endregion
 }

 #region functions
 void WallCling(RaycastHit2D ray, float playerDir)
 {
     if (ray.collider == null) return;

     Vector2 wallPos = rayPos[1] - new Vector2(rayDist * playerDir, 0);
     float wallEdge = playerDir > 0 ? Mathf.Floor(wallPos.x) : Mathf.Ceil(wallPos.x);
     float wallTop = Mathf.Ceil(wallPos.y);
     Vector2 ledgeCorner = new Vector2(wallEdge, wallTop);
     Vector2 newPos = ledgeCorner - new Vector2(ledgeHangOffset.x * playerDir, -ledgeHangOffset.y);

     transform.position = newPos;

     pm.rb.velocity = Vector2.zero;
     pm.rb.gravityScale = 0;
     pm.rb.isKinematic = true;
     pm.canJump = false;
     pm.canFlip = false;

     anim.SetBool("isWallClinging", true);
 }