How can I make diagonal collisions work with raycasts from a boxcollider? (3D, C#)

First I'd like to mention that I'm very new to unity, C# and game development. I'm currently making my own player controller using raycasts to detect collisions, but I'm stuck at how to handle diagonal collisions.

What I have is a box with a boxcollider that can move in all directions within the x and z axis. The box itself doesn't rotate, but instead raycasts are being drawn towards one of the 4 axis (x, z, xz or zx). Collisions against one of the faces of the box work, but diagonal collisions against the edges of the box don't.

Here's a gif of the raycasts being drawn in the direction the player is moving: http://i.imgur.com/QseT1Ot.gifv

Here's a gif showing that the diagonal collisions aren't working: http://i.imgur.com/wYuSm8O.gifv

When that didn't work, I tried another method. Instead of having raycasts in 4 axis, I made the box rotate and shot raycasts only forward. Here's a gif of that: http://i.imgur.com/hYHahDA.gifv

That does work better, but still not perfect. Which one of the methods would work best and can you push me into the right direction how to fix the collision system?

Here's the part of the script that handles raycasts and collisions like in the first 2 gifs:

    void XCollisions(ref Vector3 velocity) {
         float directionX = Mathf.Sign(velocity.x);
         float rayLength = Mathf.Abs(velocity.x) + skinWidth;
 
         //horizontal rays
         for(int i = 0; i < xRayCount; i++) {
             Vector3 rayOrigin = (directionX == -1) ? raycastOrigins.bottomLeftBack : raycastOrigins.bottomRightBack;
             rayOrigin += Vector3.forward * (xRaySpacing * i);
                 
             //vertical multiplier
             for(int j = 0; j < xzRayCountY; j++) {
                 Vector3 xrayOriginY = rayOrigin;
                 xrayOriginY += Vector3.up * (xzRaySpacingY * j);
 
                 RaycastHit hitInfo;
                 bool hit = Physics.Raycast(xrayOriginY, Vector3.right * directionX, out hitInfo, rayLength, collisionMask);
                 Debug.DrawRay(xrayOriginY, Vector3.right * directionX * 3, Color.red);
 
                 if(hit) {
 
                     if(hitInfo.distance == 0) {
                         continue;
                     }
 
                     float slopeAngle = Vector3.Angle(hitInfo.normal, Vector3.up);
                     
                     if(i == 0 && slopeAngle <= maxClimbAngle) {
                         if(collisions.descendingSlope) {
                             collisions.descendingSlope = false;
                             velocity = collisions.velocityOld; 
                         }
                         float distanceToSlopeStart = 0;
                         if(slopeAngle != collisions.slopeAngleOld) {
                             distanceToSlopeStart = hitInfo.distance - skinWidth;
                             velocity.x -= distanceToSlopeStart * directionX;
                         }
                         ClimbSlopeX(ref velocity, slopeAngle);
                         velocity.x += distanceToSlopeStart * directionX;
                     }
                     
                     if(!collisions.climbingSlope || slopeAngle > maxClimbAngle) {
 
                         velocity.x = (hitInfo.distance - skinWidth) * directionX;
                         rayLength = hitInfo.distance;
 
                         //collision in x axis while climbing slope
                         if(collisions.climbingSlope) {
                             velocity.y = Mathf.Tan(collisions.slopeAngle * Mathf.Deg2Rad) * Mathf.Abs(velocity.x);
                         }
 
                         collisions.left = directionX == -1;
                         collisions.right = directionX == 1;
                     }
                 }
             }
         }
     }
 
     void ZCollisions(ref Vector3 velocity) {
         float directionZ = Mathf.Sign(velocity.z);
         float rayLength = Mathf.Abs(velocity.z) + skinWidth;
 
         //horizontal rays
         for(int i = 0; i < zRayCount; i++) {
             Vector3 rayOrigin = (directionZ == -1) ? raycastOrigins.bottomLeftBack : raycastOrigins.bottomLeftFront;
             rayOrigin += Vector3.right * (zRaySpacing * i);
 
             //vertical multiplier
             for(int j = 0; j < xzRayCountY; j++) {
                 Vector3 zrayOriginY = rayOrigin;
                 zrayOriginY += Vector3.up * (xzRaySpacingY * j);
 
                 RaycastHit hitInfo;
                 bool hit = Physics.Raycast(zrayOriginY, Vector3.forward * directionZ, out hitInfo, rayLength, collisionMask);
                 Debug.DrawRay(zrayOriginY, Vector3.forward * directionZ * 3, Color.blue);
 
                 if(hit) {
 
                     if(hitInfo.distance == 0) {
                         continue;
                     }
 
                     float slopeAngle = Vector3.Angle(hitInfo.normal, Vector3.up);
 
                     if(i == 0 && slopeAngle <= maxClimbAngle) {
                         if(collisions.descendingSlope) {
                             collisions.descendingSlope = false;
                             velocity = collisions.velocityOld;
                         }
                         float distanceToSlopeStart = 0;
                         if(slopeAngle != collisions.slopeAngleOld) {
                             distanceToSlopeStart = hitInfo.distance - skinWidth;
                             velocity.z -= distanceToSlopeStart * directionZ;
                         }
                         ClimbSlopeZ(ref velocity, slopeAngle);
                         velocity.z += distanceToSlopeStart * directionZ;
                     }
 
                     if(!collisions.climbingSlope || slopeAngle > maxClimbAngle) {
 
                         velocity.z = (hitInfo.distance - skinWidth) * directionZ;
                         rayLength = hitInfo.distance;
 
                         //collision in z axis while climbing slope
                         if(collisions.climbingSlope) {
                             velocity.y = Mathf.Tan(collisions.slopeAngle * Mathf.Deg2Rad) * Mathf.Abs(velocity.z);
                         }
 
                         collisions.back = directionZ == -1;
                         collisions.front = directionZ == 1;
                     }
                 }
             }
         }
     }
 
     void XZCollisions(ref Vector3 velocity) {
         float directionX = Mathf.Sign(velocity.x);
         float directionZ = Mathf.Sign(velocity.z);
         float directionXZ = 0;
         float directionZX = 0;
 
         if(directionX == -1) {
             directionXZ = (directionZ == 1) ? 1 : 0;
             directionZX = (directionZ == -1) ? -1 : 0;
         } else if(directionX == 1) {
             directionXZ = (directionZ == -1) ? -1 : 0;
             directionZX = (directionZ == 1) ? 1 : 0;
         }
 
         float rayLength = (Mathf.Abs(velocity.z) + Mathf.Abs(velocity.x)) / 2 + skinWidth;
         Vector3 playerDirection = new Vector3(velocity.x, 0, velocity.z);
         Vector3 playerDirectionXZ = new Vector3(-1, 0, 1);
         Vector3 playerDirectionZX = new Vector3(1, 0, 1);
 
         if(directionXZ != 0) {
             Vector3 rayOrigin = (directionXZ == -1) ? raycastOrigins.bottomRightBack : raycastOrigins.bottomLeftFront;
             for(int i = 0; i < xzRayCountY; i++) {
                 Vector3 rayOriginX = (directionXZ == -1) ? raycastOrigins.bottomRightBack : raycastOrigins.bottomLeftFront;
                 Vector3 rayOriginZ = (directionXZ == -1) ? raycastOrigins.bottomRightBack : raycastOrigins.bottomLeftFront;
                 rayOriginX += Vector3.right * directionXZ * (xRaySpacing * i);
                 rayOriginZ += Vector3.back * directionXZ * (zRaySpacing * i);
 
                 for(int j = 0; j < xzRayCountY; j++) {
                     Vector3 xzrayOriginYX = rayOriginX;
                     xzrayOriginYX += Vector3.up * (xzRaySpacingY * j);
                     RaycastHit hitInfoX;
                     bool hitX = Physics.Raycast(xzrayOriginYX, playerDirectionXZ * directionXZ, out hitInfoX, rayLength, collisionMask);
                     Debug.DrawRay(xzrayOriginYX, playerDirectionXZ * directionXZ * 3, Color.magenta);
 
                     if(hitX) {
                         Debug.Log("diagonal collision");
                         velocity.x = (hitInfoX.distance - skinWidth) * directionXZ;
                         velocity.z = (hitInfoX.distance - skinWidth) * directionXZ;
                         rayLength = hitInfoX.distance;
                     }
 
                     Vector3 xzrayOriginYZ = rayOriginZ;
                     xzrayOriginYZ += Vector3.up * (xzRaySpacingY * j);
                     RaycastHit hitInfoZ;
                     bool hitZ = Physics.Raycast(xzrayOriginYZ, playerDirectionXZ * directionXZ, out hitInfoZ, rayLength, collisionMask);
                     Debug.DrawRay(xzrayOriginYZ, playerDirectionXZ * directionXZ * 3, Color.magenta);
 
                     if(hitZ) {
                         Debug.Log("diagonal collision");
                         velocity.x = (hitInfoZ.distance - skinWidth) * directionXZ;
                         velocity.z = (hitInfoZ.distance - skinWidth) * directionXZ;
                         rayLength = hitInfoZ.distance;
                     }
                 }
             }
         }
 
         if(directionZX != 0) {
             Vector3 rayOrigin = (directionZX == -1) ? raycastOrigins.bottomLeftBack : raycastOrigins.bottomRightFront;
             for(int i = 0; i < xzRayCountY; i++) {
                 Vector3 rayOriginX = (directionZX == -1) ? raycastOrigins.bottomLeftBack : raycastOrigins.bottomRightFront;
                 Vector3 rayOriginZ = (directionZX == -1) ? raycastOrigins.bottomLeftBack : raycastOrigins.bottomRightFront;
                 rayOriginX += Vector3.back * directionZX * (xRaySpacing * i);
                 rayOriginZ += Vector3.left * directionZX * (zRaySpacing * i);
 
                 for(int j = 0; j < xzRayCountY; j++) {
                     Vector3 xzrayOriginYX = rayOriginX;
                     xzrayOriginYX += Vector3.up * (xzRaySpacingY * j);
                     RaycastHit hitInfoX;
                     bool hitX = Physics.Raycast(xzrayOriginYX, playerDirectionZX * directionZX, out hitInfoX, rayLength, collisionMask);
                     Debug.DrawRay(xzrayOriginYX, playerDirectionZX * directionZX * 3, Color.yellow);
 
                     if(hitX) {
                         Debug.Log("diagonal collision");
                         velocity.x = 0;
                         velocity.z = 0;
                         rayLength = hitInfoX.distance;
                     }
 
                     Vector3 xzrayOriginYZ = rayOriginZ;
                     xzrayOriginYZ += Vector3.up * (xzRaySpacingY * j);
                     RaycastHit hitInfoZ;
                     bool hitZ = Physics.Raycast(xzrayOriginYZ, playerDirectionZX * directionZX, out hitInfoZ, rayLength, collisionMask);
                     Debug.DrawRay(xzrayOriginYZ, playerDirectionZX * directionZX * 3, Color.yellow);
 
                     if(hitZ) {
                         Debug.Log("diagonal collision");
                         velocity.x = 0;
                         velocity.z = 0;
                         rayLength = hitInfoZ.distance;
                     }
                 }
             }
 
             //player direction
             RaycastHit hitInfoPD;
             bool hitPD = Physics.Raycast(rayOrigin, playerDirection * directionZ, out hitInfoPD, rayLength, collisionMask);
             Debug.DrawRay(rayOrigin, playerDirection * directionXZ * 3, Color.cyan);
 
             if(hitPD) {
                 Debug.Log("PlayerDirection collision");
                 velocity.x = 0;
                 velocity.z = 0;
                 rayLength = hitInfoPD.distance;
             }
         }
 
     }

If you need any more information, feel free to ask.

What am I doing wrong?