Optimizing fog of war in 3D environment

So after spending a while thinking about how to do proper fog of war in a top down game with an arbitrary number of levels, I decided to implement the obvious solution (raycasting). I expected this to give me unacceptable performance, and it does (big surprise). It's not as terrible as I assumed it would be, but it still takes between 50-60ms on my machine (i5-3570k @ stock). The big killer here is finding the game objects within the camera frustum to test against. The actual raycasting only takes ~5ms, which, while not great, I'm okay with. Here's the code I ended up using:

 void UpdateGeometryLayers ()
 {

     GameObject[] allObjects = GameObject.FindGameObjectsWithTag ("SpawnedObject");
     Plane[] frustumPlanes = GeometryUtility.CalculateFrustumPlanes (camera);
     bool isViable;
     bool isVisible;

     if (!liveLayerData) {
         foreach (GameObject go in allObjects) {
             for (int l = 0; l < go.transform.GetChild (0).childCount; l++) {
                 go.transform.GetChild (0).transform.GetChild (l).gameObject.layer = 10;
             }
         }
     }

     foreach (GameObject go in allObjects) {
         isViable = false;
         isVisible = false;
         for (int l = 0; l < go.transform.GetChild (0).childCount; l++) {
             GameObject subObject = go.transform.GetChild (0).transform.GetChild (l).gameObject;
             if (GeometryUtility.TestPlanesAABB (frustumPlanes, subObject.GetComponent<MeshRenderer> ().bounds))
                 isViable = true;
         }
         if (isViable) {
             Collider[] colliders = go.GetComponentsInChildren<Collider> ();
             Collider closestCollider = colliders [0];

             foreach (Collider collider in colliders) {
                 RaycastHit hit;
                 int layers = 0;
                 for (int i = 12; i >= 0; i--) {
                     layers |= (1 << i);
                 }

                 Vector3 closestPoint = collider.ClosestPointOnBounds (head.transform.position);
                 Vector3 direction = closestPoint - head.transform.position;

                 //Debug.DrawRay (head.transform.position, direction, Color.blue);

                 if (Physics.Raycast (head.transform.position, direction, out hit, visionDistance, layers)) {
                     Bounds hitLocation = new Bounds (hit.point, new Vector3 (0.1F, 0.1F, 0.1F));
                     if (hitLocation.Contains (closestPoint))
                         isVisible = true;
                 }

                 //Increases accuracy, esp for stairs, at a large performance cost
                 if (multipassLayerCalc) {
                     Vector3 highestPointA = new Vector3 (collider.bounds.max.x, collider.bounds.max.y, collider.bounds.max.z);
                     direction = highestPointA - head.transform.position;
                     if (Physics.Raycast (head.transform.position, direction, out hit, visionDistance, layers)) {
                         Bounds hitLocation = new Bounds (hit.point, new Vector3 (0.1F, 0.1F, 0.1F));
                         if (hitLocation.Contains (highestPointA))
                             isVisible = true;
                     }

                     Vector3 highestPointB = new Vector3 (collider.bounds.min.x, collider.bounds.max.y, collider.bounds.min.z);
                     direction = highestPointB - head.transform.position;
                     if (Physics.Raycast (head.transform.position, direction, out hit, visionDistance, layers)) {
                         Bounds hitLocation = new Bounds (hit.point, new Vector3 (0.1F, 0.1F, 0.1F));
                         if (hitLocation.Contains (highestPointB))
                             isVisible = true;
                     }
                     Vector3 highestPointC = new Vector3 (collider.bounds.max.x, collider.bounds.max.y, collider.bounds.min.z);
                     direction = highestPointC - head.transform.position;
                     if (Physics.Raycast (head.transform.position, direction, out hit, visionDistance, layers)) {
                         Bounds hitLocation = new Bounds (hit.point, new Vector3 (0.1F, 0.1F, 0.1F));
                         if (hitLocation.Contains (highestPointB))
                             isVisible = true;
                     }

                     Vector3 highestPointD = new Vector3 (collider.bounds.min.x, collider.bounds.max.y, collider.bounds.max.z);
                     direction = highestPointD - head.transform.position;
                     if (Physics.Raycast (head.transform.position, direction, out hit, visionDistance, layers)) {
                         Bounds hitLocation = new Bounds (hit.point, new Vector3 (0.1F, 0.1F, 0.1F));
                         if (hitLocation.Contains (highestPointB))
                             isVisible = true;
                     }
                 }
             }
             if (isVisible) {
                 if ((go.transform.position.y < head.transform.position.y))
                     for (int l = 0; l < go.transform.GetChild (0).childCount; l++)
                         go.transform.GetChild (0).transform.GetChild (l).gameObject.layer = 8; //Visible
                 else
                     for (int l = 0; l < go.transform.GetChild (0).childCount; l++)
                         go.transform.GetChild (0).transform.GetChild (l).gameObject.layer = 11; //Culled Known
             } else {
                 if ((go.transform.position.y < head.transform.position.y))
                     for (int l = 0; l < go.transform.GetChild (0).childCount; l++) {
                         if (go.transform.GetChild (0).transform.GetChild (l).gameObject.layer == 8 ||
                             go.transform.GetChild (0).transform.GetChild (l).gameObject.layer == 9 ||
                             go.transform.GetChild (0).transform.GetChild (l).gameObject.layer == 11)
                             go.transform.GetChild (0).transform.GetChild (l).gameObject.layer = 9; //Known
                     }
                 else
                     for (int l = 0; l < go.transform.GetChild (0).childCount; l++) {
                         if (go.transform.GetChild (0).transform.GetChild (l).gameObject.layer == 8 ||
                             go.transform.GetChild (0).transform.GetChild (l).gameObject.layer == 9 ||
                             go.transform.GetChild (0).transform.GetChild (l).gameObject.layer == 11)
                             go.transform.GetChild (0).transform.GetChild (l).gameObject.layer = 11; //Culled Known
                     }
             }
         }
     }
     liveLayerData = true;
     bakedLayerData = false;
 }

And a screenshot to show what it looks like in practice:

You can see the geometry currently visible is in color, the the geometry not visible is in black and white without shadows, and the unknown geometry is invisible.

There has to be a better way to do this than literally checking every object in the scene every frame. If I can get the total update time down to < 10ms, I can do an update every 100ms or so, and that would probably be okay.