[Unity Netcode] When rejoining lobby gets 2 index out of range error

Unity 2020.3.44f1

I'm trying to create a lobby scene where players who join get placeholder UI image and have the ability to toggle ready up.

When a player leaves and then joins again I get these two errors:

How do I fix this? Does it have something to do with that I leave and join from the same PC?

Here is my code:

 using System.Collections;
 using System.Collections.Generic;
 using UnityEngine;
 using UnityEngine.UI;
 using Unity.Netcode;
 using TMPro;
 
 public class LobbyManager : NetworkBehaviour
 {
     [SerializeField] private GameObject UI_PlayerIconPrefab;
     [SerializeField] private Transform UI_PlayerLayoutGroup;
 
     [SerializeField] private TextMeshProUGUI connectedCount;
     private readonly List<GameObject> playerUIThings = new List<GameObject>();
 
     public NetworkVariable<int> playerCount = new NetworkVariable<int>();
 
     public NetworkList<bool> playerReadyingList;
 
     private void Awake()
     {
         playerReadyingList = new NetworkList<bool>();
     }
 
     private void Start()
     {
         NetworkManager.Singleton.OnClientConnectedCallback += Client_Join;
         NetworkManager.Singleton.OnClientDisconnectCallback += Client_Disconnect;
 
         if (IsServer)
         {
             playerReadyingList.Add(false);
             playerCount.Value = NetworkManager.Singleton.ConnectedClientsList.Count;
         }
     }
 
     public void Client_Join(ulong clientID)
     {
         if (IsServer)
         {
             playerCount.Value = NetworkManager.Singleton.ConnectedClientsList.Count;
             Debug.Log("Client " + clientID + " just joined the server!");
             playerReadyingList.Add(false);
         }
     }
 
     public void Client_Disconnect(ulong clientID)
     {
         if (IsServer)
         {
             playerCount.Value = NetworkManager.Singleton.ConnectedClientsList.Count;
             Debug.Log("Client " + clientID + " just left the server!");
 
             ClientDisconnectUIUpdateClientRpc(clientID);
         }
     }
 
     private void LateUpdate()
     {
         if (IsServer && playerReadyingList != null)
         {
             playerCount.Value = NetworkManager.Singleton.ConnectedClientsList.Count;
             UpdateTheUIClientRpc();
         }
     }
 
     public void OnClick_ReadyUp()
     {
         ReadyUp_ServerRpc();
     }
     [ServerRpc(RequireOwnership = false)]
     public void ReadyUp_ServerRpc(ServerRpcParams serverRpcParams = default)
     {
         var clientId = serverRpcParams.Receive.SenderClientId;
         if (playerReadyingList[(int)clientId])
         {
             playerReadyingList[(int)clientId] = false;
         }
         else
         {
             playerReadyingList[(int)clientId] = true;
         }
     }
 
     [ClientRpc]
     public void UpdateTheUIClientRpc()
     {
         connectedCount.text = "Player Count: " + playerCount.Value;
         for (int i = 0; i < playerCount.Value; i++)
         {
             if(playerUIThings.Count < playerCount.Value)
             {
                 Build_PlayerIcon();
             }
         }
 
         //So kinda funny if I comment this part out the error DOESN'T happen for THIS method. Rather it happens in ClientDisconnectUIUpdateClientRpc when you leave.
         for (int i = 0; i < playerReadyingList.Count; i++)
         {
             if (playerUIThings[i] != null)
             {
                 if (!playerReadyingList[i])
                 {
                     playerUIThings[i].GetComponent<Image>().color = Color.red;
                 }
                 else
                 {
                     playerUIThings[i].GetComponent<Image>().color = Color.green;
                 }
             }
         }
     }
 
     [ClientRpc]
     public void ClientDisconnectUIUpdateClientRpc(ulong clientID)
     {
         GameObject playerUIThingToRemove = playerUIThings[(int)clientID];
         playerUIThings[(int)clientID] = null;
         Destroy(playerUIThingToRemove);
     }
 
 
     private void Build_PlayerIcon()
     {
         playerUIThings.Add(Instantiate(UI_PlayerIconPrefab, UI_PlayerLayoutGroup));
     }
 }