Removing item from inventory not updating correct based on order

I'm trying to do a inventory system, the way it works is that I have an empty gamebject that holds in a list all items that I have collected and on my UI, I have a object that holds 20 prefabs that serve as a container for t$$anonymous$$s item on that list. When the inventory is empty, all prefabs are deactivated, they are activated based on the items on the list. Now i'm trying to delete a certain item, but it only works if I go form the $$anonymous$$ghest index to the lowest, if I do the lowest or a middle index, it does not work.

Edit1:My hypotesis is that when I remove an item, the second index will became the new first, so it cause an error in logic, but I just don't know how to fix.

My script that handles updating the UI

 using System;
 using System.Collections;
 using System.Collections.Generic;
 using UnityEngine;
 using UnityEngine.Events;
 using UnityEngine.EventSystems;
 using TMPro;
 using UnityEngine.UI;
 public class TestLoop : MonoBehaviour
 {
     public int num;
 
 
     public Inventory inventory;
     public GameObject[] holder;
     public Button[] SpecialButtons;
 
     public GameObject Selector;
 
     public bool isOnIventory;
 
     public bool inDelete;
 
     private void OnEnable()
     {
         isOnIventory = true;
     }
     private void OnDisable()
     {
         isOnIventory = false;
     }
     private void Start()
     {
         //Sets all slots to be deactivated
         for (int i = 0; i < holder.Length; i++)
         {
             holder[i].SetActive(false);
         }
     }
     //item.GetComponentInC$$anonymous$$ldren<TextMeshProUGUI>().text = item.item.displayName;
 
     //When Iventory slot is clicked call t$$anonymous$$s
     public void GoToBottom()
     {
         num = Array.IndexOf(holder, EventSystem.current.currentSelectedGameObject);
         Selector.transform.position = SpecialButtons[0].transform.position;
         SpecialButtons[0].Select();
         print(num);
         inDelete = true;
     }
 
     //Test to delete Item
     public void TestDelete()
     {
         if (inventory.inventory[num] == null)
         {
             return;
         }
         else
         {
             if (num < 0)
             {
                 return;
             }
             holder[num].SetActive(false);
             inventory.inventory.Remove(inventory.inventory[num]);
             inDelete = false;
         }
     }
 
     //Update Item UI
     void UpdateItens()
     {
         if (inventory.inventory.Count < holder.Length)
         {
             for (int i = 0; i < inventory.inventory.Count; i++)
             {
                 holder[i].SetActive(true);
                 var bruh = inventory.inventory[i].itemData;
                 holder[i].GetComponentInC$$anonymous$$ldren<TextMeshProUGUI>().text = bruh.displayName;
             }
         }
     }
 
     private void Update()
     {
         print(num);
         if (inDelete)
         {
 
         }
         else
         {
             UpdateItens();
         }
     }
 }