Conflicting Box Colliders

I have a 2D menu that has a box collider on it to allow it to scroll up and down. I then have Empty GameObjects with box colliders over the top of where I want the buttons to be (to select a level). However, when both sets of box colliders are on, if I try and click and drag the menu, it selects the Empty GameObject level button and takes me to that level. I've tried putting in script to make the Empty GameObject button inactive OnMouseDown and OnMouseDrag but that does nothing. Here's the script I have so far:

 using UnityEngine;
  
 using System.Collections;
  
 [RequireComponent(typeof(BoxCollider))]
  
 public class Drag : MonoBehaviour
 {
     private Vector3 screenPoint;
     private Vector3 offset;
     private float _lockedYPosition; 
     public Vector2 limits = new Vector2(-20, 20);
     
     private GameObject Level1;
     private GameObject Level2;
     private GameObject Level3;
     private GameObject Level4;
     private GameObject Level5;
     
     void Start () {
         Level1 = GameObject.Find("Levels/Level1");
         Level2 = GameObject.Find("Levels/Level2");
         Level3 = GameObject.Find("Levels/Level3");
         Level4 = GameObject.Find("Levels/Level4");
         Level5 = GameObject.Find("Levels/Level5");
         
         Level1.SetActive(true);
         Level2.SetActive(true);
         Level3.SetActive(true);
         Level4.SetActive(true);
         Level5.SetActive(true);
     }
      
     void OnMouseDown() {
      
        _lockedYPosition = screenPoint.y;
        offset = gameObject.transform.position - Camera.main.ScreenToWorldPoint(new Vector3(Input.mousePosition.x, Input.mousePosition.y, screenPoint.z));
            
         Level1.SetActive(false);
         Level2.SetActive(false);
         Level3.SetActive(false);
         Level4.SetActive(false);
         Level5.SetActive(false);
         
               
     }
  
     void OnMouseDrag() 
     { 
       
         
     Vector3 curScreenPoint = new Vector3(Input.mousePosition.x, Input.mousePosition.y, screenPoint.z);
     Vector3 curPosition = Camera.main.ScreenToWorldPoint(curScreenPoint) + offset;
     curPosition.x = _lockedYPosition;
     transform.position = curPosition;
    
     curPosition.y = Mathf.Clamp(curPosition.y, limits.x, limits.y);
 if(curPosition.y > limits.y){
     curPosition.y = limits.y;
 }
 else if(curPosition.x < limits.x){
     curPosition.x = limits.x;
             
         Level1.SetActive(false);
         Level2.SetActive(false);
         Level3.SetActive(false);
         Level4.SetActive(false);
         Level5.SetActive(false);
 }
     }
 }

I'm sure there must be a way of checking that if the mouse is dragging, the box colliders should not be active?