How to assign local client authority to UI button?

Hello everybody. I'm developing a multiplayer game using UNET. There are 2 players in a single game, there is a roll dice button which obviously generates random number as rolling a dice. However, when other client clicks on button it says "Trying to send command for object without authority". I know 2 things now; 1- Server/Client clicks on button and its shown in Client's local game 2- Client clicks on button and I get the above error because Client does not have authority on the button.

How can I pass some kind of an authority exception for client to the button? Below is my script attached to player object which is spawned by network manager.

 using System.Collections;
 using System.Collections.Generic;
 using UnityEngine;
 using UnityEngine.UI;
 using UnityEngine.Networking;
 
 public class Roll_Dice : NetworkBehaviour
 {
     public int Dice_Score;
     public Text Dice_Text;
     public GameObject BoardManager;
     public Button RollDice;
     public bool PlayerTurn = false;
 
 
     void Start()
     {
         RollDice.gameObject.SetActive(true);
         Dice_Text.text = " ";
         BoardManager.gameObject.SetActive(true);
         Button btn1 = RollDice.GetComponent<Button>();
         btn1.onClick.AddListener(TaskOnClick);
     }
 
     public void TaskOnClick()
     {
         
         RollDice.gameObject.SetActive(false);
         Dice_Score = Random.Range(2, 13);
         Dice_Text.gameObject.SetActive(true);
         Dice_Text.text = "Rolled" + " " + Dice_Score;
         //BoardManager.gameObject.GetComponent<Board_Manager>().diceIsClosing = true;
         Invoke("Close_Dice_Roll", 3);
         CmdRollDice(Dice_Score);
         
 
     }
 
     public void Close_Dice_Roll()
     {
         Dice_Text.gameObject.SetActive(false);
         RollDice.gameObject.SetActive(true);
         PlayerTurn = !PlayerTurn;
         
     }
 
     [Command]
     void CmdRollDice(int Dice_Score)
     {
         Debug.Log("Command");
         RollDice.gameObject.SetActive(false);
         Dice_Text.gameObject.SetActive(true);
         Dice_Text.text = "Rolled" + " " + Dice_Score;
         //BoardManager.gameObject.GetComponent<Board_Manager>().diceIsClosing = true;
         RpcRollDice(Dice_Score);
         Invoke("Close_Dice_Roll", 3);
         
     }
 
     [ClientRpc]
     void RpcRollDice(int Dice_Score)
     {
         Debug.Log("RPC");
         RollDice.gameObject.SetActive(false);
         Dice_Text.gameObject.SetActive(true);
         Dice_Text.text = "Rolled" + " " + Dice_Score;
         //BoardManager.gameObject.GetComponent<Board_Manager>().diceIsClosing = true;
         Invoke("Close_Dice_Roll", 3);
     }
 }