diff --git a/runelite-client/src/main/java/net/runelite/client/plugins/xptracker/XpInfoBox.java b/runelite-client/src/main/java/net/runelite/client/plugins/xptracker/XpInfoBox.java
index 60dc5059af..7d9dbe7723 100644
--- a/runelite-client/src/main/java/net/runelite/client/plugins/xptracker/XpInfoBox.java
+++ b/runelite-client/src/main/java/net/runelite/client/plugins/xptracker/XpInfoBox.java
@@ -41,6 +41,8 @@ import javax.swing.JPopupMenu;
 import javax.swing.SwingConstants;
 import javax.swing.SwingUtilities;
 import javax.swing.border.EmptyBorder;
+import javax.swing.event.PopupMenuEvent;
+import javax.swing.event.PopupMenuListener;
 import lombok.AccessLevel;
 import lombok.Getter;
 import net.runelite.api.Client;
@@ -139,18 +141,34 @@ class XpInfoBox extends JPanel
 		popupMenu.add(resetOthers);
 		popupMenu.add(pauseSkill);
 		popupMenu.add(canvasItem);
+		popupMenu.addPopupMenuListener(new PopupMenuListener()
+		{
+			@Override
+			public void popupMenuWillBecomeVisible(PopupMenuEvent popupMenuEvent)
+			{
+				canvasItem.setText(xpTrackerPlugin.hasOverlay(skill) ? REMOVE_STATE : ADD_STATE);
+			}
+
+			@Override
+			public void popupMenuWillBecomeInvisible(PopupMenuEvent popupMenuEvent)
+			{
+			}
+
+			@Override
+			public void popupMenuCanceled(PopupMenuEvent popupMenuEvent)
+			{
+			}
+		});
 
 		canvasItem.addActionListener(e ->
 		{
 			if (canvasItem.getText().equals(REMOVE_STATE))
 			{
 				xpTrackerPlugin.removeOverlay(skill);
-				canvasItem.setText(ADD_STATE);
 			}
 			else
 			{
 				xpTrackerPlugin.addOverlay(skill);
-				canvasItem.setText(REMOVE_STATE);
 			}
 		});
 
diff --git a/runelite-client/src/main/java/net/runelite/client/plugins/xptracker/XpTrackerPlugin.java b/runelite-client/src/main/java/net/runelite/client/plugins/xptracker/XpTrackerPlugin.java
index 56ca040a8a..a4a2f15be7 100644
--- a/runelite-client/src/main/java/net/runelite/client/plugins/xptracker/XpTrackerPlugin.java
+++ b/runelite-client/src/main/java/net/runelite/client/plugins/xptracker/XpTrackerPlugin.java
@@ -262,6 +262,17 @@ public class XpTrackerPlugin extends Plugin
 		overlayManager.removeIf(e -> e instanceof XpInfoBoxOverlay && ((XpInfoBoxOverlay) e).getSkill() == skill);
 	}
 
+	/**
+	 * Check if there is an overlay on the canvas for the skill.
+	 *
+	 * @param skill the skill which should have an overlay.
+	 * @return true if the skill has an overlay.
+	 */
+	boolean hasOverlay(final Skill skill)
+	{
+		return overlayManager.anyMatch(o -> o instanceof XpInfoBoxOverlay && ((XpInfoBoxOverlay) o).getSkill() == skill);
+	}
+
 	/**
 	 * Reset internal state and re-initialize all skills with XP currently cached by the RS client
 	 * This is called by the user manually clicking resetSkillState in the UI.
@@ -708,9 +719,4 @@ public class XpTrackerPlugin extends Plugin
 			pauseSkill(skill, pause);
 		}
 	}
-
-	private boolean hasOverlay(final Skill skill)
-	{
-		return overlayManager.anyMatch(o -> o instanceof XpInfoBoxOverlay && ((XpInfoBoxOverlay) o).getSkill() == skill);
-	}
 }