diff --git a/runelite-client/src/main/java/net/runelite/client/ui/overlay/OverlayManager.java b/runelite-client/src/main/java/net/runelite/client/ui/overlay/OverlayManager.java
index 0f2f77d6e0..d9f96daed4 100644
--- a/runelite-client/src/main/java/net/runelite/client/ui/overlay/OverlayManager.java
+++ b/runelite-client/src/main/java/net/runelite/client/ui/overlay/OverlayManager.java
@@ -114,20 +114,16 @@ public class OverlayManager
 			return false;
 		}
 
-		final boolean add = overlays.add(overlay);
-
-		if (add)
-		{
-			final Point location = loadOverlayLocation(overlay);
-			overlay.setPreferredLocation(location);
-			final Dimension size = loadOverlaySize(overlay);
-			overlay.setPreferredSize(size);
-			final OverlayPosition position = loadOverlayPosition(overlay);
-			overlay.setPreferredPosition(position);
-			rebuildOverlayLayers();
-		}
-
-		return add;
+		// Add is always true
+		overlays.add(overlay);
+		final Point location = loadOverlayLocation(overlay);
+		overlay.setPreferredLocation(location);
+		final Dimension size = loadOverlaySize(overlay);
+		overlay.setPreferredSize(size);
+		final OverlayPosition position = loadOverlayPosition(overlay);
+		overlay.setPreferredPosition(position);
+		rebuildOverlayLayers();
+		return true;
 	}
 
 	/**
@@ -157,7 +153,12 @@ public class OverlayManager
 	public synchronized boolean removeIf(Predicate<Overlay> filter)
 	{
 		final boolean removeIf = overlays.removeIf(filter);
-		rebuildOverlayLayers();
+
+		if (removeIf)
+		{
+			rebuildOverlayLayers();
+		}
+
 		return removeIf;
 	}