Merge pull request #3202 from open-osrs/scraper-update

runelite-client/src/main/java/com/openosrs/client/game/NPCManager.java

@@ -74,7 +74,7 @@ public class NPCManager
 	{
 		try (JsonReader reader = new JsonReader(new InputStreamReader(NPCManager.class.getResourceAsStream("/npc_stats.json"), StandardCharsets.UTF_8)))
 		{
-			ImmutableMap.Builder<Integer, NPCStats> builder = ImmutableMap.builderWithExpectedSize(2821);
+			ImmutableMap.Builder<Integer, NPCStats> builder = ImmutableMap.builderWithExpectedSize(3123);
 			reader.beginObject();
 
 			while (reader.hasNext())

runelite-client/src/main/java/com/openosrs/client/game/NPCStats.java

@@ -67,10 +67,23 @@ public class NPCStats
 
 	private final boolean poisonImmune;
 	private final boolean venomImmune;
+	private final boolean cannonImmune;
+	private final boolean thrallImmune;
 
-	private final boolean dragon;
 	private final boolean demon;
+	private final boolean dragon;
+	private final boolean fiery;
+	private final boolean kalphite;
+	private final boolean leafy;
+	private final boolean shade;
+	private final boolean spectral;
 	private final boolean undead;
+	private final boolean vampyre1;
+	private final boolean vampyre2;
+	private final boolean vampyre3;
+	private final boolean xerician;
+
+	private final int respawn;
 
 	/**
 	 * Based off the formula found here: http://services.runescape.com/m=forum/c=PLuJ4cy6gtA/forums.ws?317,318,712,65587452,209,337584542#209
@@ -184,15 +197,51 @@ public class NPCStats
 					case "venomImmune":
 						builder.venomImmune(in.nextBoolean());
 						break;
-					case "dragon":
+					case "cannonImmune":
-						builder.dragon(in.nextBoolean());
+						builder.cannonImmune(in.nextBoolean());
+						break;
+					case "thrallImmune":
+						builder.thrallImmune(in.nextBoolean());
 						break;
 					case "demon":
 						builder.demon(in.nextBoolean());
 						break;
+					case "dragon":
+						builder.dragon(in.nextBoolean());
+						break;
+					case "fiery":
+						builder.fiery(in.nextBoolean());
+						break;
+					case "kalphite":
+						builder.kalphite(in.nextBoolean());
+						break;
+					case "leafy":
+						builder.leafy(in.nextBoolean());
+						break;
+					case "shade":
+						builder.shade(in.nextBoolean());
+						break;
+					case "spectral":
+						builder.spectral(in.nextBoolean());
+						break;
 					case "undead":
 						builder.undead(in.nextBoolean());
 						break;
+					case "vampyre1":
+						builder.vampyre1(in.nextBoolean());
+						break;
+					case "vampyre2":
+						builder.vampyre2(in.nextBoolean());
+						break;
+					case "vampyre3":
+						builder.vampyre3(in.nextBoolean());
+						break;
+					case "xerician":
+						builder.xerician(in.nextBoolean());
+						break;
+					case "respawn":
+						builder.respawn(in.nextInt());
+						break;
 				}
 			}
 

wiki-scraper/src/main/java/net/runelite/data/App.java

@@ -68,7 +68,7 @@ public class App
 
 	private static Store cacheStore() throws IOException
 	{
-		Path path = Paths.get(System.getProperty("user.home"), "jagexcache" + File.separator + "oldschool" + File.separator + "LIVE");
+		Path path = Paths.get(System.getProperty("user.home"), ".openosrs" + File.separator + "jagexcache" + File.separator + "oldschool" + File.separator + "LIVE");
 		final File jagexcache = new File(String.valueOf(path));
 
 		if (!Files.exists(path))
@@ -80,9 +80,6 @@ public class App
 
 		cacheStore.load();
 
-		// Try to make this go faster (probably not very smart)
-		System.setProperty("java.util.concurrent.ForkJoinPool.common.parallelism", "100");
-
 		return cacheStore;
 	}
 

wiki-scraper/src/main/java/net/runelite/data/dump/MediaWiki.java

@@ -27,6 +27,8 @@ import java.io.UnsupportedEncodingException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.util.Map;
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
 import net.runelite.data.App;
 import okhttp3.HttpUrl;
 import okhttp3.OkHttpClient;
@@ -34,7 +36,10 @@ import okhttp3.Request;
 import okhttp3.Response;
 import java.net.URLDecoder;
 import java.nio.charset.StandardCharsets;
+import okhttp3.ResponseBody;
+import org.apache.commons.lang3.tuple.Pair;
 
+@Slf4j
 public class MediaWiki
 {
 	private static final class WikiInnerResponse
@@ -53,6 +58,7 @@ public class MediaWiki
 		.followSslRedirects(false)
 		.build();
 
+	@Getter
 	private final HttpUrl base;
 
 	public MediaWiki(final String base)
@@ -60,7 +66,7 @@ public class MediaWiki
 		this.base = HttpUrl.parse(base);
 	}
 
-	public String getSpecialLookupData(final String type, final int id, final int section)
+	public Pair<String, String> getSpecialLookupData(final String type, final int id, final int section)
 	{
 		final HttpUrl url = base.newBuilder()
 			.addPathSegment("w")
@@ -77,20 +83,39 @@ public class MediaWiki
 		{
 			if (response.isRedirect())
 			{
-				final String page = response.header("Location")
+				String responseHeaderLocation = response.header("Location");
+
+				if (responseHeaderLocation == null)
+				{
+					return null;
+				}
+
+				final String page = responseHeaderLocation
 					.replace(base.newBuilder().addPathSegment("w").build().toString() + "/", "");
+
 				return getPageData(page, section);
 			}
+			else
+			{
+				log.info("unsuccessful: {}", response.code());
+
+				if (response.code() == 429)
+				{
+					Thread.sleep(2500);
+					return getSpecialLookupData(type, id, section);
+				}
+			}
 		}
 		catch (Exception e)
 		{
-			return "";
+			log.info("exception: {}", e.getMessage());
+			return null;
 		}
 
-		return "";
+		return null;
 	}
 
-	public String getPageData(String page, int section)
+	public Pair<String, String> getPageData(String page, int section)
 	{
 		// decode html encoded page name
 		// ex: Mage%27s book -> Mage's_book
@@ -126,15 +151,33 @@ public class MediaWiki
 		{
 			if (response.isSuccessful())
 			{
-				final InputStream in = response.body().byteStream();
+				ResponseBody responseBody = response.body();
-				return App.GSON.fromJson(new InputStreamReader(in), WikiResponse.class).parse.wikitext.get("*");
+
+				if (responseBody == null)
+				{
+					return null;
+				}
+
+				final InputStream in = responseBody.byteStream();
+				return Pair.of(page.replaceAll(" ", "_"), App.GSON.fromJson(new InputStreamReader(in), WikiResponse.class).parse.wikitext.get("*"));
+			}
+			else
+			{
+				log.info("page data unsuccessful: {}", response.code());
+
+				if (response.code() == 429)
+				{
+					Thread.sleep(2500);
+					return getPageData(page, section);
+				}
 			}
 		}
 		catch (Exception e)
 		{
-			return "";
+			log.info("exception page data: {}", e.getMessage());
+			return null;
 		}
 
-		return "";
+		return null;
 	}
 }

wiki-scraper/src/main/java/net/runelite/data/dump/MediaWikiTemplate.java

@@ -230,6 +230,11 @@ public class MediaWikiTemplate
 		this.map = map;
 	}
 
+	public Map<String, String> getMap()
+	{
+		return map;
+	}
+
 	public String getValue(final String key)
 	{
 		String val = map.get(key);

wiki-scraper/src/main/java/net/runelite/data/dump/wiki/ItemLimitsDumper.java

@@ -43,6 +43,7 @@ import net.runelite.cache.util.Namer;
 import net.runelite.data.App;
 import net.runelite.data.dump.MediaWiki;
 import net.runelite.data.dump.MediaWikiTemplate;
+import org.apache.commons.lang3.tuple.Pair;
 
 @Slf4j
 public class ItemLimitsDumper
@@ -88,7 +89,14 @@ public class ItemLimitsDumper
 				return;
 			}
 
-			String data = wiki.getPageData("Module:Exchange/" + name, -1);
+			Pair<String, String> pageData = wiki.getPageData("Module:Exchange/" + name, -1);
+
+			if (pageData == null)
+			{
+				return;
+			}
+
+			String data = pageData.getRight();
 
 			if (Strings.isNullOrEmpty(data))
 			{

wiki-scraper/src/main/java/net/runelite/data/dump/wiki/ItemStatsDumper.java

@@ -42,6 +42,7 @@ import net.runelite.cache.util.Namer;
 import net.runelite.data.App;
 import net.runelite.data.dump.MediaWiki;
 import net.runelite.data.dump.MediaWikiTemplate;
+import org.apache.commons.lang3.tuple.Pair;
 
 @Slf4j
 public class ItemStatsDumper
@@ -57,7 +58,10 @@ public class ItemStatsDumper
 
 		final Map<Integer, ItemStats> itemStats = new TreeMap<>();
 		final Collection<ItemDefinition> items = itemManager.getItems();
-		final Stream<ItemDefinition> itemDefinitionStream = items.parallelStream();
+
+		log.info("{}", items.size());
+
+		final Stream<ItemDefinition> itemDefinitionStream = items.stream();
 
 		itemDefinitionStream.forEach(item ->
 		{
@@ -81,7 +85,14 @@ public class ItemStatsDumper
 				return;
 			}
 
-			String data = wiki.getSpecialLookupData("item", item.id, 0);
+			Pair<String, String> pageData = wiki.getSpecialLookupData("item", item.id, 0);
+
+			if (pageData == null)
+			{
+				return;
+			}
+
+			String data = pageData.getRight();
 
 			if (Strings.isNullOrEmpty(data))
 			{
@@ -109,6 +120,7 @@ public class ItemStatsDumper
 					continue;
 				}
 
+				itemStat.wiki(pageData.getLeft());
 				itemStat.name(getVarString(base, "name", offset) == null ? getVarString(base, "name1", offset) : getVarString(base, "name", offset));
 				itemStat.quest(getVarBoolean(base, "quest", offset));
 				itemStat.equipable(getVarBoolean(base, "equipable", offset) == null
@@ -122,7 +134,14 @@ public class ItemStatsDumper
 
 					if (stats == null)
 					{
-						data = wiki.getSpecialLookupData("item", item.id, 1);
+						pageData = wiki.getSpecialLookupData("item", item.id, 1);
+
+						if (pageData == null)
+						{
+							return;
+						}
+
+						data = pageData.getRight();
 
 						if (Strings.isNullOrEmpty(data))
 						{
@@ -156,7 +175,8 @@ public class ItemStatsDumper
 					equipmentStat.rstr(getVarInt(stats, "rstr", offset));
 					equipmentStat.mdmg(getVarInt(stats, "mdmg", offset));
 					equipmentStat.prayer(getVarInt(stats, "prayer", offset));
-					equipmentStat.aspeed(getVarInt(stats, "aspeed", offset));
+
+					equipmentStat.aspeed(getVarInt(stats, "speed", offset));
 
 					final ItemEquipmentStats builtEqStat = equipmentStat.build();
 
@@ -347,6 +367,7 @@ public class ItemStatsDumper
 	{
 		static final ItemStats DEFAULT = ItemStats.builder().build();
 
+		private final String wiki;
 		private final String name;
 		private final Boolean quest;
 		private final Boolean equipable;

wiki-scraper/src/main/java/net/runelite/data/dump/wiki/NpcStatsDumper.java

@@ -47,6 +47,7 @@ import net.runelite.cache.util.Namer;
 import net.runelite.data.App;
 import net.runelite.data.dump.MediaWiki;
 import net.runelite.data.dump.MediaWikiTemplate;
+import org.apache.commons.lang3.tuple.Pair;
 
 @Slf4j
 public class NpcStatsDumper
@@ -56,6 +57,8 @@ public class NpcStatsDumper
 	private static final class NpcStats
 	{
 		private String name;
+		private String wiki;
+
 		private final Integer hitpoints;
 		private final Integer hitpoints1;
 		private final Integer combatLevel;
@@ -87,10 +90,23 @@ public class NpcStatsDumper
 
 		private final Boolean poisonImmune;
 		private final Boolean venomImmune;
+		private final Boolean cannonImmune;
+		private final Boolean thrallImmune;
 
-		private final Boolean dragon;
 		private final Boolean demon;
+		private final Boolean dragon;
+		private final Boolean fiery;
+		private final Boolean kalphite;
+		private final Boolean leafy;
+		private final Boolean shade;
+		private final Boolean spectral;
 		private final Boolean undead;
+		private final Boolean vampyre1;
+		private final Boolean vampyre2;
+		private final Boolean vampyre3;
+		private final Boolean xerician;
+
+		private final Integer respawn;
 	}
 
 	private static final NpcStats DEFAULT = NpcStats.builder().build();
@@ -133,7 +149,10 @@ public class NpcStatsDumper
 
 		final Map<Integer, NpcStats> npcStats = new HashMap<>();
 		final Collection<NpcDefinition> definitions = npcManager.getNpcs();
-		final Stream<NpcDefinition> npcDefinitionStream = definitions.parallelStream();
+
+		log.info("{}", definitions.size());
+
+		final Stream<NpcDefinition> npcDefinitionStream = definitions.stream();
 
 		// Ensure variant names match cache as wiki isn't always correct
 		final Map<Integer, String> nameMap = new HashMap<>();
@@ -175,7 +194,15 @@ public class NpcStatsDumper
 				return;
 			}
 
-			final String data = wiki.getSpecialLookupData("npc", n.getId(), 0);
+			Pair<String, String> pageData = wiki.getSpecialLookupData("npc", n.getId(), 0);
+
+			if (pageData == null)
+			{
+				return;
+			}
+
+			String data = pageData.getRight();
+
 			if (Strings.isNullOrEmpty(data))
 			{
 				return;
@@ -232,6 +259,7 @@ public class NpcStatsDumper
 							// Update variant name or fall back to current name
 							final String curName = nameMap.get(curID);
 							stats.setName(curName == null ? stats.getName() : curName);
+							stats.setWiki(pageData.getLeft());
 
 							npcStats.put(curID, stats);
 							log.debug("Dumped npc stats for npc id: {}", curID);
@@ -319,40 +347,91 @@ public class NpcStatsDumper
 		stats.bonusRangeStrength(getInt("rngbns", variantKey, template));
 		stats.bonusMagicDamage(getInt("mbns", variantKey, template));
 
+		stats.respawn(getInt("respawn", variantKey, template));
+
 		final String keySuffix = getKeySuffix(variantKey);
-		boolean pImmune = "immune".equalsIgnoreCase(template.getValue("immunepoison" + keySuffix));
-		boolean vImmune = "immune".equalsIgnoreCase(template.getValue("immunevenom" + keySuffix));
 
-		stats.poisonImmune(!pImmune ? null : true);
+		stats.poisonImmune(isImmune(template, keySuffix, "immunepoison"));
-		stats.venomImmune(!vImmune ? null : true);
+		stats.venomImmune(isImmune(template, keySuffix, "immunevenom"));
+		stats.cannonImmune(isImmune(template, keySuffix, "immunecannon"));
+		stats.thrallImmune(isImmune(template, keySuffix, "immunethrall"));
 
-		final String weaknessValue = template.getValue("weakness");
+		final String attributesValues = template.getValue("attributes");
-		if (weaknessValue != null)
+		if (attributesValues != null)
 		{
-			final String[] values = weaknessValue.split(",");
+			final String[] values = attributesValues.split(",");
 			for (String value : values)
 			{
 				value = value.toLowerCase();
-				if (stats.dragon == null && (value.contains("dragonbane weapons")))
-				{
-					stats.dragon(true);
-				}
 
-				if (stats.demon == null && (value.contains("demonbane weapons") || value.contains("silverlight") || value.contains("arclight")))
+				if (stats.demon == null && containsString(value, "demon"))
 				{
 					stats.demon(true);
 				}
-
+				else if (stats.dragon == null && (containsString(value, "dragon") || containsString(value, "draconic")))
-				if (stats.undead == null && (value.contains("salve amulet") || value.contains("crumble undead")))
+				{
+					stats.dragon(true);
+				}
+				else if (stats.fiery == null && containsString(value, "fiery"))
+				{
+					stats.fiery(true);
+				}
+				else if (stats.kalphite == null && containsString(value, "kalphite"))
+				{
+					stats.kalphite(true);
+				}
+				else if (stats.leafy == null && containsString(value, "leafy"))
+				{
+					stats.leafy(true);
+				}
+				else if (stats.shade == null && containsString(value, "shade"))
+				{
+					stats.shade(true);
+				}
+				else if (stats.spectral == null && containsString(value, "spectral"))
+				{
+					stats.spectral(true);
+				}
+				else if (stats.undead == null && containsString(value, "undead"))
 				{
 					stats.undead(true);
 				}
+				else if (stats.vampyre1 == null && containsString(value, "vampyre1"))
+				{
+					stats.vampyre1(true);
+				}
+				else if (stats.vampyre2 == null && containsString(value, "vampyre2"))
+				{
+					stats.vampyre2(true);
+				}
+				else if (stats.vampyre3 == null && containsString(value, "vampyre3"))
+				{
+					stats.vampyre3(true);
+				}
+				else if (stats.xerician == null && containsString(value, "xerician"))
+				{
+					stats.xerician(true);
+				}
 			}
 		}
 
 		return stats.build();
 	}
 
+	private static boolean isImmune(final MediaWikiTemplate template, final String keySuffix, final String key)
+	{
+		if (!template.containsKey(key + keySuffix))
+		{
+			return false;
+		}
+		else
+		{
+			String immunevenom = template.getMap().get(key + keySuffix);
+
+			return immunevenom.equalsIgnoreCase("yes") || immunevenom.equalsIgnoreCase("immune");
+		}
+	}
+
 	static Integer getInt(final String mainKey, final Integer variation, final MediaWikiTemplate template)
 	{
 		final String key = mainKey + getKeySuffix(variation);
@@ -391,4 +470,10 @@ public class NpcStatsDumper
 			return null;
 		}
 	}
+
+	public static boolean containsString(String left, String right)
+	{
+		return left.toLowerCase().contains(right.toLowerCase());
+	}
+
 }

wiki-scraper/wiki-scraper.gradle.kts

@@ -33,6 +33,7 @@ dependencies {
 
     compileOnly(group = "org.projectlombok", name = "lombok", version = ProjectVersions.lombokVersion)
 
+    implementation(group = "org.apache.commons", name = "commons-text", version = "1.9")
     implementation(group = "com.google.code.gson", name = "gson", version = "2.8.5")
     implementation(group = "com.google.guava", name = "guava", version = "30.1.1-jre")
     implementation(group = "com.github.petitparser", name = "java-petitparser", version = "2.3.1")