Merge remote-tracking branch 'Owain/scrapy'

wiki-scraper/src/main/java/net/runelite/data/App.java

@@ -68,7 +68,7 @@ public class App
 
 	private static Store cacheStore() throws IOException
 	{
-		Path path = Paths.get(System.getProperty("user.home"), "jagexcache" + File.separator + "oldschool" + File.separator + "LIVE");
+		Path path = Paths.get(System.getProperty("user.home"), ".openosrs" + File.separator + "jagexcache" + File.separator + "oldschool" + File.separator + "LIVE");
 		final File jagexcache = new File(String.valueOf(path));
 
 		if (!Files.exists(path))
@@ -80,9 +80,6 @@ public class App
 
 		cacheStore.load();
 
-		// Try to make this go faster (probably not very smart)
-		System.setProperty("java.util.concurrent.ForkJoinPool.common.parallelism", "100");
-
 		return cacheStore;
 	}
 

wiki-scraper/src/main/java/net/runelite/data/dump/MediaWiki.java

@@ -27,6 +27,9 @@ import java.io.UnsupportedEncodingException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
 import net.runelite.data.App;
 import okhttp3.HttpUrl;
 import okhttp3.OkHttpClient;
@@ -35,6 +38,7 @@ import okhttp3.Response;
 import java.net.URLDecoder;
 import java.nio.charset.StandardCharsets;
 
+@Slf4j
 public class MediaWiki
 {
 	private static final class WikiInnerResponse
@@ -53,6 +57,7 @@ public class MediaWiki
 		.followSslRedirects(false)
 		.build();
 
+	@Getter
 	private final HttpUrl base;
 
 	public MediaWiki(final String base)
@@ -75,15 +80,28 @@ public class MediaWiki
 
 		try (final Response response = clientNoRedirect.newCall(request).execute())
 		{
+			log.info("original url: {}", url);
 			if (response.isRedirect())
 			{
 				final String page = response.header("Location")
 					.replace(base.newBuilder().addPathSegment("w").build().toString() + "/", "");
+				log.info("redirect url: {}", page);
 				return getPageData(page, section);
 			}
+			else
+			{
+				log.info("unsuccessful: {}", response.code());
+
+				if (response.code() == 429)
+				{
+					Thread.sleep(2500);
+					return getSpecialLookupData(type, id, section);
+				}
+			}
 		}
 		catch (Exception e)
 		{
+			log.info("exception: {}", e.getMessage());
 			return "";
 		}
 
@@ -109,7 +127,7 @@ public class MediaWiki
 			.addQueryParameter("format", "json")
 			.addQueryParameter("prop", "wikitext")
 			.addQueryParameter("redirects", "true")
-			.addQueryParameter("page", page.replaceAll(" ", "_"));
+			.addQueryParameter("page", page.replaceAll(" ", "_"))
 
 		if (section != -1)
 		{
@@ -129,9 +147,20 @@ public class MediaWiki
 				final InputStream in = response.body().byteStream();
 				return App.GSON.fromJson(new InputStreamReader(in), WikiResponse.class).parse.wikitext.get("*");
 			}
+			else
+			{
+				log.info("page data unsuccessful: {}", response.code());
+
+				if (response.code() == 429)
+				{
+					Thread.sleep(2500);
+					return getPageData(page, section);
+				}
+			}
 		}
 		catch (Exception e)
 		{
+			log.info("exception page data: {}", e.getMessage());
 			return "";
 		}
 

wiki-scraper/src/main/java/net/runelite/data/dump/wiki/ItemStatsDumper.java

@@ -57,7 +57,10 @@ public class ItemStatsDumper
 
 		final Map<Integer, ItemStats> itemStats = new TreeMap<>();
 		final Collection<ItemDefinition> items = itemManager.getItems();
-		final Stream<ItemDefinition> itemDefinitionStream = items.parallelStream();
+
+		log.info("{}", items.size());
+
+		final Stream<ItemDefinition> itemDefinitionStream = items.stream();
 
 		itemDefinitionStream.forEach(item ->
 		{
@@ -109,6 +112,12 @@ public class ItemStatsDumper
 					continue;
 				}
 
+				itemStat.wiki(wiki.getBase().newBuilder()
+					.addPathSegment("w")
+					.addPathSegment("Special:Lookup")
+					.addQueryParameter("type", "item")
+					.addQueryParameter("id", String.valueOf(item.id))
+					.build().toString());
 				itemStat.name(getVarString(base, "name", offset) == null ? getVarString(base, "name1", offset) : getVarString(base, "name", offset));
 				itemStat.quest(getVarBoolean(base, "quest", offset));
 				itemStat.equipable(getVarBoolean(base, "equipable", offset) == null
@@ -347,6 +356,7 @@ public class ItemStatsDumper
 	{
 		static final ItemStats DEFAULT = ItemStats.builder().build();
 
+		private final String wiki;
 		private final String name;
 		private final Boolean quest;
 		private final Boolean equipable;

wiki-scraper/src/main/java/net/runelite/data/dump/wiki/NpcStatsDumper.java

@@ -56,6 +56,7 @@ public class NpcStatsDumper
 	private static final class NpcStats
 	{
 		private String name;
+		private String wiki;
 		private final Integer hitpoints;
 		private final Integer hitpoints1;
 		private final Integer combatLevel;
@@ -133,7 +134,10 @@ public class NpcStatsDumper
 
 		final Map<Integer, NpcStats> npcStats = new HashMap<>();
 		final Collection<NpcDefinition> definitions = npcManager.getNpcs();
-		final Stream<NpcDefinition> npcDefinitionStream = definitions.parallelStream();
+
+		log.info("{}", definitions.size());
+
+		final Stream<NpcDefinition> npcDefinitionStream = definitions.stream();
 
 		// Ensure variant names match cache as wiki isn't always correct
 		final Map<Integer, String> nameMap = new HashMap<>();
@@ -232,6 +236,12 @@ public class NpcStatsDumper
 							// Update variant name or fall back to current name
 							final String curName = nameMap.get(curID);
 							stats.setName(curName == null ? stats.getName() : curName);
+							stats.setWiki(wiki.getBase().newBuilder()
+								.addPathSegment("w")
+								.addPathSegment("Special:Lookup")
+								.addQueryParameter("type", "npc")
+								.addQueryParameter("id", String.valueOf(n.getId()))
+								.build().toString());
 
 							npcStats.put(curID, stats);
 							log.debug("Dumped npc stats for npc id: {}", curID);