ra/runelite
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

projec(scraper)t: Fix scraper and add some new fields

Browse Source
This commit is contained in:
Owain van Brakel
2022-05-19 15:02:04 +02:00
parent b15e5d6d1f
commit d85365a77b
10 changed files with 214 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
wiki-scraper/src/main/java/net/runelite/data/dump/MediaWiki.java
View File

@@ -27,7 +27,6 @@ import java.io.UnsupportedEncodingException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import net.runelite.data.App;
@@ -37,6 +36,8 @@ import okhttp3.Request;
import okhttp3.Response;
import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import okhttp3.ResponseBody;
import org.apache.commons.lang3.tuple.Pair;
@Slf4j
public class MediaWiki
@@ -65,7 +66,7 @@ public class MediaWiki
this.base = HttpUrl.parse(base);
}
public String getSpecialLookupData(final String type, final int id, final int section)
public Pair<String, String> getSpecialLookupData(final String type, final int id, final int section)
{
final HttpUrl url = base.newBuilder()
.addPathSegment("w")
@@ -80,12 +81,18 @@ public class MediaWiki
try (final Response response = clientNoRedirect.newCall(request).execute())
{
log.info("original url: {}", url);
if (response.isRedirect())
{
final String page = response.header("Location")
String responseHeaderLocation = response.header("Location");
if (responseHeaderLocation == null)
{
return null;
}
final String page = responseHeaderLocation
.replace(base.newBuilder().addPathSegment("w").build().toString() + "/", "");
log.info("redirect url: {}", page);
return getPageData(page, section);
}
else
@@ -102,13 +109,13 @@ public class MediaWiki
catch (Exception e)
{
log.info("exception: {}", e.getMessage());
return "";
return null;
}
return "";
return null;
}
public String getPageData(String page, int section)
public Pair<String, String> getPageData(String page, int section)
{
// decode html encoded page name
// ex: Mage%27s book -> Mage's_book
@@ -127,7 +134,7 @@ public class MediaWiki
.addQueryParameter("format", "json")
.addQueryParameter("prop", "wikitext")
.addQueryParameter("redirects", "true")
.addQueryParameter("page", page.replaceAll(" ", "_"))
.addQueryParameter("page", page.replaceAll(" ", "_"));
if (section != -1)
{
@@ -144,8 +151,15 @@ public class MediaWiki
{
if (response.isSuccessful())
{
final InputStream in = response.body().byteStream();
return App.GSON.fromJson(new InputStreamReader(in), WikiResponse.class).parse.wikitext.get("*");
ResponseBody responseBody = response.body();
if (responseBody == null)
{
return null;
}
final InputStream in = responseBody.byteStream();
return Pair.of(page.replaceAll(" ", "_"), App.GSON.fromJson(new InputStreamReader(in), WikiResponse.class).parse.wikitext.get("*"));
}
else
{
@@ -161,9 +175,9 @@ public class MediaWiki
catch (Exception e)
{
log.info("exception page data: {}", e.getMessage());
return "";
return null;
}
return "";
return null;
}
}

5
wiki-scraper/src/main/java/net/runelite/data/dump/MediaWikiTemplate.java
View File

@@ -230,6 +230,11 @@ public class MediaWikiTemplate
this.map = map;
}
public Map<String, String> getMap()
{
return map;
}
public String getValue(final String key)
{
String val = map.get(key);

10
wiki-scraper/src/main/java/net/runelite/data/dump/wiki/ItemLimitsDumper.java
View File

@@ -43,6 +43,7 @@ import net.runelite.cache.util.Namer;
import net.runelite.data.App;
import net.runelite.data.dump.MediaWiki;
import net.runelite.data.dump.MediaWikiTemplate;
import org.apache.commons.lang3.tuple.Pair;
@Slf4j
public class ItemLimitsDumper
@@ -88,7 +89,14 @@ public class ItemLimitsDumper
return;
}
String data = wiki.getPageData("Module:Exchange/" + name, -1);
Pair<String, String> pageData = wiki.getPageData("Module:Exchange/" + name, -1);
if (pageData == null)
{
return;
}
String data = pageData.getRight();
if (Strings.isNullOrEmpty(data))
{

29
wiki-scraper/src/main/java/net/runelite/data/dump/wiki/ItemStatsDumper.java
View File

@@ -42,6 +42,7 @@ import net.runelite.cache.util.Namer;
import net.runelite.data.App;
import net.runelite.data.dump.MediaWiki;
import net.runelite.data.dump.MediaWikiTemplate;
import org.apache.commons.lang3.tuple.Pair;
@Slf4j
public class ItemStatsDumper
@@ -84,7 +85,14 @@ public class ItemStatsDumper
return;
}
String data = wiki.getSpecialLookupData("item", item.id, 0);
Pair<String, String> pageData = wiki.getSpecialLookupData("item", item.id, 0);
if (pageData == null)
{
return;
}
String data = pageData.getRight();
if (Strings.isNullOrEmpty(data))
{
@@ -112,12 +120,7 @@ public class ItemStatsDumper
continue;
}
itemStat.wiki(wiki.getBase().newBuilder()
.addPathSegment("w")
.addPathSegment("Special:Lookup")
.addQueryParameter("type", "item")
.addQueryParameter("id", String.valueOf(item.id))
.build().toString());
itemStat.wiki(pageData.getLeft());
itemStat.name(getVarString(base, "name", offset) == null ? getVarString(base, "name1", offset) : getVarString(base, "name", offset));
itemStat.quest(getVarBoolean(base, "quest", offset));
itemStat.equipable(getVarBoolean(base, "equipable", offset) == null
@@ -131,7 +134,14 @@ public class ItemStatsDumper
if (stats == null)
{
data = wiki.getSpecialLookupData("item", item.id, 1);
pageData = wiki.getSpecialLookupData("item", item.id, 1);
if (pageData == null)
{
return;
}
data = pageData.getRight();
if (Strings.isNullOrEmpty(data))
{
@@ -165,7 +175,8 @@ public class ItemStatsDumper
equipmentStat.rstr(getVarInt(stats, "rstr", offset));
equipmentStat.mdmg(getVarInt(stats, "mdmg", offset));
equipmentStat.prayer(getVarInt(stats, "prayer", offset));
equipmentStat.aspeed(getVarInt(stats, "aspeed", offset));
equipmentStat.aspeed(getVarInt(stats, "speed", offset));
final ItemEquipmentStats builtEqStat = equipmentStat.build();

119
wiki-scraper/src/main/java/net/runelite/data/dump/wiki/NpcStatsDumper.java
View File

@@ -47,6 +47,7 @@ import net.runelite.cache.util.Namer;
import net.runelite.data.App;
import net.runelite.data.dump.MediaWiki;
import net.runelite.data.dump.MediaWikiTemplate;
import org.apache.commons.lang3.tuple.Pair;
@Slf4j
public class NpcStatsDumper
@@ -57,6 +58,7 @@ public class NpcStatsDumper
{
private String name;
private String wiki;
private final Integer hitpoints;
private final Integer hitpoints1;
private final Integer combatLevel;
@@ -88,10 +90,23 @@ public class NpcStatsDumper
private final Boolean poisonImmune;
private final Boolean venomImmune;
private final Boolean cannonImmune;
private final Boolean thrallImmune;
private final Boolean dragon;
private final Boolean demon;
private final Boolean dragon;
private final Boolean fiery;
private final Boolean kalphite;
private final Boolean leafy;
private final Boolean shade;
private final Boolean spectral;
private final Boolean undead;
private final Boolean vampyre1;
private final Boolean vampyre2;
private final Boolean vampyre3;
private final Boolean xerician;
private final Integer respawn;
}
private static final NpcStats DEFAULT = NpcStats.builder().build();
@@ -179,7 +194,15 @@ public class NpcStatsDumper
return;
}
final String data = wiki.getSpecialLookupData("npc", n.getId(), 0);
Pair<String, String> pageData = wiki.getSpecialLookupData("npc", n.getId(), 0);
if (pageData == null)
{
return;
}
String data = pageData.getRight();
if (Strings.isNullOrEmpty(data))
{
return;
@@ -236,12 +259,7 @@ public class NpcStatsDumper
// Update variant name or fall back to current name
final String curName = nameMap.get(curID);
stats.setName(curName == null ? stats.getName() : curName);
stats.setWiki(wiki.getBase().newBuilder()
.addPathSegment("w")
.addPathSegment("Special:Lookup")
.addQueryParameter("type", "npc")
.addQueryParameter("id", String.valueOf(n.getId()))
.build().toString());
stats.setWiki(pageData.getLeft());
npcStats.put(curID, stats);
log.debug("Dumped npc stats for npc id: {}", curID);
@@ -329,40 +347,91 @@ public class NpcStatsDumper
stats.bonusRangeStrength(getInt("rngbns", variantKey, template));
stats.bonusMagicDamage(getInt("mbns", variantKey, template));
stats.respawn(getInt("respawn", variantKey, template));
final String keySuffix = getKeySuffix(variantKey);
boolean pImmune = "immune".equalsIgnoreCase(template.getValue("immunepoison" + keySuffix));
boolean vImmune = "immune".equalsIgnoreCase(template.getValue("immunevenom" + keySuffix));
stats.poisonImmune(!pImmune ? null : true);
stats.venomImmune(!vImmune ? null : true);
stats.poisonImmune(isImmune(template, keySuffix, "immunepoison"));
stats.venomImmune(isImmune(template, keySuffix, "immunevenom"));
stats.cannonImmune(isImmune(template, keySuffix, "immunecannon"));
stats.thrallImmune(isImmune(template, keySuffix, "immunethrall"));
final String weaknessValue = template.getValue("weakness");
if (weaknessValue != null)
final String attributesValues = template.getValue("attributes");
if (attributesValues != null)
{
final String[] values = weaknessValue.split(",");
final String[] values = attributesValues.split(",");
for (String value : values)
{
value = value.toLowerCase();
if (stats.dragon == null && (value.contains("dragonbane weapons")))
{
stats.dragon(true);
}
if (stats.demon == null && (value.contains("demonbane weapons") || value.contains("silverlight") || value.contains("arclight")))
if (stats.demon == null && containsString(value, "demon"))
{
stats.demon(true);
}
if (stats.undead == null && (value.contains("salve amulet") || value.contains("crumble undead")))
else if (stats.dragon == null && (containsString(value, "dragon") || containsString(value, "draconic")))
{
stats.dragon(true);
}
else if (stats.fiery == null && containsString(value, "fiery"))
{
stats.fiery(true);
}
else if (stats.kalphite == null && containsString(value, "kalphite"))
{
stats.kalphite(true);
}
else if (stats.leafy == null && containsString(value, "leafy"))
{
stats.leafy(true);
}
else if (stats.shade == null && containsString(value, "shade"))
{
stats.shade(true);
}
else if (stats.spectral == null && containsString(value, "spectral"))
{
stats.spectral(true);
}
else if (stats.undead == null && containsString(value, "undead"))
{
stats.undead(true);
}
else if (stats.vampyre1 == null && containsString(value, "vampyre1"))
{
stats.vampyre1(true);
}
else if (stats.vampyre2 == null && containsString(value, "vampyre2"))
{
stats.vampyre2(true);
}
else if (stats.vampyre3 == null && containsString(value, "vampyre3"))
{
stats.vampyre3(true);
}
else if (stats.xerician == null && containsString(value, "xerician"))
{
stats.xerician(true);
}
}
}
return stats.build();
}
private static boolean isImmune(final MediaWikiTemplate template, final String keySuffix, final String key)
{
if (!template.containsKey(key + keySuffix))
{
return false;
}
else
{
String immunevenom = template.getMap().get(key + keySuffix);
return immunevenom.equalsIgnoreCase("yes") || immunevenom.equalsIgnoreCase("immune");
}
}
static Integer getInt(final String mainKey, final Integer variation, final MediaWikiTemplate template)
{
final String key = mainKey + getKeySuffix(variation);
@@ -401,4 +470,10 @@ public class NpcStatsDumper
return null;
}
}
public static boolean containsString(String left, String right)
{
return left.toLowerCase().contains(right.toLowerCase());
}
}

1
wiki-scraper/wiki-scraper.gradle.kts
View File

@@ -33,6 +33,7 @@ dependencies {
compileOnly(group = "org.projectlombok", name = "lombok", version = ProjectVersions.lombokVersion)
implementation(group = "org.apache.commons", name = "commons-text", version = "1.9")
implementation(group = "com.google.code.gson", name = "gson", version = "2.8.5")
implementation(group = "com.google.guava", name = "guava", version = "30.1.1-jre")
implementation(group = "com.github.petitparser", name = "java-petitparser", version = "2.3.1")