ra/runelite
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Merge pull request #3202 from open-osrs/scraper-update

Browse Source
This commit is contained in:
Owain van Brakel
2022-05-19 15:08:05 +02:00
committed by GitHub
parent 7968eebfb6 d85365a77b
commit d803eef8c3
11 changed files with 250 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
runelite-client/src/main/java/com/openosrs/client/game/NPCManager.java
View File

@@ -74,7 +74,7 @@ public class NPCManager
{
try (JsonReader reader = new JsonReader(new InputStreamReader(NPCManager.class.getResourceAsStream("/npc_stats.json"), StandardCharsets.UTF_8)))
{
ImmutableMap.Builder<Integer, NPCStats> builder = ImmutableMap.builderWithExpectedSize(2821);
ImmutableMap.Builder<Integer, NPCStats> builder = ImmutableMap.builderWithExpectedSize(3123);
reader.beginObject();
while (reader.hasNext())

55
runelite-client/src/main/java/com/openosrs/client/game/NPCStats.java
View File

@@ -67,10 +67,23 @@ public class NPCStats
private final boolean poisonImmune;
private final boolean venomImmune;
private final boolean cannonImmune;
private final boolean thrallImmune;
private final boolean dragon;
private final boolean demon;
private final boolean dragon;
private final boolean fiery;
private final boolean kalphite;
private final boolean leafy;
private final boolean shade;
private final boolean spectral;
private final boolean undead;
private final boolean vampyre1;
private final boolean vampyre2;
private final boolean vampyre3;
private final boolean xerician;
private final int respawn;
/**
* Based off the formula found here: http://services.runescape.com/m=forum/c=PLuJ4cy6gtA/forums.ws?317,318,712,65587452,209,337584542#209
@@ -184,15 +197,51 @@ public class NPCStats
case "venomImmune":
builder.venomImmune(in.nextBoolean());
break;
case "dragon":
builder.dragon(in.nextBoolean());
case "cannonImmune":
builder.cannonImmune(in.nextBoolean());
break;
case "thrallImmune":
builder.thrallImmune(in.nextBoolean());
break;
case "demon":
builder.demon(in.nextBoolean());
break;
case "dragon":
builder.dragon(in.nextBoolean());
break;
case "fiery":
builder.fiery(in.nextBoolean());
break;
case "kalphite":
builder.kalphite(in.nextBoolean());
break;
case "leafy":
builder.leafy(in.nextBoolean());
break;
case "shade":
builder.shade(in.nextBoolean());
break;
case "spectral":
builder.spectral(in.nextBoolean());
break;
case "undead":
builder.undead(in.nextBoolean());
break;
case "vampyre1":
builder.vampyre1(in.nextBoolean());
break;
case "vampyre2":
builder.vampyre2(in.nextBoolean());
break;
case "vampyre3":
builder.vampyre3(in.nextBoolean());
break;
case "xerician":
builder.xerician(in.nextBoolean());
break;
case "respawn":
builder.respawn(in.nextInt());
break;
}
}

2
runelite-client/src/main/resources/item_stats.json
View File

File diff suppressed because one or more lines are too long

2
runelite-client/src/main/resources/npc_stats.json
View File

File diff suppressed because one or more lines are too long

5
wiki-scraper/src/main/java/net/runelite/data/App.java
View File

@@ -68,7 +68,7 @@ public class App
private static Store cacheStore() throws IOException
{
Path path = Paths.get(System.getProperty("user.home"), "jagexcache" + File.separator + "oldschool" + File.separator + "LIVE");
Path path = Paths.get(System.getProperty("user.home"), ".openosrs" + File.separator + "jagexcache" + File.separator + "oldschool" + File.separator + "LIVE");
final File jagexcache = new File(String.valueOf(path));
if (!Files.exists(path))
@@ -80,9 +80,6 @@ public class App
cacheStore.load();
// Try to make this go faster (probably not very smart)
System.setProperty("java.util.concurrent.ForkJoinPool.common.parallelism", "100");
return cacheStore;
}

61
wiki-scraper/src/main/java/net/runelite/data/dump/MediaWiki.java
View File

@@ -27,6 +27,8 @@ import java.io.UnsupportedEncodingException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.Map;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import net.runelite.data.App;
import okhttp3.HttpUrl;
import okhttp3.OkHttpClient;
@@ -34,7 +36,10 @@ import okhttp3.Request;
import okhttp3.Response;
import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import okhttp3.ResponseBody;
import org.apache.commons.lang3.tuple.Pair;
@Slf4j
public class MediaWiki
{
private static final class WikiInnerResponse
@@ -53,6 +58,7 @@ public class MediaWiki
.followSslRedirects(false)
.build();
@Getter
private final HttpUrl base;
public MediaWiki(final String base)
@@ -60,7 +66,7 @@ public class MediaWiki
this.base = HttpUrl.parse(base);
}
public String getSpecialLookupData(final String type, final int id, final int section)
public Pair<String, String> getSpecialLookupData(final String type, final int id, final int section)
{
final HttpUrl url = base.newBuilder()
.addPathSegment("w")
@@ -77,20 +83,39 @@ public class MediaWiki
{
if (response.isRedirect())
{
final String page = response.header("Location")
String responseHeaderLocation = response.header("Location");
if (responseHeaderLocation == null)
{
return null;
}
final String page = responseHeaderLocation
.replace(base.newBuilder().addPathSegment("w").build().toString() + "/", "");
return getPageData(page, section);
}
else
{
log.info("unsuccessful: {}", response.code());
if (response.code() == 429)
{
Thread.sleep(2500);
return getSpecialLookupData(type, id, section);
}
}
}
catch (Exception e)
{
return "";
log.info("exception: {}", e.getMessage());
return null;
}
return "";
return null;
}
public String getPageData(String page, int section)
public Pair<String, String> getPageData(String page, int section)
{
// decode html encoded page name
// ex: Mage%27s book -> Mage's_book
@@ -126,15 +151,33 @@ public class MediaWiki
{
if (response.isSuccessful())
{
final InputStream in = response.body().byteStream();
return App.GSON.fromJson(new InputStreamReader(in), WikiResponse.class).parse.wikitext.get("*");
ResponseBody responseBody = response.body();
if (responseBody == null)
{
return null;
}
final InputStream in = responseBody.byteStream();
return Pair.of(page.replaceAll(" ", "_"), App.GSON.fromJson(new InputStreamReader(in), WikiResponse.class).parse.wikitext.get("*"));
}
else
{
log.info("page data unsuccessful: {}", response.code());
if (response.code() == 429)
{
Thread.sleep(2500);
return getPageData(page, section);
}
}
}
catch (Exception e)
{
return "";
log.info("exception page data: {}", e.getMessage());
return null;
}
return "";
return null;
}
}

5
wiki-scraper/src/main/java/net/runelite/data/dump/MediaWikiTemplate.java
View File

@@ -230,6 +230,11 @@ public class MediaWikiTemplate
this.map = map;
}
public Map<String, String> getMap()
{
return map;
}
public String getValue(final String key)
{
String val = map.get(key);

10
wiki-scraper/src/main/java/net/runelite/data/dump/wiki/ItemLimitsDumper.java
View File

@@ -43,6 +43,7 @@ import net.runelite.cache.util.Namer;
import net.runelite.data.App;
import net.runelite.data.dump.MediaWiki;
import net.runelite.data.dump.MediaWikiTemplate;
import org.apache.commons.lang3.tuple.Pair;
@Slf4j
public class ItemLimitsDumper
@@ -88,7 +89,14 @@ public class ItemLimitsDumper
return;
}
String data = wiki.getPageData("Module:Exchange/" + name, -1);
Pair<String, String> pageData = wiki.getPageData("Module:Exchange/" + name, -1);
if (pageData == null)
{
return;
}
String data = pageData.getRight();
if (Strings.isNullOrEmpty(data))
{

29
wiki-scraper/src/main/java/net/runelite/data/dump/wiki/ItemStatsDumper.java
View File

@@ -42,6 +42,7 @@ import net.runelite.cache.util.Namer;
import net.runelite.data.App;
import net.runelite.data.dump.MediaWiki;
import net.runelite.data.dump.MediaWikiTemplate;
import org.apache.commons.lang3.tuple.Pair;
@Slf4j
public class ItemStatsDumper
@@ -57,7 +58,10 @@ public class ItemStatsDumper
final Map<Integer, ItemStats> itemStats = new TreeMap<>();
final Collection<ItemDefinition> items = itemManager.getItems();
final Stream<ItemDefinition> itemDefinitionStream = items.parallelStream();
log.info("{}", items.size());
final Stream<ItemDefinition> itemDefinitionStream = items.stream();
itemDefinitionStream.forEach(item ->
{
@@ -81,7 +85,14 @@ public class ItemStatsDumper
return;
}
String data = wiki.getSpecialLookupData("item", item.id, 0);
Pair<String, String> pageData = wiki.getSpecialLookupData("item", item.id, 0);
if (pageData == null)
{
return;
}
String data = pageData.getRight();
if (Strings.isNullOrEmpty(data))
{
@@ -109,6 +120,7 @@ public class ItemStatsDumper
continue;
}
itemStat.wiki(pageData.getLeft());
itemStat.name(getVarString(base, "name", offset) == null ? getVarString(base, "name1", offset) : getVarString(base, "name", offset));
itemStat.quest(getVarBoolean(base, "quest", offset));
itemStat.equipable(getVarBoolean(base, "equipable", offset) == null
@@ -122,7 +134,14 @@ public class ItemStatsDumper
if (stats == null)
{
data = wiki.getSpecialLookupData("item", item.id, 1);
pageData = wiki.getSpecialLookupData("item", item.id, 1);
if (pageData == null)
{
return;
}
data = pageData.getRight();
if (Strings.isNullOrEmpty(data))
{
@@ -156,7 +175,8 @@ public class ItemStatsDumper
equipmentStat.rstr(getVarInt(stats, "rstr", offset));
equipmentStat.mdmg(getVarInt(stats, "mdmg", offset));
equipmentStat.prayer(getVarInt(stats, "prayer", offset));
equipmentStat.aspeed(getVarInt(stats, "aspeed", offset));
equipmentStat.aspeed(getVarInt(stats, "speed", offset));
final ItemEquipmentStats builtEqStat = equipmentStat.build();
@@ -347,6 +367,7 @@ public class ItemStatsDumper
{
static final ItemStats DEFAULT = ItemStats.builder().build();
private final String wiki;
private final String name;
private final Boolean quest;
private final Boolean equipable;

119
wiki-scraper/src/main/java/net/runelite/data/dump/wiki/NpcStatsDumper.java
View File

@@ -47,6 +47,7 @@ import net.runelite.cache.util.Namer;
import net.runelite.data.App;
import net.runelite.data.dump.MediaWiki;
import net.runelite.data.dump.MediaWikiTemplate;
import org.apache.commons.lang3.tuple.Pair;
@Slf4j
public class NpcStatsDumper
@@ -56,6 +57,8 @@ public class NpcStatsDumper
private static final class NpcStats
{
private String name;
private String wiki;
private final Integer hitpoints;
private final Integer hitpoints1;
private final Integer combatLevel;
@@ -87,10 +90,23 @@ public class NpcStatsDumper
private final Boolean poisonImmune;
private final Boolean venomImmune;
private final Boolean cannonImmune;
private final Boolean thrallImmune;
private final Boolean dragon;
private final Boolean demon;
private final Boolean dragon;
private final Boolean fiery;
private final Boolean kalphite;
private final Boolean leafy;
private final Boolean shade;
private final Boolean spectral;
private final Boolean undead;
private final Boolean vampyre1;
private final Boolean vampyre2;
private final Boolean vampyre3;
private final Boolean xerician;
private final Integer respawn;
}
private static final NpcStats DEFAULT = NpcStats.builder().build();
@@ -133,7 +149,10 @@ public class NpcStatsDumper
final Map<Integer, NpcStats> npcStats = new HashMap<>();
final Collection<NpcDefinition> definitions = npcManager.getNpcs();
final Stream<NpcDefinition> npcDefinitionStream = definitions.parallelStream();
log.info("{}", definitions.size());
final Stream<NpcDefinition> npcDefinitionStream = definitions.stream();
// Ensure variant names match cache as wiki isn't always correct
final Map<Integer, String> nameMap = new HashMap<>();
@@ -175,7 +194,15 @@ public class NpcStatsDumper
return;
}
final String data = wiki.getSpecialLookupData("npc", n.getId(), 0);
Pair<String, String> pageData = wiki.getSpecialLookupData("npc", n.getId(), 0);
if (pageData == null)
{
return;
}
String data = pageData.getRight();
if (Strings.isNullOrEmpty(data))
{
return;
@@ -232,6 +259,7 @@ public class NpcStatsDumper
// Update variant name or fall back to current name
final String curName = nameMap.get(curID);
stats.setName(curName == null ? stats.getName() : curName);
stats.setWiki(pageData.getLeft());
npcStats.put(curID, stats);
log.debug("Dumped npc stats for npc id: {}", curID);
@@ -319,40 +347,91 @@ public class NpcStatsDumper
stats.bonusRangeStrength(getInt("rngbns", variantKey, template));
stats.bonusMagicDamage(getInt("mbns", variantKey, template));
stats.respawn(getInt("respawn", variantKey, template));
final String keySuffix = getKeySuffix(variantKey);
boolean pImmune = "immune".equalsIgnoreCase(template.getValue("immunepoison" + keySuffix));
boolean vImmune = "immune".equalsIgnoreCase(template.getValue("immunevenom" + keySuffix));
stats.poisonImmune(!pImmune ? null : true);
stats.venomImmune(!vImmune ? null : true);
stats.poisonImmune(isImmune(template, keySuffix, "immunepoison"));
stats.venomImmune(isImmune(template, keySuffix, "immunevenom"));
stats.cannonImmune(isImmune(template, keySuffix, "immunecannon"));
stats.thrallImmune(isImmune(template, keySuffix, "immunethrall"));
final String weaknessValue = template.getValue("weakness");
if (weaknessValue != null)
final String attributesValues = template.getValue("attributes");
if (attributesValues != null)
{
final String[] values = weaknessValue.split(",");
final String[] values = attributesValues.split(",");
for (String value : values)
{
value = value.toLowerCase();
if (stats.dragon == null && (value.contains("dragonbane weapons")))
{
stats.dragon(true);
}
if (stats.demon == null && (value.contains("demonbane weapons") || value.contains("silverlight") || value.contains("arclight")))
if (stats.demon == null && containsString(value, "demon"))
{
stats.demon(true);
}
if (stats.undead == null && (value.contains("salve amulet") || value.contains("crumble undead")))
else if (stats.dragon == null && (containsString(value, "dragon") || containsString(value, "draconic")))
{
stats.dragon(true);
}
else if (stats.fiery == null && containsString(value, "fiery"))
{
stats.fiery(true);
}
else if (stats.kalphite == null && containsString(value, "kalphite"))
{
stats.kalphite(true);
}
else if (stats.leafy == null && containsString(value, "leafy"))
{
stats.leafy(true);
}
else if (stats.shade == null && containsString(value, "shade"))
{
stats.shade(true);
}
else if (stats.spectral == null && containsString(value, "spectral"))
{
stats.spectral(true);
}
else if (stats.undead == null && containsString(value, "undead"))
{
stats.undead(true);
}
else if (stats.vampyre1 == null && containsString(value, "vampyre1"))
{
stats.vampyre1(true);
}
else if (stats.vampyre2 == null && containsString(value, "vampyre2"))
{
stats.vampyre2(true);
}
else if (stats.vampyre3 == null && containsString(value, "vampyre3"))
{
stats.vampyre3(true);
}
else if (stats.xerician == null && containsString(value, "xerician"))
{
stats.xerician(true);
}
}
}
return stats.build();
}
private static boolean isImmune(final MediaWikiTemplate template, final String keySuffix, final String key)
{
if (!template.containsKey(key + keySuffix))
{
return false;
}
else
{
String immunevenom = template.getMap().get(key + keySuffix);
return immunevenom.equalsIgnoreCase("yes") || immunevenom.equalsIgnoreCase("immune");
}
}
static Integer getInt(final String mainKey, final Integer variation, final MediaWikiTemplate template)
{
final String key = mainKey + getKeySuffix(variation);
@@ -391,4 +470,10 @@ public class NpcStatsDumper
return null;
}
}
public static boolean containsString(String left, String right)
{
return left.toLowerCase().contains(right.toLowerCase());
}
}

1
wiki-scraper/wiki-scraper.gradle.kts
View File

@@ -33,6 +33,7 @@ dependencies {
compileOnly(group = "org.projectlombok", name = "lombok", version = ProjectVersions.lombokVersion)
implementation(group = "org.apache.commons", name = "commons-text", version = "1.9")
implementation(group = "com.google.code.gson", name = "gson", version = "2.8.5")
implementation(group = "com.google.guava", name = "guava", version = "30.1.1-jre")
implementation(group = "com.github.petitparser", name = "java-petitparser", version = "2.3.1")