ra/runelite
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

project: Add wiki scraper (#1348)

Browse Source 
* wiki-scraper: Add wiki scroper to the main project

* client: Updated wiki stats

* wiki-scraper: Checkstyle

* wiki-scraper: Pull in @Ganom his changes

* client: Updated wiki stats
This commit is contained in:
Owain van Brakel
2019-08-15 23:10:25 +02:00
committed by Kyleeld
parent 4fba65d4f1
commit 8239be4b75
14 changed files with 3136 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
wiki-scraper/build.gradle Normal file
View File

@@ -0,0 +1,28 @@
apply plugin:'application'
repositories {
mavenCentral()
maven { url 'https://jitpack.io' }
}
description = 'RuneLite Wiki scraper'
mainClassName = "net.runelite.data.App"
dependencies {
api project(':cache')
api project(':runelite-api')
annotationProcessor group: 'org.projectlombok', name: 'lombok', version: lombok
compileOnly group: 'org.projectlombok', name: 'lombok', version: lombok
implementation group: 'com.google.code.gson', name: 'gson', version: gson
implementation group: 'com.google.guava', name: 'guava', version: guava
implementation group: 'com.github.petitparser', name: 'java-petitparser', version: '2.2.0'
implementation group: 'com.squareup.okhttp3', name: 'okhttp', version: okhttp3
implementation group: 'org.slf4j', name: 'slf4j-api', version: slf4j
implementation group: 'org.slf4j', name: 'slf4j-simple', version: slf4j
testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-api', version: jupiter
}

59
wiki-scraper/src/main/java/net/runelite/data/App.java Normal file
View File

@@ -0,0 +1,59 @@
/*
* MIT License
*
* Copyright (c) 2018 Tomas Slusny
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package net.runelite.data;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import java.io.File;
import java.io.IOException;
import net.runelite.cache.fs.Store;
import net.runelite.data.dump.MediaWiki;
import net.runelite.data.dump.wiki.NpcStatsDumper;
public class App
{
public static final Gson GSON = new GsonBuilder()
.setPrettyPrinting()
.disableHtmlEscaping()
.create();
public static void main(String[] args) throws IOException
{
final File home = new File(System.getProperty("user.home"));
final Store cacheStore = new Store(new File(home,
"jagexcache" + File.separator + "oldschool" + File.separator + "LIVE"));
cacheStore.load();
// Try to make this go faster (probably not very smart)
System.setProperty("java.util.concurrent.ForkJoinPool.common.parallelism", "100");
final MediaWiki wiki = new MediaWiki("https://oldschool.runescape.wiki");
// Only use this to diff current limits with scraped limits
// ItemLimitsDumper.dump(cacheStore, wiki);
// ItemStatsDumper.dump(cacheStore, wiki);
NpcStatsDumper.dump(cacheStore, wiki);
}
}

140
wiki-scraper/src/main/java/net/runelite/data/dump/MediaWiki.java Normal file
View File

@@ -0,0 +1,140 @@
/*
* MIT License
*
* Copyright (c) 2018 Tomas Slusny
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package net.runelite.data.dump;
import java.io.UnsupportedEncodingException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.Map;
import net.runelite.data.App;
import okhttp3.HttpUrl;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;
import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
public class MediaWiki
{
private static final class WikiInnerResponse
{
Map<String, String> wikitext;
}
private static final class WikiResponse
{
WikiInnerResponse parse;
}
private final OkHttpClient client = new OkHttpClient();
private final OkHttpClient clientNoRedirect = client.newBuilder()
.followRedirects(false)
.followSslRedirects(false)
.build();
private final HttpUrl base;
public MediaWiki(final String base)
{
this.base = HttpUrl.parse(base);
}
public String getSpecialLookupData(final String type, final int id, final int section)
{
final HttpUrl url = base.newBuilder()
.addPathSegment("w")
.addPathSegment("Special:Lookup")
.addQueryParameter("type", type)
.addQueryParameter("id", String.valueOf(id))
.build();
final Request request = new Request.Builder()
.url(url)
.build();
try (final Response response = clientNoRedirect.newCall(request).execute())
{
if (response.isRedirect())
{
final String page = response.header("Location")
.replace(base.newBuilder().addPathSegment("w").build().toString() + "/", "");
return getPageData(page, section);
}
}
catch (Exception e)
{
return "";
}
return "";
}
public String getPageData(String page, int section)
{
// decode html encoded page name
// ex: Mage%27s book -> Mage's_book
try
{
page = URLDecoder.decode(page, StandardCharsets.UTF_8.name());
}
catch (UnsupportedEncodingException e)
{
// do nothing, keep page the same
}
final HttpUrl.Builder urlBuilder = base.newBuilder()
.addPathSegment("api.php")
.addQueryParameter("action", "parse")
.addQueryParameter("format", "json")
.addQueryParameter("prop", "wikitext")
.addQueryParameter("redirects", "true")
.addQueryParameter("page", page.replaceAll(" ", "_"));
if (section != -1)
{
urlBuilder.addQueryParameter("section", String.valueOf(section));
}
final HttpUrl url = urlBuilder.build();
final Request request = new Request.Builder()
.url(url)
.build();
try (final Response response = client.newCall(request).execute())
{
if (response.isSuccessful())
{
final InputStream in = response.body().byteStream();
return App.GSON.fromJson(new InputStreamReader(in), WikiResponse.class).parse.wikitext.get("*");
}
}
catch (Exception e)
{
return "";
}
return "";
}
}

302
wiki-scraper/src/main/java/net/runelite/data/dump/MediaWikiTemplate.java Normal file
View File

@@ -0,0 +1,302 @@
/*
* MIT License
*
* Copyright (c) 2018 Tomas Slusny
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package net.runelite.data.dump;
import com.google.common.base.Function;
import com.google.common.base.MoreObjects;
import com.google.common.base.Strings;
import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.annotation.Nullable;
import lombok.extern.slf4j.Slf4j;
import org.petitparser.context.Result;
import org.petitparser.parser.Parser;
import org.petitparser.parser.primitive.CharacterParser;
import org.petitparser.parser.primitive.StringParser;
@Slf4j
public class MediaWikiTemplate
{
private static final Parser LUA_PARSER;
private static final Parser MEDIAWIKI_PARSER;
static
{
final Parser singleString = CharacterParser.of('\'').seq(CharacterParser.of('\'').neg().plus().flatten()).seq(CharacterParser.of('\''));
final Parser doubleString = CharacterParser.of('"').seq(CharacterParser.of('"').neg().plus().flatten()).seq(CharacterParser.of('"'));
final Parser string = singleString.or(doubleString).pick(1);
final Parser key = CharacterParser.letter().or(CharacterParser.of('-')).or(CharacterParser.of('_')).or(CharacterParser.of(' ')).or(CharacterParser.digit()).plus().flatten();
final Parser value = string.or(key);
final Parser pair = key.trim()
.seq(CharacterParser.of('=').trim())
.seq(value.trim())
.map((Function<List<String>, Map.Entry<String, String>>) input -> new AbstractMap.SimpleEntry<>(input.get(0).trim(), input.get(2).trim()));
final Parser commaLine = pair
.seq(CharacterParser.of(',').optional().trim())
.pick(0);
LUA_PARSER = StringParser.of("return").trim()
.seq(CharacterParser.of('{').trim())
.seq(commaLine.plus().trim())
.pick(2);
final Parser wikiValue = CharacterParser.of('|')
.or(StringParser.of("}}"))
.or(StringParser.of("{{"))
.or(StringParser.of("]]"))
.or(StringParser.of("[["))
.neg().plus().trim();
final Parser wikiBraceExpression = StringParser.of("{{")
.seq(StringParser.of("}}").neg().star().trim())
.seq(StringParser.of("}}"));
final Parser wikiSquareExpression = StringParser.of("[[")
.seq(StringParser.of("]]").neg().star().trim())
.seq(StringParser.of("]]"));
final Parser notOrPair = key.trim()
.seq(CharacterParser.of('=').trim())
.seq(CharacterParser.whitespace().star().seq(wikiSquareExpression.or(wikiBraceExpression).or(wikiValue)).plus().flatten().trim().optional())
.map((Function<List<String>, Map.Entry<String, String>>) input -> new AbstractMap.SimpleEntry<>(
input.get(0).trim(),
MoreObjects.firstNonNull(input.get(2), "").trim()));
final Parser orLine = CharacterParser.of('|')
.seq(notOrPair.trim().optional())
.pick(1);
MEDIAWIKI_PARSER = orLine.plus().trim().seq(StringParser.of("}}")).pick(0);
}
@Nullable
public static MediaWikiTemplate parseWikitext(final String name, final String data)
{
final Pattern exactNameTest = Pattern.compile("\\{\\{\\s*" + name + "\\s*\\|", Pattern.CASE_INSENSITIVE);
final Matcher m = exactNameTest.matcher(data.toLowerCase());
// Early exit
if (!m.find())
{
return null;
}
final Map<String, String> out = new HashMap<>();
final Parser wikiParser = StringParser.of("{{")
.seq(StringParser.ofIgnoringCase(name).trim())
.seq(MEDIAWIKI_PARSER)
.pick(2);
final List<Object> parsed = wikiParser.matchesSkipping(data);
if (parsed.isEmpty())
{
final Result parse = StringParser.of("{{")
.seq(StringParser.ofIgnoringCase(name).trim())
.neg()
.star()
.seq(wikiParser)
.seq(CharacterParser.any().star())
.parse(data);
if (!parse.isSuccess())
{
log.warn("Failed to parse: {}", data);
log.warn("Error message: {}", parse.getMessage());
}
return null;
}
final List<Map.Entry<String, String>> entries = (List<Map.Entry<String, String>>) parsed.get(0);
for (Map.Entry<String, String> entry : entries)
{
if (entry == null)
{
continue;
}
out.put(entry.getKey(), entry.getValue());
}
if (out.isEmpty())
{
return null;
}
return new MediaWikiTemplate(out);
}
@Nullable
public static MediaWikiTemplate parseLua(final String data)
{
final Map<String, String> out = new HashMap<>();
final List<Object> parsed = LUA_PARSER.matchesSkipping(data);
if (parsed.isEmpty())
{
final Result parse = StringParser.of("return")
.neg()
.star()
.seq(LUA_PARSER)
.seq(CharacterParser.any()).parse(data);
if (!parse.isSuccess())
{
log.warn("Failed to parse: {}", data);
log.warn("Error message: {}", parse.getMessage());
}
return null;
}
final List<Map.Entry<String, String>> entries = (List<Map.Entry<String, String>>) parsed.get(0);
for (Map.Entry<String, String> entry : entries)
{
out.put(entry.getKey(), entry.getValue());
}
if (out.isEmpty())
{
return null;
}
return new MediaWikiTemplate(out);
}
/**
* Looks for and parses the `Switch infobox` into a {@link MediaWikiTemplate} and then iterates over the `item#` values.
* Attempts to parse each `item#` value via `parseWikiText`, matching the `name` attribute. null values are ignored
*
* @param name only parses MediaWikiTemplates from `Switch infobox` if matches this value. (case insensitive)
* @param baseTemplate the {@link MediaWikiTemplate} representation of the `Switch infobox` to parse from
* @return List of all valid {@link MediaWikiTemplate}s matching `name` from `baseTemplate`s `item#` values
*/
public static List<MediaWikiTemplate> parseSwitchInfoboxItems(final String name, final MediaWikiTemplate baseTemplate)
{
final List<MediaWikiTemplate> templates = new ArrayList<>();
String value;
int suffix = 1;
while ((value = baseTemplate.getValue("item" + suffix)) != null)
{
final MediaWikiTemplate subTemplate = parseWikitext(name, value);
if (subTemplate != null)
{
templates.add(subTemplate);
}
suffix++;
}
return templates;
}
private final Map<String, String> map;
private MediaWikiTemplate(final Map<String, String> map)
{
this.map = map;
}
public String getValue(final String key)
{
String val = map.get(key);
if (Strings.isNullOrEmpty(val) ||
val.equalsIgnoreCase("no") ||
val.equalsIgnoreCase("n/a") ||
val.equals("nil") ||
val.equalsIgnoreCase("varies"))
{
return null;
}
val = val.replace("kg", "").replaceAll("[><]", "");
return Strings.isNullOrEmpty(val) ? null : val;
}
public Boolean getBoolean(final String key)
{
final String val = getValue(key);
return !Strings.isNullOrEmpty(val) ? true : null;
}
public Double getDouble(final String key)
{
final String val = getValue(key);
if (Strings.isNullOrEmpty(val))
{
return null;
}
try
{
double v = Double.parseDouble(val);
return v != 0 ? v : null;
}
catch (NumberFormatException e)
{
e.printStackTrace();
return null;
}
}
public Integer getInt(final String key)
{
final String val = getValue(key);
if (Strings.isNullOrEmpty(val))
{
return null;
}
try
{
int v = Integer.parseInt(val);
return v != 0 ? v : null;
}
catch (NumberFormatException e)
{
e.printStackTrace();
return null;
}
}
public boolean containsKey(final String key)
{
return map.containsKey(key);
}
}

155
wiki-scraper/src/main/java/net/runelite/data/dump/wiki/ItemLimitsDumper.java Normal file
View File

@@ -0,0 +1,155 @@
/*
* MIT License
*
* Copyright (c) 2018 Tomas Slusny
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package net.runelite.data.dump.wiki;
import com.google.common.base.Strings;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Stream;
import lombok.extern.slf4j.Slf4j;
import net.runelite.cache.ItemManager;
import net.runelite.cache.definitions.ItemDefinition;
import net.runelite.cache.fs.Store;
import net.runelite.cache.util.Namer;
import net.runelite.data.App;
import net.runelite.data.dump.MediaWiki;
import net.runelite.data.dump.MediaWikiTemplate;
@Slf4j
public class ItemLimitsDumper
{
public static void dump(final Store store, final MediaWiki wiki) throws IOException
{
final File out = new File("../runelite-client/src/main/resources/net/runelite/client/plugins/grandexchange/");
log.info("Dumping item limits to {}", out);
final ItemManager itemManager = new ItemManager(store);
itemManager.load();
final Pattern pattern = Pattern.compile("limit {6}= (.*),");
final Map<Integer, Integer> limits = new TreeMap<>();
final Collection<ItemDefinition> items = itemManager.getItems();
final Stream<ItemDefinition> itemDefinitionStream = items.parallelStream();
List<String> missing = new ArrayList<>();
itemDefinitionStream.forEach(item ->
{
if (!item.isTradeable)
{
return;
}
if (item.getNotedTemplate() != -1)
{
return;
}
if (item.name.equalsIgnoreCase("NULL"))
{
return;
}
String name = Namer
.removeTags(item.name)
.replace('\u00A0', ' ')
.replaceAll("\\+", "%2b")
.trim();
if (name.isEmpty())
{
return;
}
String data = wiki.getPageData("Module:Exchange/" + name, -1);
if (Strings.isNullOrEmpty(data))
{
log.debug("Data is null or empty: {}", name);
missing.add(name);
return;
}
final MediaWikiTemplate geStats = MediaWikiTemplate.parseLua(data);
if (geStats == null)
{
return;
}
final Integer limit = geStats.getInt("limit");
if (limit == null || limit <= 0)
{
Matcher matcher = pattern.matcher(data);
String temp = "";
while (matcher.find())
{
temp = matcher.group(1);
if (Strings.isNullOrEmpty(temp) ||
temp.equalsIgnoreCase("no") ||
temp.equalsIgnoreCase("n/a") ||
temp.equals("nil") ||
temp.equalsIgnoreCase("varies"))
{
temp = null;
}
}
if (!Strings.isNullOrEmpty(temp))
{
limits.put(item.id, Integer.valueOf(temp));
}
else
{
log.debug("Item was still null: {}", name);
missing.add(name);
}
return;
}
limits.put(item.id, limit);
log.debug("Dumped item limit for {} {}", item.id, name);
});
try (FileWriter fw = new FileWriter(new File(out, "ge_limits.json")))
{
fw.write(App.GSON.toJson(limits));
}
log.info("Dumped {} item limits", limits.size());
log.info("Total Missing: " + missing.size());
missing.forEach(str ->
{
log.info("Still Missing: {}", str);
});
}
}

359
wiki-scraper/src/main/java/net/runelite/data/dump/wiki/ItemStatsDumper.java Normal file
View File

@@ -0,0 +1,359 @@
/*
* MIT License
*
* Copyright (c) 2018 Tomas Slusny
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package net.runelite.data.dump.wiki;
import com.google.common.base.Strings;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.Collection;
import java.util.Map;
import java.util.TreeMap;
import java.util.stream.Stream;
import lombok.Builder;
import lombok.Value;
import lombok.extern.slf4j.Slf4j;
import net.runelite.api.EquipmentInventorySlot;
import net.runelite.cache.ItemManager;
import net.runelite.cache.definitions.ItemDefinition;
import net.runelite.cache.fs.Store;
import net.runelite.cache.util.Namer;
import net.runelite.data.App;
import net.runelite.data.dump.MediaWiki;
import net.runelite.data.dump.MediaWikiTemplate;
@Slf4j
public class ItemStatsDumper
{
private final static Integer MAX_ITEMS_ON_PAGE = 50;
public static void dump(final Store store, final MediaWiki wiki) throws IOException
{
final File out = new File("../runelite-client/src/main/resources/");
log.info("Dumping item stats to {}", out);
final ItemManager itemManager = new ItemManager(store);
itemManager.load();
final Map<Integer, ItemStats> itemStats = new TreeMap<>();
final Collection<ItemDefinition> items = itemManager.getItems();
final Stream<ItemDefinition> itemDefinitionStream = items.parallelStream();
itemDefinitionStream.forEach(item ->
{
if (item.getNotedTemplate() != -1)
{
return;
}
if (item.name.equalsIgnoreCase("NULL"))
{
return;
}
final String name = Namer
.removeTags(item.name)
.replace('\u00A0', ' ')
.trim();
if (name.isEmpty())
{
return;
}
String data = wiki.getSpecialLookupData("item", item.id, 0);
if (Strings.isNullOrEmpty(data))
{
return;
}
MediaWikiTemplate base = MediaWikiTemplate.parseWikitext("Infobox Item", data);
if (base == null)
{
return;
}
final int nItems = findMaxIndex(base);
final ItemStats.ItemStatsBuilder itemStat = ItemStats.builder();
for (int index = 1; index <= nItems; index++)
{
final int offset = nItems == 1 ? 0 : index;
final String wikiName = getVarString(base, "name", offset);
// Skip this index if name or itemId doesn't match with wiki
if (nItems > 1 && !wikiName.equalsIgnoreCase(name))
{
continue;
}
itemStat.name(getVarString(base, "name", offset) == null ? getVarString(base, "name1", offset) : getVarString(base, "name", offset));
itemStat.quest(getVarBoolean(base, "quest", offset));
itemStat.equipable(getVarBoolean(base, "equipable", offset) == null
? getVarBoolean(base, "equipable1", offset) : getVarBoolean(base, "equipable", offset));
itemStat.weight(getVarDouble(base, "weight", offset));
if (Boolean.TRUE.equals(itemStat.equipable))
{
MediaWikiTemplate stats = MediaWikiTemplate.parseWikitext("Infobox Bonuses", data);
if (stats == null)
{
data = wiki.getSpecialLookupData("item", item.id, 1);
if (Strings.isNullOrEmpty(data))
{
break;
}
stats = MediaWikiTemplate.parseWikitext("Infobox Bonuses", data);
}
if (stats == null)
{
break;
}
final ItemEquipmentStats.ItemEquipmentStatsBuilder equipmentStat = ItemEquipmentStats.builder();
equipmentStat.slot(toEquipmentSlot(getVarString(stats, "slot", offset)));
equipmentStat.astab(getVarInt(stats, "astab", offset));
equipmentStat.aslash(getVarInt(stats, "aslash", offset));
equipmentStat.acrush(getVarInt(stats, "acrush", offset));
equipmentStat.amagic(getVarInt(stats, "amagic", offset));
equipmentStat.arange(getVarInt(stats, "arange", offset));
equipmentStat.dstab(getVarInt(stats, "dstab", offset));
equipmentStat.dslash(getVarInt(stats, "dslash", offset));
equipmentStat.dcrush(getVarInt(stats, "dcrush", offset));
equipmentStat.dmagic(getVarInt(stats, "dmagic", offset));
equipmentStat.drange(getVarInt(stats, "drange", offset));
equipmentStat.str(getVarInt(stats, "str", offset));
equipmentStat.rstr(getVarInt(stats, "rstr", offset));
equipmentStat.mdmg(getVarInt(stats, "mdmg", offset));
equipmentStat.prayer(getVarInt(stats, "prayer", offset));
equipmentStat.aspeed(getVarInt(stats, "aspeed", offset));
final ItemEquipmentStats builtEqStat = equipmentStat.build();
if (!builtEqStat.equals(ItemEquipmentStats.builder().build()))
{
itemStat.equipment(builtEqStat);
}
}
break;
}
final ItemStats val = itemStat.build();
if (ItemStats.DEFAULT.equals(val))
{
return;
}
itemStats.put(item.id, val);
log.debug("Dumped item stat for {} {}", item.id, name);
});
try (FileWriter fw = new FileWriter(new File(out, "item_stats.json")))
{
fw.write(App.GSON.toJson(itemStats));
}
log.info("Dumped {} item stats", itemStats.size());
}
/**
* Counts how many items are on page
*
* @param template media wiki template
* @return item count
*/
private static int findMaxIndex(final MediaWikiTemplate template)
{
int nItems = 1;
if (template.getValue("version1") == null)
{
return nItems;
}
while (nItems < MAX_ITEMS_ON_PAGE)
{
if (template.getValue(fixIndex("name", nItems + 1)) != null || template.getValue(fixIndex("version", nItems + 1)) != null)
{
nItems++;
}
else
{
break;
}
}
return nItems;
}
/**
* Return fixed string version of indexed key
*
* @param base key name
* @param index current index
* @return string representation of index
*/
private static String fixIndex(final String base, final Integer index)
{
return index == 0 ? base : base + index;
}
private static String getVarString(final MediaWikiTemplate template, final String key, final Integer index)
{
final String var = template.getValue(fixIndex(key, index));
if (var != null)
{
return var;
}
return template.getValue(key);
}
private static Boolean getVarBoolean(final MediaWikiTemplate template, final String key, final Integer index)
{
final Boolean var = template.getBoolean(fixIndex(key, index));
if (var != null)
{
return var;
}
return template.getBoolean(key);
}
private static Integer getVarInt(final MediaWikiTemplate template, final String key, final Integer index)
{
final Integer var = template.getInt(fixIndex(key, index));
if (var != null)
{
return var;
}
return template.getInt(key);
}
private static Double getVarDouble(final MediaWikiTemplate template, final String key, final Integer index)
{
final Double var = template.getDouble(fixIndex(key, index));
if (var != null)
{
return var;
}
return template.getDouble(key);
}
private static Integer toEquipmentSlot(final String slotName)
{
if (slotName == null)
{
return null;
}
switch (slotName.toLowerCase())
{
case "weapon":
case "2h":
// TODO: 2h should return both weapon and shield somehow
return EquipmentInventorySlot.WEAPON.getSlotIdx();
case "body":
return EquipmentInventorySlot.BODY.getSlotIdx();
case "head":
return EquipmentInventorySlot.HEAD.getSlotIdx();
case "ammo":
return EquipmentInventorySlot.AMMO.getSlotIdx();
case "legs":
return EquipmentInventorySlot.LEGS.getSlotIdx();
case "feet":
return EquipmentInventorySlot.BOOTS.getSlotIdx();
case "hands":
return EquipmentInventorySlot.GLOVES.getSlotIdx();
case "cape":
return EquipmentInventorySlot.CAPE.getSlotIdx();
case "neck":
return EquipmentInventorySlot.AMULET.getSlotIdx();
case "ring":
return EquipmentInventorySlot.RING.getSlotIdx();
case "shield":
return EquipmentInventorySlot.SHIELD.getSlotIdx();
}
return null;
}
@Value
@Builder
private static final class ItemEquipmentStats
{
private final Integer slot;
private final Integer astab;
private final Integer aslash;
private final Integer acrush;
private final Integer amagic;
private final Integer arange;
private final Integer dstab;
private final Integer dslash;
private final Integer dcrush;
private final Integer dmagic;
private final Integer drange;
private final Integer str;
private final Integer rstr;
private final Integer mdmg;
private final Integer prayer;
private final Integer aspeed;
}
@Value
@Builder
private static final class ItemStats
{
static final ItemStats DEFAULT = ItemStats.builder().build();
private final String name;
private final Boolean quest;
private final Boolean equipable;
private final Double weight;
private final ItemEquipmentStats equipment;
}
}

393
wiki-scraper/src/main/java/net/runelite/data/dump/wiki/NpcStatsDumper.java Normal file
View File

@@ -0,0 +1,393 @@
/*
* MIT License
*
* Copyright (c) 2019 TheStonedTurtle <https://github.com/TheStonedTurtle>
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package net.runelite.data.dump.wiki;
import com.google.common.base.Strings;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import lombok.Builder;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import net.runelite.cache.NpcManager;
import net.runelite.cache.definitions.NpcDefinition;
import net.runelite.cache.fs.Store;
import net.runelite.cache.util.Namer;
import net.runelite.data.App;
import net.runelite.data.dump.MediaWiki;
import net.runelite.data.dump.MediaWikiTemplate;
@Slf4j
public class NpcStatsDumper
{
@Data
@Builder
private static final class NpcStats
{
private String name;
private final Integer hitpoints;
private final Integer hitpoints1;
private final Integer combatLevel;
private final Integer slayerLevel;
private final Integer attackSpeed;
private final Integer attackLevel;
private final Integer strengthLevel;
private final Integer defenceLevel;
private final Integer rangeLevel;
private final Integer magicLevel;
private final Integer stab;
private final Integer slash;
private final Integer crush;
private final Integer range;
private final Integer magic;
private final Integer stabDef;
private final Integer slashDef;
private final Integer crushDef;
private final Integer rangeDef;
private final Integer magicDef;
private final Integer bonusAttack;
private final Integer bonusStrength;
private final Integer bonusRangeStrength;
private final Integer bonusMagicDamage;
private final Boolean poisonImmune;
private final Boolean venomImmune;
private final Boolean dragon;
private final Boolean demon;
private final Boolean undead;
}
private static final NpcStats DEFAULT = NpcStats.builder().build();
/**
* Looks for and parses the `Switch infobox` into a {@link MediaWikiTemplate} and then iterates over the `item#` values.
* Attempts to parse each `item#` value via `parseWikiText`, matching the `name` attribute. null values are ignored
*
* @param name only parses MediaWikiTemplates from `Switch infobox` if matches this value. (case insensitive)
* @param baseTemplate the {@link MediaWikiTemplate} representation of the `Switch infobox` to parse from
* @return List of all valid {@link MediaWikiTemplate}s matching `name` from `baseTemplate`s `item#` values
*/
static List<MediaWikiTemplate> parseSwitchInfoboxItems(final String name, final MediaWikiTemplate baseTemplate)
{
final List<MediaWikiTemplate> templates = new ArrayList<>();
String value;
int suffix = 1;
while ((value = baseTemplate.getValue("item" + suffix)) != null)
{
final MediaWikiTemplate subTemplate = MediaWikiTemplate.parseWikitext(name, value);
if (subTemplate != null)
{
templates.add(subTemplate);
}
suffix++;
}
return templates;
}
public static void dump(final Store store, final MediaWiki wiki) throws IOException
{
final File out = new File("../runelite-client/src/main/resources/");
log.info("Dumping npc stats to {}", out);
final NpcManager npcManager = new NpcManager(store);
npcManager.load();
final Map<Integer, NpcStats> npcStats = new HashMap<>();
final Collection<NpcDefinition> definitions = npcManager.getNpcs();
final Stream<NpcDefinition> npcDefinitionStream = definitions.parallelStream();
// Ensure variant names match cache as wiki isn't always correct
final Map<Integer, String> nameMap = new HashMap<>();
for (NpcDefinition n : definitions)
{
if (n.getName().equalsIgnoreCase("NULL"))
{
continue;
}
final String name = Namer
.removeTags(n.getName())
.replace('\u00A0', ' ')
.trim();
if (name.isEmpty())
{
continue;
}
nameMap.put(n.getId(), name);
}
npcDefinitionStream.forEach(n ->
{
if (npcStats.containsKey(n.getId()))
{
return;
}
final String name = nameMap.get(n.getId());
if (name == null)
{
return;
}
if (!isAttackableNpc(n))
{
return;
}
final String data = wiki.getSpecialLookupData("npc", n.getId(), 0);
if (Strings.isNullOrEmpty(data))
{
return;
}
List<MediaWikiTemplate> bases = new ArrayList<>();
final MediaWikiTemplate switchBase = MediaWikiTemplate.parseWikitext("Switch infobox", data);
if (switchBase != null)
{
bases = parseSwitchInfoboxItems("Infobox Monster", switchBase);
}
else
{
final MediaWikiTemplate base = MediaWikiTemplate.parseWikitext("Infobox Monster", data);
if (base == null)
{
return;
}
bases.add(base);
}
for (final MediaWikiTemplate base : bases)
{
int variantKey = 0;
String wikiIdString = getWikiIdString(base, variantKey);
if (wikiIdString == null)
{
// Try again as `id` will be null if there are variants and `id1` is the starting key
variantKey++;
wikiIdString = getWikiIdString(base, variantKey);
}
while (wikiIdString != null)
{
if (wikiIdString.isEmpty())
{
continue;
}
final Set<Integer> ids = Arrays.stream(wikiIdString.split(","))
.map(s -> Integer.parseInt(s.trim()))
.collect(Collectors.toSet());
final NpcStats stats = buildNpcStats(base, variantKey);
if (!stats.equals(DEFAULT))
{
stats.setName(name);
for (final int curID : ids)
{
// Update variant name or fall back to current name
final String curName = nameMap.get(curID);
stats.setName(curName == null ? stats.getName() : curName);
npcStats.put(curID, stats);
log.debug("Dumped npc stats for npc id: {}", curID);
}
}
variantKey++;
wikiIdString = getWikiIdString(base, variantKey);
}
}
});
// Cast to TreeMap so sort output JSON in numerical order (npc id)
final Map<Integer, NpcStats> sorted = new TreeMap<>(npcStats);
try (FileWriter fw = new FileWriter(new File(out, "npc_stats.json")))
{
fw.write(App.GSON.toJson(sorted));
}
// try (FileWriter fw = new FileWriter(new File(out, "npc_stats.min.json")))
// {
// fw.write(new GsonBuilder().disableHtmlEscaping().create().toJson(sorted));
// }
log.info("Dumped {} npc stats", sorted.size());
}
private static boolean isAttackableNpc(final NpcDefinition n)
{
for (final String s : n.getOptions())
{
if ("attack".equalsIgnoreCase(s))
{
return true;
}
}
return false;
}
private static String getKeySuffix(final int variantKey)
{
return variantKey > 0 ? String.valueOf(variantKey) : "";
}
private static String getWikiIdString(final MediaWikiTemplate template, final int variantKey)
{
return template.getValue("id" + getKeySuffix(variantKey));
}
private static NpcStats buildNpcStats(final MediaWikiTemplate template, int variantKey)
{
final NpcStats.NpcStatsBuilder stats = NpcStats.builder();
stats.hitpoints(getInt("hitpoints", variantKey, template));
if (stats.hitpoints == null)
{
stats.hitpoints(getInt("hitpoints1", variantKey, template));
}
stats.combatLevel(getInt("combat", variantKey, template));
stats.slayerLevel(getInt("slaylvl", variantKey, template));
stats.attackSpeed(getInt("attack speed", variantKey, template));
stats.attackLevel(getInt("att", variantKey, template));
stats.strengthLevel(getInt("str", variantKey, template));
stats.defenceLevel(getInt("def", variantKey, template));
stats.rangeLevel(getInt("range", variantKey, template));
stats.magicLevel(getInt("mage", variantKey, template));
stats.stab(getInt("astab", variantKey, template));
stats.slash(getInt("aslash", variantKey, template));
stats.crush(getInt("acrush", variantKey, template));
stats.range(getInt("arange", variantKey, template));
stats.magic(getInt("amagic", variantKey, template));
stats.stabDef(getInt("dstab", variantKey, template));
stats.slashDef(getInt("dslash", variantKey, template));
stats.crushDef(getInt("dcrush", variantKey, template));
stats.rangeDef(getInt("drange", variantKey, template));
stats.magicDef(getInt("dmagic", variantKey, template));
stats.bonusAttack(getInt("attbns", variantKey, template));
stats.bonusStrength(getInt("strbns", variantKey, template));
stats.bonusRangeStrength(getInt("rngbns", variantKey, template));
stats.bonusMagicDamage(getInt("mbns", variantKey, template));
final String keySuffix = getKeySuffix(variantKey);
boolean pImmune = "immune".equalsIgnoreCase(template.getValue("immunepoison" + keySuffix));
boolean vImmune = "immune".equalsIgnoreCase(template.getValue("immunevenom" + keySuffix));
stats.poisonImmune(!pImmune ? null : true);
stats.venomImmune(!vImmune ? null : true);
final String weaknessValue = template.getValue("weakness");
if (weaknessValue != null)
{
final String[] values = weaknessValue.split(",");
for (String value : values)
{
value = value.toLowerCase();
if (stats.dragon == null && (value.contains("dragonbane weapons")))
{
stats.dragon(true);
}
if (stats.demon == null && (value.contains("demonbane weapons") || value.contains("silverlight") || value.contains("arclight")))
{
stats.demon(true);
}
if (stats.undead == null && (value.contains("salve amulet") || value.contains("crumble undead")))
{
stats.undead(true);
}
}
}
return stats.build();
}
static Integer getInt(final String mainKey, final Integer variation, final MediaWikiTemplate template)
{
final String key = mainKey + getKeySuffix(variation);
if (!template.containsKey(key))
{
if (variation >= 1)
{
// Use variation fallback via recursion
return getInt(mainKey, variation - 1, template);
}
return null;
}
final String val = template.getValue(key);
if (Strings.isNullOrEmpty(val))
{
return null;
}
try
{
// Remove everything after the first non-number character to account for any comments
final String fixedVal = val.trim().replaceAll("\\D+.*", "");
if (fixedVal.isEmpty())
{
return null;
}
int v = Integer.parseInt(fixedVal);
return v != 0 ? v : null;
}
catch (NumberFormatException e)
{
e.printStackTrace();
return null;
}
}
}

549
wiki-scraper/src/test/java/net/runelite/data/dump/MediaWikiTemplateTest.java Normal file
View File

@@ -0,0 +1,549 @@
/*
* MIT License
*
* Copyright (c) 2018 Tomas Slusny
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package net.runelite.data.dump;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import org.junit.jupiter.api.Test;
class MediaWikiTemplateTest
{
@Test
void parseInfoboxItem1()
{
final String data =
"{{Infobox Item\n" +
"|name = Dragon claws\n" +
"|image = [[File:Dragon claws.png]]\n" +
"|release = [[5 January]] [[2017]]\n" +
"|update = Dragon Claws & 3rd Birthday\n" +
"|members = Yes\n" +
"|quest = No\n" +
"|tradeable = Yes\n" +
"|equipable = Yes\n" +
"|stackable = No\n" +
"|high = 123000\n" +
"|low = 82000\n" +
"|destroy = Drop\n" +
"|store = No\n" +
"|exchange = gemw\n" +
"|examine = A set of fighting claws.\n" +
"|weight = 0\n" +
"}}\n";
final MediaWikiTemplate template = MediaWikiTemplate.parseWikitext("Infobox Item", data);
assertNotNull(template);
assertEquals(123000, (int) template.getInt("high"));
}
@Test
void parseInfoboxItem2()
{
final String data =
"{{Infobox item\n" +
"|name = Magic shortbow (i)\n" +
"|image = [[File:Magic shortbow (i).png]]\n" +
"|release = [[18 September]] [[2014]]\n" +
"|update = Bounty Hunter\n" +
"|members = Yes\n" +
"|tradeable = No\n" +
"|equipable = Yes\n" +
"|stackable = No\n" +
"|quest = No\n" +
"|low = 640\n" +
"|high = 960\n" +
"|store = No\n" +
"|examine = Short and magical, but still effective.\n" +
"|weight = 1\n" +
"|destroy = Drop\n" +
"}}\n";
final MediaWikiTemplate template = MediaWikiTemplate.parseWikitext("Infobox Item", data);
assertNotNull(template);
assertEquals((int) template.getInt("high"), 960);
}
@Test
void parseInfoboxItem3()
{
final String data =
"{{Infobox Item\n" +
"|name = Proselyte cuisse\n" +
"|image = [[File:Proselyte cuisse.png|Proselyte cuisse]]\n" +
"|release = [[20 September]] [[2006]]\n" +
"|update = Slug Menace\n" +
"|members = Yes\n" +
"|quest = [[The Slug Menace]]\n" +
"|tradeable = Yes\n" +
"|equipable = Yes\n" +
"|stackable = No\n" +
"|noteable = Yes\n" +
"|placeholder = Yes\n" +
"|destroy = Drop\n" +
"|value = 10000\n" +
"|store = 10000\n" +
"|exchange = gemw\n" +
"|weight = 7.711\n" +
"|examine = A Proselyte Temple Knight's leg armour.\n" +
"|id = 9676,20565\n" +
"}}";
final MediaWikiTemplate template = MediaWikiTemplate.parseWikitext("Infobox Item", data);
assertNotNull(template);
}
@Test
void parseInfoboxItem4()
{
final String data =
"{{Infobox Item\n" +
"|name = Explorer's ring 2\n" +
"|image = [[File:Explorer's ring 2.png]]\n" +
"|release = [[5 March]] [[2015]]\n" +
"|update = Achievement Diaries\n" +
"|members = Yes\n" +
"|quest = No\n" +
"|tradeable = No\n" +
"|equipable = Yes\n" +
"|stackable = No\n" +
"|noteable = No\n" +
"|placeholder = Yes\n" +
"|destroy = Drop\n" +
"|value = 0\n" +
"|store = No\n" +
"|weight = 0\n" +
"|examine = A Lumbridge explorer's ring.\n" +
"|id = 13126\n" +
"}}";
final MediaWikiTemplate template = MediaWikiTemplate.parseWikitext("Infobox Item", data);
assertNotNull(template);
}
@Test
void parseInfoboxItem5()
{
final String data =
"{{Infobox Item\n" +
"|version1 = Normal\n" +
"|version2 = Broken\n" +
"|name1 = Fighter hat\n" +
"|name2 = Fighter hat (broken)\n" +
"|image1 = [[File:Fighter hat.png]]\n" +
"|image2 = [[File:Fighter hat (broken).png]]\n" +
"|release1 = [[4 January]] [[2007]]\n" +
"|release2 = [[21 July]] [[2016]]\n" +
"|update1 = Barbarian Assault\n" +
"|update2 = Broken Armour & Open Beta\n" +
"|members = Yes\n" +
"|quest = No\n" +
"|tradeable = No\n" +
"|equipable1 = Yes\n" +
"|equipable2 = No\n" +
"|stackable = No\n" +
"|noteable = No\n" +
"|placeholder = Yes\n" +
"|destroy = Drop\n" +
"|value1 = 65002\n" +
"|value2 = 1\n" +
"|alchable = No\n" +
"|store1 = 275\n" +
"|store2 = No\n" +
"|currency = Honour points in each role; must have also killed [[Penance Queen|Queen]]\n" +
"|seller = Commander Connad\n" +
"|weight = 2\n" +
"|examine1 = A Penance Fighter hat.\n" +
"|examine2 = A broken Penance Fighter hat.\n" +
"|id1 = 10548\n" +
"|id2 = 20507\n" +
"}}\n";
final MediaWikiTemplate template = MediaWikiTemplate.parseWikitext("Infobox Item", data);
assertNotNull(template);
}
@Test
void parseInfoboxItem6()
{
final String data =
"{{Infobox Item\n" +
"|version1 = Unpoisoned\n" +
"|version2 = Poison\n" +
"|version3 = Poison+\n" +
"|version4 = Poison++\n" +
"|version5 = Karambwan poison\n" +
"|name1 = Iron hasta\n" +
"|name2 = Iron hasta(p)\n" +
"|name3 = Iron hasta(p+)\n" +
"|name4 = Iron hasta(p++)\n" +
"|name5 = Iron hasta(kp)\n" +
"|image1 = [[File:Iron hasta.png]]\n" +
"|image2 = [[File:Iron hasta(p).png]]\n" +
"|image3 = [[File:Iron hasta(p+).png]]\n" +
"|image4 = [[File:Iron hasta(p++).png]]\n" +
"|image5 = [[File:Iron hasta(kp).png]]\n" +
"|release = [[3 July]] [[2007]]\n" +
"|update = Barbarian Training\n" +
"|members = Yes\n" +
"|quest = No\n" +
"|tradeable1 = Yes\n" +
"|tradeable2 = Yes\n" +
"|tradeable3 = Yes\n" +
"|tradeable4 = Yes\n" +
"|tradeable5 = No\n" +
"|equipable = Yes\n" +
"|stackable = No\n" +
"|noteable1 = Yes\n" +
"|noteable2 = Yes\n" +
"|noteable3 = Yes\n" +
"|noteable4 = Yes\n" +
"|noteable5 = No\n" +
"|placeholder = Yes\n" +
"|destroy = Drop\n" +
"|value = 91\n" +
"|store = No\n" +
"|exchange1 = gemw\n" +
"|exchange2 = gemw\n" +
"|exchange3 = gemw\n" +
"|exchange4 = gemw\n" +
"|weight1 = 2.267\n" +
"|weight2 = 2.267\n" +
"|weight3 = 2\n" +
"|weight4 = 2.267\n" +
"|weight5 = 2.267\n" +
"|examine1 = An iron-tipped, one-handed hasta.\n" +
"|examine2 = A poison-tipped, one-handed iron hasta.\n" +
"|examine3 = A poison-tipped, one-handed iron hasta.\n" +
"|examine4 = A poison-tipped, one-handed iron hasta.\n" +
"|examine5 = A karambwan poison-tipped, one-handed iron hasta.\n" +
"|id1 = 11369\n" +
"|id2 = 11386\n" +
"|id3 = 11389\n" +
"|id4 = 11391\n" +
"|id5 = 11388\n" +
"}}";
final MediaWikiTemplate template = MediaWikiTemplate.parseWikitext("Infobox Item", data);
assertNotNull(template);
}
@Test
void parseInfoboxBonuses1()
{
final String data =
"{{Infobox Bonuses\n" +
"|astab = 41\n" +
"|aslash = 57\n" +
"|acrush = -4\n" +
"|amagic = 0\n" +
"|arange = 0\n" +
"|dstab = 13\n" +
"|dslash = 26\n" +
"|dcrush = 7\n" +
"|dmagic = 0\n" +
"|drange = 0\n" +
"|str = 56\n" +
"|rstr = 0\n" +
"|mdmg = 0\n" +
"|prayer = 0\n" +
"|caption = A player wearing dragon claws.\n" +
"|aspeed = 4|slot = 2h\n" +
"|image = Dragon claws equipped.png{{!}}130px}}\n";
final MediaWikiTemplate template = MediaWikiTemplate.parseWikitext("Infobox Bonuses", data);
assertNotNull(template);
assertEquals(4, (int) template.getInt("aspeed"));
assertEquals("2h", template.getValue("slot"));
assertEquals("Dragon claws equipped.png{{!}}130px", template.getValue("image"));
}
@Test
void parseInfoboxBonuses2()
{
final String data =
"{{Infobox Bonuses\n" +
"|version1 = Uncharged\n" +
"|version2 = Charged\n" +
"|image_1 = Dragonfire shield (uncharged) equipped.png{{!}}150px\n" +
"|image_2 = Dragonfire shield equipped.png{{!}}150px\n" +
"|astab = 0\n" +
"|aslash = 0\n" +
"|acrush = 0\n" +
"|amagic = -10\n" +
"|arange = -5\n" +
"|dstab1 = +20\n" +
"|dslash1 = +25\n" +
"|dcrush1 = +22\n" +
"|dmagic1 = +10\n" +
"|drange1 = +22\n" +
"|dstab2 = +70\n" +
"|dslash2 = +75\n" +
"|dcrush2 = +72\n" +
"|dmagic2 = +10\n" +
"|drange2 = +72\n" +
"|str = +7\n" +
"|rstr = 0\n" +
"|mdmg = 0\n" +
"|prayer = 0\n" +
"|slot = Shield\n" +
"}}";
final MediaWikiTemplate template = MediaWikiTemplate.parseWikitext("Infobox Bonuses", data);
assertNotNull(template);
assertEquals(70, (int) template.getInt("dstab2"));
}
@Test
void parseInfoboxBonuses3()
{
final String data =
"{{Infobox Bonuses\n" +
"|astab = 0\n" +
"|aslash = 0\n" +
"|acrush = 0\n" +
"|amagic = -21\n" +
"|arange = -7\n" +
"|dstab = +33\n" +
"|dslash = +31\n" +
"|dcrush = +29\n" +
"|dmagic = -4\n" +
"|drange = +31\n" +
"|str = 0\n" +
"|rstr = 0\n" +
"|mdmg = 0\n" +
"|prayer = +6\n" +
"|slot = Legs\n" +
"|image = Proselyte armour equipped.png{{!}}110px\n" +
"|caption = A player wearing proselyte armour.\n" +
"}}";
final MediaWikiTemplate template = MediaWikiTemplate.parseWikitext("Infobox Bonuses", data);
assertNotNull(template);
assertEquals((int) template.getInt("prayer"), 6);
}
@Test
void parseInfoboxBonuses4()
{
final String data =
"{{Infobox Bonuses\n" +
"|image = \n" +
"|caption = \n" +
"|astab =0 \n" +
"|aslash =0 \n" +
"|acrush =0 \n" +
"|amagic =0 \n" +
"|arange =0 \n" +
"|dstab =0 \n" +
"|dslash =0 \n" +
"|dcrush =0 \n" +
"|dmagic =0 \n" +
"|drange =0 \n" +
"|str =0 \n" +
"|prayer =+1 \n" +
"|slot = ring\n" +
"|rstr = 0\n" +
"|mdmg = 0\n" +
"}}";
final MediaWikiTemplate template = MediaWikiTemplate.parseWikitext("Infobox Bonuses", data);
assertNotNull(template);
}
@Test
void parseInfoboxBonuses5()
{
final String data =
"{{Infobox Bonuses|| astab = +8\n" +
"| aspeed = 5\n" +
"| aslash = -2\n" +
"| acrush = +6\n" +
"| amagic = 0\n" +
"| arange = 0\n" +
"| dstab = 0\n" +
"| dslash = +1\n" +
"| dcrush = 0\n" +
"| dmagic = 0\n" +
"| drange = 0\n" +
"| str = +9\n" +
"|rstr = 0\n" +
"|mdmg = 0\n" +
"| prayer = 0\n" +
"|image = Steel pickaxe equipped.png{{!}}150px\n" +
"|caption = A player wielding a steel pickaxe.\n" +
"||slot = Weapon}}";
final MediaWikiTemplate template = MediaWikiTemplate.parseWikitext("Infobox Bonuses", data);
assertNotNull(template);
}
@Test
void parseLua()
{
final String exchangeInfoData =
"return {\n" +
" itemId = 13652,\n" +
" price = 83173735,\n" +
" last = 83533604,\n" +
" date = '12:18, November 08, 2018 (UTC)',\n" +
" lastDate = '05:43, November 08, 2018 (UTC)',\n" +
" icon = 'Dragon claws.png',\n" +
" item = 'Dragon claws',\n" +
" value = -205000,\n" +
" limit = nil,\n" +
" members = true,\n" +
" category = nil,\n" +
" examine = 'A set of fighting claws.'\n" +
"}\n";
final MediaWikiTemplate exchangeInfo = MediaWikiTemplate.parseLua(exchangeInfoData);
assertNotNull(exchangeInfo);
assertEquals((int) exchangeInfo.getInt("value"), -205000);
}
@Test
void parseKeysWithSpaces()
{
final String data =
"{{Infobox Monster\n" +
"|name = Aberrant spectre\n" +
"|combat = 96\n" +
"|attack speed = 4\n" +
"|foo attack style= Magic\n" +
"|id = 2,3,4,5,6,7\n" +
"}}";
final MediaWikiTemplate template = MediaWikiTemplate.parseWikitext("Infobox Monster", data);
assertNotNull(template);
assertEquals(template.getInt("combat"), 96);
assertEquals(template.getInt("attack speed"), 4);
assertEquals(template.getValue("foo attack style"), "Magic");
}
@Test
void parseWikitextExactName()
{
final String data =
"{{ Infobox Monster/sandbox \n" +
"|version1 = Lv 51\n" +
"|version2 = Lv 76\n" +
"|name = Brawler\n" +
"|combat1 = 51\n" +
"|combat2 = 76\n" +
"|hitpoints1 = 53\n" +
"|hitpoints2 = 83\n" +
"|max hit1 = 7\n" +
"|max hit2 = 9\n" +
"|slaylvl = No\n" +
"|slayxp = No\n" +
"|att1 = <!--Level 51 stats-->\n" +
"|att2 = <!--Level 76 stats-->\n" +
"|id1 = 1734\n" +
"|id2 = 1735\n" +
"}}";
final MediaWikiTemplate template = MediaWikiTemplate.parseWikitext("Infobox Monster/sandbox", data);
assertNotNull(template);
final MediaWikiTemplate template2 = MediaWikiTemplate.parseWikitext("Infobox Monster", data);
assertNull(template2);
}
@Test
void parseSwitchInfobox()
{
final String data =
"{{External|rs}}\n" +
"{{Switch infobox\n" +
"|item1= \n" +
"{{Infobox Monster\n" +
"|name = Ghast\n" +
"|combat = 30\n" +
"|id = 946\n" +
"}}\n" +
"|text1 = Level 30\n" +
"|item2= \n" +
"{{Infobox Monster\n" +
"|name = Ghast\n" +
"|combat = 79\n" +
"|id = 5625\n" +
"}}\n" +
"|text2 = Level 79\n" +
"|item3= \n" +
"{{Infobox Monster\n" +
"|name = Ghast\n" +
"|combat = 109\n" +
"|id = 5626\n" +
"}}\n" +
"|text3 = Level 109\n" +
"|item4= \n" +
"{{Infobox Monster\n" +
"|name = Ghast\n" +
"|combat = 139\n" +
"|id = 5627\n" +
"}}\n" +
"|text4 = Level 139\n" +
"|item5 =\n" +
"{{Infobox non-player character\n" +
"|name = \n" +
"|update = Nature Spirit Quest\n" +
"|race = Undead\n" +
"|members = Yes\n" +
"|quest = [[Nature Spirit]]\n" +
"|location = [[Morytania]]\n" +
"|shop = No\n" +
"|gender = N/A\n" +
"|examine = \n" +
"|id = 945, 5622, 5623, 5624\n" +
"}}\n" +
"|text5 = Invisible\n" +
"}}";
final MediaWikiTemplate switchInfobox = MediaWikiTemplate.parseWikitext("Switch infobox", data);
assertNotNull(switchInfobox);
// Infobox monster
final List<MediaWikiTemplate> templates = MediaWikiTemplate.parseSwitchInfoboxItems("Infobox monster", switchInfobox);
assertEquals(templates.size(), 4);
final MediaWikiTemplate item1 = templates.get(0);
assertEquals(item1.getInt("combat"), 30);
final MediaWikiTemplate item2 = templates.get(1);
assertEquals(item2.getInt("combat"), 79);
// Infobox non-player character
final List<MediaWikiTemplate> npcs = MediaWikiTemplate.parseSwitchInfoboxItems("Infobox non-player character", switchInfobox);
assertEquals(npcs.size(), 1);
final MediaWikiTemplate npc1 = npcs.get(0);
assertEquals(npc1.getValue("race"), "Undead");
// Infobox item
final List<MediaWikiTemplate> items = MediaWikiTemplate.parseSwitchInfoboxItems("Infobox item", switchInfobox);
assertEquals(items.size(), 0);
}
}

129
wiki-scraper/src/test/java/net/runelite/data/dump/wiki/NpcStatsDumperTest.java Normal file
View File

@@ -0,0 +1,129 @@
/*
* MIT License
*
* Copyright (c) 2019 TheStonedTurtle <https://github.com/TheStonedTurtle>
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package net.runelite.data.dump.wiki;
import java.util.List;
import net.runelite.data.dump.MediaWikiTemplate;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import org.junit.jupiter.api.Test;
class NpcStatsDumperTest
{
@Test
void npcVariantFallThrough()
{
final String data =
"{{Infobox Monster\n" +
"|combat = 2\n" +
"|combat8 = \n" +
"|combat10 = 4\n" +
"}}";
final MediaWikiTemplate template = MediaWikiTemplate.parseWikitext("Infobox Monster", data);
assertNotNull(template);
assertEquals(NpcStatsDumper.getInt("combat", 0, template), 2);
assertEquals(NpcStatsDumper.getInt("combat", 7, template), 2);
assertNull(NpcStatsDumper.getInt("combat", 8, template));
assertNull(NpcStatsDumper.getInt("combat", 9, template));
assertEquals(NpcStatsDumper.getInt("combat", 10, template), 4);
}
@Test
void parseSwitchInfoboxItems()
{
final String data =
"{{Switch infobox\n" +
"|item1= \n" +
"{{Infobox Monster\n" +
"|name = Ghast\n" +
"|combat = 30\n" +
"|id = 946\n" +
"}}\n" +
"|text1 = Level 30\n" +
"|item2= \n" +
"{{Infobox Monster\n" +
"|name = Ghast\n" +
"|combat = 79\n" +
"|id = 5625\n" +
"}}\n" +
"|text2 = Level 79\n" +
"|item3= \n" +
"{{Infobox Monster\n" +
"|name = Ghast\n" +
"|combat = 109\n" +
"|id = 5626\n" +
"}}\n" +
"|text3 = Level 109\n" +
"|item4= \n" +
"{{Infobox Monster\n" +
"|name = Ghast\n" +
"|combat = 139\n" +
"|id = 5627\n" +
"}}\n" +
"|text4 = Level 139\n" +
"|item5 =\n" +
"{{Infobox non-player character\n" +
"|name = \n" +
"|update = Nature Spirit Quest\n" +
"|race = Undead\n" +
"|members = Yes\n" +
"|quest = [[Nature Spirit]]\n" +
"|location = [[Morytania]]\n" +
"|shop = No\n" +
"|gender = N/A\n" +
"|examine = \n" +
"|id = 945, 5622, 5623, 5624\n" +
"}}\n" +
"|text5 = Invisible\n" +
"}}";
final MediaWikiTemplate switchInfobox = MediaWikiTemplate.parseWikitext("Switch infobox", data);
assertNotNull(switchInfobox);
// Infobox monster
final List<MediaWikiTemplate> templates = NpcStatsDumper.parseSwitchInfoboxItems("Infobox monster", switchInfobox);
assertEquals(templates.size(), 4);
final MediaWikiTemplate item1 = templates.get(0);
assertEquals(item1.getInt("combat"), 30);
final MediaWikiTemplate item2 = templates.get(1);
assertEquals(item2.getInt("combat"), 79);
// Infobox non-player character
final List<MediaWikiTemplate> npcs = NpcStatsDumper.parseSwitchInfoboxItems("Infobox non-player character", switchInfobox);
assertEquals(npcs.size(), 1);
final MediaWikiTemplate npc1 = npcs.get(0);
assertEquals(npc1.getValue("race"), "Undead");
// Infobox item
final List<MediaWikiTemplate> items = NpcStatsDumper.parseSwitchInfoboxItems("Infobox item", switchInfobox);
assertEquals(items.size(), 0);
}
}