project: Add wiki scraper (#1348)
* wiki-scraper: Add wiki scroper to the main project * client: Updated wiki stats * wiki-scraper: Checkstyle * wiki-scraper: Pull in @Ganom his changes * client: Updated wiki stats
This commit is contained in:
Owain van Brakel
committed by
Kyleeld
Kyleeld
parent
4fba65d4f1
commit
8239be4b75
59
wiki-scraper/src/main/java/net/runelite/data/App.java
Normal file
59
wiki-scraper/src/main/java/net/runelite/data/App.java
Normal file
@@ -0,0 +1,59 @@
|
||||
/*
|
||||
* MIT License
|
||||
*
|
||||
* Copyright (c) 2018 Tomas Slusny
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
* of this software and associated documentation files (the "Software"), to deal
|
||||
* in the Software without restriction, including without limitation the rights
|
||||
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the Software is
|
||||
* furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
* SOFTWARE.
|
||||
*/
|
||||
package net.runelite.data;
|
||||
|
||||
import com.google.gson.Gson;
|
||||
import com.google.gson.GsonBuilder;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import net.runelite.cache.fs.Store;
|
||||
import net.runelite.data.dump.MediaWiki;
|
||||
import net.runelite.data.dump.wiki.NpcStatsDumper;
|
||||
|
||||
public class App
|
||||
{
|
||||
public static final Gson GSON = new GsonBuilder()
|
||||
.setPrettyPrinting()
|
||||
.disableHtmlEscaping()
|
||||
.create();
|
||||
|
||||
public static void main(String[] args) throws IOException
|
||||
{
|
||||
final File home = new File(System.getProperty("user.home"));
|
||||
final Store cacheStore = new Store(new File(home,
|
||||
"jagexcache" + File.separator + "oldschool" + File.separator + "LIVE"));
|
||||
cacheStore.load();
|
||||
|
||||
// Try to make this go faster (probably not very smart)
|
||||
System.setProperty("java.util.concurrent.ForkJoinPool.common.parallelism", "100");
|
||||
|
||||
final MediaWiki wiki = new MediaWiki("https://oldschool.runescape.wiki");
|
||||
|
||||
// Only use this to diff current limits with scraped limits
|
||||
// ItemLimitsDumper.dump(cacheStore, wiki);
|
||||
// ItemStatsDumper.dump(cacheStore, wiki);
|
||||
|
||||
NpcStatsDumper.dump(cacheStore, wiki);
|
||||
}
|
||||
}
|
||||
140
wiki-scraper/src/main/java/net/runelite/data/dump/MediaWiki.java
Normal file
140
wiki-scraper/src/main/java/net/runelite/data/dump/MediaWiki.java
Normal file
@@ -0,0 +1,140 @@
|
||||
/*
|
||||
* MIT License
|
||||
*
|
||||
* Copyright (c) 2018 Tomas Slusny
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
* of this software and associated documentation files (the "Software"), to deal
|
||||
* in the Software without restriction, including without limitation the rights
|
||||
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the Software is
|
||||
* furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
* SOFTWARE.
|
||||
*/
|
||||
package net.runelite.data.dump;
|
||||
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.util.Map;
|
||||
import net.runelite.data.App;
|
||||
import okhttp3.HttpUrl;
|
||||
import okhttp3.OkHttpClient;
|
||||
import okhttp3.Request;
|
||||
import okhttp3.Response;
|
||||
import java.net.URLDecoder;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
public class MediaWiki
|
||||
{
|
||||
private static final class WikiInnerResponse
|
||||
{
|
||||
Map<String, String> wikitext;
|
||||
}
|
||||
|
||||
private static final class WikiResponse
|
||||
{
|
||||
WikiInnerResponse parse;
|
||||
}
|
||||
|
||||
private final OkHttpClient client = new OkHttpClient();
|
||||
private final OkHttpClient clientNoRedirect = client.newBuilder()
|
||||
.followRedirects(false)
|
||||
.followSslRedirects(false)
|
||||
.build();
|
||||
|
||||
private final HttpUrl base;
|
||||
|
||||
public MediaWiki(final String base)
|
||||
{
|
||||
this.base = HttpUrl.parse(base);
|
||||
}
|
||||
|
||||
public String getSpecialLookupData(final String type, final int id, final int section)
|
||||
{
|
||||
final HttpUrl url = base.newBuilder()
|
||||
.addPathSegment("w")
|
||||
.addPathSegment("Special:Lookup")
|
||||
.addQueryParameter("type", type)
|
||||
.addQueryParameter("id", String.valueOf(id))
|
||||
.build();
|
||||
|
||||
final Request request = new Request.Builder()
|
||||
.url(url)
|
||||
.build();
|
||||
|
||||
try (final Response response = clientNoRedirect.newCall(request).execute())
|
||||
{
|
||||
if (response.isRedirect())
|
||||
{
|
||||
final String page = response.header("Location")
|
||||
.replace(base.newBuilder().addPathSegment("w").build().toString() + "/", "");
|
||||
return getPageData(page, section);
|
||||
}
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
return "";
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
public String getPageData(String page, int section)
|
||||
{
|
||||
// decode html encoded page name
|
||||
// ex: Mage%27s book -> Mage's_book
|
||||
try
|
||||
{
|
||||
page = URLDecoder.decode(page, StandardCharsets.UTF_8.name());
|
||||
}
|
||||
catch (UnsupportedEncodingException e)
|
||||
{
|
||||
// do nothing, keep page the same
|
||||
}
|
||||
|
||||
final HttpUrl.Builder urlBuilder = base.newBuilder()
|
||||
.addPathSegment("api.php")
|
||||
.addQueryParameter("action", "parse")
|
||||
.addQueryParameter("format", "json")
|
||||
.addQueryParameter("prop", "wikitext")
|
||||
.addQueryParameter("redirects", "true")
|
||||
.addQueryParameter("page", page.replaceAll(" ", "_"));
|
||||
|
||||
if (section != -1)
|
||||
{
|
||||
urlBuilder.addQueryParameter("section", String.valueOf(section));
|
||||
}
|
||||
|
||||
final HttpUrl url = urlBuilder.build();
|
||||
|
||||
final Request request = new Request.Builder()
|
||||
.url(url)
|
||||
.build();
|
||||
|
||||
try (final Response response = client.newCall(request).execute())
|
||||
{
|
||||
if (response.isSuccessful())
|
||||
{
|
||||
final InputStream in = response.body().byteStream();
|
||||
return App.GSON.fromJson(new InputStreamReader(in), WikiResponse.class).parse.wikitext.get("*");
|
||||
}
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
return "";
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,302 @@
|
||||
/*
|
||||
* MIT License
|
||||
*
|
||||
* Copyright (c) 2018 Tomas Slusny
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
* of this software and associated documentation files (the "Software"), to deal
|
||||
* in the Software without restriction, including without limitation the rights
|
||||
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the Software is
|
||||
* furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
* SOFTWARE.
|
||||
*/
|
||||
package net.runelite.data.dump;
|
||||
|
||||
import com.google.common.base.Function;
|
||||
import com.google.common.base.MoreObjects;
|
||||
import com.google.common.base.Strings;
|
||||
import java.util.AbstractMap;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import javax.annotation.Nullable;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.petitparser.context.Result;
|
||||
import org.petitparser.parser.Parser;
|
||||
import org.petitparser.parser.primitive.CharacterParser;
|
||||
import org.petitparser.parser.primitive.StringParser;
|
||||
|
||||
@Slf4j
|
||||
public class MediaWikiTemplate
|
||||
{
|
||||
private static final Parser LUA_PARSER;
|
||||
private static final Parser MEDIAWIKI_PARSER;
|
||||
|
||||
static
|
||||
{
|
||||
final Parser singleString = CharacterParser.of('\'').seq(CharacterParser.of('\'').neg().plus().flatten()).seq(CharacterParser.of('\''));
|
||||
final Parser doubleString = CharacterParser.of('"').seq(CharacterParser.of('"').neg().plus().flatten()).seq(CharacterParser.of('"'));
|
||||
final Parser string = singleString.or(doubleString).pick(1);
|
||||
|
||||
final Parser key = CharacterParser.letter().or(CharacterParser.of('-')).or(CharacterParser.of('_')).or(CharacterParser.of(' ')).or(CharacterParser.digit()).plus().flatten();
|
||||
final Parser value = string.or(key);
|
||||
|
||||
final Parser pair = key.trim()
|
||||
.seq(CharacterParser.of('=').trim())
|
||||
.seq(value.trim())
|
||||
.map((Function<List<String>, Map.Entry<String, String>>) input -> new AbstractMap.SimpleEntry<>(input.get(0).trim(), input.get(2).trim()));
|
||||
|
||||
final Parser commaLine = pair
|
||||
.seq(CharacterParser.of(',').optional().trim())
|
||||
.pick(0);
|
||||
|
||||
LUA_PARSER = StringParser.of("return").trim()
|
||||
.seq(CharacterParser.of('{').trim())
|
||||
.seq(commaLine.plus().trim())
|
||||
.pick(2);
|
||||
|
||||
final Parser wikiValue = CharacterParser.of('|')
|
||||
.or(StringParser.of("}}"))
|
||||
.or(StringParser.of("{{"))
|
||||
.or(StringParser.of("]]"))
|
||||
.or(StringParser.of("[["))
|
||||
.neg().plus().trim();
|
||||
|
||||
final Parser wikiBraceExpression = StringParser.of("{{")
|
||||
.seq(StringParser.of("}}").neg().star().trim())
|
||||
.seq(StringParser.of("}}"));
|
||||
|
||||
final Parser wikiSquareExpression = StringParser.of("[[")
|
||||
.seq(StringParser.of("]]").neg().star().trim())
|
||||
.seq(StringParser.of("]]"));
|
||||
|
||||
final Parser notOrPair = key.trim()
|
||||
.seq(CharacterParser.of('=').trim())
|
||||
.seq(CharacterParser.whitespace().star().seq(wikiSquareExpression.or(wikiBraceExpression).or(wikiValue)).plus().flatten().trim().optional())
|
||||
.map((Function<List<String>, Map.Entry<String, String>>) input -> new AbstractMap.SimpleEntry<>(
|
||||
input.get(0).trim(),
|
||||
MoreObjects.firstNonNull(input.get(2), "").trim()));
|
||||
|
||||
final Parser orLine = CharacterParser.of('|')
|
||||
.seq(notOrPair.trim().optional())
|
||||
.pick(1);
|
||||
|
||||
MEDIAWIKI_PARSER = orLine.plus().trim().seq(StringParser.of("}}")).pick(0);
|
||||
}
|
||||
|
||||
@Nullable
|
||||
public static MediaWikiTemplate parseWikitext(final String name, final String data)
|
||||
{
|
||||
final Pattern exactNameTest = Pattern.compile("\\{\\{\\s*" + name + "\\s*\\|", Pattern.CASE_INSENSITIVE);
|
||||
final Matcher m = exactNameTest.matcher(data.toLowerCase());
|
||||
|
||||
// Early exit
|
||||
if (!m.find())
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
final Map<String, String> out = new HashMap<>();
|
||||
final Parser wikiParser = StringParser.of("{{")
|
||||
.seq(StringParser.ofIgnoringCase(name).trim())
|
||||
.seq(MEDIAWIKI_PARSER)
|
||||
.pick(2);
|
||||
|
||||
final List<Object> parsed = wikiParser.matchesSkipping(data);
|
||||
|
||||
if (parsed.isEmpty())
|
||||
{
|
||||
final Result parse = StringParser.of("{{")
|
||||
.seq(StringParser.ofIgnoringCase(name).trim())
|
||||
.neg()
|
||||
.star()
|
||||
.seq(wikiParser)
|
||||
.seq(CharacterParser.any().star())
|
||||
.parse(data);
|
||||
|
||||
if (!parse.isSuccess())
|
||||
{
|
||||
log.warn("Failed to parse: {}", data);
|
||||
log.warn("Error message: {}", parse.getMessage());
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
final List<Map.Entry<String, String>> entries = (List<Map.Entry<String, String>>) parsed.get(0);
|
||||
|
||||
for (Map.Entry<String, String> entry : entries)
|
||||
{
|
||||
if (entry == null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
out.put(entry.getKey(), entry.getValue());
|
||||
}
|
||||
|
||||
if (out.isEmpty())
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return new MediaWikiTemplate(out);
|
||||
}
|
||||
|
||||
@Nullable
|
||||
public static MediaWikiTemplate parseLua(final String data)
|
||||
{
|
||||
final Map<String, String> out = new HashMap<>();
|
||||
final List<Object> parsed = LUA_PARSER.matchesSkipping(data);
|
||||
|
||||
if (parsed.isEmpty())
|
||||
{
|
||||
final Result parse = StringParser.of("return")
|
||||
.neg()
|
||||
.star()
|
||||
.seq(LUA_PARSER)
|
||||
.seq(CharacterParser.any()).parse(data);
|
||||
|
||||
if (!parse.isSuccess())
|
||||
{
|
||||
log.warn("Failed to parse: {}", data);
|
||||
log.warn("Error message: {}", parse.getMessage());
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
final List<Map.Entry<String, String>> entries = (List<Map.Entry<String, String>>) parsed.get(0);
|
||||
|
||||
for (Map.Entry<String, String> entry : entries)
|
||||
{
|
||||
out.put(entry.getKey(), entry.getValue());
|
||||
}
|
||||
|
||||
if (out.isEmpty())
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return new MediaWikiTemplate(out);
|
||||
}
|
||||
|
||||
/**
|
||||
* Looks for and parses the `Switch infobox` into a {@link MediaWikiTemplate} and then iterates over the `item#` values.
|
||||
* Attempts to parse each `item#` value via `parseWikiText`, matching the `name` attribute. null values are ignored
|
||||
*
|
||||
* @param name only parses MediaWikiTemplates from `Switch infobox` if matches this value. (case insensitive)
|
||||
* @param baseTemplate the {@link MediaWikiTemplate} representation of the `Switch infobox` to parse from
|
||||
* @return List of all valid {@link MediaWikiTemplate}s matching `name` from `baseTemplate`s `item#` values
|
||||
*/
|
||||
public static List<MediaWikiTemplate> parseSwitchInfoboxItems(final String name, final MediaWikiTemplate baseTemplate)
|
||||
{
|
||||
final List<MediaWikiTemplate> templates = new ArrayList<>();
|
||||
|
||||
String value;
|
||||
int suffix = 1;
|
||||
while ((value = baseTemplate.getValue("item" + suffix)) != null)
|
||||
{
|
||||
final MediaWikiTemplate subTemplate = parseWikitext(name, value);
|
||||
if (subTemplate != null)
|
||||
{
|
||||
templates.add(subTemplate);
|
||||
}
|
||||
|
||||
suffix++;
|
||||
}
|
||||
|
||||
return templates;
|
||||
}
|
||||
|
||||
private final Map<String, String> map;
|
||||
|
||||
private MediaWikiTemplate(final Map<String, String> map)
|
||||
{
|
||||
this.map = map;
|
||||
}
|
||||
|
||||
public String getValue(final String key)
|
||||
{
|
||||
String val = map.get(key);
|
||||
|
||||
if (Strings.isNullOrEmpty(val) ||
|
||||
val.equalsIgnoreCase("no") ||
|
||||
val.equalsIgnoreCase("n/a") ||
|
||||
val.equals("nil") ||
|
||||
val.equalsIgnoreCase("varies"))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
val = val.replace("kg", "").replaceAll("[><]", "");
|
||||
return Strings.isNullOrEmpty(val) ? null : val;
|
||||
}
|
||||
|
||||
public Boolean getBoolean(final String key)
|
||||
{
|
||||
final String val = getValue(key);
|
||||
return !Strings.isNullOrEmpty(val) ? true : null;
|
||||
}
|
||||
|
||||
public Double getDouble(final String key)
|
||||
{
|
||||
final String val = getValue(key);
|
||||
|
||||
if (Strings.isNullOrEmpty(val))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
double v = Double.parseDouble(val);
|
||||
return v != 0 ? v : null;
|
||||
}
|
||||
catch (NumberFormatException e)
|
||||
{
|
||||
e.printStackTrace();
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public Integer getInt(final String key)
|
||||
{
|
||||
final String val = getValue(key);
|
||||
|
||||
if (Strings.isNullOrEmpty(val))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
int v = Integer.parseInt(val);
|
||||
return v != 0 ? v : null;
|
||||
}
|
||||
catch (NumberFormatException e)
|
||||
{
|
||||
e.printStackTrace();
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public boolean containsKey(final String key)
|
||||
{
|
||||
return map.containsKey(key);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
/*
|
||||
* MIT License
|
||||
*
|
||||
* Copyright (c) 2018 Tomas Slusny
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
* of this software and associated documentation files (the "Software"), to deal
|
||||
* in the Software without restriction, including without limitation the rights
|
||||
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the Software is
|
||||
* furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
* SOFTWARE.
|
||||
*/
|
||||
package net.runelite.data.dump.wiki;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
import java.io.File;
|
||||
import java.io.FileWriter;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Stream;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import net.runelite.cache.ItemManager;
|
||||
import net.runelite.cache.definitions.ItemDefinition;
|
||||
import net.runelite.cache.fs.Store;
|
||||
import net.runelite.cache.util.Namer;
|
||||
import net.runelite.data.App;
|
||||
import net.runelite.data.dump.MediaWiki;
|
||||
import net.runelite.data.dump.MediaWikiTemplate;
|
||||
|
||||
@Slf4j
|
||||
public class ItemLimitsDumper
|
||||
{
|
||||
public static void dump(final Store store, final MediaWiki wiki) throws IOException
|
||||
{
|
||||
final File out = new File("../runelite-client/src/main/resources/net/runelite/client/plugins/grandexchange/");
|
||||
|
||||
log.info("Dumping item limits to {}", out);
|
||||
|
||||
final ItemManager itemManager = new ItemManager(store);
|
||||
itemManager.load();
|
||||
final Pattern pattern = Pattern.compile("limit {6}= (.*),");
|
||||
|
||||
final Map<Integer, Integer> limits = new TreeMap<>();
|
||||
final Collection<ItemDefinition> items = itemManager.getItems();
|
||||
final Stream<ItemDefinition> itemDefinitionStream = items.parallelStream();
|
||||
List<String> missing = new ArrayList<>();
|
||||
|
||||
itemDefinitionStream.forEach(item ->
|
||||
{
|
||||
if (!item.isTradeable)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (item.getNotedTemplate() != -1)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (item.name.equalsIgnoreCase("NULL"))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
String name = Namer
|
||||
.removeTags(item.name)
|
||||
.replace('\u00A0', ' ')
|
||||
.replaceAll("\\+", "%2b")
|
||||
.trim();
|
||||
|
||||
if (name.isEmpty())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
String data = wiki.getPageData("Module:Exchange/" + name, -1);
|
||||
|
||||
if (Strings.isNullOrEmpty(data))
|
||||
{
|
||||
log.debug("Data is null or empty: {}", name);
|
||||
missing.add(name);
|
||||
return;
|
||||
}
|
||||
|
||||
final MediaWikiTemplate geStats = MediaWikiTemplate.parseLua(data);
|
||||
|
||||
if (geStats == null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
final Integer limit = geStats.getInt("limit");
|
||||
|
||||
if (limit == null || limit <= 0)
|
||||
{
|
||||
Matcher matcher = pattern.matcher(data);
|
||||
String temp = "";
|
||||
while (matcher.find())
|
||||
{
|
||||
temp = matcher.group(1);
|
||||
if (Strings.isNullOrEmpty(temp) ||
|
||||
temp.equalsIgnoreCase("no") ||
|
||||
temp.equalsIgnoreCase("n/a") ||
|
||||
temp.equals("nil") ||
|
||||
temp.equalsIgnoreCase("varies"))
|
||||
{
|
||||
temp = null;
|
||||
}
|
||||
}
|
||||
if (!Strings.isNullOrEmpty(temp))
|
||||
{
|
||||
limits.put(item.id, Integer.valueOf(temp));
|
||||
}
|
||||
else
|
||||
{
|
||||
log.debug("Item was still null: {}", name);
|
||||
missing.add(name);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
limits.put(item.id, limit);
|
||||
log.debug("Dumped item limit for {} {}", item.id, name);
|
||||
});
|
||||
|
||||
try (FileWriter fw = new FileWriter(new File(out, "ge_limits.json")))
|
||||
{
|
||||
fw.write(App.GSON.toJson(limits));
|
||||
}
|
||||
|
||||
log.info("Dumped {} item limits", limits.size());
|
||||
log.info("Total Missing: " + missing.size());
|
||||
missing.forEach(str ->
|
||||
{
|
||||
log.info("Still Missing: {}", str);
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,359 @@
|
||||
/*
|
||||
* MIT License
|
||||
*
|
||||
* Copyright (c) 2018 Tomas Slusny
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
* of this software and associated documentation files (the "Software"), to deal
|
||||
* in the Software without restriction, including without limitation the rights
|
||||
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the Software is
|
||||
* furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
* SOFTWARE.
|
||||
*/
|
||||
package net.runelite.data.dump.wiki;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
import java.io.File;
|
||||
import java.io.FileWriter;
|
||||
import java.io.IOException;
|
||||
import java.util.Collection;
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
import java.util.stream.Stream;
|
||||
import lombok.Builder;
|
||||
import lombok.Value;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import net.runelite.api.EquipmentInventorySlot;
|
||||
import net.runelite.cache.ItemManager;
|
||||
import net.runelite.cache.definitions.ItemDefinition;
|
||||
import net.runelite.cache.fs.Store;
|
||||
import net.runelite.cache.util.Namer;
|
||||
import net.runelite.data.App;
|
||||
import net.runelite.data.dump.MediaWiki;
|
||||
import net.runelite.data.dump.MediaWikiTemplate;
|
||||
|
||||
@Slf4j
|
||||
public class ItemStatsDumper
|
||||
{
|
||||
private final static Integer MAX_ITEMS_ON_PAGE = 50;
|
||||
|
||||
public static void dump(final Store store, final MediaWiki wiki) throws IOException
|
||||
{
|
||||
final File out = new File("../runelite-client/src/main/resources/");
|
||||
|
||||
log.info("Dumping item stats to {}", out);
|
||||
|
||||
final ItemManager itemManager = new ItemManager(store);
|
||||
itemManager.load();
|
||||
|
||||
final Map<Integer, ItemStats> itemStats = new TreeMap<>();
|
||||
final Collection<ItemDefinition> items = itemManager.getItems();
|
||||
final Stream<ItemDefinition> itemDefinitionStream = items.parallelStream();
|
||||
|
||||
itemDefinitionStream.forEach(item ->
|
||||
{
|
||||
if (item.getNotedTemplate() != -1)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (item.name.equalsIgnoreCase("NULL"))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
final String name = Namer
|
||||
.removeTags(item.name)
|
||||
.replace('\u00A0', ' ')
|
||||
.trim();
|
||||
|
||||
if (name.isEmpty())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
String data = wiki.getSpecialLookupData("item", item.id, 0);
|
||||
|
||||
if (Strings.isNullOrEmpty(data))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
MediaWikiTemplate base = MediaWikiTemplate.parseWikitext("Infobox Item", data);
|
||||
|
||||
if (base == null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
final int nItems = findMaxIndex(base);
|
||||
final ItemStats.ItemStatsBuilder itemStat = ItemStats.builder();
|
||||
|
||||
for (int index = 1; index <= nItems; index++)
|
||||
{
|
||||
final int offset = nItems == 1 ? 0 : index;
|
||||
final String wikiName = getVarString(base, "name", offset);
|
||||
|
||||
// Skip this index if name or itemId doesn't match with wiki
|
||||
if (nItems > 1 && !wikiName.equalsIgnoreCase(name))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
itemStat.name(getVarString(base, "name", offset) == null ? getVarString(base, "name1", offset) : getVarString(base, "name", offset));
|
||||
itemStat.quest(getVarBoolean(base, "quest", offset));
|
||||
itemStat.equipable(getVarBoolean(base, "equipable", offset) == null
|
||||
? getVarBoolean(base, "equipable1", offset) : getVarBoolean(base, "equipable", offset));
|
||||
itemStat.weight(getVarDouble(base, "weight", offset));
|
||||
|
||||
|
||||
if (Boolean.TRUE.equals(itemStat.equipable))
|
||||
{
|
||||
MediaWikiTemplate stats = MediaWikiTemplate.parseWikitext("Infobox Bonuses", data);
|
||||
|
||||
if (stats == null)
|
||||
{
|
||||
data = wiki.getSpecialLookupData("item", item.id, 1);
|
||||
|
||||
if (Strings.isNullOrEmpty(data))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
stats = MediaWikiTemplate.parseWikitext("Infobox Bonuses", data);
|
||||
}
|
||||
|
||||
if (stats == null)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
final ItemEquipmentStats.ItemEquipmentStatsBuilder equipmentStat = ItemEquipmentStats.builder();
|
||||
|
||||
equipmentStat.slot(toEquipmentSlot(getVarString(stats, "slot", offset)));
|
||||
equipmentStat.astab(getVarInt(stats, "astab", offset));
|
||||
equipmentStat.aslash(getVarInt(stats, "aslash", offset));
|
||||
equipmentStat.acrush(getVarInt(stats, "acrush", offset));
|
||||
equipmentStat.amagic(getVarInt(stats, "amagic", offset));
|
||||
equipmentStat.arange(getVarInt(stats, "arange", offset));
|
||||
|
||||
equipmentStat.dstab(getVarInt(stats, "dstab", offset));
|
||||
equipmentStat.dslash(getVarInt(stats, "dslash", offset));
|
||||
equipmentStat.dcrush(getVarInt(stats, "dcrush", offset));
|
||||
equipmentStat.dmagic(getVarInt(stats, "dmagic", offset));
|
||||
equipmentStat.drange(getVarInt(stats, "drange", offset));
|
||||
|
||||
equipmentStat.str(getVarInt(stats, "str", offset));
|
||||
equipmentStat.rstr(getVarInt(stats, "rstr", offset));
|
||||
equipmentStat.mdmg(getVarInt(stats, "mdmg", offset));
|
||||
equipmentStat.prayer(getVarInt(stats, "prayer", offset));
|
||||
equipmentStat.aspeed(getVarInt(stats, "aspeed", offset));
|
||||
|
||||
final ItemEquipmentStats builtEqStat = equipmentStat.build();
|
||||
|
||||
if (!builtEqStat.equals(ItemEquipmentStats.builder().build()))
|
||||
{
|
||||
itemStat.equipment(builtEqStat);
|
||||
}
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
final ItemStats val = itemStat.build();
|
||||
|
||||
if (ItemStats.DEFAULT.equals(val))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
itemStats.put(item.id, val);
|
||||
log.debug("Dumped item stat for {} {}", item.id, name);
|
||||
});
|
||||
|
||||
try (FileWriter fw = new FileWriter(new File(out, "item_stats.json")))
|
||||
{
|
||||
fw.write(App.GSON.toJson(itemStats));
|
||||
}
|
||||
|
||||
log.info("Dumped {} item stats", itemStats.size());
|
||||
}
|
||||
|
||||
/**
|
||||
* Counts how many items are on page
|
||||
*
|
||||
* @param template media wiki template
|
||||
* @return item count
|
||||
*/
|
||||
private static int findMaxIndex(final MediaWikiTemplate template)
|
||||
{
|
||||
int nItems = 1;
|
||||
|
||||
if (template.getValue("version1") == null)
|
||||
{
|
||||
return nItems;
|
||||
}
|
||||
|
||||
while (nItems < MAX_ITEMS_ON_PAGE)
|
||||
{
|
||||
if (template.getValue(fixIndex("name", nItems + 1)) != null || template.getValue(fixIndex("version", nItems + 1)) != null)
|
||||
{
|
||||
nItems++;
|
||||
}
|
||||
else
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return nItems;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return fixed string version of indexed key
|
||||
*
|
||||
* @param base key name
|
||||
* @param index current index
|
||||
* @return string representation of index
|
||||
*/
|
||||
private static String fixIndex(final String base, final Integer index)
|
||||
{
|
||||
return index == 0 ? base : base + index;
|
||||
}
|
||||
|
||||
private static String getVarString(final MediaWikiTemplate template, final String key, final Integer index)
|
||||
{
|
||||
final String var = template.getValue(fixIndex(key, index));
|
||||
|
||||
if (var != null)
|
||||
{
|
||||
return var;
|
||||
}
|
||||
|
||||
return template.getValue(key);
|
||||
}
|
||||
|
||||
private static Boolean getVarBoolean(final MediaWikiTemplate template, final String key, final Integer index)
|
||||
{
|
||||
final Boolean var = template.getBoolean(fixIndex(key, index));
|
||||
|
||||
if (var != null)
|
||||
{
|
||||
return var;
|
||||
}
|
||||
|
||||
return template.getBoolean(key);
|
||||
}
|
||||
|
||||
private static Integer getVarInt(final MediaWikiTemplate template, final String key, final Integer index)
|
||||
{
|
||||
final Integer var = template.getInt(fixIndex(key, index));
|
||||
|
||||
if (var != null)
|
||||
{
|
||||
return var;
|
||||
}
|
||||
|
||||
return template.getInt(key);
|
||||
}
|
||||
|
||||
private static Double getVarDouble(final MediaWikiTemplate template, final String key, final Integer index)
|
||||
{
|
||||
final Double var = template.getDouble(fixIndex(key, index));
|
||||
|
||||
if (var != null)
|
||||
{
|
||||
return var;
|
||||
}
|
||||
|
||||
return template.getDouble(key);
|
||||
}
|
||||
|
||||
private static Integer toEquipmentSlot(final String slotName)
|
||||
{
|
||||
if (slotName == null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
switch (slotName.toLowerCase())
|
||||
{
|
||||
case "weapon":
|
||||
case "2h":
|
||||
// TODO: 2h should return both weapon and shield somehow
|
||||
return EquipmentInventorySlot.WEAPON.getSlotIdx();
|
||||
case "body":
|
||||
return EquipmentInventorySlot.BODY.getSlotIdx();
|
||||
case "head":
|
||||
return EquipmentInventorySlot.HEAD.getSlotIdx();
|
||||
case "ammo":
|
||||
return EquipmentInventorySlot.AMMO.getSlotIdx();
|
||||
case "legs":
|
||||
return EquipmentInventorySlot.LEGS.getSlotIdx();
|
||||
case "feet":
|
||||
return EquipmentInventorySlot.BOOTS.getSlotIdx();
|
||||
case "hands":
|
||||
return EquipmentInventorySlot.GLOVES.getSlotIdx();
|
||||
case "cape":
|
||||
return EquipmentInventorySlot.CAPE.getSlotIdx();
|
||||
case "neck":
|
||||
return EquipmentInventorySlot.AMULET.getSlotIdx();
|
||||
case "ring":
|
||||
return EquipmentInventorySlot.RING.getSlotIdx();
|
||||
case "shield":
|
||||
return EquipmentInventorySlot.SHIELD.getSlotIdx();
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
@Value
|
||||
@Builder
|
||||
private static final class ItemEquipmentStats
|
||||
{
|
||||
private final Integer slot;
|
||||
|
||||
private final Integer astab;
|
||||
private final Integer aslash;
|
||||
private final Integer acrush;
|
||||
private final Integer amagic;
|
||||
private final Integer arange;
|
||||
|
||||
private final Integer dstab;
|
||||
private final Integer dslash;
|
||||
private final Integer dcrush;
|
||||
private final Integer dmagic;
|
||||
private final Integer drange;
|
||||
|
||||
private final Integer str;
|
||||
private final Integer rstr;
|
||||
private final Integer mdmg;
|
||||
private final Integer prayer;
|
||||
private final Integer aspeed;
|
||||
}
|
||||
|
||||
@Value
|
||||
@Builder
|
||||
private static final class ItemStats
|
||||
{
|
||||
static final ItemStats DEFAULT = ItemStats.builder().build();
|
||||
|
||||
private final String name;
|
||||
private final Boolean quest;
|
||||
private final Boolean equipable;
|
||||
private final Double weight;
|
||||
|
||||
private final ItemEquipmentStats equipment;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,393 @@
|
||||
/*
|
||||
* MIT License
|
||||
*
|
||||
* Copyright (c) 2019 TheStonedTurtle <https://github.com/TheStonedTurtle>
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
* of this software and associated documentation files (the "Software"), to deal
|
||||
* in the Software without restriction, including without limitation the rights
|
||||
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the Software is
|
||||
* furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
* SOFTWARE.
|
||||
*/
|
||||
package net.runelite.data.dump.wiki;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
import java.io.File;
|
||||
import java.io.FileWriter;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.TreeMap;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import net.runelite.cache.NpcManager;
|
||||
import net.runelite.cache.definitions.NpcDefinition;
|
||||
import net.runelite.cache.fs.Store;
|
||||
import net.runelite.cache.util.Namer;
|
||||
import net.runelite.data.App;
|
||||
import net.runelite.data.dump.MediaWiki;
|
||||
import net.runelite.data.dump.MediaWikiTemplate;
|
||||
|
||||
@Slf4j
|
||||
public class NpcStatsDumper
|
||||
{
|
||||
@Data
|
||||
@Builder
|
||||
private static final class NpcStats
|
||||
{
|
||||
private String name;
|
||||
private final Integer hitpoints;
|
||||
private final Integer hitpoints1;
|
||||
private final Integer combatLevel;
|
||||
private final Integer slayerLevel;
|
||||
private final Integer attackSpeed;
|
||||
|
||||
private final Integer attackLevel;
|
||||
private final Integer strengthLevel;
|
||||
private final Integer defenceLevel;
|
||||
private final Integer rangeLevel;
|
||||
private final Integer magicLevel;
|
||||
|
||||
private final Integer stab;
|
||||
private final Integer slash;
|
||||
private final Integer crush;
|
||||
private final Integer range;
|
||||
private final Integer magic;
|
||||
|
||||
private final Integer stabDef;
|
||||
private final Integer slashDef;
|
||||
private final Integer crushDef;
|
||||
private final Integer rangeDef;
|
||||
private final Integer magicDef;
|
||||
|
||||
private final Integer bonusAttack;
|
||||
private final Integer bonusStrength;
|
||||
private final Integer bonusRangeStrength;
|
||||
private final Integer bonusMagicDamage;
|
||||
|
||||
private final Boolean poisonImmune;
|
||||
private final Boolean venomImmune;
|
||||
|
||||
private final Boolean dragon;
|
||||
private final Boolean demon;
|
||||
private final Boolean undead;
|
||||
}
|
||||
|
||||
private static final NpcStats DEFAULT = NpcStats.builder().build();
|
||||
|
||||
/**
|
||||
* Looks for and parses the `Switch infobox` into a {@link MediaWikiTemplate} and then iterates over the `item#` values.
|
||||
* Attempts to parse each `item#` value via `parseWikiText`, matching the `name` attribute. null values are ignored
|
||||
*
|
||||
* @param name only parses MediaWikiTemplates from `Switch infobox` if matches this value. (case insensitive)
|
||||
* @param baseTemplate the {@link MediaWikiTemplate} representation of the `Switch infobox` to parse from
|
||||
* @return List of all valid {@link MediaWikiTemplate}s matching `name` from `baseTemplate`s `item#` values
|
||||
*/
|
||||
static List<MediaWikiTemplate> parseSwitchInfoboxItems(final String name, final MediaWikiTemplate baseTemplate)
|
||||
{
|
||||
final List<MediaWikiTemplate> templates = new ArrayList<>();
|
||||
|
||||
String value;
|
||||
int suffix = 1;
|
||||
while ((value = baseTemplate.getValue("item" + suffix)) != null)
|
||||
{
|
||||
final MediaWikiTemplate subTemplate = MediaWikiTemplate.parseWikitext(name, value);
|
||||
if (subTemplate != null)
|
||||
{
|
||||
templates.add(subTemplate);
|
||||
}
|
||||
|
||||
suffix++;
|
||||
}
|
||||
|
||||
return templates;
|
||||
}
|
||||
|
||||
public static void dump(final Store store, final MediaWiki wiki) throws IOException
|
||||
{
|
||||
final File out = new File("../runelite-client/src/main/resources/");
|
||||
|
||||
log.info("Dumping npc stats to {}", out);
|
||||
|
||||
final NpcManager npcManager = new NpcManager(store);
|
||||
npcManager.load();
|
||||
|
||||
final Map<Integer, NpcStats> npcStats = new HashMap<>();
|
||||
final Collection<NpcDefinition> definitions = npcManager.getNpcs();
|
||||
final Stream<NpcDefinition> npcDefinitionStream = definitions.parallelStream();
|
||||
|
||||
// Ensure variant names match cache as wiki isn't always correct
|
||||
final Map<Integer, String> nameMap = new HashMap<>();
|
||||
for (NpcDefinition n : definitions)
|
||||
{
|
||||
if (n.getName().equalsIgnoreCase("NULL"))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
final String name = Namer
|
||||
.removeTags(n.getName())
|
||||
.replace('\u00A0', ' ')
|
||||
.trim();
|
||||
|
||||
if (name.isEmpty())
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
nameMap.put(n.getId(), name);
|
||||
}
|
||||
|
||||
npcDefinitionStream.forEach(n ->
|
||||
{
|
||||
if (npcStats.containsKey(n.getId()))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
final String name = nameMap.get(n.getId());
|
||||
if (name == null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (!isAttackableNpc(n))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
final String data = wiki.getSpecialLookupData("npc", n.getId(), 0);
|
||||
if (Strings.isNullOrEmpty(data))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
List<MediaWikiTemplate> bases = new ArrayList<>();
|
||||
|
||||
final MediaWikiTemplate switchBase = MediaWikiTemplate.parseWikitext("Switch infobox", data);
|
||||
if (switchBase != null)
|
||||
{
|
||||
bases = parseSwitchInfoboxItems("Infobox Monster", switchBase);
|
||||
}
|
||||
else
|
||||
{
|
||||
final MediaWikiTemplate base = MediaWikiTemplate.parseWikitext("Infobox Monster", data);
|
||||
if (base == null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
bases.add(base);
|
||||
}
|
||||
|
||||
for (final MediaWikiTemplate base : bases)
|
||||
{
|
||||
int variantKey = 0;
|
||||
String wikiIdString = getWikiIdString(base, variantKey);
|
||||
if (wikiIdString == null)
|
||||
{
|
||||
// Try again as `id` will be null if there are variants and `id1` is the starting key
|
||||
variantKey++;
|
||||
wikiIdString = getWikiIdString(base, variantKey);
|
||||
}
|
||||
|
||||
while (wikiIdString != null)
|
||||
{
|
||||
if (wikiIdString.isEmpty())
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
final Set<Integer> ids = Arrays.stream(wikiIdString.split(","))
|
||||
.map(s -> Integer.parseInt(s.trim()))
|
||||
.collect(Collectors.toSet());
|
||||
|
||||
final NpcStats stats = buildNpcStats(base, variantKey);
|
||||
if (!stats.equals(DEFAULT))
|
||||
{
|
||||
stats.setName(name);
|
||||
for (final int curID : ids)
|
||||
{
|
||||
// Update variant name or fall back to current name
|
||||
final String curName = nameMap.get(curID);
|
||||
stats.setName(curName == null ? stats.getName() : curName);
|
||||
|
||||
npcStats.put(curID, stats);
|
||||
log.debug("Dumped npc stats for npc id: {}", curID);
|
||||
}
|
||||
}
|
||||
|
||||
variantKey++;
|
||||
wikiIdString = getWikiIdString(base, variantKey);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Cast to TreeMap so sort output JSON in numerical order (npc id)
|
||||
final Map<Integer, NpcStats> sorted = new TreeMap<>(npcStats);
|
||||
|
||||
try (FileWriter fw = new FileWriter(new File(out, "npc_stats.json")))
|
||||
{
|
||||
fw.write(App.GSON.toJson(sorted));
|
||||
}
|
||||
|
||||
// try (FileWriter fw = new FileWriter(new File(out, "npc_stats.min.json")))
|
||||
// {
|
||||
// fw.write(new GsonBuilder().disableHtmlEscaping().create().toJson(sorted));
|
||||
// }
|
||||
|
||||
log.info("Dumped {} npc stats", sorted.size());
|
||||
}
|
||||
|
||||
private static boolean isAttackableNpc(final NpcDefinition n)
|
||||
{
|
||||
for (final String s : n.getOptions())
|
||||
{
|
||||
if ("attack".equalsIgnoreCase(s))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private static String getKeySuffix(final int variantKey)
|
||||
{
|
||||
return variantKey > 0 ? String.valueOf(variantKey) : "";
|
||||
}
|
||||
|
||||
private static String getWikiIdString(final MediaWikiTemplate template, final int variantKey)
|
||||
{
|
||||
return template.getValue("id" + getKeySuffix(variantKey));
|
||||
}
|
||||
|
||||
private static NpcStats buildNpcStats(final MediaWikiTemplate template, int variantKey)
|
||||
{
|
||||
final NpcStats.NpcStatsBuilder stats = NpcStats.builder();
|
||||
|
||||
stats.hitpoints(getInt("hitpoints", variantKey, template));
|
||||
if (stats.hitpoints == null)
|
||||
{
|
||||
stats.hitpoints(getInt("hitpoints1", variantKey, template));
|
||||
}
|
||||
stats.combatLevel(getInt("combat", variantKey, template));
|
||||
stats.slayerLevel(getInt("slaylvl", variantKey, template));
|
||||
stats.attackSpeed(getInt("attack speed", variantKey, template));
|
||||
|
||||
stats.attackLevel(getInt("att", variantKey, template));
|
||||
stats.strengthLevel(getInt("str", variantKey, template));
|
||||
stats.defenceLevel(getInt("def", variantKey, template));
|
||||
stats.rangeLevel(getInt("range", variantKey, template));
|
||||
stats.magicLevel(getInt("mage", variantKey, template));
|
||||
|
||||
stats.stab(getInt("astab", variantKey, template));
|
||||
stats.slash(getInt("aslash", variantKey, template));
|
||||
stats.crush(getInt("acrush", variantKey, template));
|
||||
stats.range(getInt("arange", variantKey, template));
|
||||
stats.magic(getInt("amagic", variantKey, template));
|
||||
|
||||
stats.stabDef(getInt("dstab", variantKey, template));
|
||||
stats.slashDef(getInt("dslash", variantKey, template));
|
||||
stats.crushDef(getInt("dcrush", variantKey, template));
|
||||
stats.rangeDef(getInt("drange", variantKey, template));
|
||||
stats.magicDef(getInt("dmagic", variantKey, template));
|
||||
|
||||
stats.bonusAttack(getInt("attbns", variantKey, template));
|
||||
stats.bonusStrength(getInt("strbns", variantKey, template));
|
||||
stats.bonusRangeStrength(getInt("rngbns", variantKey, template));
|
||||
stats.bonusMagicDamage(getInt("mbns", variantKey, template));
|
||||
|
||||
final String keySuffix = getKeySuffix(variantKey);
|
||||
boolean pImmune = "immune".equalsIgnoreCase(template.getValue("immunepoison" + keySuffix));
|
||||
boolean vImmune = "immune".equalsIgnoreCase(template.getValue("immunevenom" + keySuffix));
|
||||
|
||||
stats.poisonImmune(!pImmune ? null : true);
|
||||
stats.venomImmune(!vImmune ? null : true);
|
||||
|
||||
final String weaknessValue = template.getValue("weakness");
|
||||
if (weaknessValue != null)
|
||||
{
|
||||
final String[] values = weaknessValue.split(",");
|
||||
for (String value : values)
|
||||
{
|
||||
value = value.toLowerCase();
|
||||
if (stats.dragon == null && (value.contains("dragonbane weapons")))
|
||||
{
|
||||
stats.dragon(true);
|
||||
}
|
||||
|
||||
if (stats.demon == null && (value.contains("demonbane weapons") || value.contains("silverlight") || value.contains("arclight")))
|
||||
{
|
||||
stats.demon(true);
|
||||
}
|
||||
|
||||
if (stats.undead == null && (value.contains("salve amulet") || value.contains("crumble undead")))
|
||||
{
|
||||
stats.undead(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return stats.build();
|
||||
}
|
||||
|
||||
static Integer getInt(final String mainKey, final Integer variation, final MediaWikiTemplate template)
|
||||
{
|
||||
final String key = mainKey + getKeySuffix(variation);
|
||||
if (!template.containsKey(key))
|
||||
{
|
||||
if (variation >= 1)
|
||||
{
|
||||
// Use variation fallback via recursion
|
||||
return getInt(mainKey, variation - 1, template);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
final String val = template.getValue(key);
|
||||
if (Strings.isNullOrEmpty(val))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
// Remove everything after the first non-number character to account for any comments
|
||||
final String fixedVal = val.trim().replaceAll("\\D+.*", "");
|
||||
if (fixedVal.isEmpty())
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
int v = Integer.parseInt(fixedVal);
|
||||
return v != 0 ? v : null;
|
||||
}
|
||||
catch (NumberFormatException e)
|
||||
{
|
||||
e.printStackTrace();
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user