ra/runelite
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

actions: More GitHub actions (#1636)

Browse Source 
* actions: Run in parallel

* actions: NPC stats scraper

* actions: Rename actions

* actions: Auto approve NPC stats updates

* gradle: Add tasks for different wiki scrape actions

* actions: Add item stats scraper

* actions: run scraper on a schedule (twice a week)

* actions: gradle wrapper updater
This commit is contained in:
Owain van Brakel
2019-09-22 22:33:25 +02:00
committed by Ganom
parent f7cb809cd9
commit ec3b99f0a7
13 changed files with 212 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
wiki-scraper/build.gradle
View File

@@ -23,12 +23,34 @@ dependencies {
testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-api', version: jupiter
}
task scrape {
task npcStatsScrape {
doLast {
def path = sourceSets.main.runtimeClasspath
def loader = new URLClassLoader(path.collect { f -> f.toURI().toURL() } as URL[])
def scrape = loader.loadClass('net.runelite.data.App')
scrape.main();
scrape.npcStats(rootProject.file("./runelite-client/src/main/resources/"));
loader.close()
}
}
task itemStatsScrape {
doLast {
def path = sourceSets.main.runtimeClasspath
def loader = new URLClassLoader(path.collect { f -> f.toURI().toURL() } as URL[])
def scrape = loader.loadClass('net.runelite.data.App')
scrape.itemStats(rootProject.file("./runelite-client/src/main/resources/"));
loader.close()
}
}
task itemLimitsScrape {
doLast {
def path = sourceSets.main.runtimeClasspath
def loader = new URLClassLoader(path.collect { f -> f.toURI().toURL() } as URL[])
def scrape = loader.loadClass('net.runelite.data.App')
scrape.itemLimits(rootProject.file("./runelite-client/src/main/resources/net/runelite/client/plugins/grandexchange/"));
loader.close()
}

27
wiki-scraper/src/main/java/net/runelite/data/App.java
View File

@@ -32,6 +32,8 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import net.runelite.cache.fs.Store;
import net.runelite.data.dump.MediaWiki;
import net.runelite.data.dump.wiki.ItemLimitsDumper;
import net.runelite.data.dump.wiki.ItemStatsDumper;
import net.runelite.data.dump.wiki.NpcStatsDumper;
public class App
@@ -41,14 +43,16 @@ public class App
.disableHtmlEscaping()
.create();
public static void main(String[] args) throws IOException
private final static MediaWiki wiki = new MediaWiki("https://oldschool.runescape.wiki");
public static Store cacheStore() throws IOException
{
Path path = Paths.get(System.getProperty("user.home"), "jagexcache" + File.separator + "oldschool" + File.separator + "LIVE");
final File jagexcache = new File(String.valueOf(path));
if (!Files.exists(path))
{
return;
return null;
}
final Store cacheStore = new Store(jagexcache);
@@ -58,12 +62,21 @@ public class App
// Try to make this go faster (probably not very smart)
System.setProperty("java.util.concurrent.ForkJoinPool.common.parallelism", "100");
final MediaWiki wiki = new MediaWiki("https://oldschool.runescape.wiki");
return cacheStore;
}
// Only use this to diff current limits with scraped limits
// ItemLimitsDumper.dump(cacheStore, wiki);
// ItemStatsDumper.dump(cacheStore, wiki);
public static void npcStats(File path) throws IOException
{
NpcStatsDumper.dump(cacheStore(), wiki, path);
}
NpcStatsDumper.dump(cacheStore, wiki);
public static void itemStats(File path) throws IOException
{
ItemStatsDumper.dump(cacheStore(), wiki, path);
}
public static void itemLimits(File path) throws IOException
{
ItemLimitsDumper.dump(cacheStore(), wiki, path);
}
}

8
wiki-scraper/src/main/java/net/runelite/data/dump/wiki/ItemLimitsDumper.java
View File

@@ -47,11 +47,9 @@ import net.runelite.data.dump.MediaWikiTemplate;
@Slf4j
public class ItemLimitsDumper
{
public static void dump(final Store store, final MediaWiki wiki) throws IOException
public static void dump(final Store store, final MediaWiki wiki, final File path) throws IOException
{
final File out = new File("../runelite-client/src/main/resources/net/runelite/client/plugins/grandexchange/");
log.info("Dumping item limits to {}", out);
log.info("Dumping item limits to {}", path);
final ItemManager itemManager = new ItemManager(store);
itemManager.load();
@@ -140,7 +138,7 @@ public class ItemLimitsDumper
log.debug("Dumped item limit for {} {}", item.id, name);
});
try (FileWriter fw = new FileWriter(new File(out, "ge_limits.json")))
try (FileWriter fw = new FileWriter(new File(path, "ge_limits.json")))
{
fw.write(App.GSON.toJson(limits));
}

8
wiki-scraper/src/main/java/net/runelite/data/dump/wiki/ItemStatsDumper.java
View File

@@ -48,11 +48,9 @@ public class ItemStatsDumper
{
private final static Integer MAX_ITEMS_ON_PAGE = 50;
public static void dump(final Store store, final MediaWiki wiki) throws IOException
public static void dump(final Store store, final MediaWiki wiki, final File path) throws IOException
{
final File out = new File("../runelite-client/src/main/resources/");
log.info("Dumping item stats to {}", out);
log.info("Dumping item stats to {}", path);
final ItemManager itemManager = new ItemManager(store);
itemManager.load();
@@ -182,7 +180,7 @@ public class ItemStatsDumper
log.debug("Dumped item stat for {} {}", item.id, name);
});
try (FileWriter fw = new FileWriter(new File(out, "item_stats.json")))
try (FileWriter fw = new FileWriter(new File(path, "item_stats.json")))
{
fw.write(App.GSON.toJson(itemStats));
}

7
wiki-scraper/src/main/java/net/runelite/data/dump/wiki/NpcStatsDumper.java
View File

@@ -123,11 +123,10 @@ public class NpcStatsDumper
return templates;
}
public static void dump(final Store store, final MediaWiki wiki) throws IOException
public static void dump(final Store store, final MediaWiki wiki, final File path) throws IOException
{
final File out = new File("../runelite-client/src/main/resources/");
log.info("Dumping npc stats to {}", out);
log.info("Dumping npc stats to {}", path);
final NpcManager npcManager = new NpcManager(store);
npcManager.load();
@@ -246,7 +245,7 @@ public class NpcStatsDumper
// Cast to TreeMap so sort output JSON in numerical order (npc id)
final Map<Integer, NpcStats> sorted = new TreeMap<>(npcStats);
try (FileWriter fw = new FileWriter(new File(out, "npc_stats.json")))
try (FileWriter fw = new FileWriter(new File(path, "npc_stats.json")))
{
fw.write(App.GSON.toJson(sorted));
}