actions: More GitHub actions (#1636)

* actions: Run in parallel

* actions: NPC stats scraper

* actions: Rename actions

* actions: Auto approve NPC stats updates

* gradle: Add tasks for different wiki scrape actions

* actions: Add item stats scraper

* actions: run scraper on a schedule (twice a week)

* actions: gradle wrapper updater

.github/workflows/auto-approve.yml

@@ -0,0 +1,14 @@
+name: RunelitePlus - Auto approve
+
+on: pull_request
+
+jobs:
+  approve:
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: hmarr/auto-approve-action@v2.0.0
+        if: github.actor == 'github-actions[bot]'
+        with:
+          github-token: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/build.yml

@@ -1,4 +1,4 @@
-name: RunelitePlus continuous integration
+name: RunelitePlus - CI
 
 on: [pull_request, push]
 
@@ -17,7 +17,3 @@ jobs:
         run: gradlew assemble --console=plain
       - name: Building
         run: gradlew build --stacktrace -x test -x checkstyleMain --console=plain
-      - name: Testing
-        run: gradlew test --stacktrace --console=plain
-      - name: Checking code conventions
-        run: gradlew checkstyleMain --console=plain

.github/workflows/checkstyle.yml

@@ -0,0 +1,19 @@
+name: RunelitePlus - Code convention
+
+on: [pull_request, push]
+
+jobs:
+  checkstyle:
+
+    runs-on: windows-latest
+
+    steps:
+      - uses: actions/checkout@v1
+      - name: Set up JDK 11
+        uses: actions/setup-java@v1
+        with:
+          java-version: 11
+      - name: Assembling
+        run: gradlew assemble --console=plain
+      - name: Checking code conventions
+        run: gradlew checkstyleMain --console=plain

.github/workflows/schedule.yml

@@ -0,0 +1,17 @@
+name: RunelitePlus - Gradle Wrapper
+
+on:
+  schedule:
+  - cron: 0 0 * * *
+
+jobs:
+  update:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@master
+    - name: Update Gradle Wrapper
+      uses: rahulsom/gradle-up@master
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/scraper.yml

@@ -0,0 +1,47 @@
+name: RunelitePlus - Scraper
+
+on:
+  schedule:
+    - cron: "0 0 * * 2,5"
+
+jobs:
+  scrape:
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v1
+      - name: Set up JDK 11
+        uses: actions/setup-java@v1
+        with:
+          java-version: 11
+      - name: Assembling cache client
+        run: ./gradlew :cache-client:assemble --console=plain
+      - name: Assembling scraper
+        run: ./gradlew :wiki-scraper:assemble --console=plain
+      - name: Building cache client
+        run: ./gradlew :cache-client:build --console=plain
+      - name: Building scraper
+        run: ./gradlew :wiki-scraper:build --console=plain
+      - name: Downloading jagex cache
+        run: ./gradlew :cache-client:update --console=plain
+      - name: Scraping NPC stats
+        run: ./gradlew :wiki-scraper:npcStatsScrape --console=plain
+      - name: Create NPC stats Pull Request
+        uses: peter-evans/create-pull-request@v1.2.0
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PULL_REQUEST_BRANCH: NPC-UPDATE
+          PULL_REQUEST_TITLE: 'Client: Update NPC stats'
+          PULL_REQUEST_BODY: This is an auto-generated PR with changes from the OSRS wiki
+          COMMIT_MESSAGE: 'Client: Update NPC stats'
+      - name: Scraping item stats
+        run: ./gradlew :wiki-scraper:npcStatsScrape --console=plain
+      - name: Create item stats Pull Request
+        uses: peter-evans/create-pull-request@v1.2.0
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PULL_REQUEST_BRANCH: ITEM-STATS-UPDATE
+          PULL_REQUEST_TITLE: 'Client: Update item stats'
+          PULL_REQUEST_BODY: This is an auto-generated PR with changes from the OSRS wiki
+          COMMIT_MESSAGE: 'Client: Update item stats'

.github/workflows/test.yml

@@ -0,0 +1,19 @@
+name: RunelitePlus - Testing
+
+on: [pull_request, push]
+
+jobs:
+  test:
+
+    runs-on: windows-latest
+
+    steps:
+      - uses: actions/checkout@v1
+      - name: Set up JDK 11
+        uses: actions/setup-java@v1
+        with:
+          java-version: 11
+      - name: Assembling
+        run: gradlew assemble --console=plain
+      - name: Testing
+        run: gradlew test --stacktrace --console=plain

cache-client/build.gradle

@@ -12,3 +12,17 @@ dependencies {
     testImplementation group: 'org.slf4j', name: 'slf4j-simple', version: slf4j
     testImplementation project(path: ':cache', configuration: 'testArchives')
 }
+
+task update {
+    dependsOn ":cache-client:build"
+
+    doLast {
+        def path = sourceSets.main.runtimeClasspath
+        def loader = new URLClassLoader(path.collect { f -> f.toURI().toURL() } as URL[])
+        def cacheClient = loader.loadClass('net.runelite.cache.client.CacheClient')
+
+        cacheClient.getCache(rsversion);
+
+        loader.close()
+    }
+}

cache-client/src/main/java/net/runelite/cache/client/CacheClient.java

@@ -37,7 +37,10 @@ import io.netty.channel.EventLoopGroup;
 import io.netty.channel.nio.NioEventLoopGroup;
 import io.netty.channel.socket.SocketChannel;
 import io.netty.channel.socket.nio.NioSocketChannel;
+import java.io.File;
 import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.List;
@@ -82,6 +85,30 @@ public class CacheClient implements AutoCloseable
 	private CompletableFuture<HandshakeResponseType> handshakeFuture;
 	private final Queue<PendingFileRequest> requests = new ArrayDeque<>();
 
+	public static void getCache(int clientRevision)
+	{
+		Path path = Paths.get(System.getProperty("user.home"), "jagexcache" + File.separator + "oldschool" + File.separator + "LIVE");
+		final File jagexcache = new File(String.valueOf(path));
+
+		jagexcache.mkdirs();
+
+		try (Store store = new Store(jagexcache))
+		{
+			store.load();
+			CacheClient c = new CacheClient(store, clientRevision);
+			c.connect();
+			CompletableFuture<HandshakeResponseType> handshake = c.handshake();
+			handshake.get();
+			c.download();
+			c.close();
+			store.save();
+		}
+		catch (Exception ex)
+		{
+			logger.error(ex.getMessage());
+		}
+	}
+
 	public CacheClient(Store store, int clientRevision)
 	{
 		this(store, HOST, clientRevision);

wiki-scraper/build.gradle

@@ -23,12 +23,34 @@ dependencies {
     testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-api', version: jupiter
 }
 
-task scrape {
+task npcStatsScrape {
     doLast {
         def path = sourceSets.main.runtimeClasspath
         def loader = new URLClassLoader(path.collect { f -> f.toURI().toURL() } as URL[])
         def scrape = loader.loadClass('net.runelite.data.App')
-        scrape.main();
+        scrape.npcStats(rootProject.file("./runelite-client/src/main/resources/"));
+
+        loader.close()
+    }
+}
+
+task itemStatsScrape {
+    doLast {
+        def path = sourceSets.main.runtimeClasspath
+        def loader = new URLClassLoader(path.collect { f -> f.toURI().toURL() } as URL[])
+        def scrape = loader.loadClass('net.runelite.data.App')
+        scrape.itemStats(rootProject.file("./runelite-client/src/main/resources/"));
+
+        loader.close()
+    }
+}
+
+task itemLimitsScrape {
+    doLast {
+        def path = sourceSets.main.runtimeClasspath
+        def loader = new URLClassLoader(path.collect { f -> f.toURI().toURL() } as URL[])
+        def scrape = loader.loadClass('net.runelite.data.App')
+        scrape.itemLimits(rootProject.file("./runelite-client/src/main/resources/net/runelite/client/plugins/grandexchange/"));
 
         loader.close()
     }

wiki-scraper/src/main/java/net/runelite/data/App.java

@@ -32,6 +32,8 @@ import java.nio.file.Path;
 import java.nio.file.Paths;
 import net.runelite.cache.fs.Store;
 import net.runelite.data.dump.MediaWiki;
+import net.runelite.data.dump.wiki.ItemLimitsDumper;
+import net.runelite.data.dump.wiki.ItemStatsDumper;
 import net.runelite.data.dump.wiki.NpcStatsDumper;
 
 public class App
@@ -41,14 +43,16 @@ public class App
 		.disableHtmlEscaping()
 		.create();
 
-	public static void main(String[] args) throws IOException
+	private final static MediaWiki wiki = new MediaWiki("https://oldschool.runescape.wiki");
+
+	public static Store cacheStore() throws IOException
 	{
 		Path path = Paths.get(System.getProperty("user.home"), "jagexcache" + File.separator + "oldschool" + File.separator + "LIVE");
 		final File jagexcache = new File(String.valueOf(path));
 
 		if (!Files.exists(path))
 		{
-			return;
+			return null;
 		}
 
 		final Store cacheStore = new Store(jagexcache);
@@ -58,12 +62,21 @@ public class App
 		// Try to make this go faster (probably not very smart)
 		System.setProperty("java.util.concurrent.ForkJoinPool.common.parallelism", "100");
 
-		final MediaWiki wiki = new MediaWiki("https://oldschool.runescape.wiki");
+		return cacheStore;
+	}
 
-		// Only use this to diff current limits with scraped limits
-		// ItemLimitsDumper.dump(cacheStore, wiki);
-		// ItemStatsDumper.dump(cacheStore, wiki);
+	public static void npcStats(File path) throws IOException
+	{
+		NpcStatsDumper.dump(cacheStore(), wiki, path);
+	}
 
-		NpcStatsDumper.dump(cacheStore, wiki);
+	public static void itemStats(File path) throws IOException
+	{
+		ItemStatsDumper.dump(cacheStore(), wiki, path);
+	}
+
+	public static void itemLimits(File path) throws IOException
+	{
+		ItemLimitsDumper.dump(cacheStore(), wiki, path);
 	}
 }

wiki-scraper/src/main/java/net/runelite/data/dump/wiki/ItemLimitsDumper.java

@@ -47,11 +47,9 @@ import net.runelite.data.dump.MediaWikiTemplate;
 @Slf4j
 public class ItemLimitsDumper
 {
-	public static void dump(final Store store, final MediaWiki wiki) throws IOException
+	public static void dump(final Store store, final MediaWiki wiki, final File path) throws IOException
 	{
-		final File out = new File("../runelite-client/src/main/resources/net/runelite/client/plugins/grandexchange/");
-
-		log.info("Dumping item limits to {}", out);
+		log.info("Dumping item limits to {}", path);
 
 		final ItemManager itemManager = new ItemManager(store);
 		itemManager.load();
@@ -140,7 +138,7 @@ public class ItemLimitsDumper
 			log.debug("Dumped item limit for {} {}", item.id, name);
 		});
 
-		try (FileWriter fw = new FileWriter(new File(out, "ge_limits.json")))
+		try (FileWriter fw = new FileWriter(new File(path, "ge_limits.json")))
 		{
 			fw.write(App.GSON.toJson(limits));
 		}

wiki-scraper/src/main/java/net/runelite/data/dump/wiki/ItemStatsDumper.java

@@ -48,11 +48,9 @@ public class ItemStatsDumper
 {
 	private final static Integer MAX_ITEMS_ON_PAGE = 50;
 
-	public static void dump(final Store store, final MediaWiki wiki) throws IOException
+	public static void dump(final Store store, final MediaWiki wiki, final File path) throws IOException
 	{
-		final File out = new File("../runelite-client/src/main/resources/");
-
-		log.info("Dumping item stats to {}", out);
+		log.info("Dumping item stats to {}", path);
 
 		final ItemManager itemManager = new ItemManager(store);
 		itemManager.load();
@@ -182,7 +180,7 @@ public class ItemStatsDumper
 			log.debug("Dumped item stat for {} {}", item.id, name);
 		});
 
-		try (FileWriter fw = new FileWriter(new File(out, "item_stats.json")))
+		try (FileWriter fw = new FileWriter(new File(path, "item_stats.json")))
 		{
 			fw.write(App.GSON.toJson(itemStats));
 		}

wiki-scraper/src/main/java/net/runelite/data/dump/wiki/NpcStatsDumper.java

@@ -123,11 +123,10 @@ public class NpcStatsDumper
 		return templates;
 	}
 
-	public static void dump(final Store store, final MediaWiki wiki) throws IOException
+	public static void dump(final Store store, final MediaWiki wiki, final File path) throws IOException
 	{
-		final File out = new File("../runelite-client/src/main/resources/");
 
-		log.info("Dumping npc stats to {}", out);
+		log.info("Dumping npc stats to {}", path);
 
 		final NpcManager npcManager = new NpcManager(store);
 		npcManager.load();
@@ -246,7 +245,7 @@ public class NpcStatsDumper
 		// Cast to TreeMap so sort output JSON in numerical order (npc id)
 		final Map<Integer, NpcStats> sorted = new TreeMap<>(npcStats);
 
-		try (FileWriter fw = new FileWriter(new File(out, "npc_stats.json")))
+		try (FileWriter fw = new FileWriter(new File(path, "npc_stats.json")))
 		{
 			fw.write(App.GSON.toJson(sorted));
 		}