diff --git a/.github/workflows/auto-approve.yml b/.github/workflows/auto-approve.yml new file mode 100644 index 0000000000..b85578e594 --- /dev/null +++ b/.github/workflows/auto-approve.yml @@ -0,0 +1,14 @@ +name: RunelitePlus - Auto approve + +on: pull_request + +jobs: + approve: + + runs-on: ubuntu-latest + + steps: + - uses: hmarr/auto-approve-action@v2.0.0 + if: github.actor == 'github-actions[bot]' + with: + github-token: "${{ secrets.GITHUB_TOKEN }}" \ No newline at end of file diff --git a/.github/workflows/gradle.yml b/.github/workflows/build.yml similarity index 62% rename from .github/workflows/gradle.yml rename to .github/workflows/build.yml index fc6618218c..de841f14b6 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/build.yml @@ -1,4 +1,4 @@ -name: RunelitePlus continuous integration +name: RunelitePlus - CI on: [pull_request, push] @@ -16,8 +16,4 @@ jobs: - name: Assembling run: gradlew assemble --console=plain - name: Building - run: gradlew build --stacktrace -x test -x checkstyleMain --console=plain - - name: Testing - run: gradlew test --stacktrace --console=plain - - name: Checking code conventions - run: gradlew checkstyleMain --console=plain \ No newline at end of file + run: gradlew build --stacktrace -x test -x checkstyleMain --console=plain \ No newline at end of file diff --git a/.github/workflows/checkstyle.yml b/.github/workflows/checkstyle.yml new file mode 100644 index 0000000000..47663761fc --- /dev/null +++ b/.github/workflows/checkstyle.yml @@ -0,0 +1,19 @@ +name: RunelitePlus - Code convention + +on: [pull_request, push] + +jobs: + checkstyle: + + runs-on: windows-latest + + steps: + - uses: actions/checkout@v1 + - name: Set up JDK 11 + uses: actions/setup-java@v1 + with: + java-version: 11 + - name: Assembling + run: gradlew assemble --console=plain + - name: Checking code conventions + run: gradlew checkstyleMain --console=plain \ No newline at end of file diff --git a/.github/workflows/schedule.yml b/.github/workflows/schedule.yml new file mode 100644 index 0000000000..963fbdb2b8 --- /dev/null +++ b/.github/workflows/schedule.yml @@ -0,0 +1,17 @@ +name: RunelitePlus - Gradle Wrapper + +on: + schedule: + - cron: 0 0 * * * + +jobs: + update: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@master + - name: Update Gradle Wrapper + uses: rahulsom/gradle-up@master + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/scraper.yml b/.github/workflows/scraper.yml new file mode 100644 index 0000000000..fd7fa73285 --- /dev/null +++ b/.github/workflows/scraper.yml @@ -0,0 +1,47 @@ +name: RunelitePlus - Scraper + +on: + schedule: + - cron: "0 0 * * 2,5" + +jobs: + scrape: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v1 + - name: Set up JDK 11 + uses: actions/setup-java@v1 + with: + java-version: 11 + - name: Assembling cache client + run: ./gradlew :cache-client:assemble --console=plain + - name: Assembling scraper + run: ./gradlew :wiki-scraper:assemble --console=plain + - name: Building cache client + run: ./gradlew :cache-client:build --console=plain + - name: Building scraper + run: ./gradlew :wiki-scraper:build --console=plain + - name: Downloading jagex cache + run: ./gradlew :cache-client:update --console=plain + - name: Scraping NPC stats + run: ./gradlew :wiki-scraper:npcStatsScrape --console=plain + - name: Create NPC stats Pull Request + uses: peter-evans/create-pull-request@v1.2.0 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PULL_REQUEST_BRANCH: NPC-UPDATE + PULL_REQUEST_TITLE: 'Client: Update NPC stats' + PULL_REQUEST_BODY: This is an auto-generated PR with changes from the OSRS wiki + COMMIT_MESSAGE: 'Client: Update NPC stats' + - name: Scraping item stats + run: ./gradlew :wiki-scraper:npcStatsScrape --console=plain + - name: Create item stats Pull Request + uses: peter-evans/create-pull-request@v1.2.0 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PULL_REQUEST_BRANCH: ITEM-STATS-UPDATE + PULL_REQUEST_TITLE: 'Client: Update item stats' + PULL_REQUEST_BODY: This is an auto-generated PR with changes from the OSRS wiki + COMMIT_MESSAGE: 'Client: Update item stats' \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000000..f6a8f9ddaf --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,19 @@ +name: RunelitePlus - Testing + +on: [pull_request, push] + +jobs: + test: + + runs-on: windows-latest + + steps: + - uses: actions/checkout@v1 + - name: Set up JDK 11 + uses: actions/setup-java@v1 + with: + java-version: 11 + - name: Assembling + run: gradlew assemble --console=plain + - name: Testing + run: gradlew test --stacktrace --console=plain \ No newline at end of file diff --git a/cache-client/build.gradle b/cache-client/build.gradle index 024ee40b90..feed79e98e 100644 --- a/cache-client/build.gradle +++ b/cache-client/build.gradle @@ -12,3 +12,17 @@ dependencies { testImplementation group: 'org.slf4j', name: 'slf4j-simple', version: slf4j testImplementation project(path: ':cache', configuration: 'testArchives') } + +task update { + dependsOn ":cache-client:build" + + doLast { + def path = sourceSets.main.runtimeClasspath + def loader = new URLClassLoader(path.collect { f -> f.toURI().toURL() } as URL[]) + def cacheClient = loader.loadClass('net.runelite.cache.client.CacheClient') + + cacheClient.getCache(rsversion); + + loader.close() + } +} diff --git a/cache-client/src/main/java/net/runelite/cache/client/CacheClient.java b/cache-client/src/main/java/net/runelite/cache/client/CacheClient.java index 273da5d5fa..a8afe8b82d 100644 --- a/cache-client/src/main/java/net/runelite/cache/client/CacheClient.java +++ b/cache-client/src/main/java/net/runelite/cache/client/CacheClient.java @@ -37,7 +37,10 @@ import io.netty.channel.EventLoopGroup; import io.netty.channel.nio.NioEventLoopGroup; import io.netty.channel.socket.SocketChannel; import io.netty.channel.socket.nio.NioSocketChannel; +import java.io.File; import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayDeque; import java.util.ArrayList; import java.util.List; @@ -82,6 +85,30 @@ public class CacheClient implements AutoCloseable private CompletableFuture handshakeFuture; private final Queue requests = new ArrayDeque<>(); + public static void getCache(int clientRevision) + { + Path path = Paths.get(System.getProperty("user.home"), "jagexcache" + File.separator + "oldschool" + File.separator + "LIVE"); + final File jagexcache = new File(String.valueOf(path)); + + jagexcache.mkdirs(); + + try (Store store = new Store(jagexcache)) + { + store.load(); + CacheClient c = new CacheClient(store, clientRevision); + c.connect(); + CompletableFuture handshake = c.handshake(); + handshake.get(); + c.download(); + c.close(); + store.save(); + } + catch (Exception ex) + { + logger.error(ex.getMessage()); + } + } + public CacheClient(Store store, int clientRevision) { this(store, HOST, clientRevision); diff --git a/wiki-scraper/build.gradle b/wiki-scraper/build.gradle index ee2a723443..e34e73e741 100644 --- a/wiki-scraper/build.gradle +++ b/wiki-scraper/build.gradle @@ -23,12 +23,34 @@ dependencies { testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-api', version: jupiter } -task scrape { +task npcStatsScrape { doLast { def path = sourceSets.main.runtimeClasspath def loader = new URLClassLoader(path.collect { f -> f.toURI().toURL() } as URL[]) def scrape = loader.loadClass('net.runelite.data.App') - scrape.main(); + scrape.npcStats(rootProject.file("./runelite-client/src/main/resources/")); + + loader.close() + } +} + +task itemStatsScrape { + doLast { + def path = sourceSets.main.runtimeClasspath + def loader = new URLClassLoader(path.collect { f -> f.toURI().toURL() } as URL[]) + def scrape = loader.loadClass('net.runelite.data.App') + scrape.itemStats(rootProject.file("./runelite-client/src/main/resources/")); + + loader.close() + } +} + +task itemLimitsScrape { + doLast { + def path = sourceSets.main.runtimeClasspath + def loader = new URLClassLoader(path.collect { f -> f.toURI().toURL() } as URL[]) + def scrape = loader.loadClass('net.runelite.data.App') + scrape.itemLimits(rootProject.file("./runelite-client/src/main/resources/net/runelite/client/plugins/grandexchange/")); loader.close() } diff --git a/wiki-scraper/src/main/java/net/runelite/data/App.java b/wiki-scraper/src/main/java/net/runelite/data/App.java index 523fa9b6e2..981a4c7d6b 100644 --- a/wiki-scraper/src/main/java/net/runelite/data/App.java +++ b/wiki-scraper/src/main/java/net/runelite/data/App.java @@ -32,6 +32,8 @@ import java.nio.file.Path; import java.nio.file.Paths; import net.runelite.cache.fs.Store; import net.runelite.data.dump.MediaWiki; +import net.runelite.data.dump.wiki.ItemLimitsDumper; +import net.runelite.data.dump.wiki.ItemStatsDumper; import net.runelite.data.dump.wiki.NpcStatsDumper; public class App @@ -41,14 +43,16 @@ public class App .disableHtmlEscaping() .create(); - public static void main(String[] args) throws IOException + private final static MediaWiki wiki = new MediaWiki("https://oldschool.runescape.wiki"); + + public static Store cacheStore() throws IOException { Path path = Paths.get(System.getProperty("user.home"), "jagexcache" + File.separator + "oldschool" + File.separator + "LIVE"); final File jagexcache = new File(String.valueOf(path)); if (!Files.exists(path)) { - return; + return null; } final Store cacheStore = new Store(jagexcache); @@ -58,12 +62,21 @@ public class App // Try to make this go faster (probably not very smart) System.setProperty("java.util.concurrent.ForkJoinPool.common.parallelism", "100"); - final MediaWiki wiki = new MediaWiki("https://oldschool.runescape.wiki"); + return cacheStore; + } - // Only use this to diff current limits with scraped limits - // ItemLimitsDumper.dump(cacheStore, wiki); - // ItemStatsDumper.dump(cacheStore, wiki); + public static void npcStats(File path) throws IOException + { + NpcStatsDumper.dump(cacheStore(), wiki, path); + } - NpcStatsDumper.dump(cacheStore, wiki); + public static void itemStats(File path) throws IOException + { + ItemStatsDumper.dump(cacheStore(), wiki, path); + } + + public static void itemLimits(File path) throws IOException + { + ItemLimitsDumper.dump(cacheStore(), wiki, path); } } \ No newline at end of file diff --git a/wiki-scraper/src/main/java/net/runelite/data/dump/wiki/ItemLimitsDumper.java b/wiki-scraper/src/main/java/net/runelite/data/dump/wiki/ItemLimitsDumper.java index 990bac90cb..b178af6aca 100644 --- a/wiki-scraper/src/main/java/net/runelite/data/dump/wiki/ItemLimitsDumper.java +++ b/wiki-scraper/src/main/java/net/runelite/data/dump/wiki/ItemLimitsDumper.java @@ -47,11 +47,9 @@ import net.runelite.data.dump.MediaWikiTemplate; @Slf4j public class ItemLimitsDumper { - public static void dump(final Store store, final MediaWiki wiki) throws IOException + public static void dump(final Store store, final MediaWiki wiki, final File path) throws IOException { - final File out = new File("../runelite-client/src/main/resources/net/runelite/client/plugins/grandexchange/"); - - log.info("Dumping item limits to {}", out); + log.info("Dumping item limits to {}", path); final ItemManager itemManager = new ItemManager(store); itemManager.load(); @@ -140,7 +138,7 @@ public class ItemLimitsDumper log.debug("Dumped item limit for {} {}", item.id, name); }); - try (FileWriter fw = new FileWriter(new File(out, "ge_limits.json"))) + try (FileWriter fw = new FileWriter(new File(path, "ge_limits.json"))) { fw.write(App.GSON.toJson(limits)); } diff --git a/wiki-scraper/src/main/java/net/runelite/data/dump/wiki/ItemStatsDumper.java b/wiki-scraper/src/main/java/net/runelite/data/dump/wiki/ItemStatsDumper.java index dca8b5d85e..51b9580d6d 100644 --- a/wiki-scraper/src/main/java/net/runelite/data/dump/wiki/ItemStatsDumper.java +++ b/wiki-scraper/src/main/java/net/runelite/data/dump/wiki/ItemStatsDumper.java @@ -48,11 +48,9 @@ public class ItemStatsDumper { private final static Integer MAX_ITEMS_ON_PAGE = 50; - public static void dump(final Store store, final MediaWiki wiki) throws IOException + public static void dump(final Store store, final MediaWiki wiki, final File path) throws IOException { - final File out = new File("../runelite-client/src/main/resources/"); - - log.info("Dumping item stats to {}", out); + log.info("Dumping item stats to {}", path); final ItemManager itemManager = new ItemManager(store); itemManager.load(); @@ -182,7 +180,7 @@ public class ItemStatsDumper log.debug("Dumped item stat for {} {}", item.id, name); }); - try (FileWriter fw = new FileWriter(new File(out, "item_stats.json"))) + try (FileWriter fw = new FileWriter(new File(path, "item_stats.json"))) { fw.write(App.GSON.toJson(itemStats)); } diff --git a/wiki-scraper/src/main/java/net/runelite/data/dump/wiki/NpcStatsDumper.java b/wiki-scraper/src/main/java/net/runelite/data/dump/wiki/NpcStatsDumper.java index 1078ed462e..746e41542b 100644 --- a/wiki-scraper/src/main/java/net/runelite/data/dump/wiki/NpcStatsDumper.java +++ b/wiki-scraper/src/main/java/net/runelite/data/dump/wiki/NpcStatsDumper.java @@ -123,11 +123,10 @@ public class NpcStatsDumper return templates; } - public static void dump(final Store store, final MediaWiki wiki) throws IOException + public static void dump(final Store store, final MediaWiki wiki, final File path) throws IOException { - final File out = new File("../runelite-client/src/main/resources/"); - log.info("Dumping npc stats to {}", out); + log.info("Dumping npc stats to {}", path); final NpcManager npcManager = new NpcManager(store); npcManager.load(); @@ -246,7 +245,7 @@ public class NpcStatsDumper // Cast to TreeMap so sort output JSON in numerical order (npc id) final Map sorted = new TreeMap<>(npcStats); - try (FileWriter fw = new FileWriter(new File(out, "npc_stats.json"))) + try (FileWriter fw = new FileWriter(new File(path, "npc_stats.json"))) { fw.write(App.GSON.toJson(sorted)); }