From 8239be4b75c4deac97fed6b31c10eea9ec2b9117 Mon Sep 17 00:00:00 2001 From: Owain van Brakel Date: Thu, 15 Aug 2019 23:10:25 +0200 Subject: [PATCH] project: Add wiki scraper (#1348) * wiki-scraper: Add wiki scroper to the main project * client: Updated wiki stats * wiki-scraper: Checkstyle * wiki-scraper: Pull in @Ganom his changes * client: Updated wiki stats --- build.gradle | 2 + .../src/main/resources/item_stats.json | 697 +++++++++++++++++- .../plugins/grandexchange/ge_limits.json | 37 +- .../src/main/resources/npc_stats.json | 359 ++++++++- settings.gradle | 5 +- wiki-scraper/build.gradle | 28 + .../src/main/java/net/runelite/data/App.java | 59 ++ .../net/runelite/data/dump/MediaWiki.java | 140 ++++ .../runelite/data/dump/MediaWikiTemplate.java | 302 ++++++++ .../data/dump/wiki/ItemLimitsDumper.java | 155 ++++ .../data/dump/wiki/ItemStatsDumper.java | 359 +++++++++ .../data/dump/wiki/NpcStatsDumper.java | 393 ++++++++++ .../data/dump/MediaWikiTemplateTest.java | 549 ++++++++++++++ .../data/dump/wiki/NpcStatsDumperTest.java | 129 ++++ 14 files changed, 3136 insertions(+), 78 deletions(-) create mode 100644 wiki-scraper/build.gradle create mode 100644 wiki-scraper/src/main/java/net/runelite/data/App.java create mode 100644 wiki-scraper/src/main/java/net/runelite/data/dump/MediaWiki.java create mode 100644 wiki-scraper/src/main/java/net/runelite/data/dump/MediaWikiTemplate.java create mode 100644 wiki-scraper/src/main/java/net/runelite/data/dump/wiki/ItemLimitsDumper.java create mode 100644 wiki-scraper/src/main/java/net/runelite/data/dump/wiki/ItemStatsDumper.java create mode 100644 wiki-scraper/src/main/java/net/runelite/data/dump/wiki/NpcStatsDumper.java create mode 100644 wiki-scraper/src/test/java/net/runelite/data/dump/MediaWikiTemplateTest.java create mode 100644 wiki-scraper/src/test/java/net/runelite/data/dump/wiki/NpcStatsDumperTest.java diff --git a/build.gradle b/build.gradle index fb02f033d7..e99b050a9a 100644 --- a/build.gradle +++ b/build.gradle @@ -51,6 +51,7 @@ ext { jogamp = '2.3.2' jopt = '5.0.4' junit = '4.12' + jupiter = '5.5.1' logback = '1.2.3' lombok = '1.18.8' mapstruct = '1.3.0.Final' @@ -64,6 +65,7 @@ ext { netty = '4.1.37.Final' okhttp3 = '4.1.0' orangeExtensions = '1.0' + petitparser = '2.2.0' plexus = '3.2.1' rxjava = '2.2.11' rxrelay = '2.1.0' diff --git a/runelite-client/src/main/resources/item_stats.json b/runelite-client/src/main/resources/item_stats.json index 1d00ea3a24..d18109fa50 100644 --- a/runelite-client/src/main/resources/item_stats.json +++ b/runelite-client/src/main/resources/item_stats.json @@ -820,7 +820,7 @@ "weight": 0.51 }, "293": { - "name": "A key", + "name": "Key", "quest": true, "weight": 0.01 }, @@ -1022,6 +1022,9 @@ "name": "Lobster", "weight": 0.35 }, + "381": { + "name": "Burnt lobster" + }, "383": { "name": "Raw shark", "weight": 0.7 @@ -1500,6 +1503,9 @@ "561": { "name": "Nature rune" }, + "562": { + "name": "Chaos rune" + }, "563": { "name": "Law rune" }, @@ -3569,6 +3575,14 @@ "slot": 0 } }, + "1038": { + "name": "Red partyhat", + "equipable": true, + "weight": 0.056, + "equipment": { + "slot": 0 + } + }, "1040": { "name": "Yellow partyhat", "equipable": true, @@ -6832,6 +6846,10 @@ "quest": true, "weight": 0.085 }, + "1550": { + "name": "Garlic", + "weight": 0.028 + }, "1552": { "name": "Seasoned sardine", "quest": true, @@ -7820,6 +7838,10 @@ "name": "Chopped tomato", "weight": 0.2 }, + "1871": { + "name": "Chopped onion", + "weight": 0.2 + }, "1873": { "name": "Chopped ugthanki", "weight": 0.2 @@ -8596,6 +8618,11 @@ "quest": true, "weight": 0.1 }, + "2340": { + "name": "Palm leaf", + "quest": true, + "weight": 0.1 + }, "2341": { "name": "Wrapped oomlie", "quest": true, @@ -10349,6 +10376,9 @@ "2709": { "name": "Clue scroll (easy)" }, + "2710": { + "name": "Clue scroll (easy)" + }, "2711": { "name": "Clue scroll (easy)" }, @@ -14998,8 +15028,7 @@ "weight": 0.014 }, "4207": { - "name": "Crystal seed", - "quest": true, + "name": "Crystal weapon seed", "weight": 0.014 }, "4209": { @@ -16873,6 +16902,11 @@ "quest": true, "weight": 0.03 }, + "4595": { + "name": "Karidian disguise", + "quest": true, + "weight": 0.1 + }, "4597": { "name": "Note", "quest": true, @@ -17252,7 +17286,7 @@ "4708": { "name": "Ahrim's hood", "equipable": true, - "weight": 0.9, + "weight": 0.907, "equipment": { "slot": 0, "amagic": 6, @@ -17494,6 +17528,21 @@ "rstr": 55 } }, + "4745": { + "name": "Torag's helm", + "equipable": true, + "weight": 2.721, + "equipment": { + "slot": 0, + "amagic": -6, + "arange": -2, + "dstab": 55, + "dslash": 58, + "dcrush": 54, + "dmagic": -1, + "drange": 62 + } + }, "4747": { "name": "Torag's hammers", "equipable": true, @@ -17645,6 +17694,14 @@ "rstr": 45 } }, + "4803": { + "name": "Rune brutal", + "equipable": true, + "equipment": { + "slot": 13, + "rstr": 60 + } + }, "4808": { "name": "Black prism", "quest": true, @@ -17718,7 +17775,7 @@ "4827": { "name": "Comp ogre bow", "equipable": true, - "weight": 1.8, + "weight": 1.814, "equipment": { "slot": 3, "arange": 38, @@ -18421,6 +18478,21 @@ "drange": 62 } }, + "4905": { + "name": "Guthan's helm 75", + "equipable": true, + "weight": 2.721, + "equipment": { + "slot": 0, + "amagic": -6, + "arange": -2, + "dstab": 55, + "dslash": 58, + "dcrush": 54, + "dmagic": -1, + "drange": 62 + } + }, "4906": { "name": "Guthan's helm 50", "equipable": true, @@ -24609,7 +24681,7 @@ "weight": 0.005 }, "6547": { - "name": "Doctors hat", + "name": "Doctor's hat", "quest": true, "equipable": true, "weight": 0.3, @@ -24759,6 +24831,23 @@ "prayer": 5 } }, + "6587": { + "name": "White claws", + "equipable": true, + "weight": 0.907, + "equipment": { + "slot": 3, + "astab": 10, + "aslash": 14, + "acrush": -4, + "dstab": 4, + "dslash": 7, + "dcrush": 2, + "str": 14, + "prayer": 1, + "aspeed": 4 + } + }, "6589": { "name": "White battleaxe", "equipable": true, @@ -25845,6 +25934,10 @@ "quest": true, "weight": 0.005 }, + "6794": { + "name": "Choc-ice", + "weight": 0.001 + }, "6796": { "name": "Lamp", "quest": true, @@ -30781,11 +30874,11 @@ "weight": 1.0 }, "8544": { - "name": "Mahogany eagle", + "name": "Mahogany eagle lectern", "weight": 1.0 }, "8546": { - "name": "Mahogany demon", + "name": "Mahogany demon lectern", "weight": 1.0 }, "8548": { @@ -34342,6 +34435,10 @@ "name": "A green square", "quest": true }, + "9612": { + "name": "A green pentagon", + "quest": true + }, "9613": { "name": "A blue circle", "quest": true @@ -40157,6 +40254,11 @@ "quest": true, "weight": 0.004 }, + "11156": { + "name": "Astral rune shards", + "quest": true, + "weight": 0.453 + }, "11157": { "name": "Dreamy lamp", "quest": true, @@ -40414,6 +40516,16 @@ "aspeed": 3 } }, + "11231": { + "name": "Dragon dart(p)", + "equipable": true, + "equipment": { + "slot": 3, + "arange": 18, + "rstr": 20, + "aspeed": 3 + } + }, "11232": { "name": "Dragon dart tip" }, @@ -43538,6 +43650,9 @@ "12087": { "name": "Clue scroll (elite)" }, + "12088": { + "name": "Clue scroll (elite)" + }, "12089": { "name": "Clue scroll (elite)" }, @@ -46517,7 +46632,7 @@ "12797": { "name": "Dragon pickaxe", "equipable": true, - "weight": 2.2, + "weight": 2.4, "equipment": { "slot": 3, "astab": 38, @@ -47412,6 +47527,10 @@ "name": "Ancient rune armour set (lg)", "weight": 6.0 }, + "13062": { + "name": "Ancient rune armour set (sk)", + "weight": 6.0 + }, "13064": { "name": "Combat potion set" }, @@ -48838,6 +48957,9 @@ "13252": { "name": "Sack pack" }, + "13254": { + "name": "Basket pack" + }, "13256": { "name": "Saradomin's light" }, @@ -51891,6 +52013,10 @@ "19835": { "name": "Clue scroll (master)" }, + "19836": { + "name": "Reward casket (master)", + "weight": 5.0 + }, "19837": { "name": "Torn clue scroll (part 1)", "equipable": true @@ -52638,6 +52764,13 @@ "slot": 7 } }, + "20107": { + "name": "Ankou socks", + "equipable": true, + "equipment": { + "slot": 10 + } + }, "20110": { "name": "Bowl wig", "equipable": true, @@ -55626,6 +55759,9 @@ "20884": { "name": "Keystone crystal" }, + "20885": { + "name": "Cavern grubs" + }, "20886": { "name": "Creature keeper's journal" }, @@ -55644,6 +55780,9 @@ "20895": { "name": "Vanguard judgement" }, + "20897": { + "name": "Houndmaster's diary" + }, "20899": { "name": "Dark journal" }, @@ -55710,6 +55849,9 @@ "20921": { "name": "Elder (+)(1)" }, + "20922": { + "name": "Elder (+)(2)" + }, "20923": { "name": "Elder (+)(3)" }, @@ -55990,7 +56132,7 @@ "21009": { "name": "Dragon sword", "equipable": true, - "weight": 1.0, + "weight": 1.75, "equipment": { "slot": 3, "astab": 65, @@ -56575,7 +56717,7 @@ "21206": { "name": "Dragon sword", "equipable": true, - "weight": 1.0, + "weight": 1.75, "equipment": { "slot": 3, "astab": 65, @@ -60816,6 +60958,21 @@ "drange": 62 } }, + "22644": { + "name": "Morrigan's leather chaps", + "equipable": true, + "weight": 5.4, + "equipment": { + "slot": 7, + "amagic": -10, + "arange": 23, + "dstab": 35, + "dslash": 29, + "dcrush": 37, + "dmagic": 46, + "drange": 35 + } + }, "22647": { "name": "Zuriel's staff", "equipable": true, @@ -63308,7 +63465,7 @@ "23677": { "name": "Dragon pickaxe(or)", "equipable": true, - "weight": 2.2, + "weight": 2.4, "equipment": { "slot": 3, "astab": 38, @@ -63347,6 +63504,9 @@ "aspeed": 5 } }, + "23685": { + "name": "Divine super combat potion(4)" + }, "23688": { "name": "Divine super combat potion(3)" }, @@ -63448,7 +63608,8 @@ "name": "Prifddinas teleport" }, "23773": { - "name": "Scrawled notes" + "name": "Scrawled notes", + "quest": true }, "23775": { "name": "Hand mirror", @@ -63486,7 +63647,12 @@ "weight": 0.015 }, "23782": { - "name": "Black crystal" + "name": "Black crystal", + "quest": true + }, + "23783": { + "name": "Green crystal", + "quest": true }, "23784": { "name": "Fractured crystal", @@ -63584,6 +63750,11 @@ "quest": true, "weight": 0.015 }, + "23804": { + "name": "Crystal dust", + "quest": true, + "weight": 0.007 + }, "23806": { "name": "Inversion potion", "quest": true, @@ -63621,7 +63792,17 @@ "quest": true }, "23820": { - "name": "Corrupted sceptre" + "name": "Corrupted sceptre", + "equipable": true, + "weight": 2.0, + "equipment": { + "slot": 3, + "astab": 8, + "aslash": 10, + "acrush": 16, + "str": 20, + "aspeed": 5 + } }, "23821": { "name": "Corrupted axe" @@ -63659,8 +63840,8 @@ "23837": { "name": "Corrupted ore" }, - "23838": { - "name": "Phren bark" + "23858": { + "name": "Corrupted teleport crystal" }, "23859": { "name": "Gauntlet cape", @@ -63669,6 +63850,19 @@ "slot": 1 } }, + "23861": { + "name": "Crystal sceptre", + "equipable": true, + "weight": 2.0, + "equipment": { + "slot": 3, + "astab": 8, + "aslash": 10, + "acrush": 16, + "str": 20, + "aspeed": 5 + } + }, "23862": { "name": "Crystal axe", "equipable": true, @@ -63682,6 +63876,9 @@ "aspeed": 5 } }, + "23866": { + "name": "Crystal shards" + }, "23868": { "name": "Crystal spike" }, @@ -63697,23 +63894,318 @@ "23874": { "name": "Paddlefish" }, + "23877": { + "name": "Crystal ore" + }, "23878": { "name": "Phren bark" }, + "23881": { + "name": "Grym potion (unf)" + }, + "23882": { + "name": "Egniol potion (1)" + }, + "23883": { + "name": "Egniol potion (2)" + }, + "23884": { + "name": "Egniol potion (3)" + }, + "23885": { + "name": "Egniol potion (4)" + }, + "23886": { + "name": "Crystal helm (basic)", + "equipable": true, + "weight": 2.0, + "equipment": { + "slot": 0, + "dstab": 28, + "dslash": 28, + "dcrush": 28, + "dmagic": 28, + "drange": 28, + "prayer": 1 + } + }, + "23887": { + "name": "Crystal helm (attuned)", + "equipable": true, + "weight": 2.0, + "equipment": { + "slot": 0, + "dstab": 48, + "dslash": 48, + "dcrush": 48, + "dmagic": 48, + "drange": 48, + "prayer": 2 + } + }, + "23888": { + "name": "Crystal helm (perfected)", + "equipable": true, + "weight": 2.0, + "equipment": { + "slot": 0, + "dstab": 68, + "dslash": 68, + "dcrush": 68, + "dmagic": 68, + "drange": 68, + "prayer": 3 + } + }, + "23889": { + "name": "Crystal body (basic)", + "equipable": true, + "weight": 9.0, + "equipment": { + "slot": 4, + "dstab": 86, + "dslash": 86, + "dcrush": 86, + "dmagic": 86, + "drange": 86, + "prayer": 3 + } + }, + "23890": { + "name": "Crystal body (attuned)", + "equipable": true, + "weight": 2.0, + "equipment": { + "slot": 4, + "dstab": 102, + "dslash": 102, + "dcrush": 102, + "dmagic": 102, + "drange": 102, + "prayer": 4 + } + }, + "23891": { + "name": "Crystal body (perfected)", + "equipable": true, + "weight": 2.0, + "equipment": { + "slot": 4, + "dstab": 124, + "dslash": 124, + "dcrush": 124, + "dmagic": 124, + "drange": 124, + "prayer": 5 + } + }, + "23893": { + "name": "Crystal legs (attuned)", + "equipable": true, + "weight": 2.0, + "equipment": { + "slot": 7, + "dstab": 74, + "dslash": 74, + "dcrush": 74, + "dmagic": 74, + "drange": 74, + "prayer": 3 + } + }, + "23894": { + "name": "Crystal legs (perfected)", + "equipable": true, + "weight": 2.0, + "equipment": { + "slot": 7, + "dstab": 92, + "dslash": 92, + "dcrush": 92, + "dmagic": 92, + "drange": 92, + "prayer": 4 + } + }, + "23895": { + "name": "Crystal halberd (basic)", + "equipable": true, + "weight": 2.0, + "equipment": { + "slot": 3, + "astab": 68, + "aslash": 68, + "acrush": 4, + "str": 42, + "prayer": 1, + "aspeed": 4 + } + }, + "23896": { + "name": "Crystal halberd (attuned)", + "equipable": true, + "weight": 2.0, + "equipment": { + "slot": 3, + "astab": 114, + "aslash": 114, + "acrush": 12, + "str": 88, + "prayer": 2, + "aspeed": 4 + } + }, + "23897": { + "name": "Crystal halberd (perfected)", + "equipable": true, + "weight": 2.0, + "equipment": { + "slot": 3, + "astab": 166, + "aslash": 166, + "acrush": 28, + "str": 138, + "prayer": 3, + "aspeed": 4 + } + }, + "23898": { + "name": "Crystal staff (basic)", + "equipable": true, + "weight": 2.0, + "equipment": { + "slot": 3, + "amagic": 84, + "prayer": 1, + "aspeed": 4 + } + }, + "23900": { + "name": "Crystal staff (perfected)", + "equipable": true, + "weight": 2.0, + "equipment": { + "slot": 3, + "amagic": 184, + "prayer": 3, + "aspeed": 4 + } + }, + "23901": { + "name": "Crystal bow (basic)", + "equipable": true, + "weight": 2.0, + "equipment": { + "slot": 3, + "arange": 72, + "rstr": 42, + "prayer": 1, + "aspeed": 4 + } + }, + "23902": { + "name": "Crystal bow (attuned)", + "equipable": true, + "weight": 2.0, + "equipment": { + "slot": 3, + "arange": 118, + "rstr": 88, + "prayer": 1, + "aspeed": 4 + } + }, + "23903": { + "name": "Crystal bow (perfected)", + "equipable": true, + "weight": 2.0, + "equipment": { + "slot": 3, + "arange": 172, + "rstr": 138, + "prayer": 3, + "aspeed": 4 + } + }, + "23904": { + "name": "Teleport crystal" + }, "23905": { - "name": "Tephra" + "name": "Tephra", + "weight": 2.2 }, "23906": { - "name": "Refined tephra" + "name": "Refined tephra", + "weight": 2.2 }, "23907": { "name": "Imbued tephra", - "equipable": true + "equipable": true, + "equipment": { + "slot": 3, + "aspeed": 3 + } }, "23908": { "name": "Zalcano shard", "weight": 0.5 }, + "23911": { + "name": "Crystal crown", + "equipable": true, + "equipment": { + "slot": 0 + } + }, + "23913": { + "name": "Crystal crown", + "equipable": true, + "equipment": { + "slot": 0 + } + }, + "23915": { + "name": "Crystal crown", + "equipable": true, + "equipment": { + "slot": 0 + } + }, + "23917": { + "name": "Crystal crown", + "equipable": true, + "equipment": { + "slot": 0 + } + }, + "23919": { + "name": "Crystal crown", + "equipable": true, + "equipment": { + "slot": 0 + } + }, + "23921": { + "name": "Crystal crown", + "equipable": true, + "equipment": { + "slot": 0 + } + }, + "23923": { + "name": "Crystal crown", + "equipable": true, + "equipment": { + "slot": 0 + } + }, + "23925": { + "name": "Crystal crown", + "equipable": true, + "equipment": { + "slot": 0 + } + }, "23927": { "name": "Crystal of ithell" }, @@ -63740,11 +64232,14 @@ "name": "Crystal of amlodd" }, "23943": { - "name": "Elven signet" + "name": "Elven signet", + "equipable": true, + "equipment": { + "slot": 12 + } }, "23946": { - "name": "Eternal teleport crystal", - "quest": true + "name": "Eternal teleport crystal" }, "23948": { "name": "Elven dawn", @@ -63765,9 +64260,13 @@ "23962": { "name": "Crystal shard" }, + "23964": { + "name": "Crystal dust" + }, "23971": { "name": "Crystal helm", "equipable": true, + "weight": 2.0, "equipment": { "slot": 0, "amagic": -10, @@ -63783,6 +64282,7 @@ "23973": { "name": "Crystal helm (inactive)", "equipable": true, + "weight": 2.0, "equipment": { "slot": 0, "amagic": -10, @@ -63798,6 +64298,7 @@ "23975": { "name": "Crystal body", "equipable": true, + "weight": 9.0, "equipment": { "slot": 4, "amagic": -18, @@ -63813,6 +64314,7 @@ "23979": { "name": "Crystal legs", "equipable": true, + "weight": 9.0, "equipment": { "slot": 7, "amagic": -12, @@ -63828,6 +64330,7 @@ "23981": { "name": "Crystal legs (inactive)", "equipable": true, + "weight": 9.0, "equipment": { "slot": 7, "amagic": -12, @@ -63840,10 +64343,53 @@ "prayer": 2 } }, - "23995": { - "name": "Blade of Saeldor", + "23983": { + "name": "Crystal bow", + "quest": true, "equipable": true, - "weight": 1.0, + "weight": 2.0, + "equipment": { + "slot": 3, + "arange": 100, + "rstr": 70, + "aspeed": 5 + } + }, + "23987": { + "name": "Crystal halberd", + "equipable": true, + "weight": 2.0, + "equipment": { + "slot": 3, + "astab": 85, + "aslash": 110, + "acrush": 5, + "amagic": -4, + "dstab": -1, + "dslash": 4, + "dcrush": 5, + "str": 118, + "aspeed": 7 + } + }, + "23991": { + "name": "Crystal shield", + "equipable": true, + "weight": 2.7, + "equipment": { + "slot": 5, + "amagic": -10, + "arange": -10, + "dstab": 51, + "dslash": 54, + "dcrush": 53, + "drange": 80 + } + }, + "23995": { + "name": "Blade of saeldor", + "equipable": true, + "weight": 1.8, "equipment": { "slot": 3, "astab": 55, @@ -63853,14 +64399,11 @@ } }, "23997": { - "name": "Blade of Saeldor (inactive)", + "name": "Blade of saeldor (inactive)", "equipable": true, - "weight": 1.0, + "weight": 1.8, "equipment": { "slot": 3, - "astab": 55, - "aslash": 94, - "str": 89, "aspeed": 4 } }, @@ -63872,6 +64415,78 @@ "aspeed": 4 } }, + "24003": { + "name": "Elven boots", + "equipable": true, + "weight": 4.5, + "equipment": { + "slot": 10 + } + }, + "24006": { + "name": "Elven gloves", + "equipable": true, + "weight": 4.5, + "equipment": { + "slot": 9 + } + }, + "24009": { + "name": "Elven top", + "equipable": true, + "weight": 0.9, + "equipment": { + "slot": 4 + } + }, + "24012": { + "name": "Elven skirt", + "equipable": true, + "weight": 0.9, + "equipment": { + "slot": 7 + } + }, + "24015": { + "name": "Elven top", + "equipable": true, + "weight": 0.9, + "equipment": { + "slot": 4 + } + }, + "24018": { + "name": "Elven skirt", + "equipable": true, + "weight": 0.9, + "equipment": { + "slot": 7 + } + }, + "24021": { + "name": "Elven top", + "equipable": true, + "weight": 0.9, + "equipment": { + "slot": 4 + } + }, + "24024": { + "name": "Elven legwear", + "equipable": true, + "weight": 0.9, + "equipment": { + "slot": 7 + } + }, + "24027": { + "name": "Elven top", + "equipable": true, + "weight": 0.9, + "equipment": { + "slot": 4 + } + }, "24030": { "name": "Memoriam crystal (1)" }, @@ -63887,6 +64502,7 @@ "24034": { "name": "Dragonstone full helm", "equipable": true, + "weight": 2.721, "equipment": { "slot": 0, "amagic": -6, @@ -63901,6 +64517,7 @@ "24037": { "name": "Dragonstone platebody", "equipable": true, + "weight": 9.979, "equipment": { "slot": 4, "amagic": -30, @@ -63915,6 +64532,7 @@ "24040": { "name": "Dragonstone platelegs", "equipable": true, + "weight": 9.071, "equipment": { "slot": 7, "amagic": -21, @@ -63929,6 +64547,7 @@ "24043": { "name": "Dragonstone boots", "equipable": true, + "weight": 1.36, "equipment": { "slot": 10, "amagic": -3, @@ -63942,19 +64561,20 @@ "24046": { "name": "Dragonstone gauntlets", "equipable": true, + "weight": 0.226, "equipment": { "slot": 9, "astab": 8, "aslash": 8, "acrush": 8, - "amagic": 4, + "amagic": -4, "arange": 8, "dstab": 8, "dslash": 8, "dcrush": 8, - "dmagic": 4, - "drange": 8, - "str": 8 + "dmagic": -4, + "drange": 4, + "str": 4 } }, "24049": { @@ -63998,5 +64618,12 @@ }, "24075": { "name": "Legends of the mountain" + }, + "24130": { + "name": "Combat path starter kit", + "weight": 10.0 + }, + "24131": { + "name": "Combat path voucher" } } \ No newline at end of file diff --git a/runelite-client/src/main/resources/net/runelite/client/plugins/grandexchange/ge_limits.json b/runelite-client/src/main/resources/net/runelite/client/plugins/grandexchange/ge_limits.json index afc007325d..09d460f7fb 100644 --- a/runelite-client/src/main/resources/net/runelite/client/plugins/grandexchange/ge_limits.json +++ b/runelite-client/src/main/resources/net/runelite/client/plugins/grandexchange/ge_limits.json @@ -1216,9 +1216,10 @@ "4180": 70, "4207": 70, "4212": 70, + "4213": 70, "4224": 70, + "4235": 70, "4298": 15, - "4300": 15, "4242": 2000, "4243": 2000, "4245": 2000, @@ -1227,6 +1228,7 @@ "4289": 13000, "4291": 6000, "4293": 6000, + "4300": 15, "4302": 15, "4304": 15, "4306": 15, @@ -3430,6 +3432,8 @@ "13576": 8, "13652": 8, "13657": 200, + "16888": 70, + "16890": 70, "19478": 8, "19481": 8, "19484": 11000, @@ -3719,6 +3723,7 @@ "21117": 10000, "21120": 10000, "21123": 10000, + "21126": 100, "21129": 10000, "21140": 10000, "21143": 10000, @@ -3900,6 +3905,7 @@ "22305": 18000, "22324": 8, "22326": 8, + "22372": 100, "22405": 13000, "22408": 10000, "22327": 8, @@ -3933,6 +3939,8 @@ "22665": 8, "22731": 70, "22734": 70, + "22737": 70, + "22740": 70, "22743": 70, "22780": 7500, "22783": 7500, @@ -3941,6 +3949,8 @@ "22795": 10000, "22804": 11000, "22806": 7000, + "22808": 7000, + "22810": 7000, "22812": 11000, "22814": 11000, "22818": 13000, @@ -3963,11 +3973,15 @@ "22949": 10000, "22951": 70, "22954": 70, + "22957": 15, + "22960": 70, "22963": 70, + "22966": 15, "22975": 8, "22978": 8, "22983": 15, "22988": 15, + "22994": 5, "22999": 50, "23002": 50, "23037": 125, @@ -3983,6 +3997,12 @@ "23119": 70, "23124": 8, "23185": 4, + "23188": 8, + "23191": 8, + "23194": 8, + "23197": 8, + "23200": 8, + "23203": 8, "23206": 8, "23209": 8, "23212": 8, @@ -3995,18 +4015,26 @@ "23237": 4, "23242": 8, "23246": 8, + "23249": 8, "23252": 4, + "23255": 4, "23258": 8, + "23261": 8, + "23264": 8, + "23267": 8, "23270": 4, "23273": 4, "23276": 4, "23279": 4, + "23282": 4, "23285": 4, "23288": 4, "23291": 4, "23294": 4, "23297": 4, "23300": 4, + "23303": 4, + "23306": 4, "23309": 4, "23312": 4, "23315": 4, @@ -4034,6 +4062,8 @@ "23378": 70, "23381": 8, "23384": 8, + "23387": 10000, + "23389": 8, "23392": 8, "23395": 8, "23398": 8, @@ -4092,5 +4122,8 @@ "23517": 100, "23522": 70, "23525": 4, - "23528": 70 + "23528": 70, + "23839": 13000, + "23865": 40, + "23879": 13000 } diff --git a/runelite-client/src/main/resources/npc_stats.json b/runelite-client/src/main/resources/npc_stats.json index 97fe4193bd..43e0c5b00d 100644 --- a/runelite-client/src/main/resources/npc_stats.json +++ b/runelite-client/src/main/resources/npc_stats.json @@ -3713,6 +3713,7 @@ "name": "Highwayman", "hitpoints": 13, "combatLevel": 5, + "attackSpeed": 4, "attackLevel": 2, "strengthLevel": 2, "defenceLevel": 4, @@ -3728,6 +3729,7 @@ "name": "Highwayman", "hitpoints": 13, "combatLevel": 5, + "attackSpeed": 4, "attackLevel": 2, "strengthLevel": 2, "defenceLevel": 4, @@ -3849,7 +3851,7 @@ "hitpoints": 17, "combatLevel": 8, "slayerLevel": 1, - "attackSpeed": 6, + "attackSpeed": 4, "attackLevel": 5, "strengthLevel": 5, "defenceLevel": 5, @@ -4823,7 +4825,7 @@ }, "655": { "name": "Goblin", - "hitpoints": 5, + "hitpoints": 12, "combatLevel": 5, "slayerLevel": 1, "attackSpeed": 6, @@ -4837,7 +4839,7 @@ }, "656": { "name": "Goblin", - "hitpoints": 5, + "hitpoints": 12, "combatLevel": 5, "slayerLevel": 1, "attackSpeed": 6, @@ -4851,7 +4853,7 @@ }, "657": { "name": "Goblin", - "hitpoints": 5, + "hitpoints": 12, "combatLevel": 5, "slayerLevel": 1, "attackSpeed": 6, @@ -4865,7 +4867,7 @@ }, "658": { "name": "Goblin", - "hitpoints": 5, + "hitpoints": 12, "combatLevel": 5, "slayerLevel": 1, "attackSpeed": 6, @@ -4879,7 +4881,7 @@ }, "659": { "name": "Goblin", - "hitpoints": 5, + "hitpoints": 12, "combatLevel": 5, "slayerLevel": 1, "attackSpeed": 6, @@ -4893,7 +4895,7 @@ }, "660": { "name": "Goblin", - "hitpoints": 5, + "hitpoints": 12, "combatLevel": 5, "slayerLevel": 1, "attackSpeed": 6, @@ -4907,7 +4909,7 @@ }, "661": { "name": "Goblin", - "hitpoints": 5, + "hitpoints": 12, "combatLevel": 5, "slayerLevel": 1, "attackSpeed": 6, @@ -4921,7 +4923,7 @@ }, "662": { "name": "Goblin", - "hitpoints": 5, + "hitpoints": 12, "combatLevel": 5, "slayerLevel": 1, "attackSpeed": 6, @@ -4935,7 +4937,7 @@ }, "663": { "name": "Goblin", - "hitpoints": 5, + "hitpoints": 12, "combatLevel": 5, "slayerLevel": 1, "attackSpeed": 6, @@ -4949,7 +4951,7 @@ }, "664": { "name": "Goblin", - "hitpoints": 5, + "hitpoints": 12, "combatLevel": 5, "slayerLevel": 1, "attackSpeed": 6, @@ -4963,7 +4965,7 @@ }, "665": { "name": "Goblin", - "hitpoints": 5, + "hitpoints": 12, "combatLevel": 5, "slayerLevel": 1, "attackSpeed": 6, @@ -4977,7 +4979,7 @@ }, "666": { "name": "Goblin", - "hitpoints": 5, + "hitpoints": 12, "combatLevel": 5, "slayerLevel": 1, "attackSpeed": 6, @@ -4991,7 +4993,7 @@ }, "667": { "name": "Goblin", - "hitpoints": 5, + "hitpoints": 12, "combatLevel": 5, "slayerLevel": 1, "attackSpeed": 6, @@ -5005,7 +5007,7 @@ }, "668": { "name": "Goblin", - "hitpoints": 5, + "hitpoints": 12, "combatLevel": 5, "slayerLevel": 1, "attackSpeed": 6, @@ -5019,7 +5021,7 @@ }, "674": { "name": "Goblin", - "hitpoints": 5, + "hitpoints": 12, "combatLevel": 5, "slayerLevel": 1, "attackSpeed": 6, @@ -5033,7 +5035,7 @@ }, "677": { "name": "Goblin", - "hitpoints": 5, + "hitpoints": 12, "combatLevel": 5, "slayerLevel": 1, "attackSpeed": 6, @@ -5047,7 +5049,7 @@ }, "678": { "name": "Goblin", - "hitpoints": 5, + "hitpoints": 12, "combatLevel": 5, "slayerLevel": 1, "attackSpeed": 6, @@ -6160,6 +6162,14 @@ "magicLevel": 100, "demon": true }, + "924": { + "name": "Skeleton", + "hitpoints": 18, + "combatLevel": 13, + "slayerLevel": 1, + "attackSpeed": 4, + "undead": true + }, "925": { "name": "Rock", "hitpoints": 140, @@ -6960,6 +6970,18 @@ "rangeLevel": 1, "magicLevel": 1 }, + "1039": { + "name": "Albino bat", + "hitpoints": 33, + "combatLevel": 52, + "slayerLevel": 1, + "attackSpeed": 4, + "attackLevel": 57, + "strengthLevel": 57, + "defenceLevel": 30, + "rangeLevel": 1, + "magicLevel": 1 + }, "1041": { "name": "Giant mosquito", "hitpoints": 3, @@ -12282,7 +12304,7 @@ }, "2484": { "name": "Goblin", - "hitpoints": 5, + "hitpoints": 12, "combatLevel": 5, "slayerLevel": 1, "attackSpeed": 6, @@ -14848,7 +14870,7 @@ }, "3045": { "name": "Goblin", - "hitpoints": 5, + "hitpoints": 12, "combatLevel": 5, "slayerLevel": 1, "attackSpeed": 6, @@ -14879,7 +14901,7 @@ }, "3047": { "name": "Goblin", - "hitpoints": 5, + "hitpoints": 12, "combatLevel": 5, "slayerLevel": 1, "attackSpeed": 6, @@ -14893,7 +14915,7 @@ }, "3048": { "name": "Goblin", - "hitpoints": 5, + "hitpoints": 12, "combatLevel": 5, "slayerLevel": 1, "attackSpeed": 6, @@ -15254,7 +15276,7 @@ }, "3073": { "name": "Goblin", - "hitpoints": 5, + "hitpoints": 12, "combatLevel": 5, "slayerLevel": 1, "attackSpeed": 6, @@ -15268,7 +15290,7 @@ }, "3074": { "name": "Goblin", - "hitpoints": 5, + "hitpoints": 12, "combatLevel": 5, "slayerLevel": 1, "attackSpeed": 6, @@ -15282,7 +15304,7 @@ }, "3075": { "name": "Goblin", - "hitpoints": 5, + "hitpoints": 12, "combatLevel": 5, "slayerLevel": 1, "attackSpeed": 6, @@ -15296,7 +15318,7 @@ }, "3076": { "name": "Goblin", - "hitpoints": 5, + "hitpoints": 12, "combatLevel": 5, "slayerLevel": 1, "attackSpeed": 6, @@ -26090,7 +26112,7 @@ "hitpoints": 40, "combatLevel": 48, "slayerLevel": 1, - "attackSpeed": 7, + "attackSpeed": 3, "attackLevel": 40, "strengthLevel": 45, "defenceLevel": 45, @@ -26108,7 +26130,7 @@ "hitpoints": 40, "combatLevel": 48, "slayerLevel": 1, - "attackSpeed": 7, + "attackSpeed": 3, "attackLevel": 40, "strengthLevel": 45, "defenceLevel": 45, @@ -26126,7 +26148,7 @@ "hitpoints": 40, "combatLevel": 48, "slayerLevel": 1, - "attackSpeed": 7, + "attackSpeed": 3, "attackLevel": 40, "strengthLevel": 45, "defenceLevel": 45, @@ -26144,7 +26166,7 @@ "hitpoints": 40, "combatLevel": 48, "slayerLevel": 1, - "attackSpeed": 7, + "attackSpeed": 3, "attackLevel": 40, "strengthLevel": 45, "defenceLevel": 45, @@ -29459,7 +29481,7 @@ "hitpoints": 137, "combatLevel": 145, "slayerLevel": 1, - "attackSpeed": 6, + "attackSpeed": 4, "attackLevel": 75, "strengthLevel": 75, "defenceLevel": 55, @@ -29811,6 +29833,7 @@ }, "6728": { "name": "Rock Golem", + "hitpoints": 86, "combatLevel": 79, "attackSpeed": 4 }, @@ -34918,7 +34941,6 @@ "name": "Zombified Spawn", "hitpoints": 8, "combatLevel": 55, - "attackSpeed": 4, "attackLevel": 80, "strengthLevel": 80, "defenceLevel": 4, @@ -34939,7 +34961,6 @@ "name": "Zombified Spawn", "hitpoints": 38, "combatLevel": 64, - "attackSpeed": 4, "attackLevel": 82, "strengthLevel": 82, "defenceLevel": 6, @@ -36510,7 +36531,6 @@ "name": "Nylocas Matomenos", "hitpoints": 200, "combatLevel": 115, - "attackSpeed": 6, "attackLevel": 100, "strengthLevel": 100, "defenceLevel": 100, @@ -36521,7 +36541,6 @@ "name": "Blood spawn", "hitpoints": 120, "combatLevel": 55, - "attackSpeed": 4, "attackLevel": 1, "strengthLevel": 1, "rangeLevel": 1 @@ -36713,7 +36732,6 @@ "name": "Nylocas Athanatos", "hitpoints": 180, "combatLevel": 350, - "attackSpeed": 4, "attackLevel": 1, "strengthLevel": 1, "defenceLevel": 50, @@ -36724,7 +36742,6 @@ "name": "Nylocas Matomenos", "hitpoints": 200, "combatLevel": 115, - "attackSpeed": 6, "attackLevel": 100, "strengthLevel": 100, "defenceLevel": 100, @@ -36849,7 +36866,7 @@ "hitpoints": 40, "combatLevel": 48, "slayerLevel": 1, - "attackSpeed": 7, + "attackSpeed": 3, "attackLevel": 40, "strengthLevel": 45, "defenceLevel": 45, @@ -36867,7 +36884,7 @@ "hitpoints": 40, "combatLevel": 48, "slayerLevel": 1, - "attackSpeed": 7, + "attackSpeed": 3, "attackLevel": 40, "strengthLevel": 45, "defenceLevel": 45, @@ -36885,7 +36902,7 @@ "hitpoints": 40, "combatLevel": 48, "slayerLevel": 1, - "attackSpeed": 7, + "attackSpeed": 3, "attackLevel": 40, "strengthLevel": 45, "defenceLevel": 45, @@ -37524,6 +37541,20 @@ "crushDef": 50, "magicDef": 150 }, + "8736": { + "name": "Moss Giant", + "hitpoints": 120, + "combatLevel": 84, + "slayerLevel": 1, + "attackSpeed": 4, + "attackLevel": 60, + "strengthLevel": 60, + "defenceLevel": 60, + "rangeLevel": 1, + "magicLevel": 1, + "bonusAttack": 66, + "bonusStrength": 62 + }, "8917": { "name": "Fragment of Seren", "hitpoints": 1000, @@ -37627,5 +37658,253 @@ "hitpoints": 600, "combatLevel": 674, "attackSpeed": 5 + }, + "9026": { + "name": "Crystalline Rat", + "combatLevel": 24, + "attackSpeed": 4 + }, + "9028": { + "name": "Crystalline Bat", + "combatLevel": 33, + "attackSpeed": 4 + }, + "9029": { + "name": "Crystalline Unicorn", + "combatLevel": 48, + "attackSpeed": 4 + }, + "9030": { + "name": "Crystalline Scorpion", + "combatLevel": 64, + "attackSpeed": 4 + }, + "9031": { + "name": "Crystalline Wolf", + "combatLevel": 74, + "attackSpeed": 4 + }, + "9032": { + "name": "Crystalline Bear", + "combatLevel": 172, + "attackSpeed": 4 + }, + "9033": { + "name": "Crystalline Dragon", + "combatLevel": 172, + "attackSpeed": 4 + }, + "9040": { + "name": "Corrupted Rat", + "combatLevel": 34, + "attackSpeed": 4 + }, + "9041": { + "name": "Corrupted Spider", + "hitpoints": 12, + "combatLevel": 32, + "attackSpeed": 4 + }, + "9042": { + "name": "Corrupted Bat", + "combatLevel": 48, + "attackSpeed": 4 + }, + "9043": { + "name": "Corrupted Unicorn", + "combatLevel": 64, + "attackSpeed": 4 + }, + "9044": { + "name": "Corrupted Scorpion", + "combatLevel": 89, + "attackSpeed": 4 + }, + "9045": { + "name": "Corrupted Wolf", + "combatLevel": 102, + "attackSpeed": 4 + }, + "9046": { + "name": "Corrupted Bear", + "hitpoints": 100, + "combatLevel": 258, + "attackSpeed": 4 + }, + "9047": { + "name": "Corrupted Dragon", + "combatLevel": 258, + "attackSpeed": 4 + }, + "9048": { + "name": "Corrupted Dark Beast", + "hitpoints": 100, + "combatLevel": 258, + "attackSpeed": 4 + }, + "9049": { + "name": "Zalcano", + "hitpoints": 14, + "combatLevel": 64 + }, + "9182": { + "name": "Guard", + "hitpoints": 105, + "combatLevel": 108, + "slayerLevel": 1, + "attackSpeed": 4, + "attackLevel": 95, + "strengthLevel": 80, + "defenceLevel": 95, + "magicLevel": 1, + "stabDef": 50, + "slashDef": 70, + "crushDef": 70, + "rangeDef": 50, + "magicDef": 60 + }, + "9183": { + "name": "Guard", + "hitpoints": 105, + "combatLevel": 108, + "slayerLevel": 1, + "attackSpeed": 4, + "attackLevel": 95, + "strengthLevel": 80, + "defenceLevel": 95, + "magicLevel": 1, + "stabDef": 50, + "slashDef": 70, + "crushDef": 70, + "rangeDef": 50, + "magicDef": 60 + }, + "9184": { + "name": "Guard", + "hitpoints": 105, + "combatLevel": 108, + "slayerLevel": 1, + "attackSpeed": 4, + "attackLevel": 95, + "strengthLevel": 80, + "defenceLevel": 95, + "magicLevel": 1, + "stabDef": 50, + "slashDef": 70, + "crushDef": 70, + "rangeDef": 50, + "magicDef": 60 + }, + "9185": { + "name": "Guard", + "hitpoints": 105, + "combatLevel": 108, + "slayerLevel": 1, + "attackSpeed": 4, + "attackLevel": 95, + "strengthLevel": 80, + "defenceLevel": 95, + "magicLevel": 1, + "stabDef": 50, + "slashDef": 70, + "crushDef": 70, + "rangeDef": 50, + "magicDef": 60 + }, + "9186": { + "name": "Guard", + "hitpoints": 105, + "combatLevel": 108, + "slayerLevel": 1, + "attackSpeed": 4, + "attackLevel": 95, + "strengthLevel": 80, + "defenceLevel": 95, + "magicLevel": 1, + "stabDef": 50, + "slashDef": 70, + "crushDef": 70, + "rangeDef": 50, + "magicDef": 60 + }, + "9187": { + "name": "Guard", + "hitpoints": 105, + "combatLevel": 108, + "slayerLevel": 1, + "attackSpeed": 4, + "attackLevel": 95, + "strengthLevel": 80, + "defenceLevel": 95, + "magicLevel": 1, + "stabDef": 50, + "slashDef": 70, + "crushDef": 70, + "rangeDef": 50, + "magicDef": 60 + }, + "9188": { + "name": "Guard", + "hitpoints": 105, + "combatLevel": 108, + "slayerLevel": 1, + "attackSpeed": 4, + "attackLevel": 95, + "strengthLevel": 80, + "defenceLevel": 95, + "magicLevel": 1, + "stabDef": 50, + "slashDef": 70, + "crushDef": 70, + "rangeDef": 50, + "magicDef": 60 + }, + "9189": { + "name": "Guard", + "hitpoints": 105, + "combatLevel": 108, + "slayerLevel": 1, + "attackSpeed": 4, + "attackLevel": 95, + "strengthLevel": 80, + "defenceLevel": 95, + "magicLevel": 1, + "stabDef": 50, + "slashDef": 70, + "crushDef": 70, + "rangeDef": 50, + "magicDef": 60 + }, + "9190": { + "name": "Guard", + "hitpoints": 105, + "combatLevel": 108, + "slayerLevel": 1, + "attackSpeed": 4, + "attackLevel": 95, + "strengthLevel": 80, + "defenceLevel": 95, + "magicLevel": 1, + "stabDef": 50, + "slashDef": 70, + "crushDef": 70, + "rangeDef": 50, + "magicDef": 60 + }, + "9191": { + "name": "Guard", + "hitpoints": 105, + "combatLevel": 108, + "slayerLevel": 1, + "attackSpeed": 4, + "attackLevel": 95, + "strengthLevel": 80, + "defenceLevel": 95, + "magicLevel": 1, + "stabDef": 50, + "slashDef": 70, + "crushDef": 70, + "rangeDef": 50, + "magicDef": 60 } } \ No newline at end of file diff --git a/settings.gradle b/settings.gradle index 07a276ccbc..49532a3868 100644 --- a/settings.gradle +++ b/settings.gradle @@ -17,6 +17,7 @@ include ':injected-client' include ':runelite-plugin-archetype' include ':http-service' include ':http-service-plus' +include ':wiki-scraper' project(':http-api').projectDir = "$rootDir/http-api" as File project(':cache').projectDir = "$rootDir/cache" as File @@ -35,4 +36,6 @@ project(':injector-plugin').projectDir = "$rootDir/injector-plugin" as File project(':injected-client').projectDir = "$rootDir/injected-client" as File project(':runelite-plugin-archetype').projectDir = "$rootDir/runelite-plugin-archetype" as File project(':http-service').projectDir = "$rootDir/http-service" as File -project(':http-service-plus').projectDir = "$rootDir/http-service-plus" as File \ No newline at end of file +project(':http-service-plus').projectDir = "$rootDir/http-service-plus" as File +project(':wiki-scraper').projectDir = "$rootDir/wiki-scraper" as File + diff --git a/wiki-scraper/build.gradle b/wiki-scraper/build.gradle new file mode 100644 index 0000000000..1a43158e17 --- /dev/null +++ b/wiki-scraper/build.gradle @@ -0,0 +1,28 @@ +apply plugin:'application' + +repositories { + mavenCentral() + maven { url 'https://jitpack.io' } +} + +description = 'RuneLite Wiki scraper' + +mainClassName = "net.runelite.data.App" + +dependencies { + api project(':cache') + api project(':runelite-api') + + annotationProcessor group: 'org.projectlombok', name: 'lombok', version: lombok + + compileOnly group: 'org.projectlombok', name: 'lombok', version: lombok + + implementation group: 'com.google.code.gson', name: 'gson', version: gson + implementation group: 'com.google.guava', name: 'guava', version: guava + implementation group: 'com.github.petitparser', name: 'java-petitparser', version: '2.2.0' + implementation group: 'com.squareup.okhttp3', name: 'okhttp', version: okhttp3 + implementation group: 'org.slf4j', name: 'slf4j-api', version: slf4j + implementation group: 'org.slf4j', name: 'slf4j-simple', version: slf4j + + testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-api', version: jupiter +} diff --git a/wiki-scraper/src/main/java/net/runelite/data/App.java b/wiki-scraper/src/main/java/net/runelite/data/App.java new file mode 100644 index 0000000000..263bb31125 --- /dev/null +++ b/wiki-scraper/src/main/java/net/runelite/data/App.java @@ -0,0 +1,59 @@ +/* + * MIT License + * + * Copyright (c) 2018 Tomas Slusny + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package net.runelite.data; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import java.io.File; +import java.io.IOException; +import net.runelite.cache.fs.Store; +import net.runelite.data.dump.MediaWiki; +import net.runelite.data.dump.wiki.NpcStatsDumper; + +public class App +{ + public static final Gson GSON = new GsonBuilder() + .setPrettyPrinting() + .disableHtmlEscaping() + .create(); + + public static void main(String[] args) throws IOException + { + final File home = new File(System.getProperty("user.home")); + final Store cacheStore = new Store(new File(home, + "jagexcache" + File.separator + "oldschool" + File.separator + "LIVE")); + cacheStore.load(); + + // Try to make this go faster (probably not very smart) + System.setProperty("java.util.concurrent.ForkJoinPool.common.parallelism", "100"); + + final MediaWiki wiki = new MediaWiki("https://oldschool.runescape.wiki"); + + // Only use this to diff current limits with scraped limits + // ItemLimitsDumper.dump(cacheStore, wiki); + // ItemStatsDumper.dump(cacheStore, wiki); + + NpcStatsDumper.dump(cacheStore, wiki); + } +} \ No newline at end of file diff --git a/wiki-scraper/src/main/java/net/runelite/data/dump/MediaWiki.java b/wiki-scraper/src/main/java/net/runelite/data/dump/MediaWiki.java new file mode 100644 index 0000000000..f0ac4c76ea --- /dev/null +++ b/wiki-scraper/src/main/java/net/runelite/data/dump/MediaWiki.java @@ -0,0 +1,140 @@ +/* + * MIT License + * + * Copyright (c) 2018 Tomas Slusny + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package net.runelite.data.dump; + +import java.io.UnsupportedEncodingException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.Map; +import net.runelite.data.App; +import okhttp3.HttpUrl; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; + +public class MediaWiki +{ + private static final class WikiInnerResponse + { + Map wikitext; + } + + private static final class WikiResponse + { + WikiInnerResponse parse; + } + + private final OkHttpClient client = new OkHttpClient(); + private final OkHttpClient clientNoRedirect = client.newBuilder() + .followRedirects(false) + .followSslRedirects(false) + .build(); + + private final HttpUrl base; + + public MediaWiki(final String base) + { + this.base = HttpUrl.parse(base); + } + + public String getSpecialLookupData(final String type, final int id, final int section) + { + final HttpUrl url = base.newBuilder() + .addPathSegment("w") + .addPathSegment("Special:Lookup") + .addQueryParameter("type", type) + .addQueryParameter("id", String.valueOf(id)) + .build(); + + final Request request = new Request.Builder() + .url(url) + .build(); + + try (final Response response = clientNoRedirect.newCall(request).execute()) + { + if (response.isRedirect()) + { + final String page = response.header("Location") + .replace(base.newBuilder().addPathSegment("w").build().toString() + "/", ""); + return getPageData(page, section); + } + } + catch (Exception e) + { + return ""; + } + + return ""; + } + + public String getPageData(String page, int section) + { + // decode html encoded page name + // ex: Mage%27s book -> Mage's_book + try + { + page = URLDecoder.decode(page, StandardCharsets.UTF_8.name()); + } + catch (UnsupportedEncodingException e) + { + // do nothing, keep page the same + } + + final HttpUrl.Builder urlBuilder = base.newBuilder() + .addPathSegment("api.php") + .addQueryParameter("action", "parse") + .addQueryParameter("format", "json") + .addQueryParameter("prop", "wikitext") + .addQueryParameter("redirects", "true") + .addQueryParameter("page", page.replaceAll(" ", "_")); + + if (section != -1) + { + urlBuilder.addQueryParameter("section", String.valueOf(section)); + } + + final HttpUrl url = urlBuilder.build(); + + final Request request = new Request.Builder() + .url(url) + .build(); + + try (final Response response = client.newCall(request).execute()) + { + if (response.isSuccessful()) + { + final InputStream in = response.body().byteStream(); + return App.GSON.fromJson(new InputStreamReader(in), WikiResponse.class).parse.wikitext.get("*"); + } + } + catch (Exception e) + { + return ""; + } + + return ""; + } +} diff --git a/wiki-scraper/src/main/java/net/runelite/data/dump/MediaWikiTemplate.java b/wiki-scraper/src/main/java/net/runelite/data/dump/MediaWikiTemplate.java new file mode 100644 index 0000000000..f86c730327 --- /dev/null +++ b/wiki-scraper/src/main/java/net/runelite/data/dump/MediaWikiTemplate.java @@ -0,0 +1,302 @@ +/* + * MIT License + * + * Copyright (c) 2018 Tomas Slusny + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package net.runelite.data.dump; + +import com.google.common.base.Function; +import com.google.common.base.MoreObjects; +import com.google.common.base.Strings; +import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import javax.annotation.Nullable; +import lombok.extern.slf4j.Slf4j; +import org.petitparser.context.Result; +import org.petitparser.parser.Parser; +import org.petitparser.parser.primitive.CharacterParser; +import org.petitparser.parser.primitive.StringParser; + +@Slf4j +public class MediaWikiTemplate +{ + private static final Parser LUA_PARSER; + private static final Parser MEDIAWIKI_PARSER; + + static + { + final Parser singleString = CharacterParser.of('\'').seq(CharacterParser.of('\'').neg().plus().flatten()).seq(CharacterParser.of('\'')); + final Parser doubleString = CharacterParser.of('"').seq(CharacterParser.of('"').neg().plus().flatten()).seq(CharacterParser.of('"')); + final Parser string = singleString.or(doubleString).pick(1); + + final Parser key = CharacterParser.letter().or(CharacterParser.of('-')).or(CharacterParser.of('_')).or(CharacterParser.of(' ')).or(CharacterParser.digit()).plus().flatten(); + final Parser value = string.or(key); + + final Parser pair = key.trim() + .seq(CharacterParser.of('=').trim()) + .seq(value.trim()) + .map((Function, Map.Entry>) input -> new AbstractMap.SimpleEntry<>(input.get(0).trim(), input.get(2).trim())); + + final Parser commaLine = pair + .seq(CharacterParser.of(',').optional().trim()) + .pick(0); + + LUA_PARSER = StringParser.of("return").trim() + .seq(CharacterParser.of('{').trim()) + .seq(commaLine.plus().trim()) + .pick(2); + + final Parser wikiValue = CharacterParser.of('|') + .or(StringParser.of("}}")) + .or(StringParser.of("{{")) + .or(StringParser.of("]]")) + .or(StringParser.of("[[")) + .neg().plus().trim(); + + final Parser wikiBraceExpression = StringParser.of("{{") + .seq(StringParser.of("}}").neg().star().trim()) + .seq(StringParser.of("}}")); + + final Parser wikiSquareExpression = StringParser.of("[[") + .seq(StringParser.of("]]").neg().star().trim()) + .seq(StringParser.of("]]")); + + final Parser notOrPair = key.trim() + .seq(CharacterParser.of('=').trim()) + .seq(CharacterParser.whitespace().star().seq(wikiSquareExpression.or(wikiBraceExpression).or(wikiValue)).plus().flatten().trim().optional()) + .map((Function, Map.Entry>) input -> new AbstractMap.SimpleEntry<>( + input.get(0).trim(), + MoreObjects.firstNonNull(input.get(2), "").trim())); + + final Parser orLine = CharacterParser.of('|') + .seq(notOrPair.trim().optional()) + .pick(1); + + MEDIAWIKI_PARSER = orLine.plus().trim().seq(StringParser.of("}}")).pick(0); + } + + @Nullable + public static MediaWikiTemplate parseWikitext(final String name, final String data) + { + final Pattern exactNameTest = Pattern.compile("\\{\\{\\s*" + name + "\\s*\\|", Pattern.CASE_INSENSITIVE); + final Matcher m = exactNameTest.matcher(data.toLowerCase()); + + // Early exit + if (!m.find()) + { + return null; + } + + final Map out = new HashMap<>(); + final Parser wikiParser = StringParser.of("{{") + .seq(StringParser.ofIgnoringCase(name).trim()) + .seq(MEDIAWIKI_PARSER) + .pick(2); + + final List parsed = wikiParser.matchesSkipping(data); + + if (parsed.isEmpty()) + { + final Result parse = StringParser.of("{{") + .seq(StringParser.ofIgnoringCase(name).trim()) + .neg() + .star() + .seq(wikiParser) + .seq(CharacterParser.any().star()) + .parse(data); + + if (!parse.isSuccess()) + { + log.warn("Failed to parse: {}", data); + log.warn("Error message: {}", parse.getMessage()); + } + + return null; + } + + final List> entries = (List>) parsed.get(0); + + for (Map.Entry entry : entries) + { + if (entry == null) + { + continue; + } + + out.put(entry.getKey(), entry.getValue()); + } + + if (out.isEmpty()) + { + return null; + } + + return new MediaWikiTemplate(out); + } + + @Nullable + public static MediaWikiTemplate parseLua(final String data) + { + final Map out = new HashMap<>(); + final List parsed = LUA_PARSER.matchesSkipping(data); + + if (parsed.isEmpty()) + { + final Result parse = StringParser.of("return") + .neg() + .star() + .seq(LUA_PARSER) + .seq(CharacterParser.any()).parse(data); + + if (!parse.isSuccess()) + { + log.warn("Failed to parse: {}", data); + log.warn("Error message: {}", parse.getMessage()); + } + + return null; + } + + final List> entries = (List>) parsed.get(0); + + for (Map.Entry entry : entries) + { + out.put(entry.getKey(), entry.getValue()); + } + + if (out.isEmpty()) + { + return null; + } + + return new MediaWikiTemplate(out); + } + + /** + * Looks for and parses the `Switch infobox` into a {@link MediaWikiTemplate} and then iterates over the `item#` values. + * Attempts to parse each `item#` value via `parseWikiText`, matching the `name` attribute. null values are ignored + * + * @param name only parses MediaWikiTemplates from `Switch infobox` if matches this value. (case insensitive) + * @param baseTemplate the {@link MediaWikiTemplate} representation of the `Switch infobox` to parse from + * @return List of all valid {@link MediaWikiTemplate}s matching `name` from `baseTemplate`s `item#` values + */ + public static List parseSwitchInfoboxItems(final String name, final MediaWikiTemplate baseTemplate) + { + final List templates = new ArrayList<>(); + + String value; + int suffix = 1; + while ((value = baseTemplate.getValue("item" + suffix)) != null) + { + final MediaWikiTemplate subTemplate = parseWikitext(name, value); + if (subTemplate != null) + { + templates.add(subTemplate); + } + + suffix++; + } + + return templates; + } + + private final Map map; + + private MediaWikiTemplate(final Map map) + { + this.map = map; + } + + public String getValue(final String key) + { + String val = map.get(key); + + if (Strings.isNullOrEmpty(val) || + val.equalsIgnoreCase("no") || + val.equalsIgnoreCase("n/a") || + val.equals("nil") || + val.equalsIgnoreCase("varies")) + { + return null; + } + + val = val.replace("kg", "").replaceAll("[><]", ""); + return Strings.isNullOrEmpty(val) ? null : val; + } + + public Boolean getBoolean(final String key) + { + final String val = getValue(key); + return !Strings.isNullOrEmpty(val) ? true : null; + } + + public Double getDouble(final String key) + { + final String val = getValue(key); + + if (Strings.isNullOrEmpty(val)) + { + return null; + } + + try + { + double v = Double.parseDouble(val); + return v != 0 ? v : null; + } + catch (NumberFormatException e) + { + e.printStackTrace(); + return null; + } + } + + public Integer getInt(final String key) + { + final String val = getValue(key); + + if (Strings.isNullOrEmpty(val)) + { + return null; + } + + try + { + int v = Integer.parseInt(val); + return v != 0 ? v : null; + } + catch (NumberFormatException e) + { + e.printStackTrace(); + return null; + } + } + + public boolean containsKey(final String key) + { + return map.containsKey(key); + } +} diff --git a/wiki-scraper/src/main/java/net/runelite/data/dump/wiki/ItemLimitsDumper.java b/wiki-scraper/src/main/java/net/runelite/data/dump/wiki/ItemLimitsDumper.java new file mode 100644 index 0000000000..990bac90cb --- /dev/null +++ b/wiki-scraper/src/main/java/net/runelite/data/dump/wiki/ItemLimitsDumper.java @@ -0,0 +1,155 @@ +/* + * MIT License + * + * Copyright (c) 2018 Tomas Slusny + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package net.runelite.data.dump.wiki; + +import com.google.common.base.Strings; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Stream; +import lombok.extern.slf4j.Slf4j; +import net.runelite.cache.ItemManager; +import net.runelite.cache.definitions.ItemDefinition; +import net.runelite.cache.fs.Store; +import net.runelite.cache.util.Namer; +import net.runelite.data.App; +import net.runelite.data.dump.MediaWiki; +import net.runelite.data.dump.MediaWikiTemplate; + +@Slf4j +public class ItemLimitsDumper +{ + public static void dump(final Store store, final MediaWiki wiki) throws IOException + { + final File out = new File("../runelite-client/src/main/resources/net/runelite/client/plugins/grandexchange/"); + + log.info("Dumping item limits to {}", out); + + final ItemManager itemManager = new ItemManager(store); + itemManager.load(); + final Pattern pattern = Pattern.compile("limit {6}= (.*),"); + + final Map limits = new TreeMap<>(); + final Collection items = itemManager.getItems(); + final Stream itemDefinitionStream = items.parallelStream(); + List missing = new ArrayList<>(); + + itemDefinitionStream.forEach(item -> + { + if (!item.isTradeable) + { + return; + } + + if (item.getNotedTemplate() != -1) + { + return; + } + + if (item.name.equalsIgnoreCase("NULL")) + { + return; + } + + String name = Namer + .removeTags(item.name) + .replace('\u00A0', ' ') + .replaceAll("\\+", "%2b") + .trim(); + + if (name.isEmpty()) + { + return; + } + + String data = wiki.getPageData("Module:Exchange/" + name, -1); + + if (Strings.isNullOrEmpty(data)) + { + log.debug("Data is null or empty: {}", name); + missing.add(name); + return; + } + + final MediaWikiTemplate geStats = MediaWikiTemplate.parseLua(data); + + if (geStats == null) + { + return; + } + + final Integer limit = geStats.getInt("limit"); + + if (limit == null || limit <= 0) + { + Matcher matcher = pattern.matcher(data); + String temp = ""; + while (matcher.find()) + { + temp = matcher.group(1); + if (Strings.isNullOrEmpty(temp) || + temp.equalsIgnoreCase("no") || + temp.equalsIgnoreCase("n/a") || + temp.equals("nil") || + temp.equalsIgnoreCase("varies")) + { + temp = null; + } + } + if (!Strings.isNullOrEmpty(temp)) + { + limits.put(item.id, Integer.valueOf(temp)); + } + else + { + log.debug("Item was still null: {}", name); + missing.add(name); + } + return; + } + + limits.put(item.id, limit); + log.debug("Dumped item limit for {} {}", item.id, name); + }); + + try (FileWriter fw = new FileWriter(new File(out, "ge_limits.json"))) + { + fw.write(App.GSON.toJson(limits)); + } + + log.info("Dumped {} item limits", limits.size()); + log.info("Total Missing: " + missing.size()); + missing.forEach(str -> + { + log.info("Still Missing: {}", str); + }); + } +} \ No newline at end of file diff --git a/wiki-scraper/src/main/java/net/runelite/data/dump/wiki/ItemStatsDumper.java b/wiki-scraper/src/main/java/net/runelite/data/dump/wiki/ItemStatsDumper.java new file mode 100644 index 0000000000..dca8b5d85e --- /dev/null +++ b/wiki-scraper/src/main/java/net/runelite/data/dump/wiki/ItemStatsDumper.java @@ -0,0 +1,359 @@ +/* + * MIT License + * + * Copyright (c) 2018 Tomas Slusny + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package net.runelite.data.dump.wiki; + +import com.google.common.base.Strings; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.Collection; +import java.util.Map; +import java.util.TreeMap; +import java.util.stream.Stream; +import lombok.Builder; +import lombok.Value; +import lombok.extern.slf4j.Slf4j; +import net.runelite.api.EquipmentInventorySlot; +import net.runelite.cache.ItemManager; +import net.runelite.cache.definitions.ItemDefinition; +import net.runelite.cache.fs.Store; +import net.runelite.cache.util.Namer; +import net.runelite.data.App; +import net.runelite.data.dump.MediaWiki; +import net.runelite.data.dump.MediaWikiTemplate; + +@Slf4j +public class ItemStatsDumper +{ + private final static Integer MAX_ITEMS_ON_PAGE = 50; + + public static void dump(final Store store, final MediaWiki wiki) throws IOException + { + final File out = new File("../runelite-client/src/main/resources/"); + + log.info("Dumping item stats to {}", out); + + final ItemManager itemManager = new ItemManager(store); + itemManager.load(); + + final Map itemStats = new TreeMap<>(); + final Collection items = itemManager.getItems(); + final Stream itemDefinitionStream = items.parallelStream(); + + itemDefinitionStream.forEach(item -> + { + if (item.getNotedTemplate() != -1) + { + return; + } + + if (item.name.equalsIgnoreCase("NULL")) + { + return; + } + + final String name = Namer + .removeTags(item.name) + .replace('\u00A0', ' ') + .trim(); + + if (name.isEmpty()) + { + return; + } + + String data = wiki.getSpecialLookupData("item", item.id, 0); + + if (Strings.isNullOrEmpty(data)) + { + return; + } + + MediaWikiTemplate base = MediaWikiTemplate.parseWikitext("Infobox Item", data); + + if (base == null) + { + return; + } + + final int nItems = findMaxIndex(base); + final ItemStats.ItemStatsBuilder itemStat = ItemStats.builder(); + + for (int index = 1; index <= nItems; index++) + { + final int offset = nItems == 1 ? 0 : index; + final String wikiName = getVarString(base, "name", offset); + + // Skip this index if name or itemId doesn't match with wiki + if (nItems > 1 && !wikiName.equalsIgnoreCase(name)) + { + continue; + } + + itemStat.name(getVarString(base, "name", offset) == null ? getVarString(base, "name1", offset) : getVarString(base, "name", offset)); + itemStat.quest(getVarBoolean(base, "quest", offset)); + itemStat.equipable(getVarBoolean(base, "equipable", offset) == null + ? getVarBoolean(base, "equipable1", offset) : getVarBoolean(base, "equipable", offset)); + itemStat.weight(getVarDouble(base, "weight", offset)); + + + if (Boolean.TRUE.equals(itemStat.equipable)) + { + MediaWikiTemplate stats = MediaWikiTemplate.parseWikitext("Infobox Bonuses", data); + + if (stats == null) + { + data = wiki.getSpecialLookupData("item", item.id, 1); + + if (Strings.isNullOrEmpty(data)) + { + break; + } + + stats = MediaWikiTemplate.parseWikitext("Infobox Bonuses", data); + } + + if (stats == null) + { + break; + } + + final ItemEquipmentStats.ItemEquipmentStatsBuilder equipmentStat = ItemEquipmentStats.builder(); + + equipmentStat.slot(toEquipmentSlot(getVarString(stats, "slot", offset))); + equipmentStat.astab(getVarInt(stats, "astab", offset)); + equipmentStat.aslash(getVarInt(stats, "aslash", offset)); + equipmentStat.acrush(getVarInt(stats, "acrush", offset)); + equipmentStat.amagic(getVarInt(stats, "amagic", offset)); + equipmentStat.arange(getVarInt(stats, "arange", offset)); + + equipmentStat.dstab(getVarInt(stats, "dstab", offset)); + equipmentStat.dslash(getVarInt(stats, "dslash", offset)); + equipmentStat.dcrush(getVarInt(stats, "dcrush", offset)); + equipmentStat.dmagic(getVarInt(stats, "dmagic", offset)); + equipmentStat.drange(getVarInt(stats, "drange", offset)); + + equipmentStat.str(getVarInt(stats, "str", offset)); + equipmentStat.rstr(getVarInt(stats, "rstr", offset)); + equipmentStat.mdmg(getVarInt(stats, "mdmg", offset)); + equipmentStat.prayer(getVarInt(stats, "prayer", offset)); + equipmentStat.aspeed(getVarInt(stats, "aspeed", offset)); + + final ItemEquipmentStats builtEqStat = equipmentStat.build(); + + if (!builtEqStat.equals(ItemEquipmentStats.builder().build())) + { + itemStat.equipment(builtEqStat); + } + } + + break; + } + + final ItemStats val = itemStat.build(); + + if (ItemStats.DEFAULT.equals(val)) + { + return; + } + + itemStats.put(item.id, val); + log.debug("Dumped item stat for {} {}", item.id, name); + }); + + try (FileWriter fw = new FileWriter(new File(out, "item_stats.json"))) + { + fw.write(App.GSON.toJson(itemStats)); + } + + log.info("Dumped {} item stats", itemStats.size()); + } + + /** + * Counts how many items are on page + * + * @param template media wiki template + * @return item count + */ + private static int findMaxIndex(final MediaWikiTemplate template) + { + int nItems = 1; + + if (template.getValue("version1") == null) + { + return nItems; + } + + while (nItems < MAX_ITEMS_ON_PAGE) + { + if (template.getValue(fixIndex("name", nItems + 1)) != null || template.getValue(fixIndex("version", nItems + 1)) != null) + { + nItems++; + } + else + { + break; + } + } + + return nItems; + } + + /** + * Return fixed string version of indexed key + * + * @param base key name + * @param index current index + * @return string representation of index + */ + private static String fixIndex(final String base, final Integer index) + { + return index == 0 ? base : base + index; + } + + private static String getVarString(final MediaWikiTemplate template, final String key, final Integer index) + { + final String var = template.getValue(fixIndex(key, index)); + + if (var != null) + { + return var; + } + + return template.getValue(key); + } + + private static Boolean getVarBoolean(final MediaWikiTemplate template, final String key, final Integer index) + { + final Boolean var = template.getBoolean(fixIndex(key, index)); + + if (var != null) + { + return var; + } + + return template.getBoolean(key); + } + + private static Integer getVarInt(final MediaWikiTemplate template, final String key, final Integer index) + { + final Integer var = template.getInt(fixIndex(key, index)); + + if (var != null) + { + return var; + } + + return template.getInt(key); + } + + private static Double getVarDouble(final MediaWikiTemplate template, final String key, final Integer index) + { + final Double var = template.getDouble(fixIndex(key, index)); + + if (var != null) + { + return var; + } + + return template.getDouble(key); + } + + private static Integer toEquipmentSlot(final String slotName) + { + if (slotName == null) + { + return null; + } + + switch (slotName.toLowerCase()) + { + case "weapon": + case "2h": + // TODO: 2h should return both weapon and shield somehow + return EquipmentInventorySlot.WEAPON.getSlotIdx(); + case "body": + return EquipmentInventorySlot.BODY.getSlotIdx(); + case "head": + return EquipmentInventorySlot.HEAD.getSlotIdx(); + case "ammo": + return EquipmentInventorySlot.AMMO.getSlotIdx(); + case "legs": + return EquipmentInventorySlot.LEGS.getSlotIdx(); + case "feet": + return EquipmentInventorySlot.BOOTS.getSlotIdx(); + case "hands": + return EquipmentInventorySlot.GLOVES.getSlotIdx(); + case "cape": + return EquipmentInventorySlot.CAPE.getSlotIdx(); + case "neck": + return EquipmentInventorySlot.AMULET.getSlotIdx(); + case "ring": + return EquipmentInventorySlot.RING.getSlotIdx(); + case "shield": + return EquipmentInventorySlot.SHIELD.getSlotIdx(); + } + + return null; + } + + @Value + @Builder + private static final class ItemEquipmentStats + { + private final Integer slot; + + private final Integer astab; + private final Integer aslash; + private final Integer acrush; + private final Integer amagic; + private final Integer arange; + + private final Integer dstab; + private final Integer dslash; + private final Integer dcrush; + private final Integer dmagic; + private final Integer drange; + + private final Integer str; + private final Integer rstr; + private final Integer mdmg; + private final Integer prayer; + private final Integer aspeed; + } + + @Value + @Builder + private static final class ItemStats + { + static final ItemStats DEFAULT = ItemStats.builder().build(); + + private final String name; + private final Boolean quest; + private final Boolean equipable; + private final Double weight; + + private final ItemEquipmentStats equipment; + } +} diff --git a/wiki-scraper/src/main/java/net/runelite/data/dump/wiki/NpcStatsDumper.java b/wiki-scraper/src/main/java/net/runelite/data/dump/wiki/NpcStatsDumper.java new file mode 100644 index 0000000000..1078ed462e --- /dev/null +++ b/wiki-scraper/src/main/java/net/runelite/data/dump/wiki/NpcStatsDumper.java @@ -0,0 +1,393 @@ +/* + * MIT License + * + * Copyright (c) 2019 TheStonedTurtle + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package net.runelite.data.dump.wiki; + +import com.google.common.base.Strings; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import lombok.Builder; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import net.runelite.cache.NpcManager; +import net.runelite.cache.definitions.NpcDefinition; +import net.runelite.cache.fs.Store; +import net.runelite.cache.util.Namer; +import net.runelite.data.App; +import net.runelite.data.dump.MediaWiki; +import net.runelite.data.dump.MediaWikiTemplate; + +@Slf4j +public class NpcStatsDumper +{ + @Data + @Builder + private static final class NpcStats + { + private String name; + private final Integer hitpoints; + private final Integer hitpoints1; + private final Integer combatLevel; + private final Integer slayerLevel; + private final Integer attackSpeed; + + private final Integer attackLevel; + private final Integer strengthLevel; + private final Integer defenceLevel; + private final Integer rangeLevel; + private final Integer magicLevel; + + private final Integer stab; + private final Integer slash; + private final Integer crush; + private final Integer range; + private final Integer magic; + + private final Integer stabDef; + private final Integer slashDef; + private final Integer crushDef; + private final Integer rangeDef; + private final Integer magicDef; + + private final Integer bonusAttack; + private final Integer bonusStrength; + private final Integer bonusRangeStrength; + private final Integer bonusMagicDamage; + + private final Boolean poisonImmune; + private final Boolean venomImmune; + + private final Boolean dragon; + private final Boolean demon; + private final Boolean undead; + } + + private static final NpcStats DEFAULT = NpcStats.builder().build(); + + /** + * Looks for and parses the `Switch infobox` into a {@link MediaWikiTemplate} and then iterates over the `item#` values. + * Attempts to parse each `item#` value via `parseWikiText`, matching the `name` attribute. null values are ignored + * + * @param name only parses MediaWikiTemplates from `Switch infobox` if matches this value. (case insensitive) + * @param baseTemplate the {@link MediaWikiTemplate} representation of the `Switch infobox` to parse from + * @return List of all valid {@link MediaWikiTemplate}s matching `name` from `baseTemplate`s `item#` values + */ + static List parseSwitchInfoboxItems(final String name, final MediaWikiTemplate baseTemplate) + { + final List templates = new ArrayList<>(); + + String value; + int suffix = 1; + while ((value = baseTemplate.getValue("item" + suffix)) != null) + { + final MediaWikiTemplate subTemplate = MediaWikiTemplate.parseWikitext(name, value); + if (subTemplate != null) + { + templates.add(subTemplate); + } + + suffix++; + } + + return templates; + } + + public static void dump(final Store store, final MediaWiki wiki) throws IOException + { + final File out = new File("../runelite-client/src/main/resources/"); + + log.info("Dumping npc stats to {}", out); + + final NpcManager npcManager = new NpcManager(store); + npcManager.load(); + + final Map npcStats = new HashMap<>(); + final Collection definitions = npcManager.getNpcs(); + final Stream npcDefinitionStream = definitions.parallelStream(); + + // Ensure variant names match cache as wiki isn't always correct + final Map nameMap = new HashMap<>(); + for (NpcDefinition n : definitions) + { + if (n.getName().equalsIgnoreCase("NULL")) + { + continue; + } + + final String name = Namer + .removeTags(n.getName()) + .replace('\u00A0', ' ') + .trim(); + + if (name.isEmpty()) + { + continue; + } + + nameMap.put(n.getId(), name); + } + + npcDefinitionStream.forEach(n -> + { + if (npcStats.containsKey(n.getId())) + { + return; + } + + final String name = nameMap.get(n.getId()); + if (name == null) + { + return; + } + + if (!isAttackableNpc(n)) + { + return; + } + + final String data = wiki.getSpecialLookupData("npc", n.getId(), 0); + if (Strings.isNullOrEmpty(data)) + { + return; + } + + List bases = new ArrayList<>(); + + final MediaWikiTemplate switchBase = MediaWikiTemplate.parseWikitext("Switch infobox", data); + if (switchBase != null) + { + bases = parseSwitchInfoboxItems("Infobox Monster", switchBase); + } + else + { + final MediaWikiTemplate base = MediaWikiTemplate.parseWikitext("Infobox Monster", data); + if (base == null) + { + return; + } + + bases.add(base); + } + + for (final MediaWikiTemplate base : bases) + { + int variantKey = 0; + String wikiIdString = getWikiIdString(base, variantKey); + if (wikiIdString == null) + { + // Try again as `id` will be null if there are variants and `id1` is the starting key + variantKey++; + wikiIdString = getWikiIdString(base, variantKey); + } + + while (wikiIdString != null) + { + if (wikiIdString.isEmpty()) + { + continue; + } + + final Set ids = Arrays.stream(wikiIdString.split(",")) + .map(s -> Integer.parseInt(s.trim())) + .collect(Collectors.toSet()); + + final NpcStats stats = buildNpcStats(base, variantKey); + if (!stats.equals(DEFAULT)) + { + stats.setName(name); + for (final int curID : ids) + { + // Update variant name or fall back to current name + final String curName = nameMap.get(curID); + stats.setName(curName == null ? stats.getName() : curName); + + npcStats.put(curID, stats); + log.debug("Dumped npc stats for npc id: {}", curID); + } + } + + variantKey++; + wikiIdString = getWikiIdString(base, variantKey); + } + } + }); + + // Cast to TreeMap so sort output JSON in numerical order (npc id) + final Map sorted = new TreeMap<>(npcStats); + + try (FileWriter fw = new FileWriter(new File(out, "npc_stats.json"))) + { + fw.write(App.GSON.toJson(sorted)); + } + +// try (FileWriter fw = new FileWriter(new File(out, "npc_stats.min.json"))) +// { +// fw.write(new GsonBuilder().disableHtmlEscaping().create().toJson(sorted)); +// } + + log.info("Dumped {} npc stats", sorted.size()); + } + + private static boolean isAttackableNpc(final NpcDefinition n) + { + for (final String s : n.getOptions()) + { + if ("attack".equalsIgnoreCase(s)) + { + return true; + } + } + + return false; + } + + private static String getKeySuffix(final int variantKey) + { + return variantKey > 0 ? String.valueOf(variantKey) : ""; + } + + private static String getWikiIdString(final MediaWikiTemplate template, final int variantKey) + { + return template.getValue("id" + getKeySuffix(variantKey)); + } + + private static NpcStats buildNpcStats(final MediaWikiTemplate template, int variantKey) + { + final NpcStats.NpcStatsBuilder stats = NpcStats.builder(); + + stats.hitpoints(getInt("hitpoints", variantKey, template)); + if (stats.hitpoints == null) + { + stats.hitpoints(getInt("hitpoints1", variantKey, template)); + } + stats.combatLevel(getInt("combat", variantKey, template)); + stats.slayerLevel(getInt("slaylvl", variantKey, template)); + stats.attackSpeed(getInt("attack speed", variantKey, template)); + + stats.attackLevel(getInt("att", variantKey, template)); + stats.strengthLevel(getInt("str", variantKey, template)); + stats.defenceLevel(getInt("def", variantKey, template)); + stats.rangeLevel(getInt("range", variantKey, template)); + stats.magicLevel(getInt("mage", variantKey, template)); + + stats.stab(getInt("astab", variantKey, template)); + stats.slash(getInt("aslash", variantKey, template)); + stats.crush(getInt("acrush", variantKey, template)); + stats.range(getInt("arange", variantKey, template)); + stats.magic(getInt("amagic", variantKey, template)); + + stats.stabDef(getInt("dstab", variantKey, template)); + stats.slashDef(getInt("dslash", variantKey, template)); + stats.crushDef(getInt("dcrush", variantKey, template)); + stats.rangeDef(getInt("drange", variantKey, template)); + stats.magicDef(getInt("dmagic", variantKey, template)); + + stats.bonusAttack(getInt("attbns", variantKey, template)); + stats.bonusStrength(getInt("strbns", variantKey, template)); + stats.bonusRangeStrength(getInt("rngbns", variantKey, template)); + stats.bonusMagicDamage(getInt("mbns", variantKey, template)); + + final String keySuffix = getKeySuffix(variantKey); + boolean pImmune = "immune".equalsIgnoreCase(template.getValue("immunepoison" + keySuffix)); + boolean vImmune = "immune".equalsIgnoreCase(template.getValue("immunevenom" + keySuffix)); + + stats.poisonImmune(!pImmune ? null : true); + stats.venomImmune(!vImmune ? null : true); + + final String weaknessValue = template.getValue("weakness"); + if (weaknessValue != null) + { + final String[] values = weaknessValue.split(","); + for (String value : values) + { + value = value.toLowerCase(); + if (stats.dragon == null && (value.contains("dragonbane weapons"))) + { + stats.dragon(true); + } + + if (stats.demon == null && (value.contains("demonbane weapons") || value.contains("silverlight") || value.contains("arclight"))) + { + stats.demon(true); + } + + if (stats.undead == null && (value.contains("salve amulet") || value.contains("crumble undead"))) + { + stats.undead(true); + } + } + } + + return stats.build(); + } + + static Integer getInt(final String mainKey, final Integer variation, final MediaWikiTemplate template) + { + final String key = mainKey + getKeySuffix(variation); + if (!template.containsKey(key)) + { + if (variation >= 1) + { + // Use variation fallback via recursion + return getInt(mainKey, variation - 1, template); + } + + return null; + } + + final String val = template.getValue(key); + if (Strings.isNullOrEmpty(val)) + { + return null; + } + + try + { + // Remove everything after the first non-number character to account for any comments + final String fixedVal = val.trim().replaceAll("\\D+.*", ""); + if (fixedVal.isEmpty()) + { + return null; + } + + int v = Integer.parseInt(fixedVal); + return v != 0 ? v : null; + } + catch (NumberFormatException e) + { + e.printStackTrace(); + return null; + } + } +} diff --git a/wiki-scraper/src/test/java/net/runelite/data/dump/MediaWikiTemplateTest.java b/wiki-scraper/src/test/java/net/runelite/data/dump/MediaWikiTemplateTest.java new file mode 100644 index 0000000000..89ab058c1a --- /dev/null +++ b/wiki-scraper/src/test/java/net/runelite/data/dump/MediaWikiTemplateTest.java @@ -0,0 +1,549 @@ +/* + * MIT License + * + * Copyright (c) 2018 Tomas Slusny + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package net.runelite.data.dump; + +import java.util.List; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import org.junit.jupiter.api.Test; + +class MediaWikiTemplateTest +{ + @Test + void parseInfoboxItem1() + { + final String data = + "{{Infobox Item\n" + + "|name = Dragon claws\n" + + "|image = [[File:Dragon claws.png]]\n" + + "|release = [[5 January]] [[2017]]\n" + + "|update = Dragon Claws & 3rd Birthday\n" + + "|members = Yes\n" + + "|quest = No\n" + + "|tradeable = Yes\n" + + "|equipable = Yes\n" + + "|stackable = No\n" + + "|high = 123000\n" + + "|low = 82000\n" + + "|destroy = Drop\n" + + "|store = No\n" + + "|exchange = gemw\n" + + "|examine = A set of fighting claws.\n" + + "|weight = 0\n" + + "}}\n"; + + final MediaWikiTemplate template = MediaWikiTemplate.parseWikitext("Infobox Item", data); + assertNotNull(template); + assertEquals(123000, (int) template.getInt("high")); + } + + @Test + void parseInfoboxItem2() + { + final String data = + "{{Infobox item\n" + + "|name = Magic shortbow (i)\n" + + "|image = [[File:Magic shortbow (i).png]]\n" + + "|release = [[18 September]] [[2014]]\n" + + "|update = Bounty Hunter\n" + + "|members = Yes\n" + + "|tradeable = No\n" + + "|equipable = Yes\n" + + "|stackable = No\n" + + "|quest = No\n" + + "|low = 640\n" + + "|high = 960\n" + + "|store = No\n" + + "|examine = Short and magical, but still effective.\n" + + "|weight = 1\n" + + "|destroy = Drop\n" + + "}}\n"; + + final MediaWikiTemplate template = MediaWikiTemplate.parseWikitext("Infobox Item", data); + assertNotNull(template); + assertEquals((int) template.getInt("high"), 960); + } + + @Test + void parseInfoboxItem3() + { + final String data = + "{{Infobox Item\n" + + "|name = Proselyte cuisse\n" + + "|image = [[File:Proselyte cuisse.png|Proselyte cuisse]]\n" + + "|release = [[20 September]] [[2006]]\n" + + "|update = Slug Menace\n" + + "|members = Yes\n" + + "|quest = [[The Slug Menace]]\n" + + "|tradeable = Yes\n" + + "|equipable = Yes\n" + + "|stackable = No\n" + + "|noteable = Yes\n" + + "|placeholder = Yes\n" + + "|destroy = Drop\n" + + "|value = 10000\n" + + "|store = 10000\n" + + "|exchange = gemw\n" + + "|weight = 7.711\n" + + "|examine = A Proselyte Temple Knight's leg armour.\n" + + "|id = 9676,20565\n" + + "}}"; + + final MediaWikiTemplate template = MediaWikiTemplate.parseWikitext("Infobox Item", data); + assertNotNull(template); + } + + @Test + void parseInfoboxItem4() + { + final String data = + "{{Infobox Item\n" + + "|name = Explorer's ring 2\n" + + "|image = [[File:Explorer's ring 2.png]]\n" + + "|release = [[5 March]] [[2015]]\n" + + "|update = Achievement Diaries\n" + + "|members = Yes\n" + + "|quest = No\n" + + "|tradeable = No\n" + + "|equipable = Yes\n" + + "|stackable = No\n" + + "|noteable = No\n" + + "|placeholder = Yes\n" + + "|destroy = Drop\n" + + "|value = 0\n" + + "|store = No\n" + + "|weight = 0\n" + + "|examine = A Lumbridge explorer's ring.\n" + + "|id = 13126\n" + + "}}"; + + final MediaWikiTemplate template = MediaWikiTemplate.parseWikitext("Infobox Item", data); + assertNotNull(template); + } + + @Test + void parseInfoboxItem5() + { + final String data = + "{{Infobox Item\n" + + "|version1 = Normal\n" + + "|version2 = Broken\n" + + "|name1 = Fighter hat\n" + + "|name2 = Fighter hat (broken)\n" + + "|image1 = [[File:Fighter hat.png]]\n" + + "|image2 = [[File:Fighter hat (broken).png]]\n" + + "|release1 = [[4 January]] [[2007]]\n" + + "|release2 = [[21 July]] [[2016]]\n" + + "|update1 = Barbarian Assault\n" + + "|update2 = Broken Armour & Open Beta\n" + + "|members = Yes\n" + + "|quest = No\n" + + "|tradeable = No\n" + + "|equipable1 = Yes\n" + + "|equipable2 = No\n" + + "|stackable = No\n" + + "|noteable = No\n" + + "|placeholder = Yes\n" + + "|destroy = Drop\n" + + "|value1 = 65002\n" + + "|value2 = 1\n" + + "|alchable = No\n" + + "|store1 = 275\n" + + "|store2 = No\n" + + "|currency = Honour points in each role; must have also killed [[Penance Queen|Queen]]\n" + + "|seller = Commander Connad\n" + + "|weight = 2\n" + + "|examine1 = A Penance Fighter hat.\n" + + "|examine2 = A broken Penance Fighter hat.\n" + + "|id1 = 10548\n" + + "|id2 = 20507\n" + + "}}\n"; + + final MediaWikiTemplate template = MediaWikiTemplate.parseWikitext("Infobox Item", data); + assertNotNull(template); + } + + @Test + void parseInfoboxItem6() + { + final String data = + "{{Infobox Item\n" + + "|version1 = Unpoisoned\n" + + "|version2 = Poison\n" + + "|version3 = Poison+\n" + + "|version4 = Poison++\n" + + "|version5 = Karambwan poison\n" + + "|name1 = Iron hasta\n" + + "|name2 = Iron hasta(p)\n" + + "|name3 = Iron hasta(p+)\n" + + "|name4 = Iron hasta(p++)\n" + + "|name5 = Iron hasta(kp)\n" + + "|image1 = [[File:Iron hasta.png]]\n" + + "|image2 = [[File:Iron hasta(p).png]]\n" + + "|image3 = [[File:Iron hasta(p+).png]]\n" + + "|image4 = [[File:Iron hasta(p++).png]]\n" + + "|image5 = [[File:Iron hasta(kp).png]]\n" + + "|release = [[3 July]] [[2007]]\n" + + "|update = Barbarian Training\n" + + "|members = Yes\n" + + "|quest = No\n" + + "|tradeable1 = Yes\n" + + "|tradeable2 = Yes\n" + + "|tradeable3 = Yes\n" + + "|tradeable4 = Yes\n" + + "|tradeable5 = No\n" + + "|equipable = Yes\n" + + "|stackable = No\n" + + "|noteable1 = Yes\n" + + "|noteable2 = Yes\n" + + "|noteable3 = Yes\n" + + "|noteable4 = Yes\n" + + "|noteable5 = No\n" + + "|placeholder = Yes\n" + + "|destroy = Drop\n" + + "|value = 91\n" + + "|store = No\n" + + "|exchange1 = gemw\n" + + "|exchange2 = gemw\n" + + "|exchange3 = gemw\n" + + "|exchange4 = gemw\n" + + "|weight1 = 2.267\n" + + "|weight2 = 2.267\n" + + "|weight3 = 2\n" + + "|weight4 = 2.267\n" + + "|weight5 = 2.267\n" + + "|examine1 = An iron-tipped, one-handed hasta.\n" + + "|examine2 = A poison-tipped, one-handed iron hasta.\n" + + "|examine3 = A poison-tipped, one-handed iron hasta.\n" + + "|examine4 = A poison-tipped, one-handed iron hasta.\n" + + "|examine5 = A karambwan poison-tipped, one-handed iron hasta.\n" + + "|id1 = 11369\n" + + "|id2 = 11386\n" + + "|id3 = 11389\n" + + "|id4 = 11391\n" + + "|id5 = 11388\n" + + "}}"; + + final MediaWikiTemplate template = MediaWikiTemplate.parseWikitext("Infobox Item", data); + assertNotNull(template); + } + + @Test + void parseInfoboxBonuses1() + { + final String data = + "{{Infobox Bonuses\n" + + "|astab = 41\n" + + "|aslash = 57\n" + + "|acrush = -4\n" + + "|amagic = 0\n" + + "|arange = 0\n" + + "|dstab = 13\n" + + "|dslash = 26\n" + + "|dcrush = 7\n" + + "|dmagic = 0\n" + + "|drange = 0\n" + + "|str = 56\n" + + "|rstr = 0\n" + + "|mdmg = 0\n" + + "|prayer = 0\n" + + "|caption = A player wearing dragon claws.\n" + + "|aspeed = 4|slot = 2h\n" + + "|image = Dragon claws equipped.png{{!}}130px}}\n"; + + final MediaWikiTemplate template = MediaWikiTemplate.parseWikitext("Infobox Bonuses", data); + assertNotNull(template); + assertEquals(4, (int) template.getInt("aspeed")); + assertEquals("2h", template.getValue("slot")); + assertEquals("Dragon claws equipped.png{{!}}130px", template.getValue("image")); + } + + @Test + void parseInfoboxBonuses2() + { + final String data = + "{{Infobox Bonuses\n" + + "|version1 = Uncharged\n" + + "|version2 = Charged\n" + + "|image_1 = Dragonfire shield (uncharged) equipped.png{{!}}150px\n" + + "|image_2 = Dragonfire shield equipped.png{{!}}150px\n" + + "|astab = 0\n" + + "|aslash = 0\n" + + "|acrush = 0\n" + + "|amagic = -10\n" + + "|arange = -5\n" + + "|dstab1 = +20\n" + + "|dslash1 = +25\n" + + "|dcrush1 = +22\n" + + "|dmagic1 = +10\n" + + "|drange1 = +22\n" + + "|dstab2 = +70\n" + + "|dslash2 = +75\n" + + "|dcrush2 = +72\n" + + "|dmagic2 = +10\n" + + "|drange2 = +72\n" + + "|str = +7\n" + + "|rstr = 0\n" + + "|mdmg = 0\n" + + "|prayer = 0\n" + + "|slot = Shield\n" + + "}}"; + + final MediaWikiTemplate template = MediaWikiTemplate.parseWikitext("Infobox Bonuses", data); + assertNotNull(template); + assertEquals(70, (int) template.getInt("dstab2")); + } + + @Test + void parseInfoboxBonuses3() + { + final String data = + "{{Infobox Bonuses\n" + + "|astab = 0\n" + + "|aslash = 0\n" + + "|acrush = 0\n" + + "|amagic = -21\n" + + "|arange = -7\n" + + "|dstab = +33\n" + + "|dslash = +31\n" + + "|dcrush = +29\n" + + "|dmagic = -4\n" + + "|drange = +31\n" + + "|str = 0\n" + + "|rstr = 0\n" + + "|mdmg = 0\n" + + "|prayer = +6\n" + + "|slot = Legs\n" + + "|image = Proselyte armour equipped.png{{!}}110px\n" + + "|caption = A player wearing proselyte armour.\n" + + "}}"; + + final MediaWikiTemplate template = MediaWikiTemplate.parseWikitext("Infobox Bonuses", data); + assertNotNull(template); + assertEquals((int) template.getInt("prayer"), 6); + } + + @Test + void parseInfoboxBonuses4() + { + final String data = + "{{Infobox Bonuses\n" + + "|image = \n" + + "|caption = \n" + + "|astab =0 \n" + + "|aslash =0 \n" + + "|acrush =0 \n" + + "|amagic =0 \n" + + "|arange =0 \n" + + "|dstab =0 \n" + + "|dslash =0 \n" + + "|dcrush =0 \n" + + "|dmagic =0 \n" + + "|drange =0 \n" + + "|str =0 \n" + + "|prayer =+1 \n" + + "|slot = ring\n" + + "|rstr = 0\n" + + "|mdmg = 0\n" + + "}}"; + + final MediaWikiTemplate template = MediaWikiTemplate.parseWikitext("Infobox Bonuses", data); + assertNotNull(template); + } + + @Test + void parseInfoboxBonuses5() + { + final String data = + "{{Infobox Bonuses|| astab = +8\n" + + "| aspeed = 5\n" + + "| aslash = -2\n" + + "| acrush = +6\n" + + "| amagic = 0\n" + + "| arange = 0\n" + + "| dstab = 0\n" + + "| dslash = +1\n" + + "| dcrush = 0\n" + + "| dmagic = 0\n" + + "| drange = 0\n" + + "| str = +9\n" + + "|rstr = 0\n" + + "|mdmg = 0\n" + + "| prayer = 0\n" + + "|image = Steel pickaxe equipped.png{{!}}150px\n" + + "|caption = A player wielding a steel pickaxe.\n" + + "||slot = Weapon}}"; + + final MediaWikiTemplate template = MediaWikiTemplate.parseWikitext("Infobox Bonuses", data); + assertNotNull(template); + } + + @Test + void parseLua() + { + final String exchangeInfoData = + "return {\n" + + " itemId = 13652,\n" + + " price = 83173735,\n" + + " last = 83533604,\n" + + " date = '12:18, November 08, 2018 (UTC)',\n" + + " lastDate = '05:43, November 08, 2018 (UTC)',\n" + + " icon = 'Dragon claws.png',\n" + + " item = 'Dragon claws',\n" + + " value = -205000,\n" + + " limit = nil,\n" + + " members = true,\n" + + " category = nil,\n" + + " examine = 'A set of fighting claws.'\n" + + "}\n"; + + final MediaWikiTemplate exchangeInfo = MediaWikiTemplate.parseLua(exchangeInfoData); + assertNotNull(exchangeInfo); + assertEquals((int) exchangeInfo.getInt("value"), -205000); + } + + @Test + void parseKeysWithSpaces() + { + final String data = + "{{Infobox Monster\n" + + "|name = Aberrant spectre\n" + + "|combat = 96\n" + + "|attack speed = 4\n" + + "|foo attack style= Magic\n" + + "|id = 2,3,4,5,6,7\n" + + "}}"; + + final MediaWikiTemplate template = MediaWikiTemplate.parseWikitext("Infobox Monster", data); + assertNotNull(template); + assertEquals(template.getInt("combat"), 96); + assertEquals(template.getInt("attack speed"), 4); + assertEquals(template.getValue("foo attack style"), "Magic"); + } + + @Test + void parseWikitextExactName() + { + final String data = + "{{ Infobox Monster/sandbox \n" + + "|version1 = Lv 51\n" + + "|version2 = Lv 76\n" + + "|name = Brawler\n" + + "|combat1 = 51\n" + + "|combat2 = 76\n" + + "|hitpoints1 = 53\n" + + "|hitpoints2 = 83\n" + + "|max hit1 = 7\n" + + "|max hit2 = 9\n" + + "|slaylvl = No\n" + + "|slayxp = No\n" + + "|att1 = \n" + + "|att2 = \n" + + "|id1 = 1734\n" + + "|id2 = 1735\n" + + "}}"; + + final MediaWikiTemplate template = MediaWikiTemplate.parseWikitext("Infobox Monster/sandbox", data); + assertNotNull(template); + + final MediaWikiTemplate template2 = MediaWikiTemplate.parseWikitext("Infobox Monster", data); + assertNull(template2); + } + + @Test + void parseSwitchInfobox() + { + final String data = + "{{External|rs}}\n" + + "{{Switch infobox\n" + + "|item1= \n" + + "{{Infobox Monster\n" + + "|name = Ghast\n" + + "|combat = 30\n" + + "|id = 946\n" + + "}}\n" + + "|text1 = Level 30\n" + + "|item2= \n" + + "{{Infobox Monster\n" + + "|name = Ghast\n" + + "|combat = 79\n" + + "|id = 5625\n" + + "}}\n" + + "|text2 = Level 79\n" + + "|item3= \n" + + "{{Infobox Monster\n" + + "|name = Ghast\n" + + "|combat = 109\n" + + "|id = 5626\n" + + "}}\n" + + "|text3 = Level 109\n" + + "|item4= \n" + + "{{Infobox Monster\n" + + "|name = Ghast\n" + + "|combat = 139\n" + + "|id = 5627\n" + + "}}\n" + + "|text4 = Level 139\n" + + "|item5 =\n" + + "{{Infobox non-player character\n" + + "|name = \n" + + "|update = Nature Spirit Quest\n" + + "|race = Undead\n" + + "|members = Yes\n" + + "|quest = [[Nature Spirit]]\n" + + "|location = [[Morytania]]\n" + + "|shop = No\n" + + "|gender = N/A\n" + + "|examine = \n" + + "|id = 945, 5622, 5623, 5624\n" + + "}}\n" + + "|text5 = Invisible\n" + + "}}"; + + final MediaWikiTemplate switchInfobox = MediaWikiTemplate.parseWikitext("Switch infobox", data); + assertNotNull(switchInfobox); + + // Infobox monster + final List templates = MediaWikiTemplate.parseSwitchInfoboxItems("Infobox monster", switchInfobox); + assertEquals(templates.size(), 4); + + final MediaWikiTemplate item1 = templates.get(0); + assertEquals(item1.getInt("combat"), 30); + + final MediaWikiTemplate item2 = templates.get(1); + assertEquals(item2.getInt("combat"), 79); + + // Infobox non-player character + final List npcs = MediaWikiTemplate.parseSwitchInfoboxItems("Infobox non-player character", switchInfobox); + assertEquals(npcs.size(), 1); + + final MediaWikiTemplate npc1 = npcs.get(0); + assertEquals(npc1.getValue("race"), "Undead"); + + // Infobox item + final List items = MediaWikiTemplate.parseSwitchInfoboxItems("Infobox item", switchInfobox); + assertEquals(items.size(), 0); + } +} \ No newline at end of file diff --git a/wiki-scraper/src/test/java/net/runelite/data/dump/wiki/NpcStatsDumperTest.java b/wiki-scraper/src/test/java/net/runelite/data/dump/wiki/NpcStatsDumperTest.java new file mode 100644 index 0000000000..68b6ff5cca --- /dev/null +++ b/wiki-scraper/src/test/java/net/runelite/data/dump/wiki/NpcStatsDumperTest.java @@ -0,0 +1,129 @@ +/* + * MIT License + * + * Copyright (c) 2019 TheStonedTurtle + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package net.runelite.data.dump.wiki; + +import java.util.List; +import net.runelite.data.dump.MediaWikiTemplate; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import org.junit.jupiter.api.Test; + +class NpcStatsDumperTest +{ + @Test + void npcVariantFallThrough() + { + final String data = + "{{Infobox Monster\n" + + "|combat = 2\n" + + "|combat8 = \n" + + "|combat10 = 4\n" + + "}}"; + + + final MediaWikiTemplate template = MediaWikiTemplate.parseWikitext("Infobox Monster", data); + assertNotNull(template); + + assertEquals(NpcStatsDumper.getInt("combat", 0, template), 2); + assertEquals(NpcStatsDumper.getInt("combat", 7, template), 2); + assertNull(NpcStatsDumper.getInt("combat", 8, template)); + assertNull(NpcStatsDumper.getInt("combat", 9, template)); + assertEquals(NpcStatsDumper.getInt("combat", 10, template), 4); + } + + @Test + void parseSwitchInfoboxItems() + { + final String data = + "{{Switch infobox\n" + + "|item1= \n" + + "{{Infobox Monster\n" + + "|name = Ghast\n" + + "|combat = 30\n" + + "|id = 946\n" + + "}}\n" + + "|text1 = Level 30\n" + + "|item2= \n" + + "{{Infobox Monster\n" + + "|name = Ghast\n" + + "|combat = 79\n" + + "|id = 5625\n" + + "}}\n" + + "|text2 = Level 79\n" + + "|item3= \n" + + "{{Infobox Monster\n" + + "|name = Ghast\n" + + "|combat = 109\n" + + "|id = 5626\n" + + "}}\n" + + "|text3 = Level 109\n" + + "|item4= \n" + + "{{Infobox Monster\n" + + "|name = Ghast\n" + + "|combat = 139\n" + + "|id = 5627\n" + + "}}\n" + + "|text4 = Level 139\n" + + "|item5 =\n" + + "{{Infobox non-player character\n" + + "|name = \n" + + "|update = Nature Spirit Quest\n" + + "|race = Undead\n" + + "|members = Yes\n" + + "|quest = [[Nature Spirit]]\n" + + "|location = [[Morytania]]\n" + + "|shop = No\n" + + "|gender = N/A\n" + + "|examine = \n" + + "|id = 945, 5622, 5623, 5624\n" + + "}}\n" + + "|text5 = Invisible\n" + + "}}"; + + final MediaWikiTemplate switchInfobox = MediaWikiTemplate.parseWikitext("Switch infobox", data); + assertNotNull(switchInfobox); + + // Infobox monster + final List templates = NpcStatsDumper.parseSwitchInfoboxItems("Infobox monster", switchInfobox); + assertEquals(templates.size(), 4); + + final MediaWikiTemplate item1 = templates.get(0); + assertEquals(item1.getInt("combat"), 30); + + final MediaWikiTemplate item2 = templates.get(1); + assertEquals(item2.getInt("combat"), 79); + + // Infobox non-player character + final List npcs = NpcStatsDumper.parseSwitchInfoboxItems("Infobox non-player character", switchInfobox); + assertEquals(npcs.size(), 1); + + final MediaWikiTemplate npc1 = npcs.get(0); + assertEquals(npc1.getValue("race"), "Undead"); + + // Infobox item + final List items = NpcStatsDumper.parseSwitchInfoboxItems("Infobox item", switchInfobox); + assertEquals(items.size(), 0); + } +}