diff --git a/http-service/src/main/java/net/runelite/http/service/xp/XpTrackerService.java b/http-service/src/main/java/net/runelite/http/service/xp/XpTrackerService.java index 391d37d1bf..5ef7e829e4 100644 --- a/http-service/src/main/java/net/runelite/http/service/xp/XpTrackerService.java +++ b/http-service/src/main/java/net/runelite/http/service/xp/XpTrackerService.java @@ -29,8 +29,8 @@ import com.google.common.hash.Funnels; import java.nio.charset.Charset; import java.time.Duration; import java.time.Instant; +import java.util.ArrayDeque; import java.util.Queue; -import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.ExecutionException; import lombok.extern.slf4j.Slf4j; import net.runelite.http.api.hiscore.HiscoreEndpoint; @@ -50,8 +50,8 @@ import org.sql2o.Sql2o; @Slf4j public class XpTrackerService { - private static final int QUEUE_LIMIT = 100_000; - private static final Duration UPDATE_TIME = Duration.ofMinutes(5); + private static final int QUEUE_LIMIT = 32768; + private static final int BLOOMFILTER_EXPECTED_INSERTIONS = 100_000; @Autowired @Qualifier("Runelite XP Tracker SQL2O") @@ -60,7 +60,7 @@ public class XpTrackerService @Autowired private HiscoreService hiscoreService; - private final Queue usernameUpdateQueue = new ConcurrentLinkedDeque<>(); + private final Queue usernameUpdateQueue = new ArrayDeque<>(); private BloomFilter usernameFilter = createFilter(); public void update(String username) throws ExecutionException @@ -76,13 +76,37 @@ public class XpTrackerService return; } - if (usernameUpdateQueue.size() >= QUEUE_LIMIT) + try (Connection con = sql2o.open()) { - log.warn("Username update queue is full ({})", QUEUE_LIMIT); - return; + PlayerEntity playerEntity = findOrCreatePlayer(con, username); + Duration frequency = updateFrequency(playerEntity); + Instant now = Instant.now(); + Duration timeSinceLastUpdate = Duration.between(playerEntity.getLast_updated(), now); + if (timeSinceLastUpdate.toMillis() < frequency.toMillis()) + { + log.debug("User {} updated too recently", username); + usernameFilter.put(username); + return; + } + + synchronized (usernameUpdateQueue) + { + if (usernameUpdateQueue.size() >= QUEUE_LIMIT) + { + log.warn("Username update queue is full ({})", QUEUE_LIMIT); + return; + } + } + + con.createQuery("update player set last_updated = CURRENT_TIMESTAMP where id = :id") + .addParameter("id", playerEntity.getId()) + .executeUpdate(); } - usernameUpdateQueue.add(username); + synchronized (usernameUpdateQueue) + { + usernameUpdateQueue.add(username); + } usernameFilter.put(username); } @@ -104,13 +128,6 @@ public class XpTrackerService log.debug("Hiscore for {} already up to date", username); return; } - - Duration difference = Duration.between(currentXp.getTime(), now); - if (difference.compareTo(UPDATE_TIME) <= 0) - { - log.debug("Updated {} too recently", username); - return; - } } con.createQuery("insert into xp (player,attack_xp,defence_xp,strength_xp,hitpoints_xp,ranged_xp,prayer_xp,magic_xp,cooking_xp,woodcutting_xp," @@ -172,6 +189,11 @@ public class XpTrackerService .addParameter("construction_rank", hiscoreResult.getConstruction().getRank()) .addParameter("overall_rank", hiscoreResult.getOverall().getRank()) .executeUpdate(); + + con.createQuery("update player set rank = :rank where id = :id") + .addParameter("id", playerEntity.getId()) + .addParameter("rank", hiscoreResult.getOverall().getRank()) + .executeUpdate(); } } @@ -197,6 +219,7 @@ public class XpTrackerService playerEntity.setId(id); playerEntity.setName(username); playerEntity.setTracked_since(now); + playerEntity.setLast_updated(now); return playerEntity; } @@ -220,18 +243,21 @@ public class XpTrackerService @Scheduled(fixedDelay = 1000) public void update() throws ExecutionException { - String next = usernameUpdateQueue.poll(); + String next; + synchronized (usernameUpdateQueue) + { + next = usernameUpdateQueue.poll(); + } if (next == null) { return; } - HiscoreResult hiscoreResult = hiscoreService.lookupUsername(next, HiscoreEndpoint.NORMAL); - update(next, hiscoreResult); + update(next); } - @Scheduled(fixedDelay = 3 * 60 * 60 * 1000) // 3 hours + @Scheduled(fixedDelay = 6 * 60 * 60 * 1000) // 6 hours public void clearFilter() { usernameFilter = createFilter(); @@ -241,14 +267,47 @@ public class XpTrackerService { final BloomFilter filter = BloomFilter.create( Funnels.stringFunnel(Charset.defaultCharset()), - 100_000 + BLOOMFILTER_EXPECTED_INSERTIONS ); - for (String toUpdate : usernameUpdateQueue) + synchronized (usernameUpdateQueue) { - filter.put(toUpdate); + for (String toUpdate : usernameUpdateQueue) + { + filter.put(toUpdate); + } } return filter; } + + /** + * scale how often to check hiscore updates for players based on their rank + * @param playerEntity + * @return + */ + private static Duration updateFrequency(PlayerEntity playerEntity) + { + Integer rank = playerEntity.getRank(); + if (rank == null) + { + return Duration.ofDays(7); + } + else if (rank < 10_000) + { + return Duration.ofHours(6); + } + else if (rank < 50_000) + { + return Duration.ofDays(2); + } + else if (rank < 100_000) + { + return Duration.ofDays(5); + } + else + { + return Duration.ofDays(7); + } + } } diff --git a/http-service/src/main/java/net/runelite/http/service/xp/beans/PlayerEntity.java b/http-service/src/main/java/net/runelite/http/service/xp/beans/PlayerEntity.java index d07d1d4640..11bec532d7 100644 --- a/http-service/src/main/java/net/runelite/http/service/xp/beans/PlayerEntity.java +++ b/http-service/src/main/java/net/runelite/http/service/xp/beans/PlayerEntity.java @@ -33,4 +33,6 @@ public class PlayerEntity private Integer id; private String name; private Instant tracked_since; + private Instant last_updated; + private Integer rank; } diff --git a/http-service/src/main/resources/net/runelite/http/service/xp/schema.sql b/http-service/src/main/resources/net/runelite/http/service/xp/schema.sql index ba4c395d62..70f83a30fd 100644 --- a/http-service/src/main/resources/net/runelite/http/service/xp/schema.sql +++ b/http-service/src/main/resources/net/runelite/http/service/xp/schema.sql @@ -1,8 +1,8 @@ --- MySQL dump 10.16 Distrib 10.2.9-MariaDB, for Linux (x86_64) +-- MySQL dump 10.16 Distrib 10.2.18-MariaDB, for Linux (x86_64) -- -- Host: localhost Database: xptracker -- ------------------------------------------------------ --- Server version 10.2.9-MariaDB +-- Server version 10.2.18-MariaDB /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; @@ -26,6 +26,8 @@ CREATE TABLE `player` ( `id` int(11) NOT NULL AUTO_INCREMENT, `name` varchar(32) NOT NULL, `tracked_since` timestamp NOT NULL DEFAULT current_timestamp(), + `last_updated` timestamp NOT NULL DEFAULT current_timestamp(), + `rank` int(11) DEFAULT NULL, PRIMARY KEY (`id`), UNIQUE KEY `name` (`name`) ) ENGINE=InnoDB DEFAULT CHARSET=latin1; @@ -116,7 +118,7 @@ CREATE TABLE `xp` ( `overall_rank` int(11) NOT NULL, PRIMARY KEY (`id`), UNIQUE KEY `player_time` (`player`,`time`), - INDEX `idx_time` (`time`), + KEY `idx_time` (`time`), CONSTRAINT `fk_player` FOREIGN KEY (`player`) REFERENCES `player` (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=latin1; /*!40101 SET character_set_client = @saved_cs_client */; @@ -130,4 +132,4 @@ CREATE TABLE `xp` ( /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; --- Dump completed on 2018-01-20 18:37:09 +-- Dump completed on 2019-02-15 21:01:17 \ No newline at end of file