ra/runelite
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

xptracker: limit how often the same players are looked up

Browse Source 
Use player rank to set how often the same player is allowed to be looked
up. Replace the ConcurrentLinkedDeque with a synchronized ArrayDeque
which has a constant time size().
This commit is contained in:
Adam
2019-02-15 21:04:42 -05:00
parent a0cdd64719
commit bc0ec093dc
3 changed files with 89 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

103
http-service/src/main/java/net/runelite/http/service/xp/XpTrackerService.java
View File

@@ -29,8 +29,8 @@ import com.google.common.hash.Funnels;
import java.nio.charset.Charset;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayDeque;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedDeque;
import java.util.concurrent.ExecutionException;
import lombok.extern.slf4j.Slf4j;
import net.runelite.http.api.hiscore.HiscoreEndpoint;
@@ -50,8 +50,8 @@ import org.sql2o.Sql2o;
@Slf4j
public class XpTrackerService
{
private static final int QUEUE_LIMIT = 100_000;
private static final Duration UPDATE_TIME = Duration.ofMinutes(5);
private static final int QUEUE_LIMIT = 32768;
private static final int BLOOMFILTER_EXPECTED_INSERTIONS = 100_000;
@Autowired
@Qualifier("Runelite XP Tracker SQL2O")
@@ -60,7 +60,7 @@ public class XpTrackerService
@Autowired
private HiscoreService hiscoreService;
private final Queue<String> usernameUpdateQueue = new ConcurrentLinkedDeque<>();
private final Queue<String> usernameUpdateQueue = new ArrayDeque<>();
private BloomFilter<String> usernameFilter = createFilter();
public void update(String username) throws ExecutionException
@@ -76,13 +76,37 @@ public class XpTrackerService
return;
}
if (usernameUpdateQueue.size() >= QUEUE_LIMIT)
try (Connection con = sql2o.open())
{
log.warn("Username update queue is full ({})", QUEUE_LIMIT);
return;
PlayerEntity playerEntity = findOrCreatePlayer(con, username);
Duration frequency = updateFrequency(playerEntity);
Instant now = Instant.now();
Duration timeSinceLastUpdate = Duration.between(playerEntity.getLast_updated(), now);
if (timeSinceLastUpdate.toMillis() < frequency.toMillis())
{
log.debug("User {} updated too recently", username);
usernameFilter.put(username);
return;
}
synchronized (usernameUpdateQueue)
{
if (usernameUpdateQueue.size() >= QUEUE_LIMIT)
{
log.warn("Username update queue is full ({})", QUEUE_LIMIT);
return;
}
}
con.createQuery("update player set last_updated = CURRENT_TIMESTAMP where id = :id")
.addParameter("id", playerEntity.getId())
.executeUpdate();
}
usernameUpdateQueue.add(username);
synchronized (usernameUpdateQueue)
{
usernameUpdateQueue.add(username);
}
usernameFilter.put(username);
}
@@ -104,13 +128,6 @@ public class XpTrackerService
log.debug("Hiscore for {} already up to date", username);
return;
}
Duration difference = Duration.between(currentXp.getTime(), now);
if (difference.compareTo(UPDATE_TIME) <= 0)
{
log.debug("Updated {} too recently", username);
return;
}
}
con.createQuery("insert into xp (player,attack_xp,defence_xp,strength_xp,hitpoints_xp,ranged_xp,prayer_xp,magic_xp,cooking_xp,woodcutting_xp,"
@@ -172,6 +189,11 @@ public class XpTrackerService
.addParameter("construction_rank", hiscoreResult.getConstruction().getRank())
.addParameter("overall_rank", hiscoreResult.getOverall().getRank())
.executeUpdate();
con.createQuery("update player set rank = :rank where id = :id")
.addParameter("id", playerEntity.getId())
.addParameter("rank", hiscoreResult.getOverall().getRank())
.executeUpdate();
}
}
@@ -197,6 +219,7 @@ public class XpTrackerService
playerEntity.setId(id);
playerEntity.setName(username);
playerEntity.setTracked_since(now);
playerEntity.setLast_updated(now);
return playerEntity;
}
@@ -220,18 +243,21 @@ public class XpTrackerService
@Scheduled(fixedDelay = 1000)
public void update() throws ExecutionException
{
String next = usernameUpdateQueue.poll();
String next;
synchronized (usernameUpdateQueue)
{
next = usernameUpdateQueue.poll();
}
if (next == null)
{
return;
}
HiscoreResult hiscoreResult = hiscoreService.lookupUsername(next, HiscoreEndpoint.NORMAL);
update(next, hiscoreResult);
update(next);
}
@Scheduled(fixedDelay = 3 * 60 * 60 * 1000) // 3 hours
@Scheduled(fixedDelay = 6 * 60 * 60 * 1000) // 6 hours
public void clearFilter()
{
usernameFilter = createFilter();
@@ -241,14 +267,47 @@ public class XpTrackerService
{
final BloomFilter<String> filter = BloomFilter.create(
Funnels.stringFunnel(Charset.defaultCharset()),
100_000
BLOOMFILTER_EXPECTED_INSERTIONS
);
for (String toUpdate : usernameUpdateQueue)
synchronized (usernameUpdateQueue)
{
filter.put(toUpdate);
for (String toUpdate : usernameUpdateQueue)
{
filter.put(toUpdate);
}
}
return filter;
}
/**
* scale how often to check hiscore updates for players based on their rank
* @param playerEntity
* @return
*/
private static Duration updateFrequency(PlayerEntity playerEntity)
{
Integer rank = playerEntity.getRank();
if (rank == null)
{
return Duration.ofDays(7);
}
else if (rank < 10_000)
{
return Duration.ofHours(6);
}
else if (rank < 50_000)
{
return Duration.ofDays(2);
}
else if (rank < 100_000)
{
return Duration.ofDays(5);
}
else
{
return Duration.ofDays(7);
}
}
}

2
http-service/src/main/java/net/runelite/http/service/xp/beans/PlayerEntity.java
View File

@@ -33,4 +33,6 @@ public class PlayerEntity
private Integer id;
private String name;
private Instant tracked_since;
private Instant last_updated;
private Integer rank;
}

10
http-service/src/main/resources/net/runelite/http/service/xp/schema.sql
View File

@@ -1,8 +1,8 @@
-- MySQL dump 10.16 Distrib 10.2.9-MariaDB, for Linux (x86_64)
-- MySQL dump 10.16 Distrib 10.2.18-MariaDB, for Linux (x86_64)
--
-- Host: localhost Database: xptracker
-- ------------------------------------------------------
-- Server version 10.2.9-MariaDB
-- Server version 10.2.18-MariaDB
/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
@@ -26,6 +26,8 @@ CREATE TABLE `player` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`name` varchar(32) NOT NULL,
`tracked_since` timestamp NOT NULL DEFAULT current_timestamp(),
`last_updated` timestamp NOT NULL DEFAULT current_timestamp(),
`rank` int(11) DEFAULT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY `name` (`name`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
@@ -116,7 +118,7 @@ CREATE TABLE `xp` (
`overall_rank` int(11) NOT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY `player_time` (`player`,`time`),
INDEX `idx_time` (`time`),
KEY `idx_time` (`time`),
CONSTRAINT `fk_player` FOREIGN KEY (`player`) REFERENCES `player` (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
/*!40101 SET character_set_client = @saved_cs_client */;
@@ -130,4 +132,4 @@ CREATE TABLE `xp` (
/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
-- Dump completed on 2018-01-20 18:37:09
-- Dump completed on 2019-02-15 21:01:17