ra/runelite
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Add pluginhub user count tracking

Browse Source
This commit is contained in:
Adam
2020-10-16 16:06:54 -04:00
parent 901f294e06
commit 7a6c4ecc51
4 changed files with 216 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

144
http-service/src/main/java/net/runelite/http/service/pluginhub/PluginHubController.java Normal file
View File

@@ -0,0 +1,144 @@
/*
* Copyright (c) 2020, Adam <Adam@sigterm.info>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package net.runelite.http.service.pluginhub;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import java.time.Duration;
import java.time.Instant;
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.stream.IntStream;
import javax.servlet.http.HttpServletRequest;
import net.runelite.http.service.util.redis.RedisPool;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.CacheControl;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import redis.clients.jedis.Jedis;
@RestController
@RequestMapping("/pluginhub")
public class PluginHubController
{
@Value("${pluginhub.stats.days:7}")
private int days;
@Value("${pluginhub.stats.expire:90}")
private int expireDays;
@Autowired
private RedisPool redisPool;
private final Cache<String, String> pluginCache = CacheBuilder.newBuilder()
.maximumSize(512L)
.build();
private Map<String, Long> pluginCounts = Collections.emptyMap();
@GetMapping
public ResponseEntity<Map<String, Long>> get()
{
if (pluginCounts.isEmpty())
{
return ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE)
.cacheControl(CacheControl.noCache())
.build();
}
return ResponseEntity.ok()
.cacheControl(CacheControl.maxAge(30, TimeUnit.MINUTES).cachePublic())
.body(pluginCounts);
}
@PostMapping
public void submit(HttpServletRequest request, @RequestBody String[] plugins)
{
final String date = Instant.now().atZone(ZoneOffset.UTC).format(DateTimeFormatter.ISO_LOCAL_DATE);
final String ip = request.getHeader("X-Forwarded-For");
try (Jedis jedis = redisPool.getResource())
{
for (String plugin : plugins)
{
if (!plugin.matches("[a-z0-9-]+"))
{
continue;
}
jedis.pfadd("pluginhub." + plugin + "." + date, ip);
if (pluginCache.getIfPresent(plugin) == null)
{
jedis.sadd("pluginhub.plugins", plugin);
// additionally set the ttl on the hyperloglog since it might be a new key
jedis.expire("pluginhub." + plugin + "." + date, (int) (Duration.ofDays(expireDays).toMillis() / 1000L));
pluginCache.put(plugin, plugin);
}
}
}
}
@Scheduled(fixedDelay = 1_8000_000, initialDelay = 30_000) // 30 minutes with 30 second initial delay
public void rebuildCounts()
{
Map<String, Long> counts = new HashMap<>();
try (Jedis jedis = redisPool.getResource())
{
Set<String> plugins = jedis.smembers("pluginhub.plugins");
ZonedDateTime time = Instant.now().atZone(ZoneOffset.UTC);
for (String plugin : plugins)
{
// When called with multiple keys, pfcount returns the approximated
// cardinality of the union of the HyperLogLogs. We use this to determine
// the number of users in the last N days.
String[] keys = IntStream.range(0, days - 1)
.mapToObj(time::minusDays)
.map(zdt -> "pluginhub." + plugin + "." + zdt.format(DateTimeFormatter.ISO_LOCAL_DATE))
.toArray(String[]::new);
long cnt = jedis.pfcount(keys);
if (cnt > 0)
{
counts.put(plugin, cnt);
}
}
}
pluginCounts = counts;
}
}

4
http-service/src/main/resources/application.yaml
View File

@@ -25,10 +25,10 @@ minio:
secretkey: /PZCxzmsJzwCHYlogcymuprniGCaaLUOET2n6yMP
bucket: runelite
# Redis client for temporary data storage
# Redis client
redis:
pool.size: 10
host: http://localhost:6379
host: tcp://localhost:6379
mongo:
jndiName: java:comp/env/mongodb/runelite

50
runelite-client/src/main/java/net/runelite/client/externalplugins/ExternalPluginClient.java
View File

@@ -39,23 +39,28 @@ import java.security.cert.CertificateFactory;
import java.util.List;
import javax.imageio.ImageIO;
import javax.inject.Inject;
import lombok.extern.slf4j.Slf4j;
import net.runelite.client.RuneLiteProperties;
import net.runelite.http.api.RuneLiteAPI;
import net.runelite.client.util.VerificationException;
import net.runelite.http.api.RuneLiteAPI;
import okhttp3.Call;
import okhttp3.Callback;
import okhttp3.HttpUrl;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.RequestBody;
import okhttp3.Response;
import okio.BufferedSource;
@Slf4j
public class ExternalPluginClient
{
private final OkHttpClient cachingClient;
private final OkHttpClient okHttpClient;
@Inject
public ExternalPluginClient(OkHttpClient cachingClient)
private ExternalPluginClient(OkHttpClient okHttpClient)
{
this.cachingClient = cachingClient;
this.okHttpClient = okHttpClient;
}
public List<ExternalPluginManifest> downloadManifest() throws IOException, VerificationException
@@ -64,7 +69,7 @@ public class ExternalPluginClient
.newBuilder()
.addPathSegments("manifest.js")
.build();
try (Response res = cachingClient.newCall(new Request.Builder().url(manifest).build()).execute())
try (Response res = okHttpClient.newCall(new Request.Builder().url(manifest).build()).execute())
{
if (res.code() != 200)
{
@@ -110,7 +115,7 @@ public class ExternalPluginClient
.addPathSegment(plugin.getCommit() + ".png")
.build();
try (Response res = cachingClient.newCall(new Request.Builder().url(url).build()).execute())
try (Response res = okHttpClient.newCall(new Request.Builder().url(url).build()).execute())
{
byte[] bytes = res.body().bytes();
// We don't stream so the lock doesn't block the edt trying to load something at the same time
@@ -134,4 +139,37 @@ public class ExternalPluginClient
throw new RuntimeException(e);
}
}
void submitPlugins(List<String> plugins)
{
if (plugins.isEmpty())
{
return;
}
HttpUrl url = RuneLiteAPI.getApiBase().newBuilder()
.addPathSegment("pluginhub")
.build();
Request request = new Request.Builder()
.url(url)
.post(RequestBody.create(RuneLiteAPI.JSON, RuneLiteAPI.GSON.toJson(plugins)))
.build();
okHttpClient.newCall(request).enqueue(new Callback()
{
@Override
public void onFailure(Call call, IOException e)
{
log.debug("Error submitting plugins", e);
}
@Override
public void onResponse(Call call, Response response)
{
log.debug("Submitted plugin list");
response.close();
}
});
}
}

40
runelite-client/src/main/java/net/runelite/client/externalplugins/ExternalPluginManager.java
View File

@@ -40,8 +40,10 @@ import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import javax.inject.Inject;
import javax.inject.Named;
@@ -80,23 +82,33 @@ public class ExternalPluginManager
@Named("safeMode")
private boolean safeMode;
@Inject
private ConfigManager configManager;
private final ConfigManager configManager;
private final ExternalPluginClient externalPluginClient;
private final ScheduledExecutorService executor;
private final PluginManager pluginManager;
private final EventBus eventBus;
private final OkHttpClient okHttpClient;
@Inject
private ExternalPluginClient externalPluginClient;
private ExternalPluginManager(
ConfigManager configManager,
ExternalPluginClient externalPluginClient,
ScheduledExecutorService executor,
PluginManager pluginManager,
EventBus eventBus,
OkHttpClient okHttpClient
)
{
this.configManager = configManager;
this.externalPluginClient = externalPluginClient;
this.executor = executor;
this.pluginManager = pluginManager;
this.eventBus = eventBus;
this.okHttpClient = okHttpClient;
@Inject
private PluginManager pluginManager;
@Inject
private ScheduledExecutorService executor;
@Inject
private EventBus eventBus;
@Inject
private OkHttpClient okHttpClient;
executor.scheduleWithFixedDelay(() -> externalPluginClient.submitPlugins(getInstalledExternalPlugins()),
new Random().nextInt(60), 180, TimeUnit.MINUTES);
}
public void loadExternalPlugins() throws PluginInstantiationException
{