diff --git a/http-service/src/main/java/net/runelite/http/service/pluginhub/PluginHubController.java b/http-service/src/main/java/net/runelite/http/service/pluginhub/PluginHubController.java new file mode 100644 index 0000000000..fde79591e1 --- /dev/null +++ b/http-service/src/main/java/net/runelite/http/service/pluginhub/PluginHubController.java @@ -0,0 +1,144 @@ +/* + * Copyright (c) 2020, Adam + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package net.runelite.http.service.pluginhub; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.IntStream; +import javax.servlet.http.HttpServletRequest; +import net.runelite.http.service.util.redis.RedisPool; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.CacheControl; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import redis.clients.jedis.Jedis; + +@RestController +@RequestMapping("/pluginhub") +public class PluginHubController +{ + @Value("${pluginhub.stats.days:7}") + private int days; + + @Value("${pluginhub.stats.expire:90}") + private int expireDays; + + @Autowired + private RedisPool redisPool; + + private final Cache pluginCache = CacheBuilder.newBuilder() + .maximumSize(512L) + .build(); + + private Map pluginCounts = Collections.emptyMap(); + + @GetMapping + public ResponseEntity> get() + { + if (pluginCounts.isEmpty()) + { + return ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE) + .cacheControl(CacheControl.noCache()) + .build(); + } + + return ResponseEntity.ok() + .cacheControl(CacheControl.maxAge(30, TimeUnit.MINUTES).cachePublic()) + .body(pluginCounts); + } + + @PostMapping + public void submit(HttpServletRequest request, @RequestBody String[] plugins) + { + final String date = Instant.now().atZone(ZoneOffset.UTC).format(DateTimeFormatter.ISO_LOCAL_DATE); + final String ip = request.getHeader("X-Forwarded-For"); + try (Jedis jedis = redisPool.getResource()) + { + for (String plugin : plugins) + { + if (!plugin.matches("[a-z0-9-]+")) + { + continue; + } + + jedis.pfadd("pluginhub." + plugin + "." + date, ip); + + if (pluginCache.getIfPresent(plugin) == null) + { + jedis.sadd("pluginhub.plugins", plugin); + // additionally set the ttl on the hyperloglog since it might be a new key + jedis.expire("pluginhub." + plugin + "." + date, (int) (Duration.ofDays(expireDays).toMillis() / 1000L)); + + pluginCache.put(plugin, plugin); + } + } + } + } + + @Scheduled(fixedDelay = 1_8000_000, initialDelay = 30_000) // 30 minutes with 30 second initial delay + public void rebuildCounts() + { + Map counts = new HashMap<>(); + try (Jedis jedis = redisPool.getResource()) + { + Set plugins = jedis.smembers("pluginhub.plugins"); + ZonedDateTime time = Instant.now().atZone(ZoneOffset.UTC); + + for (String plugin : plugins) + { + // When called with multiple keys, pfcount returns the approximated + // cardinality of the union of the HyperLogLogs. We use this to determine + // the number of users in the last N days. + String[] keys = IntStream.range(0, days - 1) + .mapToObj(time::minusDays) + .map(zdt -> "pluginhub." + plugin + "." + zdt.format(DateTimeFormatter.ISO_LOCAL_DATE)) + .toArray(String[]::new); + long cnt = jedis.pfcount(keys); + if (cnt > 0) + { + counts.put(plugin, cnt); + } + } + } + pluginCounts = counts; + } +} diff --git a/http-service/src/main/resources/application.yaml b/http-service/src/main/resources/application.yaml index e3b8b07425..3e56fb066d 100644 --- a/http-service/src/main/resources/application.yaml +++ b/http-service/src/main/resources/application.yaml @@ -25,10 +25,10 @@ minio: secretkey: /PZCxzmsJzwCHYlogcymuprniGCaaLUOET2n6yMP bucket: runelite -# Redis client for temporary data storage +# Redis client redis: pool.size: 10 - host: http://localhost:6379 + host: tcp://localhost:6379 mongo: jndiName: java:comp/env/mongodb/runelite diff --git a/runelite-client/src/main/java/net/runelite/client/externalplugins/ExternalPluginClient.java b/runelite-client/src/main/java/net/runelite/client/externalplugins/ExternalPluginClient.java index cd023c8c3a..0df12bf1ed 100644 --- a/runelite-client/src/main/java/net/runelite/client/externalplugins/ExternalPluginClient.java +++ b/runelite-client/src/main/java/net/runelite/client/externalplugins/ExternalPluginClient.java @@ -39,23 +39,28 @@ import java.security.cert.CertificateFactory; import java.util.List; import javax.imageio.ImageIO; import javax.inject.Inject; +import lombok.extern.slf4j.Slf4j; import net.runelite.client.RuneLiteProperties; -import net.runelite.http.api.RuneLiteAPI; import net.runelite.client.util.VerificationException; +import net.runelite.http.api.RuneLiteAPI; +import okhttp3.Call; +import okhttp3.Callback; import okhttp3.HttpUrl; import okhttp3.OkHttpClient; import okhttp3.Request; +import okhttp3.RequestBody; import okhttp3.Response; import okio.BufferedSource; +@Slf4j public class ExternalPluginClient { - private final OkHttpClient cachingClient; + private final OkHttpClient okHttpClient; @Inject - public ExternalPluginClient(OkHttpClient cachingClient) + private ExternalPluginClient(OkHttpClient okHttpClient) { - this.cachingClient = cachingClient; + this.okHttpClient = okHttpClient; } public List downloadManifest() throws IOException, VerificationException @@ -64,7 +69,7 @@ public class ExternalPluginClient .newBuilder() .addPathSegments("manifest.js") .build(); - try (Response res = cachingClient.newCall(new Request.Builder().url(manifest).build()).execute()) + try (Response res = okHttpClient.newCall(new Request.Builder().url(manifest).build()).execute()) { if (res.code() != 200) { @@ -110,7 +115,7 @@ public class ExternalPluginClient .addPathSegment(plugin.getCommit() + ".png") .build(); - try (Response res = cachingClient.newCall(new Request.Builder().url(url).build()).execute()) + try (Response res = okHttpClient.newCall(new Request.Builder().url(url).build()).execute()) { byte[] bytes = res.body().bytes(); // We don't stream so the lock doesn't block the edt trying to load something at the same time @@ -134,4 +139,37 @@ public class ExternalPluginClient throw new RuntimeException(e); } } + + void submitPlugins(List plugins) + { + if (plugins.isEmpty()) + { + return; + } + + HttpUrl url = RuneLiteAPI.getApiBase().newBuilder() + .addPathSegment("pluginhub") + .build(); + + Request request = new Request.Builder() + .url(url) + .post(RequestBody.create(RuneLiteAPI.JSON, RuneLiteAPI.GSON.toJson(plugins))) + .build(); + + okHttpClient.newCall(request).enqueue(new Callback() + { + @Override + public void onFailure(Call call, IOException e) + { + log.debug("Error submitting plugins", e); + } + + @Override + public void onResponse(Call call, Response response) + { + log.debug("Submitted plugin list"); + response.close(); + } + }); + } } diff --git a/runelite-client/src/main/java/net/runelite/client/externalplugins/ExternalPluginManager.java b/runelite-client/src/main/java/net/runelite/client/externalplugins/ExternalPluginManager.java index bd6baf39ac..504331f166 100644 --- a/runelite-client/src/main/java/net/runelite/client/externalplugins/ExternalPluginManager.java +++ b/runelite-client/src/main/java/net/runelite/client/externalplugins/ExternalPluginManager.java @@ -40,8 +40,10 @@ import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Random; import java.util.Set; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import java.util.function.Function; import javax.inject.Inject; import javax.inject.Named; @@ -80,23 +82,33 @@ public class ExternalPluginManager @Named("safeMode") private boolean safeMode; - @Inject - private ConfigManager configManager; + private final ConfigManager configManager; + private final ExternalPluginClient externalPluginClient; + private final ScheduledExecutorService executor; + private final PluginManager pluginManager; + private final EventBus eventBus; + private final OkHttpClient okHttpClient; @Inject - private ExternalPluginClient externalPluginClient; + private ExternalPluginManager( + ConfigManager configManager, + ExternalPluginClient externalPluginClient, + ScheduledExecutorService executor, + PluginManager pluginManager, + EventBus eventBus, + OkHttpClient okHttpClient + ) + { + this.configManager = configManager; + this.externalPluginClient = externalPluginClient; + this.executor = executor; + this.pluginManager = pluginManager; + this.eventBus = eventBus; + this.okHttpClient = okHttpClient; - @Inject - private PluginManager pluginManager; - - @Inject - private ScheduledExecutorService executor; - - @Inject - private EventBus eventBus; - - @Inject - private OkHttpClient okHttpClient; + executor.scheduleWithFixedDelay(() -> externalPluginClient.submitPlugins(getInstalledExternalPlugins()), + new Random().nextInt(60), 180, TimeUnit.MINUTES); + } public void loadExternalPlugins() throws PluginInstantiationException {