Add pluginhub user count tracking

http-service/src/main/java/net/runelite/http/service/pluginhub/PluginHubController.java

@@ -0,0 +1,144 @@
+/*
+ * Copyright (c) 2020, Adam <Adam@sigterm.info>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package net.runelite.http.service.pluginhub;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.IntStream;
+import javax.servlet.http.HttpServletRequest;
+import net.runelite.http.service.util.redis.RedisPool;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.http.CacheControl;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+import redis.clients.jedis.Jedis;
+
+@RestController
+@RequestMapping("/pluginhub")
+public class PluginHubController
+{
+	@Value("${pluginhub.stats.days:7}")
+	private int days;
+
+	@Value("${pluginhub.stats.expire:90}")
+	private int expireDays;
+
+	@Autowired
+	private RedisPool redisPool;
+
+	private final Cache<String, String> pluginCache = CacheBuilder.newBuilder()
+		.maximumSize(512L)
+		.build();
+
+	private Map<String, Long> pluginCounts = Collections.emptyMap();
+
+	@GetMapping
+	public ResponseEntity<Map<String, Long>> get()
+	{
+		if (pluginCounts.isEmpty())
+		{
+			return ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE)
+				.cacheControl(CacheControl.noCache())
+				.build();
+		}
+
+		return ResponseEntity.ok()
+			.cacheControl(CacheControl.maxAge(30, TimeUnit.MINUTES).cachePublic())
+			.body(pluginCounts);
+	}
+
+	@PostMapping
+	public void submit(HttpServletRequest request, @RequestBody String[] plugins)
+	{
+		final String date = Instant.now().atZone(ZoneOffset.UTC).format(DateTimeFormatter.ISO_LOCAL_DATE);
+		final String ip = request.getHeader("X-Forwarded-For");
+		try (Jedis jedis = redisPool.getResource())
+		{
+			for (String plugin : plugins)
+			{
+				if (!plugin.matches("[a-z0-9-]+"))
+				{
+					continue;
+				}
+
+				jedis.pfadd("pluginhub." + plugin + "." + date, ip);
+
+				if (pluginCache.getIfPresent(plugin) == null)
+				{
+					jedis.sadd("pluginhub.plugins", plugin);
+					// additionally set the ttl on the hyperloglog since it might be a new key
+					jedis.expire("pluginhub." + plugin + "." + date, (int) (Duration.ofDays(expireDays).toMillis() / 1000L));
+
+					pluginCache.put(plugin, plugin);
+				}
+			}
+		}
+	}
+
+	@Scheduled(fixedDelay = 1_8000_000, initialDelay = 30_000) // 30 minutes with 30 second initial delay
+	public void rebuildCounts()
+	{
+		Map<String, Long> counts = new HashMap<>();
+		try (Jedis jedis = redisPool.getResource())
+		{
+			Set<String> plugins = jedis.smembers("pluginhub.plugins");
+			ZonedDateTime time = Instant.now().atZone(ZoneOffset.UTC);
+
+			for (String plugin : plugins)
+			{
+				// When called with multiple keys, pfcount returns the approximated
+				// cardinality of the union of the HyperLogLogs. We use this to determine
+				// the number of users in the last N days.
+				String[] keys = IntStream.range(0, days - 1)
+					.mapToObj(time::minusDays)
+					.map(zdt -> "pluginhub." + plugin + "." + zdt.format(DateTimeFormatter.ISO_LOCAL_DATE))
+					.toArray(String[]::new);
+				long cnt = jedis.pfcount(keys);
+				if (cnt > 0)
+				{
+					counts.put(plugin, cnt);
+				}
+			}
+		}
+		pluginCounts = counts;
+	}
+}

http-service/src/main/resources/application.yaml

@@ -25,10 +25,10 @@ minio:
   secretkey: /PZCxzmsJzwCHYlogcymuprniGCaaLUOET2n6yMP
   bucket: runelite
 
-# Redis client for temporary data storage
+# Redis client
 redis:
   pool.size: 10
-  host: http://localhost:6379
+  host: tcp://localhost:6379
 
 mongo:
   jndiName: java:comp/env/mongodb/runelite

runelite-client/src/main/java/net/runelite/client/externalplugins/ExternalPluginClient.java

@@ -39,23 +39,28 @@ import java.security.cert.CertificateFactory;
 import java.util.List;
 import javax.imageio.ImageIO;
 import javax.inject.Inject;
+import lombok.extern.slf4j.Slf4j;
 import net.runelite.client.RuneLiteProperties;
-import net.runelite.http.api.RuneLiteAPI;
 import net.runelite.client.util.VerificationException;
+import net.runelite.http.api.RuneLiteAPI;
+import okhttp3.Call;
+import okhttp3.Callback;
 import okhttp3.HttpUrl;
 import okhttp3.OkHttpClient;
 import okhttp3.Request;
+import okhttp3.RequestBody;
 import okhttp3.Response;
 import okio.BufferedSource;
 
+@Slf4j
 public class ExternalPluginClient
 {
-	private final OkHttpClient cachingClient;
+	private final OkHttpClient okHttpClient;
 
 	@Inject
-	public ExternalPluginClient(OkHttpClient cachingClient)
+	private ExternalPluginClient(OkHttpClient okHttpClient)
 	{
-		this.cachingClient = cachingClient;
+		this.okHttpClient = okHttpClient;
 	}
 
 	public List<ExternalPluginManifest> downloadManifest() throws IOException, VerificationException
@@ -64,7 +69,7 @@ public class ExternalPluginClient
 			.newBuilder()
 			.addPathSegments("manifest.js")
 			.build();
-		try (Response res = cachingClient.newCall(new Request.Builder().url(manifest).build()).execute())
+		try (Response res = okHttpClient.newCall(new Request.Builder().url(manifest).build()).execute())
 		{
 			if (res.code() != 200)
 			{
@@ -110,7 +115,7 @@ public class ExternalPluginClient
 			.addPathSegment(plugin.getCommit() + ".png")
 			.build();
 
-		try (Response res = cachingClient.newCall(new Request.Builder().url(url).build()).execute())
+		try (Response res = okHttpClient.newCall(new Request.Builder().url(url).build()).execute())
 		{
 			byte[] bytes = res.body().bytes();
 			// We don't stream so the lock doesn't block the edt trying to load something at the same time
@@ -134,4 +139,37 @@ public class ExternalPluginClient
 			throw new RuntimeException(e);
 		}
 	}
+
+	void submitPlugins(List<String> plugins)
+	{
+		if (plugins.isEmpty())
+		{
+			return;
+		}
+
+		HttpUrl url = RuneLiteAPI.getApiBase().newBuilder()
+			.addPathSegment("pluginhub")
+			.build();
+
+		Request request = new Request.Builder()
+			.url(url)
+			.post(RequestBody.create(RuneLiteAPI.JSON, RuneLiteAPI.GSON.toJson(plugins)))
+			.build();
+
+		okHttpClient.newCall(request).enqueue(new Callback()
+		{
+			@Override
+			public void onFailure(Call call, IOException e)
+			{
+				log.debug("Error submitting plugins", e);
+			}
+
+			@Override
+			public void onResponse(Call call, Response response)
+			{
+				log.debug("Submitted plugin list");
+				response.close();
+			}
+		});
+	}
 }

runelite-client/src/main/java/net/runelite/client/externalplugins/ExternalPluginManager.java

@@ -40,8 +40,10 @@ import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Random;
 import java.util.Set;
 import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
 import javax.inject.Inject;
 import javax.inject.Named;
@@ -80,23 +82,33 @@ public class ExternalPluginManager
 	@Named("safeMode")
 	private boolean safeMode;
 
-	@Inject
+	private final ConfigManager configManager;
-	private ConfigManager configManager;
+	private final ExternalPluginClient externalPluginClient;
+	private final ScheduledExecutorService executor;
+	private final PluginManager pluginManager;
+	private final EventBus eventBus;
+	private final OkHttpClient okHttpClient;
 
 	@Inject
-	private ExternalPluginClient externalPluginClient;
+	private ExternalPluginManager(
+		ConfigManager configManager,
+		ExternalPluginClient externalPluginClient,
+		ScheduledExecutorService executor,
+		PluginManager pluginManager,
+		EventBus eventBus,
+		OkHttpClient okHttpClient
+	)
+	{
+		this.configManager = configManager;
+		this.externalPluginClient = externalPluginClient;
+		this.executor = executor;
+		this.pluginManager = pluginManager;
+		this.eventBus = eventBus;
+		this.okHttpClient = okHttpClient;
 
-	@Inject
+		executor.scheduleWithFixedDelay(() -> externalPluginClient.submitPlugins(getInstalledExternalPlugins()),
-	private PluginManager pluginManager;
+			new Random().nextInt(60), 180, TimeUnit.MINUTES);
-
+	}
-	@Inject
-	private ScheduledExecutorService executor;
-
-	@Inject
-	private EventBus eventBus;
-
-	@Inject
-	private OkHttpClient okHttpClient;
 
 	public void loadExternalPlugins() throws PluginInstantiationException
 	{