config service: convert backing database to use mongodb

This is a much more natural fit for the config service which is really a
key value store, and has been abusing SQL. This will let us expand the
config stuff later to support profiles and per-account config values.

I have left in the SQL code for now so config changes are still being
sent there in the event of catastrophic mongodb failure.

http-service/pom.xml

@@ -116,6 +116,11 @@
 				</exclusion>
 			</exclusions>
 		</dependency>
+		<dependency>
+			<groupId>org.mongodb</groupId>
+			<artifactId>mongodb-driver-sync</artifactId>
+			<version>3.10.1</version>
+		</dependency>
 
 		<dependency>
 			<groupId>org.springframework.boot</groupId>

http-service/src/main/java/net/runelite/http/service/SpringBootWebApplication.java

@@ -26,6 +26,8 @@ package net.runelite.http.service;
 
 import ch.qos.logback.classic.LoggerContext;
 import com.google.common.base.Strings;
+import com.mongodb.client.MongoClient;
+import com.mongodb.client.MongoClients;
 import java.io.IOException;
 import java.time.Instant;
 import java.util.HashMap;
@@ -43,6 +45,7 @@ import okhttp3.OkHttpClient;
 import org.slf4j.ILoggerFactory;
 import org.slf4j.impl.StaticLoggerBinder;
 import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
@@ -156,6 +159,12 @@ public class SpringBootWebApplication extends SpringBootServletInitializer
 		return createSql2oFromDataSource(dataSource);
 	}
 
+	@Bean
+	public MongoClient mongoClient(@Value("${mongo.host}") String host)
+	{
+		return MongoClients.create(host);
+	}
+
 	private static DataSource getDataSource(DataSourceProperties dataSourceProperties)
 	{
 		if (!Strings.isNullOrEmpty(dataSourceProperties.getJndiName()))

http-service/src/main/java/net/runelite/http/service/config/ConfigService.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, Adam <Adam@sigterm.info>
+ * Copyright (c) 2017-2019, Adam <Adam@sigterm.info>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -24,10 +24,26 @@
  */
 package net.runelite.http.service.config;
 
+import com.google.gson.Gson;
+import com.google.gson.JsonSyntaxException;
+import com.mongodb.client.MongoClient;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.MongoDatabase;
+import static com.mongodb.client.model.Filters.eq;
+import com.mongodb.client.model.IndexOptions;
+import com.mongodb.client.model.Indexes;
+import static com.mongodb.client.model.Updates.set;
+import static com.mongodb.client.model.Updates.unset;
+import java.util.ArrayList;
+import java.util.Collection;
 import java.util.List;
+import java.util.Map;
 import javax.annotation.Nullable;
+import lombok.extern.slf4j.Slf4j;
+import net.runelite.http.api.RuneLiteAPI;
 import net.runelite.http.api.config.ConfigEntry;
 import net.runelite.http.api.config.Configuration;
+import org.bson.Document;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.stereotype.Service;
@@ -36,6 +52,7 @@ import org.sql2o.Sql2o;
 import org.sql2o.Sql2oException;
 
 @Service
+@Slf4j
 public class ConfigService
 {
 	private static final String CREATE_CONFIG = "CREATE TABLE IF NOT EXISTS `config` (\n"
@@ -49,10 +66,14 @@ public class ConfigService
 		+ "  ADD CONSTRAINT `user_fk` FOREIGN KEY (`user`) REFERENCES `users` (`id`) ON DELETE CASCADE ON UPDATE CASCADE;";
 
 	private final Sql2o sql2o;
+	private final Gson GSON = RuneLiteAPI.GSON;
+
+	private final MongoCollection<Document> mongoCollection;
 
 	@Autowired
 	public ConfigService(
-		@Qualifier("Runelite SQL2O") Sql2o sql2o
+		@Qualifier("Runelite SQL2O") Sql2o sql2o,
+		MongoClient mongoClient
 	)
 	{
 		this.sql2o = sql2o;
@@ -72,17 +93,61 @@ public class ConfigService
 				// Ignore, happens when index already exists
 			}
 		}
+
+		MongoDatabase database = mongoClient.getDatabase("config");
+		MongoCollection<Document> collection = database.getCollection("config");
+		this.mongoCollection = collection;
+
+		// Create unique index on _userId
+		IndexOptions indexOptions = new IndexOptions().unique(true);
+		collection.createIndex(Indexes.ascending("_userId"), indexOptions);
+	}
+
+	private Document getConfig(int userId)
+	{
+		return mongoCollection.find(eq("_userId", userId)).first();
 	}
 
 	public Configuration get(int userId)
 	{
-		List<ConfigEntry> config;
+		Map<String, Object> configMap = getConfig(userId);
 
-		try (Connection con = sql2o.open())
+		if (configMap == null || configMap.isEmpty())
 		{
-			config = con.createQuery("select `key`, value from config where user = :user")
-				.addParameter("user", userId)
-				.executeAndFetch(ConfigEntry.class);
+			return null;
+		}
+
+		List<ConfigEntry> config = new ArrayList<>();
+
+		for (String group : configMap.keySet())
+		{
+			// Reserved keys
+			if (group.startsWith("_") || group.startsWith("$"))
+			{
+				continue;
+			}
+
+			Map<String, Object> groupMap = (Map) configMap.get(group);
+
+			for (Map.Entry<String, Object> entry : groupMap.entrySet())
+			{
+				String key = entry.getKey();
+				Object value = entry.getValue();
+
+				if (value instanceof Map || value instanceof Collection)
+				{
+					value = GSON.toJson(entry.getValue());
+				}
+				else if (value == null)
+				{
+					continue;
+				}
+
+				ConfigEntry configEntry = new ConfigEntry();
+				configEntry.setKey(group + "." + key.replace(':', '.'));
+				configEntry.setValue(value.toString());
+				config.add(configEntry);
+			}
 		}
 
 		return new Configuration(config);
@@ -102,6 +167,21 @@ public class ConfigService
 				.addParameter("value", value != null ? value : "")
 				.executeUpdate();
 		}
+
+		if (key.startsWith("$") || key.startsWith("_"))
+		{
+			return;
+		}
+
+		String[] split = key.split("\\.", 2);
+		if (split.length != 2)
+		{
+			return;
+		}
+
+		Object jsonValue = parseJsonString(value);
+		mongoCollection.updateOne(eq("_userId", userId),
+			set(split[0] + "." + split[1].replace('.', ':'), jsonValue));
 	}
 
 	public void unsetKey(
@@ -116,5 +196,57 @@ public class ConfigService
 				.addParameter("key", key)
 				.executeUpdate();
 		}
+
+		if (key.startsWith("$") || key.startsWith("_"))
+		{
+			return;
+		}
+
+		String[] split = key.split("\\.", 2);
+		if (split.length != 2)
+		{
+			return;
+		}
+
+		mongoCollection.updateOne(eq("_userId", userId),
+			unset(split[0] + "." + split[1].replace('.', ':')));
+	}
+
+	private static Object parseJsonString(String value)
+	{
+		Object jsonValue;
+		try
+		{
+			jsonValue = RuneLiteAPI.GSON.fromJson(value, Object.class);
+
+			if (jsonValue instanceof Double || jsonValue instanceof Float)
+			{
+				Number number = (Number) jsonValue;
+				if (Math.floor(number.doubleValue()) == number.doubleValue() && !Double.isInfinite(number.doubleValue()))
+				{
+					// value is an int or long. 'number' might be truncated so parse it from 'value'
+					try
+					{
+						jsonValue = Integer.parseInt(value);
+					}
+					catch (NumberFormatException ex)
+					{
+						try
+						{
+							jsonValue = Long.parseLong(value);
+						}
+						catch (NumberFormatException ex2)
+						{
+
+						}
+					}
+				}
+			}
+		}
+		catch (JsonSyntaxException ex)
+		{
+			jsonValue = value;
+		}
+		return jsonValue;
 	}
 }

http-service/src/main/resources/application.yaml

@@ -29,6 +29,9 @@ redis:
   pool.size: 10
   host: http://localhost:6379
 
+mongo:
+  host: mongodb://localhost:27017
+
 # Twitter client for feed
 runelite:
   twitter: