item service: change search to only search database

Add fulltext index to item.name and use in searches

http-service/src/main/java/net/runelite/http/service/item/ItemController.java

@@ -26,7 +26,6 @@ package net.runelite.http.service.item;
 
 import com.google.common.cache.Cache;
 import com.google.common.cache.CacheBuilder;
-import java.io.IOException;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.List;
@@ -53,9 +52,6 @@ public class ItemController
 	private static final Duration CACHE_DUATION = Duration.ofMinutes(30);
 	private static final String RUNELITE_CACHE = "RuneLite-Cache";
 
-	private final Cache<String, SearchResult> cachedSearches = CacheBuilder.newBuilder()
-		.maximumSize(1024L)
-		.build();
 	private final Cache<Integer, Integer> cachedEmpty = CacheBuilder.newBuilder()
 		.maximumSize(1024L)
 		.build();
@@ -217,37 +213,14 @@ public class ItemController
 	@RequestMapping("/search")
 	public SearchResult search(HttpServletResponse response, @RequestParam String query)
 	{
-		// rs api seems to require lowercase
+		List<ItemEntry> result = itemService.search(query);
-		query = query.toLowerCase();
 
-		SearchResult searchResult = cachedSearches.getIfPresent(query);
+		itemService.queueSearch(query);
-		if (searchResult != null)
-		{
-			response.setHeader(RUNELITE_CACHE, "HIT");
-			return searchResult;
-		}
 
-		try
+		SearchResult searchResult = new SearchResult();
-		{
+		searchResult.setItems(result.stream()
-			RSSearch search = itemService.fetchRSSearch(query);
+			.map(ItemEntry::toItem)
-
+			.collect(Collectors.toList()));
-			searchResult = new SearchResult();
+		return searchResult;
-			List<Item> items = search.getItems().stream()
-				.map(rsi -> rsi.toItem())
-				.collect(Collectors.toList());
-			searchResult.setItems(items);
-
-			cachedSearches.put(query, searchResult);
-
-			itemService.batchInsertItems(search);
-
-			response.setHeader(RUNELITE_CACHE, "MISS");
-			return searchResult;
-		}
-		catch (IOException ex)
-		{
-			log.warn("error while searching items", ex);
-			return null;
-		}
 	}
 }

http-service/src/main/java/net/runelite/http/service/item/ItemService.java

@@ -65,7 +65,8 @@ public class ItemService
 		+ "  `icon` blob,\n"
 		+ "  `icon_large` blob,\n"
 		+ "  `timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,\n"
-		+ "  PRIMARY KEY (`id`)\n"
+		+ "  PRIMARY KEY (`id`),\n"
+		+ "  FULLTEXT idx_name (name)\n"
 		+ ") ENGINE=InnoDB";
 
 	private static final String CREATE_PRICES = "CREATE TABLE IF NOT EXISTS `prices` (\n"
@@ -80,10 +81,11 @@ public class ItemService
 	private static final String CREATE_PRICES_FK = "ALTER TABLE `prices`\n"
 		+ "  ADD CONSTRAINT `item` FOREIGN KEY (`item`) REFERENCES `items` (`id`);";
 
-	private static final int MAX_PENDING_LOOKUPS = 512;
+	private static final int MAX_PENDING = 512;
 
 	private final Sql2o sql2o;
 	private final ConcurrentLinkedQueue<Integer> pendingPriceLookups = new ConcurrentLinkedQueue<>();
+	private final ConcurrentLinkedQueue<String> pendingSearches = new ConcurrentLinkedQueue<>();
 
 	@Autowired
 	public ItemService(@Qualifier("Runelite SQL2O") Sql2o sql2o)
@@ -142,6 +144,18 @@ public class ItemService
 		}
 	}
 
+	public List<ItemEntry> search(String search)
+	{
+		try (Connection con = sql2o.open())
+		{
+			return con.createQuery("select id, name, description, type, match (name) against (:search) as score from items "
+				+ "where match (name) against (:search) order by score desc limit 10")
+				.throwOnMappingFailure(false) // otherwise it tries to map 'score'
+				.addParameter("search", search)
+				.executeAndFetch(ItemEntry.class);
+		}
+	}
+
 	public ItemEntry fetchItem(int itemId)
 	{
 		try
@@ -274,6 +288,9 @@ public class ItemService
 
 	public RSSearch fetchRSSearch(String query) throws IOException
 	{
+		// rs api seems to require lowercase
+		query = query.toLowerCase();
+
 		HttpUrl searchUrl = RS_SEARCH_URL
 			.newBuilder()
 			.addQueryParameter("alpha", query)
@@ -286,7 +303,7 @@ public class ItemService
 		return fetchJson(request, RSSearch.class);
 	}
 
-	public void batchInsertItems(RSSearch search)
+	private void batchInsertItems(RSSearch search)
 	{
 		try (Connection con = sql2o.beginTransaction())
 		{
@@ -347,7 +364,7 @@ public class ItemService
 
 	public void queueLookup(int itemId)
 	{
-		if (pendingPriceLookups.size() >= MAX_PENDING_LOOKUPS)
+		if (pendingPriceLookups.size() >= MAX_PENDING)
 		{
 			return;
 		}
@@ -355,6 +372,16 @@ public class ItemService
 		pendingPriceLookups.add(itemId);
 	}
 
+	public void queueSearch(String search)
+	{
+		if (pendingSearches.size() >= MAX_PENDING)
+		{
+			return;
+		}
+
+		pendingSearches.add(search);
+	}
+
 	@Scheduled(fixedDelay = 5000)
 	public void checkPrices()
 	{
@@ -366,4 +393,25 @@ public class ItemService
 
 		fetchPrice(itemId);
 	}
+
+	@Scheduled(fixedDelay = 5000)
+	public void checkSearches()
+	{
+		String search = pendingSearches.poll();
+		if (search == null)
+		{
+			return;
+		}
+
+		try
+		{
+			RSSearch reSearch = fetchRSSearch(search);
+
+			batchInsertItems(reSearch);
+		}
+		catch (IOException ex)
+		{
+			log.warn("error while searching items", ex);
+		}
+	}
 }