http-service: crawl tradable item prices instead of queueing on demand

http-service/src/main/java/net/runelite/http/service/cache/CacheService.java

@@ -43,7 +43,10 @@ import java.security.NoSuchAlgorithmException;
 import java.util.ArrayList;
 import java.util.List;
 import lombok.extern.slf4j.Slf4j;
+import net.runelite.cache.ConfigType;
 import net.runelite.cache.IndexType;
+import net.runelite.cache.definitions.ItemDefinition;
+import net.runelite.cache.definitions.loaders.ItemLoader;
 import net.runelite.cache.fs.ArchiveFiles;
 import net.runelite.cache.fs.Container;
 import net.runelite.cache.fs.FSFile;
@@ -226,4 +229,20 @@ public class CacheService
 			return cacheDao.findArchiveByName(con, cache, index, nameHash);
 		}
 	}
+
+	public List<ItemDefinition> getItems() throws IOException
+	{
+		CacheEntry cache = findMostRecent();
+		IndexEntry indexEntry = findIndexForCache(cache, IndexType.CONFIGS.getNumber());
+		ArchiveEntry archiveEntry = findArchiveForIndex(indexEntry, ConfigType.ITEM.getId());
+		ArchiveFiles archiveFiles = getArchiveFiles(archiveEntry);
+		final ItemLoader itemLoader = new ItemLoader();
+		final List<ItemDefinition> result = new ArrayList<>(archiveFiles.getFiles().size());
+		for (FSFile file : archiveFiles.getFiles())
+		{
+			ItemDefinition itemDef = itemLoader.load(file.getFileId(), file.getContents());
+			result.add(itemDef);
+		}
+		return result;
+	}
 }

http-service/src/main/java/net/runelite/http/service/item/ItemController.java

@@ -168,20 +168,12 @@ public class ItemController
 		else if (priceEntry == null)
 		{
 			// Price is unknown
-			itemService.queuePriceLookup(itemId); // queue lookup
 			cachedEmpty.put(itemId, itemId);
 			return ResponseEntity.notFound()
 				.header(RUNELITE_CACHE, "MISS")
 				.build();
 		}
 
-		Instant cacheTime = now.minus(CACHE_DUATION);
-		if (priceEntry.getFetched_time().isBefore(cacheTime))
-		{
-			// Queue a check for the price
-			itemService.queuePriceLookup(itemId);
-		}
-
 		ItemPrice itemPrice = new ItemPrice();
 		itemPrice.setItem(item.toItem());
 		itemPrice.setPrice(priceEntry.getPrice());

http-service/src/main/java/net/runelite/http/service/item/ItemService.java

@@ -33,11 +33,14 @@ import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Random;
 import java.util.Set;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import lombok.extern.slf4j.Slf4j;
+import net.runelite.cache.definitions.ItemDefinition;
 import net.runelite.http.api.RuneLiteAPI;
 import net.runelite.http.api.item.ItemType;
+import net.runelite.http.service.cache.CacheService;
 import okhttp3.HttpUrl;
 import okhttp3.Request;
 import okhttp3.Response;
@@ -48,7 +51,6 @@ import org.springframework.stereotype.Service;
 import org.sql2o.Connection;
 import org.sql2o.Query;
 import org.sql2o.Sql2o;
-import org.sql2o.Sql2oException;
 
 @Service
 @Slf4j
@@ -80,18 +82,21 @@ public class ItemService
 		+ "  KEY `item_fetched_time` (`item`,`fetched_time`)\n"
 		+ ") ENGINE=InnoDB";
 
-	private static final String CREATE_PRICES_FK = "ALTER TABLE `prices`\n"
-		+ "  ADD CONSTRAINT `item` FOREIGN KEY (`item`) REFERENCES `items` (`id`);";
-
 	private static final int MAX_PENDING = 512;
 
 	private final Sql2o sql2o;
+	private final CacheService cacheService;
+
 	private final ConcurrentLinkedQueue<PendingLookup> pendingLookups = new ConcurrentLinkedQueue<PendingLookup>();
+	private int[] tradeableItems;
+	private final Random random = new Random();
 
 	@Autowired
-	public ItemService(@Qualifier("Runelite SQL2O") Sql2o sql2o)
+	public ItemService(@Qualifier("Runelite SQL2O") Sql2o sql2o,
+		CacheService cacheService)
 	{
 		this.sql2o = sql2o;
+		this.cacheService = cacheService;
 
 		try (Connection con = sql2o.open())
 		{
@@ -100,16 +105,6 @@ public class ItemService
 
 			con.createQuery(CREATE_PRICES)
 				.executeUpdate();
-
-			try
-			{
-				con.createQuery(CREATE_PRICES_FK)
-					.executeUpdate();
-			}
-			catch (Sql2oException ex)
-			{
-				// Ignore, happens when index already exists
-			}
 		}
 	}
 
@@ -248,9 +243,19 @@ public class ItemService
 
 	public List<PriceEntry> fetchPrice(int itemId)
 	{
+		RSPrices rsprice;
+		try
+		{
+			rsprice = fetchRSPrices(itemId);
+		}
+		catch (IOException ex)
+		{
+			log.warn("unable to fetch price for item {}", itemId, ex);
+			return null;
+		}
+
 		try (Connection con = sql2o.beginTransaction())
 		{
-			RSPrices rsprice = fetchRSPrices(itemId);
 			List<PriceEntry> entries = new ArrayList<>();
 			Instant now = Instant.now();
 
@@ -284,11 +289,6 @@ public class ItemService
 
 			return entries;
 		}
-		catch (IOException ex)
-		{
-			log.warn("unable to fetch price for item {}", itemId, ex);
-			return null;
-		}
 	}
 
 	public List<PriceEntry> fetchPrices()
@@ -407,18 +407,6 @@ public class ItemService
 		}
 	}
 
-	public void queuePriceLookup(int itemId)
-	{
-		if (pendingLookups.size() < MAX_PENDING)
-		{
-			pendingLookups.add(new PendingLookup(itemId, PendingLookup.Type.PRICE));
-		}
-		else
-		{
-			log.debug("Dropping pending price lookup for {}", itemId);
-		}
-	}
-
 	public void queueSearch(String search)
 	{
 		if (pendingLookups.size() < MAX_PENDING)
@@ -454,9 +442,6 @@ public class ItemService
 
 		switch (pendingLookup.getType())
 		{
-			case PRICE:
-				fetchPrice(pendingLookup.getItemId());
-				break;
 			case SEARCH:
 				try
 				{
@@ -475,4 +460,31 @@ public class ItemService
 		}
 	}
 
+	@Scheduled(fixedDelay = 20_000)
+	public void crawlPrices()
+	{
+		if (tradeableItems == null || tradeableItems.length == 0)
+		{
+			return;
+		}
+
+		int idx = random.nextInt(tradeableItems.length);
+		int id = tradeableItems[idx];
+
+		log.debug("Fetching price for {}", id);
+
+		fetchPrice(id);
+	}
+
+	@Scheduled(fixedDelay = 1_8000_000) // 30 minutes
+	public void reloadItems() throws IOException
+	{
+		List<ItemDefinition> items = cacheService.getItems();
+		tradeableItems = items.stream()
+			.filter(item -> item.isTradeable)
+			.mapToInt(item -> item.id)
+			.toArray();
+		log.debug("Loaded {} tradeable items", tradeableItems.length);
+	}
+
 }

http-service/src/main/java/net/runelite/http/service/item/PendingLookup.java

@@ -31,7 +31,6 @@ class PendingLookup
 {
 	enum Type
 	{
-		PRICE,
 		SEARCH,
 		ITEM;
 	}

http-service/src/test/resources/application.properties

@@ -7,3 +7,4 @@ minio.bucket=runelite
 runelite.twitter.consumerkey=moo
 runelite.twitter.secretkey=cow
 runelite.twitter.listid=968949795153948673
+logging.level.net.runelite=DEBUG