From 7f25ec82385e929b0bced62758b6cb004780eccc Mon Sep 17 00:00:00 2001 From: Adam Date: Sat, 9 Jun 2018 21:27:11 -0400 Subject: [PATCH] http-service: crawl tradable item prices instead of queueing on demand --- .../http/service/cache/CacheService.java | 19 +++++ .../http/service/item/ItemController.java | 8 -- .../http/service/item/ItemService.java | 84 +++++++++++-------- .../http/service/item/PendingLookup.java | 1 - .../src/test/resources/application.properties | 1 + 5 files changed, 68 insertions(+), 45 deletions(-) diff --git a/http-service/src/main/java/net/runelite/http/service/cache/CacheService.java b/http-service/src/main/java/net/runelite/http/service/cache/CacheService.java index eb06fd9069..882e90c460 100644 --- a/http-service/src/main/java/net/runelite/http/service/cache/CacheService.java +++ b/http-service/src/main/java/net/runelite/http/service/cache/CacheService.java @@ -43,7 +43,10 @@ import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.List; import lombok.extern.slf4j.Slf4j; +import net.runelite.cache.ConfigType; import net.runelite.cache.IndexType; +import net.runelite.cache.definitions.ItemDefinition; +import net.runelite.cache.definitions.loaders.ItemLoader; import net.runelite.cache.fs.ArchiveFiles; import net.runelite.cache.fs.Container; import net.runelite.cache.fs.FSFile; @@ -226,4 +229,20 @@ public class CacheService return cacheDao.findArchiveByName(con, cache, index, nameHash); } } + + public List getItems() throws IOException + { + CacheEntry cache = findMostRecent(); + IndexEntry indexEntry = findIndexForCache(cache, IndexType.CONFIGS.getNumber()); + ArchiveEntry archiveEntry = findArchiveForIndex(indexEntry, ConfigType.ITEM.getId()); + ArchiveFiles archiveFiles = getArchiveFiles(archiveEntry); + final ItemLoader itemLoader = new ItemLoader(); + final List result = new ArrayList<>(archiveFiles.getFiles().size()); + for (FSFile file : archiveFiles.getFiles()) + { + ItemDefinition itemDef = itemLoader.load(file.getFileId(), file.getContents()); + result.add(itemDef); + } + return result; + } } diff --git a/http-service/src/main/java/net/runelite/http/service/item/ItemController.java b/http-service/src/main/java/net/runelite/http/service/item/ItemController.java index 137539f2d4..a18615a29d 100644 --- a/http-service/src/main/java/net/runelite/http/service/item/ItemController.java +++ b/http-service/src/main/java/net/runelite/http/service/item/ItemController.java @@ -168,20 +168,12 @@ public class ItemController else if (priceEntry == null) { // Price is unknown - itemService.queuePriceLookup(itemId); // queue lookup cachedEmpty.put(itemId, itemId); return ResponseEntity.notFound() .header(RUNELITE_CACHE, "MISS") .build(); } - Instant cacheTime = now.minus(CACHE_DUATION); - if (priceEntry.getFetched_time().isBefore(cacheTime)) - { - // Queue a check for the price - itemService.queuePriceLookup(itemId); - } - ItemPrice itemPrice = new ItemPrice(); itemPrice.setItem(item.toItem()); itemPrice.setPrice(priceEntry.getPrice()); diff --git a/http-service/src/main/java/net/runelite/http/service/item/ItemService.java b/http-service/src/main/java/net/runelite/http/service/item/ItemService.java index cb36d35ef2..1cb9ca88bd 100644 --- a/http-service/src/main/java/net/runelite/http/service/item/ItemService.java +++ b/http-service/src/main/java/net/runelite/http/service/item/ItemService.java @@ -33,11 +33,14 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Random; import java.util.Set; import java.util.concurrent.ConcurrentLinkedQueue; import lombok.extern.slf4j.Slf4j; +import net.runelite.cache.definitions.ItemDefinition; import net.runelite.http.api.RuneLiteAPI; import net.runelite.http.api.item.ItemType; +import net.runelite.http.service.cache.CacheService; import okhttp3.HttpUrl; import okhttp3.Request; import okhttp3.Response; @@ -48,7 +51,6 @@ import org.springframework.stereotype.Service; import org.sql2o.Connection; import org.sql2o.Query; import org.sql2o.Sql2o; -import org.sql2o.Sql2oException; @Service @Slf4j @@ -80,18 +82,21 @@ public class ItemService + " KEY `item_fetched_time` (`item`,`fetched_time`)\n" + ") ENGINE=InnoDB"; - private static final String CREATE_PRICES_FK = "ALTER TABLE `prices`\n" - + " ADD CONSTRAINT `item` FOREIGN KEY (`item`) REFERENCES `items` (`id`);"; - private static final int MAX_PENDING = 512; private final Sql2o sql2o; + private final CacheService cacheService; + private final ConcurrentLinkedQueue pendingLookups = new ConcurrentLinkedQueue(); + private int[] tradeableItems; + private final Random random = new Random(); @Autowired - public ItemService(@Qualifier("Runelite SQL2O") Sql2o sql2o) + public ItemService(@Qualifier("Runelite SQL2O") Sql2o sql2o, + CacheService cacheService) { this.sql2o = sql2o; + this.cacheService = cacheService; try (Connection con = sql2o.open()) { @@ -100,16 +105,6 @@ public class ItemService con.createQuery(CREATE_PRICES) .executeUpdate(); - - try - { - con.createQuery(CREATE_PRICES_FK) - .executeUpdate(); - } - catch (Sql2oException ex) - { - // Ignore, happens when index already exists - } } } @@ -248,9 +243,19 @@ public class ItemService public List fetchPrice(int itemId) { + RSPrices rsprice; + try + { + rsprice = fetchRSPrices(itemId); + } + catch (IOException ex) + { + log.warn("unable to fetch price for item {}", itemId, ex); + return null; + } + try (Connection con = sql2o.beginTransaction()) { - RSPrices rsprice = fetchRSPrices(itemId); List entries = new ArrayList<>(); Instant now = Instant.now(); @@ -284,11 +289,6 @@ public class ItemService return entries; } - catch (IOException ex) - { - log.warn("unable to fetch price for item {}", itemId, ex); - return null; - } } public List fetchPrices() @@ -407,18 +407,6 @@ public class ItemService } } - public void queuePriceLookup(int itemId) - { - if (pendingLookups.size() < MAX_PENDING) - { - pendingLookups.add(new PendingLookup(itemId, PendingLookup.Type.PRICE)); - } - else - { - log.debug("Dropping pending price lookup for {}", itemId); - } - } - public void queueSearch(String search) { if (pendingLookups.size() < MAX_PENDING) @@ -454,9 +442,6 @@ public class ItemService switch (pendingLookup.getType()) { - case PRICE: - fetchPrice(pendingLookup.getItemId()); - break; case SEARCH: try { @@ -475,4 +460,31 @@ public class ItemService } } + @Scheduled(fixedDelay = 20_000) + public void crawlPrices() + { + if (tradeableItems == null || tradeableItems.length == 0) + { + return; + } + + int idx = random.nextInt(tradeableItems.length); + int id = tradeableItems[idx]; + + log.debug("Fetching price for {}", id); + + fetchPrice(id); + } + + @Scheduled(fixedDelay = 1_8000_000) // 30 minutes + public void reloadItems() throws IOException + { + List items = cacheService.getItems(); + tradeableItems = items.stream() + .filter(item -> item.isTradeable) + .mapToInt(item -> item.id) + .toArray(); + log.debug("Loaded {} tradeable items", tradeableItems.length); + } + } diff --git a/http-service/src/main/java/net/runelite/http/service/item/PendingLookup.java b/http-service/src/main/java/net/runelite/http/service/item/PendingLookup.java index 4e0e224868..d23ff5c1d9 100644 --- a/http-service/src/main/java/net/runelite/http/service/item/PendingLookup.java +++ b/http-service/src/main/java/net/runelite/http/service/item/PendingLookup.java @@ -31,7 +31,6 @@ class PendingLookup { enum Type { - PRICE, SEARCH, ITEM; } diff --git a/http-service/src/test/resources/application.properties b/http-service/src/test/resources/application.properties index 62f152bff5..6c9bc73c2f 100644 --- a/http-service/src/test/resources/application.properties +++ b/http-service/src/test/resources/application.properties @@ -7,3 +7,4 @@ minio.bucket=runelite runelite.twitter.consumerkey=moo runelite.twitter.secretkey=cow runelite.twitter.listid=968949795153948673 +logging.level.net.runelite=DEBUG