diff --git a/http-service/src/main/java/net/runelite/http/service/SpringBootWebApplication.java b/http-service/src/main/java/net/runelite/http/service/SpringBootWebApplication.java index 4ff7fd3b1d..1ec97c0e0c 100644 --- a/http-service/src/main/java/net/runelite/http/service/SpringBootWebApplication.java +++ b/http-service/src/main/java/net/runelite/http/service/SpringBootWebApplication.java @@ -44,11 +44,13 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.builder.SpringApplicationBuilder; import org.springframework.boot.web.support.SpringBootServletInitializer; import org.springframework.context.annotation.Bean; +import org.springframework.scheduling.annotation.EnableScheduling; import org.sql2o.Sql2o; import org.sql2o.converters.Converter; import org.sql2o.quirks.NoQuirks; @SpringBootApplication +@EnableScheduling @Slf4j public class SpringBootWebApplication extends SpringBootServletInitializer { diff --git a/http-service/src/main/java/net/runelite/http/service/item/ItemController.java b/http-service/src/main/java/net/runelite/http/service/item/ItemController.java new file mode 100644 index 0000000000..859540b4df --- /dev/null +++ b/http-service/src/main/java/net/runelite/http/service/item/ItemController.java @@ -0,0 +1,253 @@ +/* + * Copyright (c) 2017-2018, Adam + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package net.runelite.http.service.item; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import java.io.IOException; +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import javax.servlet.http.HttpServletResponse; +import lombok.extern.slf4j.Slf4j; +import net.runelite.http.api.item.Item; +import net.runelite.http.api.item.ItemPrice; +import net.runelite.http.api.item.SearchResult; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.CacheControl; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequestMapping("/item") +@Slf4j +public class ItemController +{ + private static final Duration CACHE_DUATION = Duration.ofMinutes(30); + private static final String RUNELITE_CACHE = "RuneLite-Cache"; + + private final Cache cachedSearches = CacheBuilder.newBuilder() + .maximumSize(1024L) + .build(); + private final Cache cachedEmpty = CacheBuilder.newBuilder() + .maximumSize(1024L) + .build(); + + private final ItemService itemService; + + @Autowired + public ItemController(ItemService itemService) + { + this.itemService = itemService; + } + + @RequestMapping("/{itemId}") + public Item getItem(HttpServletResponse response, @PathVariable int itemId) + { + ItemEntry item = itemService.getItem(itemId); + if (item != null) + { + response.setHeader(RUNELITE_CACHE, "HIT"); + return item.toItem(); + } + + item = itemService.fetchItem(itemId); + if (item != null) + { + response.setHeader(RUNELITE_CACHE, "MISS"); + return item.toItem(); + } + + return null; + } + + @RequestMapping(path = "/{itemId}/icon", produces = "image/gif") + public byte[] getIcon(HttpServletResponse response, @PathVariable int itemId) + { + ItemEntry item = itemService.getItem(itemId); + if (item != null && item.getIcon() != null) + { + response.setHeader(RUNELITE_CACHE, "HIT"); + return item.getIcon(); + } + + item = itemService.fetchItem(itemId); + if (item != null) + { + response.setHeader(RUNELITE_CACHE, "MISS"); + return item.getIcon(); + } + + return null; + } + + @RequestMapping(path = "/{itemId}/icon/large", produces = "image/gif") + public byte[] getIconLarge(HttpServletResponse response, @PathVariable int itemId) + { + ItemEntry item = itemService.getItem(itemId); + if (item != null && item.getIcon_large() != null) + { + response.setHeader(RUNELITE_CACHE, "HIT"); + return item.getIcon_large(); + } + + item = itemService.fetchItem(itemId); + if (item != null) + { + response.setHeader(RUNELITE_CACHE, "MISS"); + return item.getIcon_large(); + } + + return null; + } + + @RequestMapping("/{itemId}/price") + public ResponseEntity itemPrice( + @PathVariable int itemId, + @RequestParam(required = false) Instant time + ) + { + if (cachedEmpty.getIfPresent(itemId) != null) + { + return ResponseEntity.notFound() + .header(RUNELITE_CACHE, "HIT") + .build(); + } + + Instant now = Instant.now(); + boolean hit = true; + + if (time != null && time.isAfter(now)) + { + time = now; + } + + ItemEntry item = itemService.getItem(itemId); + if (item == null) + { + item = itemService.fetchItem(itemId); + hit = false; + + if (item == null) + { + cachedEmpty.put(itemId, itemId); + return ResponseEntity.notFound() + .header(RUNELITE_CACHE, "MISS") + .build(); + } + } + + PriceEntry priceEntry = itemService.getPrice(itemId, time); + + if (time != null) + { + if (priceEntry == null) + { + // we maybe can't backfill this + return ResponseEntity.notFound() + .header(RUNELITE_CACHE, "MISS") + .build(); + } + } + else if (priceEntry == null) + { + // Price is unknown + List prices = itemService.fetchPrice(itemId); + + if (prices == null || prices.isEmpty()) + { + cachedEmpty.put(itemId, itemId); + return ResponseEntity.notFound() + .header(RUNELITE_CACHE, "MISS") + .build(); + } + + // Get the most recent price + priceEntry = prices.get(prices.size() - 1); + hit = false; + } + else + { + Instant cacheTime = now.minus(CACHE_DUATION); + if (priceEntry.getFetched_time().isBefore(cacheTime)) + { + // Queue a check for the price + itemService.queueLookup(itemId); + } + } + + ItemPrice itemPrice = new ItemPrice(); + itemPrice.setItem(item.toItem()); + itemPrice.setPrice(priceEntry.getPrice()); + itemPrice.setTime(priceEntry.getTime()); + + return ResponseEntity.ok() + .header(RUNELITE_CACHE, hit ? "HIT" : "MISS") + .cacheControl(CacheControl.maxAge(30, TimeUnit.MINUTES).cachePublic()) + .body(itemPrice); + } + + @RequestMapping("/search") + public SearchResult search(HttpServletResponse response, @RequestParam String query) + { + // rs api seems to require lowercase + query = query.toLowerCase(); + + SearchResult searchResult = cachedSearches.getIfPresent(query); + if (searchResult != null) + { + response.setHeader(RUNELITE_CACHE, "HIT"); + return searchResult; + } + + try + { + RSSearch search = itemService.fetchRSSearch(query); + + searchResult = new SearchResult(); + List items = search.getItems().stream() + .map(rsi -> rsi.toItem()) + .collect(Collectors.toList()); + searchResult.setItems(items); + + cachedSearches.put(query, searchResult); + + itemService.batchInsertItems(search); + + response.setHeader(RUNELITE_CACHE, "MISS"); + return searchResult; + } + catch (IOException ex) + { + log.warn("error while searching items", ex); + return null; + } + } +} diff --git a/http-service/src/main/java/net/runelite/http/service/item/ItemService.java b/http-service/src/main/java/net/runelite/http/service/item/ItemService.java index 0a6b7b2e62..9138df358c 100644 --- a/http-service/src/main/java/net/runelite/http/service/item/ItemService.java +++ b/http-service/src/main/java/net/runelite/http/service/item/ItemService.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, Adam + * Copyright (c) 2017-2018, Adam * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -24,56 +24,39 @@ */ package net.runelite.http.service.item; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; import com.google.gson.JsonParseException; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; -import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.List; -import java.util.Map.Entry; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import javax.servlet.http.HttpServletResponse; +import java.util.Map; +import java.util.concurrent.ConcurrentLinkedQueue; +import lombok.extern.slf4j.Slf4j; import net.runelite.http.api.RuneLiteAPI; -import net.runelite.http.api.item.Item; -import net.runelite.http.api.item.ItemPrice; import net.runelite.http.api.item.ItemType; -import net.runelite.http.api.item.SearchResult; import okhttp3.HttpUrl; import okhttp3.Request; import okhttp3.Response; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.http.CacheControl; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; import org.sql2o.Connection; import org.sql2o.Query; import org.sql2o.Sql2o; import org.sql2o.Sql2oException; -@RestController -@RequestMapping("/item") +@Service +@Slf4j public class ItemService { - private static final Logger logger = LoggerFactory.getLogger(ItemService.class); - private static final String BASE = "https://services.runescape.com/m=itemdb_oldschool"; private static final HttpUrl RS_ITEM_URL = HttpUrl.parse(BASE + "/api/catalogue/detail.json"); private static final HttpUrl RS_PRICE_URL = HttpUrl.parse(BASE + "/api/graph"); private static final HttpUrl RS_SEARCH_URL = HttpUrl.parse(BASE + "/api/catalogue/items.json?category=1"); - private static final Duration CACHE_DUATION = Duration.ofMinutes(30); - private static final String CREATE_ITEMS = "CREATE TABLE IF NOT EXISTS `items` (\n" + " `id` int(11) NOT NULL,\n" + " `name` tinytext NOT NULL,\n" @@ -97,12 +80,10 @@ public class ItemService private static final String CREATE_PRICES_FK = "ALTER TABLE `prices`\n" + " ADD CONSTRAINT `item` FOREIGN KEY (`item`) REFERENCES `items` (`id`);"; - private static final String RUNELITE_CACHE = "RuneLite-Cache"; + private static final int MAX_PENDING_LOOKUPS = 512; private final Sql2o sql2o; - private final Cache cachedSearches = CacheBuilder.newBuilder() - .maximumSize(1024L) - .build(); + private final ConcurrentLinkedQueue pendingPriceLookups = new ConcurrentLinkedQueue<>(); @Autowired public ItemService(@Qualifier("Runelite SQL2O") Sql2o sql2o) @@ -129,187 +110,7 @@ public class ItemService } } - @RequestMapping("/{itemId}") - public Item getItem(HttpServletResponse response, @PathVariable int itemId) - { - ItemEntry item = getItem(itemId); - if (item != null) - { - response.setHeader(RUNELITE_CACHE, "HIT"); - return item.toItem(); - } - - item = fetchItem(itemId); - if (item != null) - { - response.setHeader(RUNELITE_CACHE, "MISS"); - return item.toItem(); - } - - return null; - } - - @RequestMapping(path = "/{itemId}/icon", produces = "image/gif") - public byte[] getIcon(HttpServletResponse response, @PathVariable int itemId) - { - ItemEntry item = getItem(itemId); - if (item != null && item.getIcon() != null) - { - response.setHeader(RUNELITE_CACHE, "HIT"); - return item.getIcon(); - } - - item = fetchItem(itemId); - if (item != null) - { - response.setHeader(RUNELITE_CACHE, "MISS"); - return item.getIcon(); - } - - return null; - } - - @RequestMapping(path = "/{itemId}/icon/large", produces = "image/gif") - public byte[] getIconLarge(HttpServletResponse response, @PathVariable int itemId) - { - ItemEntry item = getItem(itemId); - if (item != null && item.getIcon_large() != null) - { - response.setHeader(RUNELITE_CACHE, "HIT"); - return item.getIcon_large(); - } - - item = fetchItem(itemId); - if (item != null) - { - response.setHeader(RUNELITE_CACHE, "MISS"); - return item.getIcon_large(); - } - - return null; - } - - @RequestMapping("/{itemId}/price") - public ResponseEntity itemPrice( - @PathVariable int itemId, - @RequestParam(required = false) Instant time - ) - { - Instant now = Instant.now(); - boolean hit = true; - - if (time != null && time.isAfter(now)) - { - time = now; - } - - ItemEntry item = getItem(itemId); - if (item == null) - { - item = fetchItem(itemId); - hit = false; - - if (item == null) - { - return ResponseEntity.notFound().build(); - } - } - - PriceEntry priceEntry = getPrice(itemId, time); - - if (time != null) - { - if (priceEntry == null) - { - // we maybe can't backfill this - return ResponseEntity.notFound().build(); - } - } - else - { - Instant cacheTime = now.minus(CACHE_DUATION); - if (priceEntry == null || priceEntry.getFetched_time().isBefore(cacheTime)) - { - // Price is unknown, or was fetched too long ago - List prices = fetchPrice(itemId); - - if (prices == null || prices.isEmpty()) - { - return ResponseEntity.notFound().build(); - } - - // Get the most recent price - priceEntry = prices.get(prices.size() - 1); - hit = false; - } - } - - ItemPrice itemPrice = new ItemPrice(); - itemPrice.setItem(item.toItem()); - itemPrice.setPrice(priceEntry.getPrice()); - itemPrice.setTime(priceEntry.getTime()); - - return ResponseEntity.ok() - .header(RUNELITE_CACHE, hit ? "HIT" : "MISS") - .cacheControl(CacheControl.maxAge(30, TimeUnit.MINUTES).cachePublic()) - .body(itemPrice); - } - - @RequestMapping("/search") - public SearchResult search(HttpServletResponse response, @RequestParam String query) - { - // rs api seems to require lowercase - query = query.toLowerCase(); - - SearchResult searchResult = cachedSearches.getIfPresent(query); - if (searchResult != null) - { - response.setHeader(RUNELITE_CACHE, "HIT"); - return searchResult; - } - - try - { - RSSearch search = fetchRSSearch(query); - - searchResult = new SearchResult(); - List items = search.getItems().stream() - .map(rsi -> rsi.toItem()) - .collect(Collectors.toList()); - searchResult.setItems(items); - - cachedSearches.put(query, searchResult); - - try (Connection con = sql2o.beginTransaction()) - { - Query q = con.createQuery("insert into items (id, name, description, type) values (:id," - + " :name, :description, :type) ON DUPLICATE KEY UPDATE name = :name," - + " description = :description, type = :type"); - - for (RSItem rsItem : search.getItems()) - { - q.addParameter("id", rsItem.getId()) - .addParameter("name", rsItem.getName()) - .addParameter("description", rsItem.getDescription()) - .addParameter("type", rsItem.getType()) - .addToBatch(); - } - - q.executeBatch(); - con.commit(); - } - - response.setHeader(RUNELITE_CACHE, "MISS"); - return searchResult; - } - catch (IOException ex) - { - logger.warn("error while searching items", ex); - return null; - } - } - - private ItemEntry getItem(int itemId) + public ItemEntry getItem(int itemId) { try (Connection con = sql2o.open()) { @@ -321,7 +122,7 @@ public class ItemService } } - private PriceEntry getPrice(int itemId, Instant time) + public PriceEntry getPrice(int itemId, Instant time) { try (Connection con = sql2o.open()) { @@ -341,7 +142,7 @@ public class ItemService } } - private ItemEntry fetchItem(int itemId) + public ItemEntry fetchItem(int itemId) { try { @@ -354,7 +155,7 @@ public class ItemService } catch (IOException ex) { - logger.warn("error fetching image", ex); + log.warn("error fetching image", ex); } try @@ -363,7 +164,7 @@ public class ItemService } catch (IOException ex) { - logger.warn("error fetching image", ex); + log.warn("error fetching image", ex); } try (Connection con = sql2o.open()) @@ -391,12 +192,12 @@ public class ItemService } catch (IOException ex) { - logger.warn("unable to fetch item {}", itemId, ex); + log.warn("unable to fetch item {}", itemId, ex); return null; } } - private List fetchPrice(int itemId) + public List fetchPrice(int itemId) { try (Connection con = sql2o.beginTransaction()) { @@ -407,7 +208,7 @@ public class ItemService Query query = con.createQuery("insert into prices (item, price, time, fetched_time) values (:item, :price, :time, :fetched_time) " + "ON DUPLICATE KEY UPDATE price = VALUES(price), fetched_time = VALUES(fetched_time)"); - for (Entry entry : rsprice.getDaily().entrySet()) + for (Map.Entry entry : rsprice.getDaily().entrySet()) { long ts = entry.getKey(); // ms since epoch int price = entry.getValue(); // gp @@ -436,7 +237,7 @@ public class ItemService } catch (IOException ex) { - logger.warn("unable to fetch price for item {}", itemId, ex); + log.warn("unable to fetch price for item {}", itemId, ex); return null; } } @@ -471,7 +272,7 @@ public class ItemService return fetchJson(request, RSPrices.class); } - private RSSearch fetchRSSearch(String query) throws IOException + public RSSearch fetchRSSearch(String query) throws IOException { HttpUrl searchUrl = RS_SEARCH_URL .newBuilder() @@ -485,6 +286,28 @@ public class ItemService return fetchJson(request, RSSearch.class); } + public void batchInsertItems(RSSearch search) + { + try (Connection con = sql2o.beginTransaction()) + { + Query q = con.createQuery("insert into items (id, name, description, type) values (:id," + + " :name, :description, :type) ON DUPLICATE KEY UPDATE name = :name," + + " description = :description, type = :type"); + + for (RSItem rsItem : search.getItems()) + { + q.addParameter("id", rsItem.getId()) + .addParameter("name", rsItem.getName()) + .addParameter("description", rsItem.getDescription()) + .addParameter("type", rsItem.getType()) + .addToBatch(); + } + + q.executeBatch(); + con.commit(); + } + } + private T fetchJson(Request request, Class clazz) throws IOException { try (Response response = RuneLiteAPI.CLIENT.newCall(request).execute()) @@ -521,4 +344,26 @@ public class ItemService return response.body().bytes(); } } + + public void queueLookup(int itemId) + { + if (pendingPriceLookups.size() >= MAX_PENDING_LOOKUPS) + { + return; + } + + pendingPriceLookups.add(itemId); + } + + @Scheduled(fixedDelay = 5000) + public void checkPrices() + { + Integer itemId = pendingPriceLookups.poll(); + if (itemId == null) + { + return; + } + + fetchPrice(itemId); + } } diff --git a/http-service/src/test/java/net/runelite/http/service/SpringBootWebApplicationTest.java b/http-service/src/test/java/net/runelite/http/service/SpringBootWebApplicationTest.java index 0edc296898..0503c075d8 100644 --- a/http-service/src/test/java/net/runelite/http/service/SpringBootWebApplicationTest.java +++ b/http-service/src/test/java/net/runelite/http/service/SpringBootWebApplicationTest.java @@ -34,11 +34,13 @@ import org.junit.Test; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.context.annotation.Bean; +import org.springframework.scheduling.annotation.EnableScheduling; import org.sql2o.Sql2o; import org.sql2o.converters.Converter; import org.sql2o.quirks.NoQuirks; @SpringBootApplication +@EnableScheduling public class SpringBootWebApplicationTest { @Bean("Runelite SQL2O")