Merge pull request #611 from Adam-/prices

item service: background price fetching

http-service/src/main/java/net/runelite/http/service/SpringBootWebApplication.java

@@ -44,11 +44,13 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.builder.SpringApplicationBuilder;
 import org.springframework.boot.web.support.SpringBootServletInitializer;
 import org.springframework.context.annotation.Bean;
+import org.springframework.scheduling.annotation.EnableScheduling;
 import org.sql2o.Sql2o;
 import org.sql2o.converters.Converter;
 import org.sql2o.quirks.NoQuirks;
 
 @SpringBootApplication
+@EnableScheduling
 @Slf4j
 public class SpringBootWebApplication extends SpringBootServletInitializer
 {

http-service/src/main/java/net/runelite/http/service/item/ItemController.java

@@ -0,0 +1,253 @@
+/*
+ * Copyright (c) 2017-2018, Adam <Adam@sigterm.info>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package net.runelite.http.service.item;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import java.io.IOException;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import javax.servlet.http.HttpServletResponse;
+import lombok.extern.slf4j.Slf4j;
+import net.runelite.http.api.item.Item;
+import net.runelite.http.api.item.ItemPrice;
+import net.runelite.http.api.item.SearchResult;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.CacheControl;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+@RestController
+@RequestMapping("/item")
+@Slf4j
+public class ItemController
+{
+	private static final Duration CACHE_DUATION = Duration.ofMinutes(30);
+	private static final String RUNELITE_CACHE = "RuneLite-Cache";
+
+	private final Cache<String, SearchResult> cachedSearches = CacheBuilder.newBuilder()
+		.maximumSize(1024L)
+		.build();
+	private final Cache<Integer, Integer> cachedEmpty = CacheBuilder.newBuilder()
+		.maximumSize(1024L)
+		.build();
+
+	private final ItemService itemService;
+
+	@Autowired
+	public ItemController(ItemService itemService)
+	{
+		this.itemService = itemService;
+	}
+
+	@RequestMapping("/{itemId}")
+	public Item getItem(HttpServletResponse response, @PathVariable int itemId)
+	{
+		ItemEntry item = itemService.getItem(itemId);
+		if (item != null)
+		{
+			response.setHeader(RUNELITE_CACHE, "HIT");
+			return item.toItem();
+		}
+
+		item = itemService.fetchItem(itemId);
+		if (item != null)
+		{
+			response.setHeader(RUNELITE_CACHE, "MISS");
+			return item.toItem();
+		}
+
+		return null;
+	}
+
+	@RequestMapping(path = "/{itemId}/icon", produces = "image/gif")
+	public byte[] getIcon(HttpServletResponse response, @PathVariable int itemId)
+	{
+		ItemEntry item = itemService.getItem(itemId);
+		if (item != null && item.getIcon() != null)
+		{
+			response.setHeader(RUNELITE_CACHE, "HIT");
+			return item.getIcon();
+		}
+
+		item = itemService.fetchItem(itemId);
+		if (item != null)
+		{
+			response.setHeader(RUNELITE_CACHE, "MISS");
+			return item.getIcon();
+		}
+
+		return null;
+	}
+
+	@RequestMapping(path = "/{itemId}/icon/large", produces = "image/gif")
+	public byte[] getIconLarge(HttpServletResponse response, @PathVariable int itemId)
+	{
+		ItemEntry item = itemService.getItem(itemId);
+		if (item != null && item.getIcon_large() != null)
+		{
+			response.setHeader(RUNELITE_CACHE, "HIT");
+			return item.getIcon_large();
+		}
+
+		item = itemService.fetchItem(itemId);
+		if (item != null)
+		{
+			response.setHeader(RUNELITE_CACHE, "MISS");
+			return item.getIcon_large();
+		}
+
+		return null;
+	}
+
+	@RequestMapping("/{itemId}/price")
+	public ResponseEntity<ItemPrice> itemPrice(
+		@PathVariable int itemId,
+		@RequestParam(required = false) Instant time
+	)
+	{
+		if (cachedEmpty.getIfPresent(itemId) != null)
+		{
+			return ResponseEntity.notFound()
+				.header(RUNELITE_CACHE, "HIT")
+				.build();
+		}
+
+		Instant now = Instant.now();
+		boolean hit = true;
+
+		if (time != null && time.isAfter(now))
+		{
+			time = now;
+		}
+
+		ItemEntry item = itemService.getItem(itemId);
+		if (item == null)
+		{
+			item = itemService.fetchItem(itemId);
+			hit = false;
+
+			if (item == null)
+			{
+				cachedEmpty.put(itemId, itemId);
+				return ResponseEntity.notFound()
+					.header(RUNELITE_CACHE, "MISS")
+					.build();
+			}
+		}
+
+		PriceEntry priceEntry = itemService.getPrice(itemId, time);
+
+		if (time != null)
+		{
+			if (priceEntry == null)
+			{
+				// we maybe can't backfill this
+				return ResponseEntity.notFound()
+					.header(RUNELITE_CACHE, "MISS")
+					.build();
+			}
+		}
+		else if (priceEntry == null)
+		{
+			// Price is unknown
+			List<PriceEntry> prices = itemService.fetchPrice(itemId);
+
+			if (prices == null || prices.isEmpty())
+			{
+				cachedEmpty.put(itemId, itemId);
+				return ResponseEntity.notFound()
+					.header(RUNELITE_CACHE, "MISS")
+					.build();
+			}
+
+			// Get the most recent price
+			priceEntry = prices.get(prices.size() - 1);
+			hit = false;
+		}
+		else
+		{
+			Instant cacheTime = now.minus(CACHE_DUATION);
+			if (priceEntry.getFetched_time().isBefore(cacheTime))
+			{
+				// Queue a check for the price
+				itemService.queueLookup(itemId);
+			}
+		}
+
+		ItemPrice itemPrice = new ItemPrice();
+		itemPrice.setItem(item.toItem());
+		itemPrice.setPrice(priceEntry.getPrice());
+		itemPrice.setTime(priceEntry.getTime());
+
+		return ResponseEntity.ok()
+			.header(RUNELITE_CACHE, hit ? "HIT" : "MISS")
+			.cacheControl(CacheControl.maxAge(30, TimeUnit.MINUTES).cachePublic())
+			.body(itemPrice);
+	}
+
+	@RequestMapping("/search")
+	public SearchResult search(HttpServletResponse response, @RequestParam String query)
+	{
+		// rs api seems to require lowercase
+		query = query.toLowerCase();
+
+		SearchResult searchResult = cachedSearches.getIfPresent(query);
+		if (searchResult != null)
+		{
+			response.setHeader(RUNELITE_CACHE, "HIT");
+			return searchResult;
+		}
+
+		try
+		{
+			RSSearch search = itemService.fetchRSSearch(query);
+
+			searchResult = new SearchResult();
+			List<Item> items = search.getItems().stream()
+				.map(rsi -> rsi.toItem())
+				.collect(Collectors.toList());
+			searchResult.setItems(items);
+
+			cachedSearches.put(query, searchResult);
+
+			itemService.batchInsertItems(search);
+
+			response.setHeader(RUNELITE_CACHE, "MISS");
+			return searchResult;
+		}
+		catch (IOException ex)
+		{
+			log.warn("error while searching items", ex);
+			return null;
+		}
+	}
+}

http-service/src/main/java/net/runelite/http/service/item/ItemService.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, Adam <Adam@sigterm.info>
+ * Copyright (c) 2017-2018, Adam <Adam@sigterm.info>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -24,56 +24,39 @@
  */
 package net.runelite.http.service.item;
 
-import com.google.common.cache.Cache;
-import com.google.common.cache.CacheBuilder;
 import com.google.gson.JsonParseException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
-import java.time.Duration;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Map.Entry;
+import java.util.Map;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.stream.Collectors;
+import lombok.extern.slf4j.Slf4j;
-import javax.servlet.http.HttpServletResponse;
 import net.runelite.http.api.RuneLiteAPI;
-import net.runelite.http.api.item.Item;
-import net.runelite.http.api.item.ItemPrice;
 import net.runelite.http.api.item.ItemType;
-import net.runelite.http.api.item.SearchResult;
 import okhttp3.HttpUrl;
 import okhttp3.Request;
 import okhttp3.Response;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.http.CacheControl;
+import org.springframework.scheduling.annotation.Scheduled;
-import org.springframework.http.ResponseEntity;
+import org.springframework.stereotype.Service;
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RequestParam;
-import org.springframework.web.bind.annotation.RestController;
 import org.sql2o.Connection;
 import org.sql2o.Query;
 import org.sql2o.Sql2o;
 import org.sql2o.Sql2oException;
 
-@RestController
+@Service
-@RequestMapping("/item")
+@Slf4j
 public class ItemService
 {
-	private static final Logger logger = LoggerFactory.getLogger(ItemService.class);
-
 	private static final String BASE = "https://services.runescape.com/m=itemdb_oldschool";
 	private static final HttpUrl RS_ITEM_URL = HttpUrl.parse(BASE + "/api/catalogue/detail.json");
 	private static final HttpUrl RS_PRICE_URL = HttpUrl.parse(BASE + "/api/graph");
 	private static final HttpUrl RS_SEARCH_URL = HttpUrl.parse(BASE + "/api/catalogue/items.json?category=1");
 
-	private static final Duration CACHE_DUATION = Duration.ofMinutes(30);
-
 	private static final String CREATE_ITEMS = "CREATE TABLE IF NOT EXISTS `items` (\n"
 		+ "  `id` int(11) NOT NULL,\n"
 		+ "  `name` tinytext NOT NULL,\n"
@@ -97,12 +80,10 @@ public class ItemService
 	private static final String CREATE_PRICES_FK = "ALTER TABLE `prices`\n"
 		+ "  ADD CONSTRAINT `item` FOREIGN KEY (`item`) REFERENCES `items` (`id`);";
 
-	private static final String RUNELITE_CACHE = "RuneLite-Cache";
+	private static final int MAX_PENDING_LOOKUPS = 512;
 
 	private final Sql2o sql2o;
-	private final Cache<String, SearchResult> cachedSearches = CacheBuilder.newBuilder()
+	private final ConcurrentLinkedQueue<Integer> pendingPriceLookups = new ConcurrentLinkedQueue<>();
-		.maximumSize(1024L)
-		.build();
 
 	@Autowired
 	public ItemService(@Qualifier("Runelite SQL2O") Sql2o sql2o)
@@ -129,187 +110,7 @@ public class ItemService
 		}
 	}
 
-	@RequestMapping("/{itemId}")
+	public ItemEntry getItem(int itemId)
-	public Item getItem(HttpServletResponse response, @PathVariable int itemId)
-	{
-		ItemEntry item = getItem(itemId);
-		if (item != null)
-		{
-			response.setHeader(RUNELITE_CACHE, "HIT");
-			return item.toItem();
-		}
-
-		item = fetchItem(itemId);
-		if (item != null)
-		{
-			response.setHeader(RUNELITE_CACHE, "MISS");
-			return item.toItem();
-		}
-
-		return null;
-	}
-
-	@RequestMapping(path = "/{itemId}/icon", produces = "image/gif")
-	public byte[] getIcon(HttpServletResponse response, @PathVariable int itemId)
-	{
-		ItemEntry item = getItem(itemId);
-		if (item != null && item.getIcon() != null)
-		{
-			response.setHeader(RUNELITE_CACHE, "HIT");
-			return item.getIcon();
-		}
-
-		item = fetchItem(itemId);
-		if (item != null)
-		{
-			response.setHeader(RUNELITE_CACHE, "MISS");
-			return item.getIcon();
-		}
-
-		return null;
-	}
-
-	@RequestMapping(path = "/{itemId}/icon/large", produces = "image/gif")
-	public byte[] getIconLarge(HttpServletResponse response, @PathVariable int itemId)
-	{
-		ItemEntry item = getItem(itemId);
-		if (item != null && item.getIcon_large() != null)
-		{
-			response.setHeader(RUNELITE_CACHE, "HIT");
-			return item.getIcon_large();
-		}
-
-		item = fetchItem(itemId);
-		if (item != null)
-		{
-			response.setHeader(RUNELITE_CACHE, "MISS");
-			return item.getIcon_large();
-		}
-
-		return null;
-	}
-
-	@RequestMapping("/{itemId}/price")
-	public ResponseEntity<ItemPrice> itemPrice(
-		@PathVariable int itemId,
-		@RequestParam(required = false) Instant time
-	)
-	{
-		Instant now = Instant.now();
-		boolean hit = true;
-
-		if (time != null && time.isAfter(now))
-		{
-			time = now;
-		}
-
-		ItemEntry item = getItem(itemId);
-		if (item == null)
-		{
-			item = fetchItem(itemId);
-			hit = false;
-
-			if (item == null)
-			{
-				return ResponseEntity.notFound().build();
-			}
-		}
-
-		PriceEntry priceEntry = getPrice(itemId, time);
-
-		if (time != null)
-		{
-			if (priceEntry == null)
-			{
-				// we maybe can't backfill this
-				return ResponseEntity.notFound().build();
-			}
-		}
-		else
-		{
-			Instant cacheTime = now.minus(CACHE_DUATION);
-			if (priceEntry == null || priceEntry.getFetched_time().isBefore(cacheTime))
-			{
-				// Price is unknown, or was fetched too long ago
-				List<PriceEntry> prices = fetchPrice(itemId);
-
-				if (prices == null || prices.isEmpty())
-				{
-					return ResponseEntity.notFound().build();
-				}
-
-				// Get the most recent price
-				priceEntry = prices.get(prices.size() - 1);
-				hit = false;
-			}
-		}
-
-		ItemPrice itemPrice = new ItemPrice();
-		itemPrice.setItem(item.toItem());
-		itemPrice.setPrice(priceEntry.getPrice());
-		itemPrice.setTime(priceEntry.getTime());
-
-		return ResponseEntity.ok()
-			.header(RUNELITE_CACHE, hit ? "HIT" : "MISS")
-			.cacheControl(CacheControl.maxAge(30, TimeUnit.MINUTES).cachePublic())
-			.body(itemPrice);
-	}
-
-	@RequestMapping("/search")
-	public SearchResult search(HttpServletResponse response, @RequestParam String query)
-	{
-		// rs api seems to require lowercase
-		query = query.toLowerCase();
-
-		SearchResult searchResult = cachedSearches.getIfPresent(query);
-		if (searchResult != null)
-		{
-			response.setHeader(RUNELITE_CACHE, "HIT");
-			return searchResult;
-		}
-
-		try
-		{
-			RSSearch search = fetchRSSearch(query);
-
-			searchResult = new SearchResult();
-			List<Item> items = search.getItems().stream()
-				.map(rsi -> rsi.toItem())
-				.collect(Collectors.toList());
-			searchResult.setItems(items);
-
-			cachedSearches.put(query, searchResult);
-
-			try (Connection con = sql2o.beginTransaction())
-			{
-				Query q = con.createQuery("insert into items (id, name, description, type) values (:id,"
-					+ " :name, :description, :type) ON DUPLICATE KEY UPDATE name = :name,"
-					+ " description = :description, type = :type");
-
-				for (RSItem rsItem : search.getItems())
-				{
-					q.addParameter("id", rsItem.getId())
-						.addParameter("name", rsItem.getName())
-						.addParameter("description", rsItem.getDescription())
-						.addParameter("type", rsItem.getType())
-						.addToBatch();
-				}
-
-				q.executeBatch();
-				con.commit();
-			}
-
-			response.setHeader(RUNELITE_CACHE, "MISS");
-			return searchResult;
-		}
-		catch (IOException ex)
-		{
-			logger.warn("error while searching items", ex);
-			return null;
-		}
-	}
-
-	private ItemEntry getItem(int itemId)
 	{
 		try (Connection con = sql2o.open())
 		{
@@ -321,7 +122,7 @@ public class ItemService
 		}
 	}
 
-	private PriceEntry getPrice(int itemId, Instant time)
+	public PriceEntry getPrice(int itemId, Instant time)
 	{
 		try (Connection con = sql2o.open())
 		{
@@ -341,7 +142,7 @@ public class ItemService
 		}
 	}
 
-	private ItemEntry fetchItem(int itemId)
+	public ItemEntry fetchItem(int itemId)
 	{
 		try
 		{
@@ -354,7 +155,7 @@ public class ItemService
 			}
 			catch (IOException ex)
 			{
-				logger.warn("error fetching image", ex);
+				log.warn("error fetching image", ex);
 			}
 
 			try
@@ -363,7 +164,7 @@ public class ItemService
 			}
 			catch (IOException ex)
 			{
-				logger.warn("error fetching image", ex);
+				log.warn("error fetching image", ex);
 			}
 
 			try (Connection con = sql2o.open())
@@ -391,12 +192,12 @@ public class ItemService
 		}
 		catch (IOException ex)
 		{
-			logger.warn("unable to fetch item {}", itemId, ex);
+			log.warn("unable to fetch item {}", itemId, ex);
 			return null;
 		}
 	}
 
-	private List<PriceEntry> fetchPrice(int itemId)
+	public List<PriceEntry> fetchPrice(int itemId)
 	{
 		try (Connection con = sql2o.beginTransaction())
 		{
@@ -407,7 +208,7 @@ public class ItemService
 			Query query = con.createQuery("insert into prices (item, price, time, fetched_time) values (:item, :price, :time, :fetched_time) "
 				+ "ON DUPLICATE KEY UPDATE price = VALUES(price), fetched_time = VALUES(fetched_time)");
 
-			for (Entry<Long, Integer> entry : rsprice.getDaily().entrySet())
+			for (Map.Entry<Long, Integer> entry : rsprice.getDaily().entrySet())
 			{
 				long ts = entry.getKey(); // ms since epoch
 				int price = entry.getValue(); // gp
@@ -436,7 +237,7 @@ public class ItemService
 		}
 		catch (IOException ex)
 		{
-			logger.warn("unable to fetch price for item {}", itemId, ex);
+			log.warn("unable to fetch price for item {}", itemId, ex);
 			return null;
 		}
 	}
@@ -471,7 +272,7 @@ public class ItemService
 		return fetchJson(request, RSPrices.class);
 	}
 
-	private RSSearch fetchRSSearch(String query) throws IOException
+	public RSSearch fetchRSSearch(String query) throws IOException
 	{
 		HttpUrl searchUrl = RS_SEARCH_URL
 			.newBuilder()
@@ -485,6 +286,28 @@ public class ItemService
 		return fetchJson(request, RSSearch.class);
 	}
 
+	public void batchInsertItems(RSSearch search)
+	{
+		try (Connection con = sql2o.beginTransaction())
+		{
+			Query q = con.createQuery("insert into items (id, name, description, type) values (:id,"
+				+ " :name, :description, :type) ON DUPLICATE KEY UPDATE name = :name,"
+				+ " description = :description, type = :type");
+
+			for (RSItem rsItem : search.getItems())
+			{
+				q.addParameter("id", rsItem.getId())
+					.addParameter("name", rsItem.getName())
+					.addParameter("description", rsItem.getDescription())
+					.addParameter("type", rsItem.getType())
+					.addToBatch();
+			}
+
+			q.executeBatch();
+			con.commit();
+		}
+	}
+
 	private <T> T fetchJson(Request request, Class<T> clazz) throws IOException
 	{
 		try (Response response = RuneLiteAPI.CLIENT.newCall(request).execute())
@@ -521,4 +344,26 @@ public class ItemService
 			return response.body().bytes();
 		}
 	}
+
+	public void queueLookup(int itemId)
+	{
+		if (pendingPriceLookups.size() >= MAX_PENDING_LOOKUPS)
+		{
+			return;
+		}
+
+		pendingPriceLookups.add(itemId);
+	}
+
+	@Scheduled(fixedDelay = 5000)
+	public void checkPrices()
+	{
+		Integer itemId = pendingPriceLookups.poll();
+		if (itemId == null)
+		{
+			return;
+		}
+
+		fetchPrice(itemId);
+	}
 }

http-service/src/test/java/net/runelite/http/service/SpringBootWebApplicationTest.java

@@ -34,11 +34,13 @@ import org.junit.Test;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.context.annotation.Bean;
+import org.springframework.scheduling.annotation.EnableScheduling;
 import org.sql2o.Sql2o;
 import org.sql2o.converters.Converter;
 import org.sql2o.quirks.NoQuirks;
 
 @SpringBootApplication
+@EnableScheduling
 public class SpringBootWebApplicationTest
 {
 	@Bean("Runelite SQL2O")