ra/runelite
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

item service: remove old item searching and item lookup methods

Browse Source 
We haven't used this code in awhile. Additionally modify the price
crawler to re-fetch known items in the event the name or examine
changes.
This commit is contained in:
Adam
2020-04-06 23:25:45 -04:00
parent 81914b7c95
commit 670e62b6b9
10 changed files with 19 additions and 589 deletions
Download Patch File Download Diff File Expand all files Collapse all files

90
http-service/src/main/java/net/runelite/http/service/item/ItemController.java
View File

@@ -26,8 +26,6 @@ package net.runelite.http.service.item;
import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.hash.HashCode;
import com.google.common.hash.Hasher;
import com.google.common.hash.Hashing;
@@ -35,11 +33,7 @@ import java.time.Instant;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import javax.servlet.http.HttpServletResponse;
import net.runelite.http.api.item.Item;
import net.runelite.http.api.item.ItemPrice;
import net.runelite.http.api.item.SearchResult;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.CacheControl;
import org.springframework.http.ResponseEntity;
@@ -53,7 +47,6 @@ import org.springframework.web.bind.annotation.RestController;
@RequestMapping("/item")
public class ItemController
{
private static final String RUNELITE_CACHE = "RuneLite-Cache";
private static final int MAX_BATCH_LOOKUP = 1024;
private static class MemoizedPrices
@@ -75,10 +68,6 @@ public class ItemController
}
}
private final Cache<Integer, Integer> cachedEmpty = CacheBuilder.newBuilder()
.maximumSize(1024L)
.build();
private final ItemService itemService;
private final Supplier<MemoizedPrices> memoizedPrices;
@@ -101,58 +90,12 @@ public class ItemController
.toArray(ItemPrice[]::new)), 30, TimeUnit.MINUTES);
}
@GetMapping("/{itemId}")
public Item getItem(HttpServletResponse response, @PathVariable int itemId)
{
ItemEntry item = itemService.getItem(itemId);
if (item != null)
{
return item.toItem();
}
itemService.queueItem(itemId);
return null;
}
@GetMapping(path = "/{itemId}/icon", produces = "image/gif")
public ResponseEntity<byte[]> getIcon(@PathVariable int itemId)
{
ItemEntry item = itemService.getItem(itemId);
if (item != null && item.getIcon() != null)
{
return ResponseEntity.ok(item.getIcon());
}
itemService.queueItem(itemId);
return ResponseEntity.notFound().build();
}
@GetMapping(path = "/{itemId}/icon/large", produces = "image/gif")
public ResponseEntity<byte[]> getIconLarge(HttpServletResponse response, @PathVariable int itemId)
{
ItemEntry item = itemService.getItem(itemId);
if (item != null && item.getIcon_large() != null)
{
return ResponseEntity.ok(item.getIcon_large());
}
itemService.queueItem(itemId);
return ResponseEntity.notFound().build();
}
@GetMapping("/{itemId}/price")
public ResponseEntity<ItemPrice> itemPrice(
@PathVariable int itemId,
@RequestParam(required = false) Instant time
)
{
if (cachedEmpty.getIfPresent(itemId) != null)
{
return ResponseEntity.notFound()
.header(RUNELITE_CACHE, "HIT")
.build();
}
Instant now = Instant.now();
if (time != null && time.isAfter(now))
@@ -160,16 +103,6 @@ public class ItemController
time = now;
}
ItemEntry item = itemService.getItem(itemId);
if (item == null)
{
itemService.queueItem(itemId); // queue lookup
cachedEmpty.put(itemId, itemId); // cache empty
return ResponseEntity.notFound()
.header(RUNELITE_CACHE, "MISS")
.build();
}
PriceEntry priceEntry = itemService.getPrice(itemId, time);
if (time != null)
@@ -178,22 +111,21 @@ public class ItemController
{
// we maybe can't backfill this
return ResponseEntity.notFound()
.header(RUNELITE_CACHE, "MISS")
.cacheControl(CacheControl.maxAge(30, TimeUnit.MINUTES).cachePublic())
.build();
}
}
else if (priceEntry == null)
{
// Price is unknown
cachedEmpty.put(itemId, itemId);
return ResponseEntity.notFound()
.header(RUNELITE_CACHE, "MISS")
.cacheControl(CacheControl.maxAge(30, TimeUnit.MINUTES).cachePublic())
.build();
}
ItemPrice itemPrice = new ItemPrice();
itemPrice.setId(item.getId());
itemPrice.setName(item.getName());
itemPrice.setId(itemId);
itemPrice.setName(priceEntry.getName());
itemPrice.setPrice(priceEntry.getPrice());
itemPrice.setTime(priceEntry.getTime());
@@ -202,20 +134,6 @@ public class ItemController
.body(itemPrice);
}
@GetMapping("/search")
public SearchResult search(@RequestParam String query)
{
List<ItemEntry> result = itemService.search(query);
itemService.queueSearch(query);
SearchResult searchResult = new SearchResult();
searchResult.setItems(result.stream()
.map(ItemEntry::toItem)
.collect(Collectors.toList()));
return searchResult;
}
@GetMapping("/price")
public ItemPrice[] prices(@RequestParam("id") int[] itemIds)
{

13
http-service/src/main/java/net/runelite/http/service/item/ItemEntry.java
View File

@@ -26,7 +26,6 @@ package net.runelite.http.service.item;
import java.time.Instant;
import lombok.Data;
import net.runelite.http.api.item.Item;
import net.runelite.http.api.item.ItemType;
@Data
@@ -36,17 +35,5 @@ public class ItemEntry
private String name;
private String description;
private ItemType type;
private byte[] icon;
private byte[] icon_large;
private Instant timestamp;
public Item toItem()
{
Item item = new Item();
item.setId(id);
item.setName(name);
item.setDescription(description);
item.setType(type);
return item;
}
}

189
http-service/src/main/java/net/runelite/http/service/item/ItemService.java
View File

@@ -35,7 +35,6 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.ConcurrentLinkedQueue;
import lombok.extern.slf4j.Slf4j;
import net.runelite.cache.definitions.ItemDefinition;
import net.runelite.http.api.RuneLiteAPI;
@@ -59,18 +58,14 @@ public class ItemService
private static final String BASE = "https://services.runescape.com/m=itemdb_oldschool";
private static final HttpUrl RS_ITEM_URL = HttpUrl.parse(BASE + "/api/catalogue/detail.json");
private static final HttpUrl RS_PRICE_URL = HttpUrl.parse(BASE + "/api/graph");
private static final HttpUrl RS_SEARCH_URL = HttpUrl.parse(BASE + "/api/catalogue/items.json?category=1");
private static final String CREATE_ITEMS = "CREATE TABLE IF NOT EXISTS `items` (\n"
+ " `id` int(11) NOT NULL,\n"
+ " `name` tinytext NOT NULL,\n"
+ " `description` tinytext NOT NULL,\n"
+ " `type` enum('DEFAULT') NOT NULL,\n"
+ " `icon` blob,\n"
+ " `icon_large` blob,\n"
+ " `timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,\n"
+ " PRIMARY KEY (`id`),\n"
+ " FULLTEXT idx_name (name)\n"
+ " PRIMARY KEY (`id`)\n"
+ ") ENGINE=InnoDB";
private static final String CREATE_PRICES = "CREATE TABLE IF NOT EXISTS `prices` (\n"
@@ -82,12 +77,9 @@ public class ItemService
+ " KEY `item_fetched_time` (`item`,`fetched_time`)\n"
+ ") ENGINE=InnoDB";
private static final int MAX_PENDING = 512;
private final Sql2o sql2o;
private final CacheService cacheService;
private final ConcurrentLinkedQueue<PendingLookup> pendingLookups = new ConcurrentLinkedQueue<PendingLookup>();
private int[] tradeableItems;
private final Random random = new Random();
@@ -112,11 +104,9 @@ public class ItemService
{
try (Connection con = sql2o.open())
{
ItemEntry item = con.createQuery("select id, name, description, type, icon, icon_large from items where id = :id")
return con.createQuery("select id, name, description, type from items where id = :id")
.addParameter("id", itemId)
.executeAndFetchFirst(ItemEntry.class);
return item;
}
}
@@ -174,54 +164,21 @@ public class ItemService
}
}
public List<ItemEntry> search(String search)
{
try (Connection con = sql2o.open())
{
return con.createQuery("select id, name, description, type, match (name) against (:search) as score from items "
+ "where match (name) against (:search) order by score desc limit 10")
.throwOnMappingFailure(false) // otherwise it tries to map 'score'
.addParameter("search", search)
.executeAndFetch(ItemEntry.class);
}
}
public ItemEntry fetchItem(int itemId)
{
try
{
RSItem rsItem = fetchRSItem(itemId);
byte[] icon = null, iconLarge = null;
try
{
icon = fetchImage(rsItem.getIcon());
}
catch (IOException ex)
{
log.warn("error fetching image", ex);
}
try
{
iconLarge = fetchImage(rsItem.getIcon_large());
}
catch (IOException ex)
{
log.warn("error fetching image", ex);
}
try (Connection con = sql2o.open())
{
con.createQuery("insert into items (id, name, description, type, icon, icon_large) values (:id,"
+ " :name, :description, :type, :icon, :icon_large) ON DUPLICATE KEY UPDATE name = :name,"
+ " description = :description, type = :type, icon = :icon, icon_large = :icon_large")
con.createQuery("insert into items (id, name, description, type) values (:id,"
+ " :name, :description, :type) ON DUPLICATE KEY UPDATE name = :name,"
+ " description = :description, type = :type")
.addParameter("id", rsItem.getId())
.addParameter("name", rsItem.getName())
.addParameter("description", rsItem.getDescription())
.addParameter("type", rsItem.getType())
.addParameter("icon", icon)
.addParameter("icon_large", iconLarge)
.executeUpdate();
}
@@ -230,8 +187,6 @@ public class ItemService
item.setName(rsItem.getName());
item.setDescription(rsItem.getDescription());
item.setType(ItemType.of(rsItem.getType()));
item.setIcon(icon);
item.setIcon_large(iconLarge);
return item;
}
catch (IOException ex)
@@ -241,7 +196,7 @@ public class ItemService
}
}
public List<PriceEntry> fetchPrice(int itemId)
private List<PriceEntry> fetchPrice(int itemId)
{
RSPrices rsprice;
try
@@ -296,8 +251,8 @@ public class ItemService
try (Connection con = sql2o.beginTransaction())
{
Query query = con.createQuery("select t2.item, t3.name, t2.time, prices.price, prices.fetched_time from (select t1.item as item, max(t1.time) as time from prices t1 group by item) t2 " +
" join prices on t2.item=prices.item and t2.time=prices.time" +
" join items t3 on t2.item=t3.id");
" join prices on t2.item=prices.item and t2.time=prices.time" +
" join items t3 on t2.item=t3.id");
return query.executeAndFetch(PriceEntry.class);
}
}
@@ -332,46 +287,7 @@ public class ItemService
return fetchJson(request, RSPrices.class);
}
public RSSearch fetchRSSearch(String query) throws IOException
{
// rs api seems to require lowercase
query = query.toLowerCase();
HttpUrl searchUrl = RS_SEARCH_URL
.newBuilder()
.addQueryParameter("alpha", query)
.build();
Request request = new Request.Builder()
.url(searchUrl)
.build();
return fetchJson(request, RSSearch.class);
}
private void batchInsertItems(RSSearch search)
{
try (Connection con = sql2o.beginTransaction())
{
Query q = con.createQuery("insert into items (id, name, description, type) values (:id,"
+ " :name, :description, :type) ON DUPLICATE KEY UPDATE name = :name,"
+ " description = :description, type = :type");
for (RSItem rsItem : search.getItems())
{
q.addParameter("id", rsItem.getId())
.addParameter("name", rsItem.getName())
.addParameter("description", rsItem.getDescription())
.addParameter("type", rsItem.getType())
.addToBatch();
}
q.executeBatch();
con.commit(false);
}
}
private <T> T fetchJson(Request request, Class<T> clazz) throws IOException
private static <T> T fetchJson(Request request, Class<T> clazz) throws IOException
{
try (Response response = RuneLiteAPI.CLIENT.newCall(request).execute())
{
@@ -389,78 +305,6 @@ public class ItemService
}
}
private byte[] fetchImage(String url) throws IOException
{
HttpUrl httpUrl = HttpUrl.parse(url);
Request request = new Request.Builder()
.url(httpUrl)
.build();
try (Response response = RuneLiteAPI.CLIENT.newCall(request).execute())
{
if (!response.isSuccessful())
{
throw new IOException("Unsuccessful http response: " + response);
}
return response.body().bytes();
}
}
public void queueSearch(String search)
{
if (pendingLookups.size() < MAX_PENDING)
{
pendingLookups.add(new PendingLookup(search, PendingLookup.Type.SEARCH));
}
else
{
log.debug("Dropping pending search for {}", search);
}
}
public void queueItem(int itemId)
{
if (pendingLookups.size() < MAX_PENDING)
{
pendingLookups.add(new PendingLookup(itemId, PendingLookup.Type.ITEM));
}
else
{
log.debug("Dropping pending item lookup for {}", itemId);
}
}
@Scheduled(fixedDelay = 5000)
public void check()
{
PendingLookup pendingLookup = pendingLookups.poll();
if (pendingLookup == null)
{
return;
}
switch (pendingLookup.getType())
{
case SEARCH:
try
{
RSSearch reSearch = fetchRSSearch(pendingLookup.getSearch());
batchInsertItems(reSearch);
}
catch (IOException ex)
{
log.warn("error while searching items", ex);
}
break;
case ITEM:
fetchItem(pendingLookup.getItemId());
break;
}
}
@Scheduled(fixedDelay = 20_000)
public void crawlPrices()
{
@@ -472,16 +316,10 @@ public class ItemService
int idx = random.nextInt(tradeableItems.length);
int id = tradeableItems[idx];
if (getItem(id) == null)
{
// This is a new item..
log.debug("Fetching new item {}", id);
queueItem(id);
return;
}
log.debug("Fetching price for {}", id);
// check if the item name or description has changed
fetchItem(id);
fetchPrice(id);
}
@@ -495,11 +333,10 @@ public class ItemService
}
tradeableItems = items.stream()
.filter(item -> item.isTradeable)
.mapToInt(item -> item.id)
.filter(ItemDefinition::isTradeable)
.mapToInt(ItemDefinition::getId)
.toArray();
log.debug("Loaded {} tradeable items", tradeableItems.length);
}
}

55
http-service/src/main/java/net/runelite/http/service/item/PendingLookup.java
View File

@@ -1,55 +0,0 @@
/*
* Copyright (c) 2018, Adam <Adam@sigterm.info>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package net.runelite.http.service.item;
import lombok.Value;
@Value
class PendingLookup
{
enum Type
{
SEARCH,
ITEM;
}
private final int itemId;
private final String search;
private final Type type;
public PendingLookup(int itemId, Type type)
{
this.itemId = itemId;
this.search = null;
this.type = type;
}
public PendingLookup(String search, Type type)
{
this.itemId = -1;
this.search = search;
this.type = type;
}
}

16
http-service/src/main/java/net/runelite/http/service/item/RSItem.java
View File

@@ -25,26 +25,12 @@
package net.runelite.http.service.item;
import lombok.Data;
import net.runelite.http.api.item.Item;
import net.runelite.http.api.item.ItemType;
@Data
public class RSItem
class RSItem
{
private int id;
private String name;
private String description;
private String type;
private String icon;
private String icon_large;
public Item toItem()
{
Item item = new Item();
item.setId(id);
item.setName(name);
item.setType(ItemType.of(type));
item.setDescription(description);
return item;
}
}

2
http-service/src/main/java/net/runelite/http/service/item/RSItemResponse.java
View File

@@ -27,7 +27,7 @@ package net.runelite.http.service.item;
import lombok.Data;
@Data
public class RSItemResponse
class RSItemResponse
{
private RSItem item;
}

34
http-service/src/main/java/net/runelite/http/service/item/RSSearch.java
View File

@@ -1,34 +0,0 @@
/*
* Copyright (c) 2017, Adam <Adam@sigterm.info>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package net.runelite.http.service.item;
import java.util.List;
import lombok.Data;
@Data
public class RSSearch
{
private List<RSItem> items;
}