From e1a6fb13a2a09cc773bef54c4aad0847ed4a10a1 Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 18 Jan 2018 17:08:18 -0500 Subject: [PATCH] cache service: lazily load store --- .../runelite/http/service/cache/CacheDAO.java | 9 ++- .../http/service/cache/CacheService.java | 78 ++++++++++--------- .../http/service/cache/CacheStorage.java | 43 ++++++---- 3 files changed, 73 insertions(+), 57 deletions(-) diff --git a/http-service/src/main/java/net/runelite/http/service/cache/CacheDAO.java b/http-service/src/main/java/net/runelite/http/service/cache/CacheDAO.java index 29dd02f05f..6138c089e2 100644 --- a/http-service/src/main/java/net/runelite/http/service/cache/CacheDAO.java +++ b/http-service/src/main/java/net/runelite/http/service/cache/CacheDAO.java @@ -33,6 +33,7 @@ import net.runelite.http.service.cache.beans.FileEntry; import net.runelite.http.service.cache.beans.IndexEntry; import org.sql2o.Connection; import org.sql2o.Query; +import org.sql2o.ResultSetIterable; public class CacheDAO { @@ -77,14 +78,14 @@ public class CacheDAO .executeAndFetchFirst(IndexEntry.class); } - public List findArchivesForIndex(Connection con, IndexEntry indexEntry) + public ResultSetIterable findArchivesForIndex(Connection con, IndexEntry indexEntry) { return con.createQuery("select archive.id, archive.archiveId, archive.nameHash," + " archive.crc, archive.revision, archive.hash from index_archive " + "join archive on index_archive.archive = archive.id " + "where index_archive.index = :id") .addParameter("id", indexEntry.getId()) - .executeAndFetch(ArchiveEntry.class); + .executeAndFetchLazy(ArchiveEntry.class); } public ArchiveEntry findArchiveForIndex(Connection con, IndexEntry indexEntry, int archiveId) @@ -135,7 +136,7 @@ public class CacheDAO .executeAndFetchFirst(ArchiveEntry.class); } - public List findFilesForArchive(Connection con, ArchiveEntry archiveEntry) + public ResultSetIterable findFilesForArchive(Connection con, ArchiveEntry archiveEntry) { if (findFilesForArchive == null) { @@ -145,7 +146,7 @@ public class CacheDAO return findFilesForArchive .addParameter("archive", archiveEntry.getId()) - .executeAndFetch(FileEntry.class); + .executeAndFetchLazy(FileEntry.class); } public CacheEntry createCache(Connection con, int revision, Instant date) diff --git a/http-service/src/main/java/net/runelite/http/service/cache/CacheService.java b/http-service/src/main/java/net/runelite/http/service/cache/CacheService.java index 362948f724..39c4f3b124 100644 --- a/http-service/src/main/java/net/runelite/http/service/cache/CacheService.java +++ b/http-service/src/main/java/net/runelite/http/service/cache/CacheService.java @@ -24,6 +24,7 @@ */ package net.runelite.http.service.cache; +import com.google.common.collect.Iterables; import com.google.common.io.BaseEncoding; import com.google.common.io.ByteStreams; import io.minio.MinioClient; @@ -39,6 +40,7 @@ import java.io.IOException; import java.io.InputStream; import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; import net.runelite.cache.ConfigType; @@ -70,6 +72,7 @@ import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; import org.sql2o.Connection; +import org.sql2o.ResultSetIterable; import org.sql2o.Sql2o; import org.xmlpull.v1.XmlPullParserException; @@ -85,7 +88,7 @@ public class CacheService @Value("${minio.bucket}") private String minioBucket; - + private final MinioClient minioClient; @Autowired @@ -106,6 +109,7 @@ public class CacheService /** * retrieve archive from storage + * * @param archiveEntry * @return */ @@ -131,41 +135,38 @@ public class CacheService } } - private ArchiveFiles getArchiveFiles(IndexType index, ConfigType config, - ArchiveEntry archiveEntry) throws IOException + private ArchiveFiles getArchiveFiles(ArchiveEntry archiveEntry) throws IOException { - List files; + CacheDAO cacheDao = new CacheDAO(); - try (Connection con = sql2o.open()) + try (Connection con = sql2o.open(); + ResultSetIterable files = cacheDao.findFilesForArchive(con, archiveEntry)) { - CacheDAO cacheDao = new CacheDAO(); - files = cacheDao.findFilesForArchive(con, archiveEntry); + byte[] archiveData = getArchive(archiveEntry); + + if (archiveData == null) + { + return null; + } + + Container result = Container.decompress(archiveData, null); + if (result == null) + { + return null; + } + + byte[] decompressedData = result.data; + + ArchiveFiles archiveFiles = new ArchiveFiles(); + for (FileEntry fileEntry : files) + { + FSFile file = new FSFile(fileEntry.getFileId()); + archiveFiles.addFile(file); + file.setNameHash(fileEntry.getNameHash()); + } + archiveFiles.loadContents(decompressedData); + return archiveFiles; } - - byte[] archiveData = getArchive(archiveEntry); - - if (archiveData == null) - { - return null; - } - - Container result = Container.decompress(archiveData, null); - if (result == null) - { - return null; - } - - byte[] decompressedData = result.data; - - ArchiveFiles archiveFiles = new ArchiveFiles(); - for (FileEntry fileEntry : files) - { - FSFile file = new FSFile(fileEntry.getFileId()); - archiveFiles.addFile(file); - file.setNameHash(fileEntry.getNameHash()); - } - archiveFiles.loadContents(decompressedData); - return archiveFiles; } @RequestMapping("/") @@ -208,7 +209,7 @@ public class CacheService public List listArchives(@PathVariable int cacheId, @PathVariable int indexId) { - List archives; + List archives = new ArrayList<>(); try (Connection con = sql2o.open()) { @@ -225,7 +226,10 @@ public class CacheService throw new NotFoundException(); } - archives = cacheDao.findArchivesForIndex(con, indexEntry); + try (ResultSetIterable archiveEntries = cacheDao.findArchivesForIndex(con, indexEntry)) + { + Iterables.addAll(archives, archiveEntries); + } } return archives.stream() @@ -306,7 +310,7 @@ public class CacheService } } - ArchiveFiles archiveFiles = getArchiveFiles(IndexType.CONFIGS, ConfigType.ITEM, archiveEntry); + ArchiveFiles archiveFiles = getArchiveFiles(archiveEntry); if (archiveFiles == null) { throw new NotFoundException(); @@ -340,7 +344,7 @@ public class CacheService } } - ArchiveFiles archiveFiles = getArchiveFiles(IndexType.CONFIGS, ConfigType.OBJECT, archiveEntry); + ArchiveFiles archiveFiles = getArchiveFiles(archiveEntry); if (archiveFiles == null) { throw new NotFoundException(); @@ -374,7 +378,7 @@ public class CacheService } } - ArchiveFiles archiveFiles = getArchiveFiles(IndexType.CONFIGS, ConfigType.NPC, archiveEntry); + ArchiveFiles archiveFiles = getArchiveFiles(archiveEntry); if (archiveFiles == null) { throw new NotFoundException(); diff --git a/http-service/src/main/java/net/runelite/http/service/cache/CacheStorage.java b/http-service/src/main/java/net/runelite/http/service/cache/CacheStorage.java index 83b0eaffa1..371b89e93d 100644 --- a/http-service/src/main/java/net/runelite/http/service/cache/CacheStorage.java +++ b/http-service/src/main/java/net/runelite/http/service/cache/CacheStorage.java @@ -25,6 +25,7 @@ package net.runelite.http.service.cache; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import net.runelite.cache.fs.Archive; import net.runelite.cache.fs.Index; @@ -36,6 +37,7 @@ import net.runelite.http.service.cache.beans.CacheEntry; import net.runelite.http.service.cache.beans.FileEntry; import net.runelite.http.service.cache.beans.IndexEntry; import org.sql2o.Connection; +import org.sql2o.ResultSetIterable; public class CacheStorage implements Storage { @@ -74,30 +76,39 @@ public class CacheStorage implements Storage public void load(Store store) throws IOException { List indexes = cacheDao.findIndexesForCache(con, cacheEntry); + List fileData = new ArrayList<>(); for (IndexEntry indexEntry : indexes) { Index index = store.addIndex(indexEntry.getIndexId()); index.setCrc(indexEntry.getCrc()); index.setRevision(indexEntry.getRevision()); - List archives = cacheDao.findArchivesForIndex(con, indexEntry); - for (ArchiveEntry archiveEntry : archives) + try (ResultSetIterable archives = cacheDao.findArchivesForIndex(con, indexEntry)) { - Archive archive = index.addArchive(archiveEntry.getArchiveId()); - archive.setNameHash(archiveEntry.getNameHash()); - archive.setCrc(archiveEntry.getCrc()); - archive.setRevision(archiveEntry.getRevision()); - archive.setHash(archiveEntry.getHash()); - - List files = cacheDao.findFilesForArchive(con, archiveEntry); - FileData[] fileData = new FileData[files.size()]; - archive.setFileData(fileData); - int idx = 0; - for (FileEntry fileEntry : files) + for (ArchiveEntry archiveEntry : archives) { - FileData file = fileData[idx++] = new FileData(); - file.setId(fileEntry.getFileId()); - file.setNameHash(fileEntry.getNameHash()); + Archive archive = index.addArchive(archiveEntry.getArchiveId()); + archive.setNameHash(archiveEntry.getNameHash()); + archive.setCrc(archiveEntry.getCrc()); + archive.setRevision(archiveEntry.getRevision()); + archive.setHash(archiveEntry.getHash()); + + try (ResultSetIterable files = cacheDao.findFilesForArchive(con, archiveEntry)) + { + fileData.clear(); + + for (FileEntry fileEntry : files) + { + FileData file = new FileData(); + file.setId(fileEntry.getFileId()); + file.setNameHash(fileEntry.getNameHash()); + fileData.add(file); + } + + FileData[] fileDataArray = new FileData[fileData.size()]; + fileData.toArray(fileDataArray); + archive.setFileData(fileDataArray); + } } } }