cache service: lazily load store

http-service/src/main/java/net/runelite/http/service/cache/CacheDAO.java

@@ -33,6 +33,7 @@ import net.runelite.http.service.cache.beans.FileEntry;
 import net.runelite.http.service.cache.beans.IndexEntry;
 import org.sql2o.Connection;
 import org.sql2o.Query;
+import org.sql2o.ResultSetIterable;
 
 public class CacheDAO
 {
@@ -77,14 +78,14 @@ public class CacheDAO
 			.executeAndFetchFirst(IndexEntry.class);
 	}
 
-	public List<ArchiveEntry> findArchivesForIndex(Connection con, IndexEntry indexEntry)
+	public ResultSetIterable<ArchiveEntry> findArchivesForIndex(Connection con, IndexEntry indexEntry)
 	{
 		return con.createQuery("select archive.id, archive.archiveId, archive.nameHash," +
 			" archive.crc, archive.revision, archive.hash from index_archive "
 			+ "join archive on index_archive.archive = archive.id "
 			+ "where index_archive.index = :id")
 			.addParameter("id", indexEntry.getId())
-			.executeAndFetch(ArchiveEntry.class);
+			.executeAndFetchLazy(ArchiveEntry.class);
 	}
 	
 	public ArchiveEntry findArchiveForIndex(Connection con, IndexEntry indexEntry, int archiveId)
@@ -135,7 +136,7 @@ public class CacheDAO
 			.executeAndFetchFirst(ArchiveEntry.class);
 	}
 
-	public List<FileEntry> findFilesForArchive(Connection con, ArchiveEntry archiveEntry)
+	public ResultSetIterable<FileEntry> findFilesForArchive(Connection con, ArchiveEntry archiveEntry)
 	{
 		if (findFilesForArchive == null)
 		{
@@ -145,7 +146,7 @@ public class CacheDAO
 
 		return findFilesForArchive
 			.addParameter("archive", archiveEntry.getId())
-			.executeAndFetch(FileEntry.class);
+			.executeAndFetchLazy(FileEntry.class);
 	}
 
 	public CacheEntry createCache(Connection con, int revision, Instant date)

http-service/src/main/java/net/runelite/http/service/cache/CacheService.java

@@ -24,6 +24,7 @@
  */
 package net.runelite.http.service.cache;
 
+import com.google.common.collect.Iterables;
 import com.google.common.io.BaseEncoding;
 import com.google.common.io.ByteStreams;
 import io.minio.MinioClient;
@@ -39,6 +40,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.security.InvalidKeyException;
 import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.stream.Collectors;
 import net.runelite.cache.ConfigType;
@@ -70,6 +72,7 @@ import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RestController;
 import org.sql2o.Connection;
+import org.sql2o.ResultSetIterable;
 import org.sql2o.Sql2o;
 import org.xmlpull.v1.XmlPullParserException;
 
@@ -106,6 +109,7 @@ public class CacheService
 
 	/**
 	 * retrieve archive from storage
+	 *
 	 * @param archiveEntry
 	 * @return
 	 */
@@ -131,41 +135,38 @@ public class CacheService
 		}
 	}
 
-	private ArchiveFiles getArchiveFiles(IndexType index, ConfigType config,
+	private ArchiveFiles getArchiveFiles(ArchiveEntry archiveEntry) throws IOException
-		ArchiveEntry archiveEntry) throws IOException
 	{
-		List<FileEntry> files;
+		CacheDAO cacheDao = new CacheDAO();
 
-		try (Connection con = sql2o.open())
+		try (Connection con = sql2o.open();
+			ResultSetIterable<FileEntry> files = cacheDao.findFilesForArchive(con, archiveEntry))
 		{
-			CacheDAO cacheDao = new CacheDAO();
+			byte[] archiveData = getArchive(archiveEntry);
-			files = cacheDao.findFilesForArchive(con, archiveEntry);
+
+			if (archiveData == null)
+			{
+				return null;
+			}
+
+			Container result = Container.decompress(archiveData, null);
+			if (result == null)
+			{
+				return null;
+			}
+
+			byte[] decompressedData = result.data;
+
+			ArchiveFiles archiveFiles = new ArchiveFiles();
+			for (FileEntry fileEntry : files)
+			{
+				FSFile file = new FSFile(fileEntry.getFileId());
+				archiveFiles.addFile(file);
+				file.setNameHash(fileEntry.getNameHash());
+			}
+			archiveFiles.loadContents(decompressedData);
+			return archiveFiles;
 		}
-
-		byte[] archiveData = getArchive(archiveEntry);
-
-		if (archiveData == null)
-		{
-			return null;
-		}
-
-		Container result = Container.decompress(archiveData, null);
-		if (result == null)
-		{
-			return null;
-		}
-
-		byte[] decompressedData = result.data;
-
-		ArchiveFiles archiveFiles = new ArchiveFiles();
-		for (FileEntry fileEntry : files)
-		{
-			FSFile file = new FSFile(fileEntry.getFileId());
-			archiveFiles.addFile(file);
-			file.setNameHash(fileEntry.getNameHash());
-		}
-		archiveFiles.loadContents(decompressedData);
-		return archiveFiles;
 	}
 
 	@RequestMapping("/")
@@ -208,7 +209,7 @@ public class CacheService
 	public List<CacheArchive> listArchives(@PathVariable int cacheId,
 		@PathVariable int indexId)
 	{
-		List<ArchiveEntry> archives;
+		List<ArchiveEntry> archives = new ArrayList<>();
 
 		try (Connection con = sql2o.open())
 		{
@@ -225,7 +226,10 @@ public class CacheService
 				throw new NotFoundException();
 			}
 
-			archives = cacheDao.findArchivesForIndex(con, indexEntry);
+			try (ResultSetIterable<ArchiveEntry> archiveEntries = cacheDao.findArchivesForIndex(con, indexEntry))
+			{
+				Iterables.addAll(archives, archiveEntries);
+			}
 		}
 
 		return archives.stream()
@@ -306,7 +310,7 @@ public class CacheService
 			}
 		}
 
-		ArchiveFiles archiveFiles = getArchiveFiles(IndexType.CONFIGS, ConfigType.ITEM, archiveEntry);
+		ArchiveFiles archiveFiles = getArchiveFiles(archiveEntry);
 		if (archiveFiles == null)
 		{
 			throw new NotFoundException();
@@ -340,7 +344,7 @@ public class CacheService
 			}
 		}
 
-		ArchiveFiles archiveFiles = getArchiveFiles(IndexType.CONFIGS, ConfigType.OBJECT, archiveEntry);
+		ArchiveFiles archiveFiles = getArchiveFiles(archiveEntry);
 		if (archiveFiles == null)
 		{
 			throw new NotFoundException();
@@ -374,7 +378,7 @@ public class CacheService
 			}
 		}
 
-		ArchiveFiles archiveFiles = getArchiveFiles(IndexType.CONFIGS, ConfigType.NPC, archiveEntry);
+		ArchiveFiles archiveFiles = getArchiveFiles(archiveEntry);
 		if (archiveFiles == null)
 		{
 			throw new NotFoundException();

http-service/src/main/java/net/runelite/http/service/cache/CacheStorage.java

@@ -25,6 +25,7 @@
 package net.runelite.http.service.cache;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
 import net.runelite.cache.fs.Archive;
 import net.runelite.cache.fs.Index;
@@ -36,6 +37,7 @@ import net.runelite.http.service.cache.beans.CacheEntry;
 import net.runelite.http.service.cache.beans.FileEntry;
 import net.runelite.http.service.cache.beans.IndexEntry;
 import org.sql2o.Connection;
+import org.sql2o.ResultSetIterable;
 
 public class CacheStorage implements Storage
 {
@@ -74,30 +76,39 @@ public class CacheStorage implements Storage
 	public void load(Store store) throws IOException
 	{
 		List<IndexEntry> indexes = cacheDao.findIndexesForCache(con, cacheEntry);
+		List<FileData> fileData = new ArrayList<>();
 		for (IndexEntry indexEntry : indexes)
 		{
 			Index index = store.addIndex(indexEntry.getIndexId());
 			index.setCrc(indexEntry.getCrc());
 			index.setRevision(indexEntry.getRevision());
 
-			List<ArchiveEntry> archives = cacheDao.findArchivesForIndex(con, indexEntry);
+			try (ResultSetIterable<ArchiveEntry> archives = cacheDao.findArchivesForIndex(con, indexEntry))
-			for (ArchiveEntry archiveEntry : archives)
 			{
-				Archive archive = index.addArchive(archiveEntry.getArchiveId());
+				for (ArchiveEntry archiveEntry : archives)
-				archive.setNameHash(archiveEntry.getNameHash());
-				archive.setCrc(archiveEntry.getCrc());
-				archive.setRevision(archiveEntry.getRevision());
-				archive.setHash(archiveEntry.getHash());
-
-				List<FileEntry> files = cacheDao.findFilesForArchive(con, archiveEntry);
-				FileData[] fileData = new FileData[files.size()];
-				archive.setFileData(fileData);
-				int idx = 0;
-				for (FileEntry fileEntry : files)
 				{
-					FileData file = fileData[idx++] = new FileData();
+					Archive archive = index.addArchive(archiveEntry.getArchiveId());
-					file.setId(fileEntry.getFileId());
+					archive.setNameHash(archiveEntry.getNameHash());
-					file.setNameHash(fileEntry.getNameHash());
+					archive.setCrc(archiveEntry.getCrc());
+					archive.setRevision(archiveEntry.getRevision());
+					archive.setHash(archiveEntry.getHash());
+
+					try (ResultSetIterable<FileEntry> files = cacheDao.findFilesForArchive(con, archiveEntry))
+					{
+						fileData.clear();
+
+						for (FileEntry fileEntry : files)
+						{
+							FileData file = new FileData();
+							file.setId(fileEntry.getFileId());
+							file.setNameHash(fileEntry.getNameHash());
+							fileData.add(file);
+						}
+
+						FileData[] fileDataArray = new FileData[fileData.size()];
+						fileData.toArray(fileDataArray);
+						archive.setFileData(fileDataArray);
+					}
 				}
 			}
 		}