ra/runelite
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

cache service: lazily load store

Browse Source
This commit is contained in:
Adam
2018-01-18 17:08:18 -05:00
parent 08c2464f06
commit e1a6fb13a2
3 changed files with 73 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
http-service/src/main/java/net/runelite/http/service/cache/CacheDAO.java vendored
View File

@@ -33,6 +33,7 @@ import net.runelite.http.service.cache.beans.FileEntry;
import net.runelite.http.service.cache.beans.IndexEntry;
import org.sql2o.Connection;
import org.sql2o.Query;
import org.sql2o.ResultSetIterable;
public class CacheDAO
{
@@ -77,14 +78,14 @@ public class CacheDAO
.executeAndFetchFirst(IndexEntry.class);
}
public List<ArchiveEntry> findArchivesForIndex(Connection con, IndexEntry indexEntry)
public ResultSetIterable<ArchiveEntry> findArchivesForIndex(Connection con, IndexEntry indexEntry)
{
return con.createQuery("select archive.id, archive.archiveId, archive.nameHash," +
" archive.crc, archive.revision, archive.hash from index_archive "
+ "join archive on index_archive.archive = archive.id "
+ "where index_archive.index = :id")
.addParameter("id", indexEntry.getId())
.executeAndFetch(ArchiveEntry.class);
.executeAndFetchLazy(ArchiveEntry.class);
}
public ArchiveEntry findArchiveForIndex(Connection con, IndexEntry indexEntry, int archiveId)
@@ -135,7 +136,7 @@ public class CacheDAO
.executeAndFetchFirst(ArchiveEntry.class);
}
public List<FileEntry> findFilesForArchive(Connection con, ArchiveEntry archiveEntry)
public ResultSetIterable<FileEntry> findFilesForArchive(Connection con, ArchiveEntry archiveEntry)
{
if (findFilesForArchive == null)
{
@@ -145,7 +146,7 @@ public class CacheDAO
return findFilesForArchive
.addParameter("archive", archiveEntry.getId())
.executeAndFetch(FileEntry.class);
.executeAndFetchLazy(FileEntry.class);
}
public CacheEntry createCache(Connection con, int revision, Instant date)

78
http-service/src/main/java/net/runelite/http/service/cache/CacheService.java vendored
View File

@@ -24,6 +24,7 @@
*/
package net.runelite.http.service.cache;
import com.google.common.collect.Iterables;
import com.google.common.io.BaseEncoding;
import com.google.common.io.ByteStreams;
import io.minio.MinioClient;
@@ -39,6 +40,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import net.runelite.cache.ConfigType;
@@ -70,6 +72,7 @@ import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import org.sql2o.Connection;
import org.sql2o.ResultSetIterable;
import org.sql2o.Sql2o;
import org.xmlpull.v1.XmlPullParserException;
@@ -85,7 +88,7 @@ public class CacheService
@Value("${minio.bucket}")
private String minioBucket;
private final MinioClient minioClient;
@Autowired
@@ -106,6 +109,7 @@ public class CacheService
/**
* retrieve archive from storage
*
* @param archiveEntry
* @return
*/
@@ -131,41 +135,38 @@ public class CacheService
}
}
private ArchiveFiles getArchiveFiles(IndexType index, ConfigType config,
ArchiveEntry archiveEntry) throws IOException
private ArchiveFiles getArchiveFiles(ArchiveEntry archiveEntry) throws IOException
{
List<FileEntry> files;
CacheDAO cacheDao = new CacheDAO();
try (Connection con = sql2o.open())
try (Connection con = sql2o.open();
ResultSetIterable<FileEntry> files = cacheDao.findFilesForArchive(con, archiveEntry))
{
CacheDAO cacheDao = new CacheDAO();
files = cacheDao.findFilesForArchive(con, archiveEntry);
byte[] archiveData = getArchive(archiveEntry);
if (archiveData == null)
{
return null;
}
Container result = Container.decompress(archiveData, null);
if (result == null)
{
return null;
}
byte[] decompressedData = result.data;
ArchiveFiles archiveFiles = new ArchiveFiles();
for (FileEntry fileEntry : files)
{
FSFile file = new FSFile(fileEntry.getFileId());
archiveFiles.addFile(file);
file.setNameHash(fileEntry.getNameHash());
}
archiveFiles.loadContents(decompressedData);
return archiveFiles;
}
byte[] archiveData = getArchive(archiveEntry);
if (archiveData == null)
{
return null;
}
Container result = Container.decompress(archiveData, null);
if (result == null)
{
return null;
}
byte[] decompressedData = result.data;
ArchiveFiles archiveFiles = new ArchiveFiles();
for (FileEntry fileEntry : files)
{
FSFile file = new FSFile(fileEntry.getFileId());
archiveFiles.addFile(file);
file.setNameHash(fileEntry.getNameHash());
}
archiveFiles.loadContents(decompressedData);
return archiveFiles;
}
@RequestMapping("/")
@@ -208,7 +209,7 @@ public class CacheService
public List<CacheArchive> listArchives(@PathVariable int cacheId,
@PathVariable int indexId)
{
List<ArchiveEntry> archives;
List<ArchiveEntry> archives = new ArrayList<>();
try (Connection con = sql2o.open())
{
@@ -225,7 +226,10 @@ public class CacheService
throw new NotFoundException();
}
archives = cacheDao.findArchivesForIndex(con, indexEntry);
try (ResultSetIterable<ArchiveEntry> archiveEntries = cacheDao.findArchivesForIndex(con, indexEntry))
{
Iterables.addAll(archives, archiveEntries);
}
}
return archives.stream()
@@ -306,7 +310,7 @@ public class CacheService
}
}
ArchiveFiles archiveFiles = getArchiveFiles(IndexType.CONFIGS, ConfigType.ITEM, archiveEntry);
ArchiveFiles archiveFiles = getArchiveFiles(archiveEntry);
if (archiveFiles == null)
{
throw new NotFoundException();
@@ -340,7 +344,7 @@ public class CacheService
}
}
ArchiveFiles archiveFiles = getArchiveFiles(IndexType.CONFIGS, ConfigType.OBJECT, archiveEntry);
ArchiveFiles archiveFiles = getArchiveFiles(archiveEntry);
if (archiveFiles == null)
{
throw new NotFoundException();
@@ -374,7 +378,7 @@ public class CacheService
}
}
ArchiveFiles archiveFiles = getArchiveFiles(IndexType.CONFIGS, ConfigType.NPC, archiveEntry);
ArchiveFiles archiveFiles = getArchiveFiles(archiveEntry);
if (archiveFiles == null)
{
throw new NotFoundException();

43
http-service/src/main/java/net/runelite/http/service/cache/CacheStorage.java vendored
View File

@@ -25,6 +25,7 @@
package net.runelite.http.service.cache;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import net.runelite.cache.fs.Archive;
import net.runelite.cache.fs.Index;
@@ -36,6 +37,7 @@ import net.runelite.http.service.cache.beans.CacheEntry;
import net.runelite.http.service.cache.beans.FileEntry;
import net.runelite.http.service.cache.beans.IndexEntry;
import org.sql2o.Connection;
import org.sql2o.ResultSetIterable;
public class CacheStorage implements Storage
{
@@ -74,30 +76,39 @@ public class CacheStorage implements Storage
public void load(Store store) throws IOException
{
List<IndexEntry> indexes = cacheDao.findIndexesForCache(con, cacheEntry);
List<FileData> fileData = new ArrayList<>();
for (IndexEntry indexEntry : indexes)
{
Index index = store.addIndex(indexEntry.getIndexId());
index.setCrc(indexEntry.getCrc());
index.setRevision(indexEntry.getRevision());
List<ArchiveEntry> archives = cacheDao.findArchivesForIndex(con, indexEntry);
for (ArchiveEntry archiveEntry : archives)
try (ResultSetIterable<ArchiveEntry> archives = cacheDao.findArchivesForIndex(con, indexEntry))
{
Archive archive = index.addArchive(archiveEntry.getArchiveId());
archive.setNameHash(archiveEntry.getNameHash());
archive.setCrc(archiveEntry.getCrc());
archive.setRevision(archiveEntry.getRevision());
archive.setHash(archiveEntry.getHash());
List<FileEntry> files = cacheDao.findFilesForArchive(con, archiveEntry);
FileData[] fileData = new FileData[files.size()];
archive.setFileData(fileData);
int idx = 0;
for (FileEntry fileEntry : files)
for (ArchiveEntry archiveEntry : archives)
{
FileData file = fileData[idx++] = new FileData();
file.setId(fileEntry.getFileId());
file.setNameHash(fileEntry.getNameHash());
Archive archive = index.addArchive(archiveEntry.getArchiveId());
archive.setNameHash(archiveEntry.getNameHash());
archive.setCrc(archiveEntry.getCrc());
archive.setRevision(archiveEntry.getRevision());
archive.setHash(archiveEntry.getHash());
try (ResultSetIterable<FileEntry> files = cacheDao.findFilesForArchive(con, archiveEntry))
{
fileData.clear();
for (FileEntry fileEntry : files)
{
FileData file = new FileData();
file.setId(fileEntry.getFileId());
file.setNameHash(fileEntry.getNameHash());
fileData.add(file);
}
FileData[] fileDataArray = new FileData[fileData.size()];
fileData.toArray(fileDataArray);
archive.setFileData(fileDataArray);
}
}
}
}