ra/runelite
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

cache: split loading archive files from archive loading

Browse Source 
Also no longer store archive contents in memory and instead read it from
storage on demand.
This commit is contained in:
Adam
2017-12-17 18:36:05 -05:00
parent 801a907f15
commit 033cf3bb01
53 changed files with 853 additions and 708 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
http-service/src/main/java/net/runelite/http/service/cache/CacheService.java vendored
View File

@@ -50,9 +50,8 @@ import net.runelite.cache.definitions.loaders.ItemLoader;
import net.runelite.cache.definitions.loaders.NpcLoader;
import net.runelite.cache.definitions.loaders.ObjectLoader;
import net.runelite.cache.fs.ArchiveFiles;
import net.runelite.cache.fs.Container;
import net.runelite.cache.fs.FSFile;
import net.runelite.cache.fs.jagex.DataFile;
import net.runelite.cache.fs.jagex.DataFileReadResult;
import net.runelite.http.api.cache.Cache;
import net.runelite.http.api.cache.CacheArchive;
import net.runelite.http.api.cache.CacheIndex;
@@ -150,7 +149,7 @@ public class CacheService
return null;
}
DataFileReadResult result = DataFile.decompress(archiveData, null);
Container result = Container.decompress(archiveData, null);
if (result == null)
{
return null;

29
http-service/src/main/java/net/runelite/http/service/cache/CacheStorage.java vendored
View File

@@ -24,14 +24,13 @@
*/
package net.runelite.http.service.cache;
import com.google.common.hash.Hashing;
import java.io.IOException;
import java.util.List;
import net.runelite.cache.fs.Archive;
import net.runelite.cache.fs.FSFile;
import net.runelite.cache.fs.Index;
import net.runelite.cache.fs.Storage;
import net.runelite.cache.fs.Store;
import net.runelite.cache.index.FileData;
import net.runelite.http.service.cache.beans.ArchiveEntry;
import net.runelite.http.service.cache.beans.CacheEntry;
import net.runelite.http.service.cache.beans.FileEntry;
@@ -88,13 +87,17 @@ public class CacheStorage implements Storage
archive.setNameHash(archiveEntry.getNameHash());
archive.setCrc(archiveEntry.getCrc());
archive.setRevision(archiveEntry.getRevision());
archive.setHash(archiveEntry.getHash());
List<FileEntry> files = cacheDao.findFilesForArchive(con, archiveEntry);
FileData[] fileData = new FileData[files.size()];
archive.setFileData(fileData);
int idx = 0;
for (FileEntry fileEntry : files)
{
FSFile file = new FSFile(fileEntry.getFileId());
FileData file = fileData[idx++] = new FileData();
file.setId(fileEntry.getFileId());
file.setNameHash(fileEntry.getNameHash());
archive.addFile(file);
}
}
}
@@ -115,13 +118,13 @@ public class CacheStorage implements Storage
archive.getNameHash(), archive.getCrc(), archive.getRevision());
if (archiveEntry == null)
{
byte[] hash = Hashing.sha256().hashBytes(archive.getData()).asBytes();
byte[] hash = archive.getHash();
archiveEntry = cacheDao.createArchive(con, entry, archive.getArchiveId(),
archive.getNameHash(), archive.getCrc(), archive.getRevision(), hash);
for (FSFile file : archive.getFiles())
for (FileData file : archive.getFileData())
{
cacheDao.associateFileToArchive(con, archiveEntry, file.getFileId(), file.getNameHash());
cacheDao.associateFileToArchive(con, archiveEntry, file.getId(), file.getNameHash());
}
}
cacheDao.associateArchiveToIndex(con, archiveEntry, entry);
@@ -129,4 +132,16 @@ public class CacheStorage implements Storage
}
}
@Override
public byte[] loadArchive(Archive archive) throws IOException
{
throw new UnsupportedOperationException();
}
@Override
public void saveArchive(Archive archive, byte[] data) throws IOException
{
throw new UnsupportedOperationException();
}
}

3
http-service/src/main/java/net/runelite/http/service/cache/CacheUpdater.java vendored
View File

@@ -96,7 +96,7 @@ public class CacheUpdater
ExecutorService executor = Executors.newSingleThreadExecutor();
CacheClient client = new CacheClient(store, rsVersion,
(Archive archive) -> executor.submit(new CacheUploader(minioClient, minioBucket, archive)));
(Archive archive, byte[] data) -> executor.submit(new CacheUploader(minioClient, minioBucket, archive, data)));
client.connect();
HandshakeResponseType result = client.handshake().join();
@@ -111,6 +111,7 @@ public class CacheUpdater
if (!checkOutOfDate(indexes, entries))
{
logger.info("All up to date.");
return;
}

7
http-service/src/main/java/net/runelite/http/service/cache/CacheUploader.java vendored
View File

@@ -50,21 +50,24 @@ public class CacheUploader implements Runnable
private final MinioClient minioClient;
private final String minioBucket;
private final Archive archive;
private final byte[] data;
public CacheUploader(MinioClient minioClient, String minioBucket, Archive archive)
public CacheUploader(MinioClient minioClient, String minioBucket, Archive archive, byte[] data)
{
this.minioClient = minioClient;
this.minioBucket = minioBucket;
this.archive = archive;
this.data = data;
}
@Override
public void run()
{
byte[] data = archive.getData();
byte[] hash = Hashing.sha256().hashBytes(data).asBytes();
String hashStr = BaseEncoding.base16().encode(hash);
archive.setHash(hash);
String path = new StringBuilder()
.append(hashStr.substring(0, 2))
.append('/')

135
http-service/src/main/java/net/runelite/http/service/cache/schema.sql vendored
View File

@@ -1,135 +0,0 @@
-- MySQL dump 10.16 Distrib 10.1.20-MariaDB, for Linux (x86_64)
--
-- Host: localhost Database: localhost
-- ------------------------------------------------------
-- Server version 10.1.20-MariaDB
/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
/*!40101 SET NAMES utf8 */;
/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
/*!40103 SET TIME_ZONE='+00:00' */;
/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
--
-- Table structure for table `archive`
--
DROP TABLE IF EXISTS `archive`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `archive` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`archiveId` int(11) NOT NULL,
`nameHash` int(11) NOT NULL,
`crc` int(11) NOT NULL,
`revision` int(11) NOT NULL,
`hash` binary(32) NOT NULL,
PRIMARY KEY (`id`),
KEY `archive_revision` (`archiveId`,`revision`) USING BTREE
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `cache`
--
DROP TABLE IF EXISTS `cache`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `cache` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`revision` int(11) NOT NULL,
`date` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
UNIQUE KEY `revision_date` (`revision`,`date`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `cache_index`
--
DROP TABLE IF EXISTS `cache_index`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `cache_index` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`cache` int(11) NOT NULL,
`index` int(11) NOT NULL,
PRIMARY KEY (`id`),
KEY `cacheId` (`cache`),
KEY `indexId` (`index`),
CONSTRAINT `cache_index_ibfk_1` FOREIGN KEY (`cache`) REFERENCES `cache` (`id`),
CONSTRAINT `cache_index_ibfk_2` FOREIGN KEY (`index`) REFERENCES `index` (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `file`
--
DROP TABLE IF EXISTS `file`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `file` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`archive` int(11) NOT NULL,
`fileId` int(11) NOT NULL,
`nameHash` int(11) NOT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY `archive_file` (`archive`,`fileId`),
CONSTRAINT `file_ibfk_1` FOREIGN KEY (`archive`) REFERENCES `archive` (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `index`
--
DROP TABLE IF EXISTS `index`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `index` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`indexId` int(11) NOT NULL,
`crc` int(11) NOT NULL,
`revision` int(11) NOT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY `indexId` (`indexId`,`revision`,`crc`) USING BTREE
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `index_archive`
--
DROP TABLE IF EXISTS `index_archive`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `index_archive` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`index` int(11) NOT NULL,
`archive` int(11) NOT NULL,
PRIMARY KEY (`id`),
KEY `index` (`index`) USING BTREE,
KEY `archive` (`archive`) USING BTREE,
CONSTRAINT `index_archive_ibfk_1` FOREIGN KEY (`index`) REFERENCES `index` (`id`),
CONSTRAINT `index_archive_ibfk_2` FOREIGN KEY (`archive`) REFERENCES `archive` (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
/*!40101 SET character_set_client = @saved_cs_client */;
/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
-- Dump completed on 2017-09-24 15:38:31

4
http-service/src/main/java/net/runelite/http/service/xtea/XteaService.java
View File

@@ -28,7 +28,7 @@ import java.io.IOException;
import java.util.List;
import java.util.stream.Collectors;
import net.runelite.cache.IndexType;
import net.runelite.cache.fs.jagex.DataFile;
import net.runelite.cache.fs.Container;
import net.runelite.cache.util.Djb2;
import net.runelite.http.api.xtea.XteaKey;
import net.runelite.http.api.xtea.XteaRequest;
@@ -226,7 +226,7 @@ public class XteaService
try
{
DataFile.decompress(data, keys);
Container.decompress(data, keys);
return true;
}
catch (IOException ex)