ra/runelite
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Merges Injector

Browse Source 
Welcome to the new world boys.
This commit is contained in:
zeruth
2019-06-06 20:47:41 -04:00
parent 79ed69ccdf
commit 882be3cb71
3613 changed files with 193663 additions and 158070 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
cache-updater/pom.xml
View File

@@ -1,104 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright (c) 2018, Adam <Adam@sigterm.info>
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>net.runelite</groupId>
<artifactId>runelite-parent</artifactId>
<version>1.5.27-SNAPSHOT</version>
</parent>
<name>Cache Updater</name>
<artifactId>cache-updater</artifactId>
<properties>
<spring.boot.version>1.5.6.RELEASE</spring.boot.version>
<mysql.connector.version>5.1.45</mysql.connector.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
<version>${spring.boot.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-jdbc</artifactId>
<version>${spring.boot.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-devtools</artifactId>
<version>${spring.boot.version}</version>
<optional>true</optional>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>${mysql.connector.version}</version>
</dependency>
<dependency>
<groupId>net.runelite</groupId>
<artifactId>cache-client</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.sql2o</groupId>
<artifactId>sql2o</artifactId>
</dependency>
<dependency>
<groupId>io.minio</groupId>
<artifactId>minio</artifactId>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<scope>provided</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<version>${spring.boot.version}</version>
<executions>
<execution>
<goals>
<goal>repackage</goal>
</goals>
<configuration>
<classifier>spring-boot</classifier>
<mainClass>net.runelite.cache.updater.CacheUpdater</mainClass>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

118
cache-updater/schema.sql
View File

@@ -1,118 +0,0 @@
-- MySQL dump 10.16 Distrib 10.2.9-MariaDB, for Linux (x86_64)
--
-- Host: localhost Database: cache
-- ------------------------------------------------------
-- Server version 10.2.9-MariaDB
/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
/*!40101 SET NAMES utf8 */;
/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
/*!40103 SET TIME_ZONE='+00:00' */;
/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
--
-- Table structure for table `archive`
--
DROP TABLE IF EXISTS `archive`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `archive` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`archiveId` int(11) NOT NULL,
`nameHash` int(11) NOT NULL,
`crc` int(11) NOT NULL,
`revision` int(11) NOT NULL,
`hash` binary(32) NOT NULL,
PRIMARY KEY (`id`),
KEY `archive_revision` (`archiveId`,`revision`) USING BTREE
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `cache`
--
DROP TABLE IF EXISTS `cache`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `cache` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`revision` int(11) NOT NULL,
`date` timestamp NOT NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
PRIMARY KEY (`id`),
UNIQUE KEY `revision_date` (`revision`,`date`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `file`
--
DROP TABLE IF EXISTS `file`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `file` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`archive` int(11) NOT NULL,
`fileId` int(11) NOT NULL,
`nameHash` int(11) NOT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY `archive_file` (`archive`,`fileId`),
CONSTRAINT `file_ibfk_1` FOREIGN KEY (`archive`) REFERENCES `archive` (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `index`
--
DROP TABLE IF EXISTS `index`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `index` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`cache` int(11) NOT NULL,
`indexId` int(11) NOT NULL,
`crc` int(11) NOT NULL,
`revision` int(11) NOT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY `indexId` (`cache`,`indexId`,`revision`,`crc`) USING BTREE,
CONSTRAINT `index_ibfk_1` FOREIGN KEY (`cache`) REFERENCES `cache` (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `index_archive`
--
DROP TABLE IF EXISTS `index_archive`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `index_archive` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`index` int(11) NOT NULL,
`archive` int(11) NOT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY `idx_index_archive` (`index`,`archive`) USING BTREE,
KEY `archive` (`archive`) USING BTREE,
CONSTRAINT `index_archive_ibfk_1` FOREIGN KEY (`index`) REFERENCES `index` (`id`),
CONSTRAINT `index_archive_ibfk_2` FOREIGN KEY (`archive`) REFERENCES `archive` (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
/*!40101 SET character_set_client = @saved_cs_client */;
/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
-- Dump completed on 2018-02-02 21:55:48

77
cache-updater/src/main/java/net/runelite/cache/updater/CacheConfiguration.java vendored
View File

@@ -1,77 +0,0 @@
/*
* Copyright (c) 2018, Adam <Adam@sigterm.info>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package net.runelite.cache.updater;
import io.minio.MinioClient;
import io.minio.errors.InvalidEndpointException;
import io.minio.errors.InvalidPortException;
import java.time.Instant;
import java.util.HashMap;
import java.util.Map;
import javax.sql.DataSource;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.jdbc.DataSourceBuilder;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.sql2o.Sql2o;
import org.sql2o.converters.Converter;
import org.sql2o.quirks.NoQuirks;
@Configuration
public class CacheConfiguration
{
@Value("${minio.endpoint}")
private String minioUrl;
@Value("${minio.accesskey}")
private String minioAccessKey;
@Value("${minio.secretkey}")
private String minioSecretKey;
@Bean
@ConfigurationProperties(prefix = "datasource.runelite-cache")
public DataSource dataSource()
{
return DataSourceBuilder.create().build();
}
@Bean
@Qualifier("Runelite Cache SQL2O")
public Sql2o sql2o(DataSource dataSource)
{
Map<Class, Converter> converters = new HashMap<>();
converters.put(Instant.class, new InstantConverter());
return new Sql2o(dataSource, new NoQuirks(converters));
}
@Bean
public MinioClient minioClient() throws InvalidEndpointException, InvalidPortException
{
return new MinioClient(minioUrl, minioAccessKey, minioSecretKey);
}
}

177
cache-updater/src/main/java/net/runelite/cache/updater/CacheDAO.java vendored
View File

@@ -1,177 +0,0 @@
/*
* Copyright (c) 2017, Adam <Adam@sigterm.info>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package net.runelite.cache.updater;
import java.time.Instant;
import java.util.List;
import net.runelite.cache.updater.beans.ArchiveEntry;
import net.runelite.cache.updater.beans.CacheEntry;
import net.runelite.cache.updater.beans.IndexEntry;
import org.sql2o.Connection;
import org.sql2o.Query;
import org.sql2o.ResultSetIterable;
class CacheDAO
{
// cache prepared statements for high volume queries
private Query associateArchive;
private Query findArchive, insertArchive;
private Query associateFile;
public CacheEntry findMostRecent(Connection con)
{
return con.createQuery("select id, revision, date from cache order by revision desc, date desc limit 1")
.executeAndFetchFirst(CacheEntry.class);
}
public List<IndexEntry> findIndexesForCache(Connection con, CacheEntry cache)
{
return con.createQuery("select id, indexId, crc, revision from `index` where cache = :cache")
.addParameter("cache", cache.getId())
.executeAndFetch(IndexEntry.class);
}
public ResultSetIterable<ArchiveEntry> findArchivesForIndex(Connection con, IndexEntry indexEntry)
{
return con.createQuery("select archive.id, archive.archiveId, archive.nameHash,"
+ " archive.crc, archive.revision, archive.hash from index_archive "
+ "join archive on index_archive.archive = archive.id "
+ "where index_archive.index = :id")
.addParameter("id", indexEntry.getId())
.executeAndFetchLazy(ArchiveEntry.class);
}
public CacheEntry createCache(Connection con, int revision, Instant date)
{
int cacheId = con.createQuery("insert into cache (revision, date) values (:revision, :date)")
.addParameter("revision", revision)
.addParameter("date", date)
.executeUpdate()
.getKey(int.class);
CacheEntry entry = new CacheEntry();
entry.setId(cacheId);
entry.setRevision(revision);
entry.setDate(date);
return entry;
}
public IndexEntry createIndex(Connection con, CacheEntry cache, int indexId, int crc, int revision)
{
int id = con.createQuery("insert into `index` (cache, indexId, crc, revision) values (:cache, :indexId, :crc, :revision)")
.addParameter("cache", cache.getId())
.addParameter("indexId", indexId)
.addParameter("crc", crc)
.addParameter("revision", revision)
.executeUpdate()
.getKey(int.class);
IndexEntry entry = new IndexEntry();
entry.setId(id);
entry.setIndexId(indexId);
entry.setCrc(crc);
entry.setRevision(revision);
return entry;
}
public void associateArchiveToIndex(Connection con, ArchiveEntry archive, IndexEntry index)
{
if (associateArchive == null)
{
associateArchive = con.createQuery("insert into index_archive (`index`, archive) values (:index, :archive)");
}
associateArchive
.addParameter("index", index.getId())
.addParameter("archive", archive.getId())
.executeUpdate();
}
public ArchiveEntry findArchive(Connection con, IndexEntry index,
int archiveId, int nameHash, int crc, int revision)
{
if (findArchive == null)
{
findArchive = con.createQuery("select distinct archive.id, archive.archiveId, archive.nameHash,"
+ " archive.crc, archive.revision, archive.hash from archive "
+ " join index_archive on index_archive.archive = archive.id"
+ " join `index` on index.id = index_archive.index"
+ " where archive.archiveId = :archiveId"
+ " and archive.nameHash = :nameHash"
+ " and archive.crc = :crc"
+ " and archive.revision = :revision"
+ " and index.indexId = :indexId");
}
ArchiveEntry entry = findArchive
.addParameter("archiveId", archiveId)
.addParameter("nameHash", nameHash)
.addParameter("crc", crc)
.addParameter("revision", revision)
.addParameter("indexId", index.getIndexId())
.executeAndFetchFirst(ArchiveEntry.class);
return entry;
}
public ArchiveEntry createArchive(Connection con, IndexEntry index,
int archiveId, int nameHash, int crc, int revision, byte[] hash)
{
if (insertArchive == null)
{
insertArchive = con.createQuery("insert into archive (archiveId, nameHash, crc, revision, hash) values "
+ "(:archiveId, :nameHash, :crc, :revision, :hash)");
}
int id = insertArchive
.addParameter("archiveId", archiveId)
.addParameter("nameHash", nameHash)
.addParameter("crc", crc)
.addParameter("revision", revision)
.addParameter("hash", hash)
.executeUpdate()
.getKey(int.class);
ArchiveEntry entry = new ArchiveEntry();
entry.setId(id);
entry.setArchiveId(archiveId);
entry.setNameHash(nameHash);
entry.setCrc(crc);
entry.setRevision(revision);
entry.setHash(hash);
return entry;
}
public void associateFileToArchive(Connection con, ArchiveEntry archive, int fileId, int nameHash)
{
if (associateFile == null)
{
associateFile = con.createQuery("insert into file (archive, fileId, nameHash) values (:archive, :fileId, :nameHash)");
}
associateFile
.addParameter("archive", archive.getId())
.addParameter("fileId", fileId)
.addParameter("nameHash", nameHash)
.executeUpdate();
}
}

144
cache-updater/src/main/java/net/runelite/cache/updater/CacheStorage.java vendored
View File

@@ -1,144 +0,0 @@
/*
* Copyright (c) 2017, Adam <Adam@sigterm.info>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package net.runelite.cache.updater;
import java.io.IOException;
import java.util.List;
import net.runelite.cache.fs.Archive;
import net.runelite.cache.fs.Index;
import net.runelite.cache.fs.Storage;
import net.runelite.cache.fs.Store;
import net.runelite.cache.index.FileData;
import net.runelite.cache.updater.beans.ArchiveEntry;
import net.runelite.cache.updater.beans.CacheEntry;
import net.runelite.cache.updater.beans.IndexEntry;
import org.sql2o.Connection;
import org.sql2o.ResultSetIterable;
public class CacheStorage implements Storage
{
private CacheEntry cacheEntry;
private final CacheDAO cacheDao;
private final Connection con;
public CacheStorage(CacheEntry cacheEntry, CacheDAO cacheDao, Connection con)
{
this.cacheEntry = cacheEntry;
this.cacheDao = cacheDao;
this.con = con;
}
public CacheEntry getCacheEntry()
{
return cacheEntry;
}
public void setCacheEntry(CacheEntry cacheEntry)
{
this.cacheEntry = cacheEntry;
}
@Override
public void init(Store store) throws IOException
{
}
@Override
public void close() throws IOException
{
}
@Override
public void load(Store store) throws IOException
{
List<IndexEntry> indexes = cacheDao.findIndexesForCache(con, cacheEntry);
for (IndexEntry indexEntry : indexes)
{
Index index = store.addIndex(indexEntry.getIndexId());
index.setCrc(indexEntry.getCrc());
index.setRevision(indexEntry.getRevision());
try (ResultSetIterable<ArchiveEntry> archives = cacheDao.findArchivesForIndex(con, indexEntry))
{
for (ArchiveEntry archiveEntry : archives)
{
if (index.getArchive(archiveEntry.getArchiveId()) != null)
{
throw new IOException("Duplicate archive " + archiveEntry + " on " + indexEntry);
}
Archive archive = index.addArchive(archiveEntry.getArchiveId());
archive.setNameHash(archiveEntry.getNameHash());
archive.setCrc(archiveEntry.getCrc());
archive.setRevision(archiveEntry.getRevision());
archive.setHash(archiveEntry.getHash());
// File data is not necessary for cache updating
}
}
}
}
@Override
public void save(Store store) throws IOException
{
for (Index index : store.getIndexes())
{
IndexEntry entry = cacheDao.createIndex(con, cacheEntry, index.getId(), index.getCrc(), index.getRevision());
for (Archive archive : index.getArchives())
{
ArchiveEntry archiveEntry = cacheDao.findArchive(con, entry, archive.getArchiveId(),
archive.getNameHash(), archive.getCrc(), archive.getRevision());
if (archiveEntry == null)
{
byte[] hash = archive.getHash();
archiveEntry = cacheDao.createArchive(con, entry, archive.getArchiveId(),
archive.getNameHash(), archive.getCrc(), archive.getRevision(), hash);
for (FileData file : archive.getFileData())
{
cacheDao.associateFileToArchive(con, archiveEntry, file.getId(), file.getNameHash());
}
}
cacheDao.associateArchiveToIndex(con, archiveEntry, entry);
}
}
}
@Override
public byte[] loadArchive(Archive archive) throws IOException
{
throw new UnsupportedOperationException();
}
@Override
public void saveArchive(Archive archive, byte[] data) throws IOException
{
throw new UnsupportedOperationException();
}
}

173
cache-updater/src/main/java/net/runelite/cache/updater/CacheUpdater.java vendored
View File

@@ -1,173 +0,0 @@
/*
* Copyright (c) 2017, Adam <Adam@sigterm.info>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package net.runelite.cache.updater;
import io.minio.MinioClient;
import io.minio.errors.InvalidEndpointException;
import io.minio.errors.InvalidPortException;
import java.io.IOException;
import java.time.Instant;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import net.runelite.cache.client.CacheClient;
import net.runelite.cache.client.IndexInfo;
import net.runelite.cache.fs.Archive;
import net.runelite.cache.fs.Store;
import net.runelite.cache.updater.beans.CacheEntry;
import net.runelite.cache.updater.beans.IndexEntry;
import net.runelite.http.api.RuneLiteAPI;
import net.runelite.protocol.api.login.HandshakeResponseType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.CommandLineRunner;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.sql2o.Connection;
import org.sql2o.Sql2o;
@SpringBootApplication
public class CacheUpdater implements CommandLineRunner
{
private static final Logger logger = LoggerFactory.getLogger(CacheUpdater.class);
private final Sql2o sql2o;
private final MinioClient minioClient;
@Value("${minio.bucket}")
private String minioBucket;
@Autowired
public CacheUpdater(
@Qualifier("Runelite Cache SQL2O") Sql2o sql2o,
MinioClient minioClient
)
{
this.sql2o = sql2o;
this.minioClient = minioClient;
}
public void update() throws IOException, InvalidEndpointException, InvalidPortException, InterruptedException
{
int rsVersion = RuneLiteAPI.getRsVersion();
try (Connection con = sql2o.beginTransaction())
{
CacheDAO cacheDao = new CacheDAO();
CacheEntry cache = cacheDao.findMostRecent(con);
boolean created = false;
if (cache == null)
{
created = true;
cache = cacheDao.createCache(con, rsVersion, Instant.now());
}
CacheStorage storage = new CacheStorage(cache, cacheDao, con);
Store store = new Store(storage);
store.load();
ExecutorService executor = Executors.newSingleThreadExecutor();
CacheClient client = new CacheClient(store, rsVersion,
(Archive archive, byte[] data) -> executor.submit(new CacheUploader(minioClient, minioBucket, archive, data)));
client.connect();
HandshakeResponseType result = client.handshake().join();
if (result != HandshakeResponseType.RESPONSE_OK)
{
logger.warn("Out of date!");
return;
}
List<IndexInfo> indexes = client.requestIndexes();
List<IndexEntry> entries = cacheDao.findIndexesForCache(con, cache);
if (!checkOutOfDate(indexes, entries))
{
logger.info("All up to date.");
return;
}
client.download();
CacheEntry newCache = created ? cache : cacheDao.createCache(con, rsVersion, Instant.now());
storage.setCacheEntry(newCache);
store.save();
// ensure objects are added to the store before they become
// visible in the database
executor.shutdown();
while (!executor.awaitTermination(1, TimeUnit.SECONDS))
{
logger.debug("Waiting for termination of executor...");
}
// commit database
con.commit();
}
}
private boolean checkOutOfDate(List<IndexInfo> indexes, List<IndexEntry> dbIndexes)
{
if (indexes.size() != dbIndexes.size())
{
return true;
}
for (int i = 0; i < indexes.size(); ++i)
{
IndexInfo ii = indexes.get(i);
IndexEntry ie = dbIndexes.get(i);
if (ii.getId() != ie.getIndexId()
|| ii.getRevision() != ie.getRevision()
|| ii.getCrc() != ie.getCrc())
{
return true;
}
}
return false;
}
@Override
public void run(String... args) throws Exception
{
update();
}
public static void main(String[] args)
{
SpringApplication.run(CacheUpdater.class, args).close();
System.exit(0);
}
}

96
cache-updater/src/main/java/net/runelite/cache/updater/CacheUploader.java vendored
View File

@@ -1,96 +0,0 @@
/*
* Copyright (c) 2017, Adam <Adam@sigterm.info>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package net.runelite.cache.updater;
import com.google.common.hash.Hashing;
import com.google.common.io.BaseEncoding;
import io.minio.MinioClient;
import io.minio.errors.ErrorResponseException;
import io.minio.errors.InsufficientDataException;
import io.minio.errors.InternalException;
import io.minio.errors.InvalidArgumentException;
import io.minio.errors.InvalidBucketNameException;
import io.minio.errors.NoResponseException;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import net.runelite.cache.fs.Archive;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xmlpull.v1.XmlPullParserException;
public class CacheUploader implements Runnable
{
private static final Logger logger = LoggerFactory.getLogger(CacheUploader.class);
private final MinioClient minioClient;
private final String minioBucket;
private final Archive archive;
private final byte[] data;
public CacheUploader(MinioClient minioClient, String minioBucket, Archive archive, byte[] data)
{
this.minioClient = minioClient;
this.minioBucket = minioBucket;
this.archive = archive;
this.data = data;
}
@Override
public void run()
{
byte[] hash = Hashing.sha256().hashBytes(data).asBytes();
String hashStr = BaseEncoding.base16().encode(hash);
archive.setHash(hash);
String path = new StringBuilder()
.append(hashStr, 0, 2)
.append('/')
.append(hashStr.substring(2))
.toString();
try
{
try (InputStream in = minioClient.getObject(minioBucket, path))
{
return; // already exists
}
catch (ErrorResponseException ex)
{
// doesn't exist
}
minioClient.putObject(minioBucket, path, new ByteArrayInputStream(data), data.length, "binary/octet-stream");
}
catch (ErrorResponseException | InsufficientDataException | InternalException | InvalidArgumentException | InvalidBucketNameException | NoResponseException | IOException | InvalidKeyException | NoSuchAlgorithmException | XmlPullParserException ex)
{
logger.warn("unable to upload data to store", ex);
}
}
}

48
cache-updater/src/main/java/net/runelite/cache/updater/InstantConverter.java vendored
View File

@@ -1,48 +0,0 @@
/*
* Copyright (c) 2017, Adam <Adam@sigterm.info>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package net.runelite.cache.updater;
import java.sql.Timestamp;
import java.time.Instant;
import org.sql2o.converters.Converter;
import org.sql2o.converters.ConverterException;
public class InstantConverter implements Converter<Instant>
{
@Override
public Instant convert(Object val) throws ConverterException
{
Timestamp ts = (Timestamp) val;
return ts.toInstant();
}
@Override
public Object toDatabaseParam(Instant val)
{
return Timestamp.from(val);
}
}

38
cache-updater/src/main/java/net/runelite/cache/updater/beans/ArchiveEntry.java vendored
View File

@@ -1,38 +0,0 @@
/*
* Copyright (c) 2017, Adam <Adam@sigterm.info>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package net.runelite.cache.updater.beans;
import lombok.Data;
@Data
public class ArchiveEntry
{
private int id;
private int archiveId;
private int nameHash;
private int crc;
private int revision;
private byte[] hash;
}

36
cache-updater/src/main/java/net/runelite/cache/updater/beans/CacheEntry.java vendored
View File

@@ -1,36 +0,0 @@
/*
* Copyright (c) 2017, Adam <Adam@sigterm.info>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package net.runelite.cache.updater.beans;
import java.time.Instant;
import lombok.Data;
@Data
public class CacheEntry
{
private int id;
private int revision;
private Instant date;
}

36
cache-updater/src/main/java/net/runelite/cache/updater/beans/FileEntry.java vendored
View File

@@ -1,36 +0,0 @@
/*
* Copyright (c) 2017, Adam <Adam@sigterm.info>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package net.runelite.cache.updater.beans;
import lombok.Data;
@Data
public class FileEntry
{
private int id;
private int archiveId;
private int fileId;
private int nameHash;
}

36
cache-updater/src/main/java/net/runelite/cache/updater/beans/IndexEntry.java vendored
View File

@@ -1,36 +0,0 @@
/*
* Copyright (c) 2017, Adam <Adam@sigterm.info>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package net.runelite.cache.updater.beans;
import lombok.Data;
@Data
public class IndexEntry
{
private int id;
private int indexId;
private int crc;
private int revision;
}

16
cache-updater/src/main/resources/application.yaml
View File

@@ -1,16 +0,0 @@
---
# Database
datasource:
runelite-cache:
driverClassName: com.mysql.jdbc.Driver
type: com.mysql.jdbc.jdbc2.optional.MysqlDataSource
url: jdbc:mysql://localhost/runelite-cache
username: runelite
password: runelite
# Minio client storage for cache
minio:
endpoint: http://localhost:9000
accesskey: AM54M27O4WZK65N6F8IP
secretkey: /PZCxzmsJzwCHYlogcymuprniGCaaLUOET2n6yMP
bucket: runelite