ra/runelite
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

cache: store index and archive crcs, and check them for updates too

Browse Source 
Switch archive storage to CAS as sometimes archives change content without changing revision
This commit is contained in:
Adam
2017-09-24 15:44:16 -04:00
parent 52d46c1272
commit 7ce893af7b
10 changed files with 161 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
http-service/src/main/java/net/runelite/http/service/cache/CacheDAO.java vendored
View File

@@ -55,7 +55,7 @@ public class CacheDAO
public List<IndexEntry> findIndexesForCache(Connection con, CacheEntry cache)
{
return con.createQuery("select index.id, index.indexId, index.revision from cache "
return con.createQuery("select index.id, index.indexId, index.crc, index.revision from cache "
+ "join cache_index on cache_index.cache = cache.id "
+ "join `index` on cache_index.index = index.id "
+ "where cache.id = :id "
@@ -66,7 +66,7 @@ public class CacheDAO
public IndexEntry findIndexForCache(Connection con, CacheEntry cache, int indexId)
{
return con.createQuery("select index.id, index.indexId, index.revision from cache "
return con.createQuery("select index.id, index.indexId, index.crc, index.revision from cache "
+ "join cache_index on cache_index.cache = cache.id "
+ "join `index` on cache_index.index = index.id "
+ "where cache.id = :id "
@@ -78,7 +78,8 @@ public class CacheDAO
public List<ArchiveEntry> findArchivesForIndex(Connection con, IndexEntry indexEntry)
{
return con.createQuery("select archive.id, archive.archiveId, archive.nameHash, archive.revision from index_archive "
return con.createQuery("select archive.id, archive.archiveId, archive.nameHash," +
" archive.crc, archive.revision, archive.hash from index_archive "
+ "join archive on index_archive.archive = archive.id "
+ "where index_archive.index = :id")
.addParameter("id", indexEntry.getId())
@@ -87,7 +88,8 @@ public class CacheDAO
public ArchiveEntry findArchiveForIndex(Connection con, IndexEntry indexEntry, int archiveId)
{
return con.createQuery("select archive.id, archive.archiveId, archive.nameHash, archive.revision from index_archive "
return con.createQuery("select archive.id, archive.archiveId, archive.nameHash," +
" archive.crc, archive.revision, archive.hash from index_archive "
+ "join archive on index_archive.archive = archive.id "
+ "where index_archive.index = :id "
+ "and archive.archiveId = :archiveId")
@@ -105,7 +107,8 @@ public class CacheDAO
*/
public ArchiveEntry findMostRecentArchive(Connection con, int indexId, int archiveId)
{
return con.createQuery("select archive.id, archive.archiveId, archive.nameHash, archive.revision from archive "
return con.createQuery("select archive.id, archive.archiveId, archive.nameHash," +
" archive.crc, archive.revision, archive.hash from archive "
+ "join index_archive on index_archive.archive = archive.id "
+ "join `index` on index.id = index_archive.index "
+ "where index.indexId = :indexId and archive.archiveId = :archiveId "
@@ -153,12 +156,14 @@ public class CacheDAO
.executeAndFetchFirst(CacheEntry.class);
}
public IndexEntry findIndex(Connection con, int indexId, int revision)
public IndexEntry findIndex(Connection con, int indexId, int crc, int revision)
{
return con.createQuery("select id, indexId, revision from `index` "
return con.createQuery("select id, indexId, crc, revision from `index` "
+ "where indexId = :indexId "
+ "and crc = :crc "
+ "and revision = :revision")
.addParameter("indexId", indexId)
.addParameter("crc", crc)
.addParameter("revision", revision)
.executeAndFetchFirst(IndexEntry.class);
}
@@ -171,10 +176,14 @@ public class CacheDAO
.executeUpdate();
}
public IndexEntry findOrCreateIndex(Connection con, CacheEntry cache, int indexId, int revision)
public IndexEntry findOrCreateIndex(Connection con, CacheEntry cache, int indexId, int crc, int revision)
{
IndexEntry entry = con.createQuery("select id, indexId, revision from `index` where indexId = :indexId and revision = :revision")
IndexEntry entry = con.createQuery("select id, indexId, crc, revision from `index`"
+ "where indexId = :indexId "
+ "and crc = :crc "
+ "and revision = :revision")
.addParameter("indexId", indexId)
.addParameter("crc", crc)
.addParameter("revision", revision)
.executeAndFetchFirst(IndexEntry.class);
@@ -183,8 +192,9 @@ public class CacheDAO
return entry;
}
int id = con.createQuery("insert into `index` (indexId, revision) values (:indexId, :revision)")
int id = con.createQuery("insert into `index` (indexId, crc, revision) values (:indexId, :crc, :revision)")
.addParameter("indexId", indexId)
.addParameter("crc", crc)
.addParameter("revision", revision)
.executeUpdate()
.getKey(int.class);
@@ -192,6 +202,7 @@ public class CacheDAO
entry = new IndexEntry();
entry.setId(id);
entry.setIndexId(indexId);
entry.setCrc(crc);
entry.setRevision(revision);
return entry;
}
@@ -209,18 +220,25 @@ public class CacheDAO
}
public ArchiveEntry findArchive(Connection con, IndexEntry index,
int archiveId, int nameHash, int revision)
int archiveId, int nameHash, int crc, int revision)
{
if (findArchive == null)
{
findArchive = con.createQuery("select archive.id, archive.archiveId, archive.nameHash, archive.revision from archive "
findArchive = con.createQuery("select archive.id, archive.archiveId, archive.nameHash,"
+ " archive.crc, archive.revision, archive.hash from archive "
+ " join index_archive on index_archive.archive = archive.id"
+ " join `index` on index.id = index_archive.index"
+ " where archive.archiveId = :archiveId and archive.revision = :revision and index.indexId = :indexId");
+ " where archive.archiveId = :archiveId"
+ " and archive.nameHash = :nameHash"
+ " and archive.crc = :crc"
+ " and archive.revision = :revision"
+ " and index.indexId = :indexId");
}
ArchiveEntry entry = findArchive
.addParameter("archiveId", archiveId)
.addParameter("nameHash", nameHash)
.addParameter("crc", crc)
.addParameter("revision", revision)
.addParameter("indexId", index.getIndexId())
.executeAndFetchFirst(ArchiveEntry.class);
@@ -228,18 +246,20 @@ public class CacheDAO
}
public ArchiveEntry createArchive(Connection con, IndexEntry index,
int archiveId, int nameHash, int revision)
int archiveId, int nameHash, int crc, int revision, byte[] hash)
{
if (insertArchive == null)
{
insertArchive = con.createQuery("insert into archive (archiveId, nameHash, revision) values "
+ "(:archiveId, :nameHash, :revision)");
insertArchive = con.createQuery("insert into archive (archiveId, nameHash, crc, revision, hash) values "
+ "(:archiveId, :nameHash, :crc, :revision, :hash)");
}
int id = insertArchive
.addParameter("archiveId", archiveId)
.addParameter("nameHash", nameHash)
.addParameter("crc", crc)
.addParameter("revision", revision)
.addParameter("hash", hash)
.executeUpdate()
.getKey(int.class);
@@ -247,7 +267,9 @@ public class CacheDAO
entry.setId(id);
entry.setArchiveId(archiveId);
entry.setNameHash(nameHash);
entry.setCrc(crc);
entry.setRevision(revision);
entry.setHash(hash);
return entry;
}

22
http-service/src/main/java/net/runelite/http/service/cache/CacheService.java vendored
View File

@@ -24,6 +24,7 @@
*/
package net.runelite.http.service.cache;
import com.google.common.io.BaseEncoding;
import com.google.common.io.ByteStreams;
import io.minio.MinioClient;
import io.minio.errors.ErrorResponseException;
@@ -106,27 +107,24 @@ public class CacheService
/**
* retrieve archive from storage
*
* @param indexId
* @param archiveId
* @param revision
* @return
*/
private byte[] getArchive(int indexId, int archiveId, int revision)
private byte[] getArchive(ArchiveEntry archiveEntry)
{
String hashStr = BaseEncoding.base16().encode(archiveEntry.getHash());
String path = new StringBuilder()
.append(indexId)
.append(hashStr.substring(0, 2))
.append('/')
.append(archiveId)
.append('/')
.append(revision)
.append(hashStr.substring(2))
.toString();
try (InputStream in = minioClient.getObject(minioBucket, path))
{
return ByteStreams.toByteArray(in);
}
catch (InvalidBucketNameException | NoSuchAlgorithmException | InsufficientDataException | IOException | InvalidKeyException | NoResponseException | XmlPullParserException | ErrorResponseException | InternalException | InvalidArgumentException ex)
catch (InvalidBucketNameException | NoSuchAlgorithmException | InsufficientDataException
| IOException | InvalidKeyException | NoResponseException | XmlPullParserException
| ErrorResponseException | InternalException | InvalidArgumentException ex)
{
logger.warn(null, ex);
return null;
@@ -144,7 +142,7 @@ public class CacheService
files = cacheDao.findFilesForArchive(con, archiveEntry);
}
byte[] archiveData = getArchive(index.getNumber(), config.getId(), archiveEntry.getRevision());
byte[] archiveData = getArchive(archiveEntry);
if (archiveData == null)
{
@@ -289,7 +287,7 @@ public class CacheService
archiveEntry = cacheDao.findArchiveForIndex(con, indexEntry, archiveId);
}
return getArchive(indexId, archiveId, archiveEntry.getRevision());
return getArchive(archiveEntry);
}
@RequestMapping("item/{itemId}")

13
http-service/src/main/java/net/runelite/http/service/cache/CacheStorage.java vendored
View File

@@ -24,6 +24,7 @@
*/
package net.runelite.http.service.cache;
import com.google.common.hash.Hashing;
import java.io.IOException;
import java.util.List;
import net.runelite.cache.fs.Archive;
@@ -77,14 +78,16 @@ public class CacheStorage implements Storage
for (IndexEntry indexEntry : indexes)
{
Index index = store.addIndex(indexEntry.getIndexId());
index.setCrc(indexEntry.getCrc());
index.setRevision(indexEntry.getRevision());
List<ArchiveEntry> archives = cacheDao.findArchivesForIndex(con, indexEntry);
for (ArchiveEntry archiveEntry : archives)
{
Archive archive = index.addArchive(archiveEntry.getArchiveId());
archive.setRevision(archiveEntry.getRevision());
archive.setNameHash(archiveEntry.getNameHash());
archive.setCrc(archiveEntry.getCrc());
archive.setRevision(archiveEntry.getRevision());
List<FileEntry> files = cacheDao.findFilesForArchive(con, archiveEntry);
for (FileEntry fileEntry : files)
@@ -102,17 +105,19 @@ public class CacheStorage implements Storage
{
for (Index index : store.getIndexes())
{
IndexEntry entry = cacheDao.findOrCreateIndex(con, cacheEntry, index.getId(), index.getRevision());
IndexEntry entry = cacheDao.findOrCreateIndex(con, cacheEntry, index.getId(), index.getCrc(), index.getRevision());
// this assumes nothing is associated to the cache yet
cacheDao.associateIndexToCache(con, cacheEntry, entry);
for (Archive archive : index.getArchives())
{
ArchiveEntry archiveEntry = cacheDao.findArchive(con, entry, archive.getArchiveId(),
archive.getNameHash(), archive.getRevision());
archive.getNameHash(), archive.getCrc(), archive.getRevision());
if (archiveEntry == null)
{
archiveEntry = cacheDao.createArchive(con, entry, archive.getArchiveId(), archive.getNameHash(), archive.getRevision());
byte[] hash = Hashing.sha256().hashBytes(archive.getData()).asBytes();
archiveEntry = cacheDao.createArchive(con, entry, archive.getArchiveId(),
archive.getNameHash(), archive.getCrc(), archive.getRevision(), hash);
for (FSFile file : archive.getFiles())
{

3
http-service/src/main/java/net/runelite/http/service/cache/CacheUpdater.java vendored
View File

@@ -147,7 +147,8 @@ public class CacheUpdater
IndexEntry ie = dbIndexes.get(i);
if (ii.getId() != ie.getIndexId()
|| ii.getRevision() != ie.getRevision())
|| ii.getRevision() != ie.getRevision()
|| ii.getCrc() != ie.getCrc())
{
return true;
}

11
http-service/src/main/java/net/runelite/http/service/cache/CacheUploader.java vendored
View File

@@ -24,6 +24,8 @@
*/
package net.runelite.http.service.cache;
import com.google.common.hash.Hashing;
import com.google.common.io.BaseEncoding;
import io.minio.MinioClient;
import io.minio.errors.ErrorResponseException;
import io.minio.errors.InsufficientDataException;
@@ -60,12 +62,13 @@ public class CacheUploader implements Runnable
public void run()
{
byte[] data = archive.getData();
byte[] hash = Hashing.sha256().hashBytes(data).asBytes();
String hashStr = BaseEncoding.base16().encode(hash);
String path = new StringBuilder()
.append(archive.getIndex().getId())
.append(hashStr.substring(0, 2))
.append('/')
.append(archive.getArchiveId())
.append('/')
.append(archive.getRevision())
.append(hashStr.substring(2))
.toString();
try

48
http-service/src/main/java/net/runelite/http/service/cache/beans/ArchiveEntry.java vendored
View File

@@ -24,27 +24,33 @@
*/
package net.runelite.http.service.cache.beans;
import java.util.Arrays;
public class ArchiveEntry
{
private int id;
private int archiveId;
private int nameHash;
private int crc;
private int revision;
private byte[] hash;
@Override
public String toString()
{
return "ArchiveEntry{" + "id=" + id + ", archiveId=" + archiveId + ", nameHash=" + nameHash + ", revision=" + revision + '}';
return "ArchiveEntry{" + "id=" + id + ", archiveId=" + archiveId + ", nameHash=" + nameHash + ", crc=" + crc + ", revision=" + revision + '}';
}
@Override
public int hashCode()
{
int hash = 7;
hash = 89 * hash + this.id;
hash = 89 * hash + this.archiveId;
hash = 89 * hash + this.nameHash;
hash = 89 * hash + this.revision;
int hash = 3;
hash = 53 * hash + this.id;
hash = 53 * hash + this.archiveId;
hash = 53 * hash + this.nameHash;
hash = 53 * hash + this.crc;
hash = 53 * hash + this.revision;
hash = 53 * hash + Arrays.hashCode(this.hash);
return hash;
}
@@ -76,10 +82,18 @@ public class ArchiveEntry
{
return false;
}
if (this.crc != other.crc)
{
return false;
}
if (this.revision != other.revision)
{
return false;
}
if (!Arrays.equals(this.hash, other.hash))
{
return false;
}
return true;
}
@@ -113,6 +127,16 @@ public class ArchiveEntry
this.nameHash = nameHash;
}
public int getCrc()
{
return crc;
}
public void setCrc(int crc)
{
this.crc = crc;
}
public int getRevision()
{
return revision;
@@ -122,5 +146,15 @@ public class ArchiveEntry
{
this.revision = revision;
}
public byte[] getHash()
{
return hash;
}
public void setHash(byte[] hash)
{
this.hash = hash;
}
}

24
http-service/src/main/java/net/runelite/http/service/cache/beans/IndexEntry.java vendored
View File

@@ -28,21 +28,23 @@ public class IndexEntry
{
private int id;
private int indexId;
private int crc;
private int revision;
@Override
public String toString()
{
return "IndexEntry{" + "id=" + id + ", indexId=" + indexId + ", revision=" + revision + '}';
return "IndexEntry{" + "id=" + id + ", indexId=" + indexId + ", crc=" + crc + ", revision=" + revision + '}';
}
@Override
public int hashCode()
{
int hash = 5;
hash = 83 * hash + this.id;
hash = 83 * hash + this.indexId;
hash = 83 * hash + this.revision;
hash = 17 * hash + this.id;
hash = 17 * hash + this.indexId;
hash = 17 * hash + this.crc;
hash = 17 * hash + this.revision;
return hash;
}
@@ -70,6 +72,10 @@ public class IndexEntry
{
return false;
}
if (this.crc != other.crc)
{
return false;
}
if (this.revision != other.revision)
{
return false;
@@ -97,6 +103,16 @@ public class IndexEntry
this.indexId = indexId;
}
public int getCrc()
{
return crc;
}
public void setCrc(int crc)
{
this.crc = crc;
}
public int getRevision()
{
return revision;

15
http-service/src/main/java/net/runelite/http/service/cache/schema.sql vendored
View File

@@ -1,8 +1,8 @@
-- MySQL dump 10.16 Distrib 10.2.8-MariaDB, for osx10.12 (x86_64)
-- MySQL dump 10.16 Distrib 10.1.20-MariaDB, for Linux (x86_64)
--
-- Host: localhost Database: cache
-- Host: localhost Database: localhost
-- ------------------------------------------------------
-- Server version 10.2.8-MariaDB
-- Server version 10.1.20-MariaDB
/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
@@ -26,7 +26,9 @@ CREATE TABLE `archive` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`archiveId` int(11) NOT NULL,
`nameHash` int(11) NOT NULL,
`crc` int(11) NOT NULL,
`revision` int(11) NOT NULL,
`hash` binary(32) NOT NULL,
PRIMARY KEY (`id`),
KEY `archive_revision` (`archiveId`,`revision`) USING BTREE
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
@@ -42,7 +44,7 @@ DROP TABLE IF EXISTS `cache`;
CREATE TABLE `cache` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`revision` int(11) NOT NULL,
`date` timestamp NOT NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
`date` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
UNIQUE KEY `revision_date` (`revision`,`date`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
@@ -95,9 +97,10 @@ DROP TABLE IF EXISTS `index`;
CREATE TABLE `index` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`indexId` int(11) NOT NULL,
`crc` int(11) NOT NULL,
`revision` int(11) NOT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY `indexId` (`indexId`,`revision`)
UNIQUE KEY `indexId` (`indexId`,`revision`,`crc`) USING BTREE
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
/*!40101 SET character_set_client = @saved_cs_client */;
@@ -129,4 +132,4 @@ CREATE TABLE `index_archive` (
/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
-- Dump completed on 2017-09-10 13:11:23
-- Dump completed on 2017-09-24 15:38:31