From 7ce893af7bbf9c2a2cd65fc13bfa552a6bc06505 Mon Sep 17 00:00:00 2001 From: Adam Date: Sun, 24 Sep 2017 15:44:16 -0400 Subject: [PATCH] cache: store index and archive crcs, and check them for updates too Switch archive storage to CAS as sometimes archives change content without changing revision --- .../runelite/cache/client/CacheClient.java | 27 ++++++++-- .../decoders/ArchiveResponseDecoder.java | 2 + .../runelite/http/service/cache/CacheDAO.java | 54 +++++++++++++------ .../http/service/cache/CacheService.java | 22 ++++---- .../http/service/cache/CacheStorage.java | 13 +++-- .../http/service/cache/CacheUpdater.java | 3 +- .../http/service/cache/CacheUploader.java | 11 ++-- .../service/cache/beans/ArchiveEntry.java | 48 ++++++++++++++--- .../http/service/cache/beans/IndexEntry.java | 24 +++++++-- .../runelite/http/service/cache/schema.sql | 15 +++--- 10 files changed, 161 insertions(+), 58 deletions(-) diff --git a/cache/src/main/java/net/runelite/cache/client/CacheClient.java b/cache/src/main/java/net/runelite/cache/client/CacheClient.java index 212bc30f3c..cb7533c187 100644 --- a/cache/src/main/java/net/runelite/cache/client/CacheClient.java +++ b/cache/src/main/java/net/runelite/cache/client/CacheClient.java @@ -58,6 +58,7 @@ import net.runelite.cache.protocol.packets.ArchiveRequestPacket; import net.runelite.cache.protocol.packets.HandshakePacket; import net.runelite.cache.protocol.packets.HandshakeResponseType; import net.runelite.cache.protocol.packets.HandshakeType; +import net.runelite.cache.util.Crc32; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -268,7 +269,9 @@ public class CacheClient implements AutoCloseable { Archive existing = index.getArchive(ad.getId()); - if (existing != null && existing.getRevision() == ad.getRevision()) + if (existing != null && existing.getRevision() == ad.getRevision() + && existing.getCrc() == ad.getCrc() + && existing.getNameHash() == ad.getNameHash()) { logger.info("Archive {}/{} in index {} is up to date", ad.getId(), indexData.getArchives().length, index.getId()); @@ -288,9 +291,12 @@ public class CacheClient implements AutoCloseable } else { - logger.info("Archive {}/{} in index {} is out of date ({} != {}), downloading", + logger.info("Archive {}/{} in index {} is out of date, downloading. " + + "revision: ours: {} theirs: {}, crc: ours: {} theirs {}, name: ours {} theirs {}", ad.getId(), indexData.getArchives().length, index.getId(), - existing.getRevision(), ad.getRevision()); + existing.getRevision(), ad.getRevision(), + existing.getCrc(), ad.getCrc(), + existing.getNameHash(), ad.getNameHash()); } final Archive archive = existing == null @@ -313,7 +319,20 @@ public class CacheClient implements AutoCloseable CompletableFuture future = requestFile(index.getId(), ad.getId(), false); future.handle((fr, ex) -> { - archive.setData(fr.getCompressedData()); + byte[] data = fr.getCompressedData(); + + Crc32 crc32 = new Crc32(); + crc32.update(data, 0, data.length); + int hash = crc32.getHash(); + + if (hash != archive.getCrc()) + { + logger.warn("crc mismatch on downloaded archive {}/{}: {} != {}", + archive.getIndex().getId(), archive.getArchiveId(), + hash, archive.getCrc()); + } + + archive.setData(data); if (watcher != null) { diff --git a/cache/src/main/java/net/runelite/cache/protocol/decoders/ArchiveResponseDecoder.java b/cache/src/main/java/net/runelite/cache/protocol/decoders/ArchiveResponseDecoder.java index e439513fc4..b5bfb5f0d6 100644 --- a/cache/src/main/java/net/runelite/cache/protocol/decoders/ArchiveResponseDecoder.java +++ b/cache/src/main/java/net/runelite/cache/protocol/decoders/ArchiveResponseDecoder.java @@ -115,6 +115,8 @@ public class ArchiveResponseDecoder extends ByteToMessageDecoder archiveResponse.setArchive(file); archiveResponse.setData(compressedData.array()); out.add(archiveResponse); + + compressedData.release(); } /** diff --git a/http-service/src/main/java/net/runelite/http/service/cache/CacheDAO.java b/http-service/src/main/java/net/runelite/http/service/cache/CacheDAO.java index 8db5f727ba..3476272dfc 100644 --- a/http-service/src/main/java/net/runelite/http/service/cache/CacheDAO.java +++ b/http-service/src/main/java/net/runelite/http/service/cache/CacheDAO.java @@ -55,7 +55,7 @@ public class CacheDAO public List findIndexesForCache(Connection con, CacheEntry cache) { - return con.createQuery("select index.id, index.indexId, index.revision from cache " + return con.createQuery("select index.id, index.indexId, index.crc, index.revision from cache " + "join cache_index on cache_index.cache = cache.id " + "join `index` on cache_index.index = index.id " + "where cache.id = :id " @@ -66,7 +66,7 @@ public class CacheDAO public IndexEntry findIndexForCache(Connection con, CacheEntry cache, int indexId) { - return con.createQuery("select index.id, index.indexId, index.revision from cache " + return con.createQuery("select index.id, index.indexId, index.crc, index.revision from cache " + "join cache_index on cache_index.cache = cache.id " + "join `index` on cache_index.index = index.id " + "where cache.id = :id " @@ -78,7 +78,8 @@ public class CacheDAO public List findArchivesForIndex(Connection con, IndexEntry indexEntry) { - return con.createQuery("select archive.id, archive.archiveId, archive.nameHash, archive.revision from index_archive " + return con.createQuery("select archive.id, archive.archiveId, archive.nameHash," + + " archive.crc, archive.revision, archive.hash from index_archive " + "join archive on index_archive.archive = archive.id " + "where index_archive.index = :id") .addParameter("id", indexEntry.getId()) @@ -87,7 +88,8 @@ public class CacheDAO public ArchiveEntry findArchiveForIndex(Connection con, IndexEntry indexEntry, int archiveId) { - return con.createQuery("select archive.id, archive.archiveId, archive.nameHash, archive.revision from index_archive " + return con.createQuery("select archive.id, archive.archiveId, archive.nameHash," + + " archive.crc, archive.revision, archive.hash from index_archive " + "join archive on index_archive.archive = archive.id " + "where index_archive.index = :id " + "and archive.archiveId = :archiveId") @@ -105,7 +107,8 @@ public class CacheDAO */ public ArchiveEntry findMostRecentArchive(Connection con, int indexId, int archiveId) { - return con.createQuery("select archive.id, archive.archiveId, archive.nameHash, archive.revision from archive " + return con.createQuery("select archive.id, archive.archiveId, archive.nameHash," + + " archive.crc, archive.revision, archive.hash from archive " + "join index_archive on index_archive.archive = archive.id " + "join `index` on index.id = index_archive.index " + "where index.indexId = :indexId and archive.archiveId = :archiveId " @@ -153,12 +156,14 @@ public class CacheDAO .executeAndFetchFirst(CacheEntry.class); } - public IndexEntry findIndex(Connection con, int indexId, int revision) + public IndexEntry findIndex(Connection con, int indexId, int crc, int revision) { - return con.createQuery("select id, indexId, revision from `index` " + return con.createQuery("select id, indexId, crc, revision from `index` " + "where indexId = :indexId " + + "and crc = :crc " + "and revision = :revision") .addParameter("indexId", indexId) + .addParameter("crc", crc) .addParameter("revision", revision) .executeAndFetchFirst(IndexEntry.class); } @@ -171,10 +176,14 @@ public class CacheDAO .executeUpdate(); } - public IndexEntry findOrCreateIndex(Connection con, CacheEntry cache, int indexId, int revision) + public IndexEntry findOrCreateIndex(Connection con, CacheEntry cache, int indexId, int crc, int revision) { - IndexEntry entry = con.createQuery("select id, indexId, revision from `index` where indexId = :indexId and revision = :revision") + IndexEntry entry = con.createQuery("select id, indexId, crc, revision from `index`" + + "where indexId = :indexId " + + "and crc = :crc " + + "and revision = :revision") .addParameter("indexId", indexId) + .addParameter("crc", crc) .addParameter("revision", revision) .executeAndFetchFirst(IndexEntry.class); @@ -183,8 +192,9 @@ public class CacheDAO return entry; } - int id = con.createQuery("insert into `index` (indexId, revision) values (:indexId, :revision)") + int id = con.createQuery("insert into `index` (indexId, crc, revision) values (:indexId, :crc, :revision)") .addParameter("indexId", indexId) + .addParameter("crc", crc) .addParameter("revision", revision) .executeUpdate() .getKey(int.class); @@ -192,6 +202,7 @@ public class CacheDAO entry = new IndexEntry(); entry.setId(id); entry.setIndexId(indexId); + entry.setCrc(crc); entry.setRevision(revision); return entry; } @@ -209,18 +220,25 @@ public class CacheDAO } public ArchiveEntry findArchive(Connection con, IndexEntry index, - int archiveId, int nameHash, int revision) + int archiveId, int nameHash, int crc, int revision) { if (findArchive == null) { - findArchive = con.createQuery("select archive.id, archive.archiveId, archive.nameHash, archive.revision from archive " + findArchive = con.createQuery("select archive.id, archive.archiveId, archive.nameHash," + + " archive.crc, archive.revision, archive.hash from archive " + " join index_archive on index_archive.archive = archive.id" + " join `index` on index.id = index_archive.index" - + " where archive.archiveId = :archiveId and archive.revision = :revision and index.indexId = :indexId"); + + " where archive.archiveId = :archiveId" + + " and archive.nameHash = :nameHash" + + " and archive.crc = :crc" + + " and archive.revision = :revision" + + " and index.indexId = :indexId"); } ArchiveEntry entry = findArchive .addParameter("archiveId", archiveId) + .addParameter("nameHash", nameHash) + .addParameter("crc", crc) .addParameter("revision", revision) .addParameter("indexId", index.getIndexId()) .executeAndFetchFirst(ArchiveEntry.class); @@ -228,18 +246,20 @@ public class CacheDAO } public ArchiveEntry createArchive(Connection con, IndexEntry index, - int archiveId, int nameHash, int revision) + int archiveId, int nameHash, int crc, int revision, byte[] hash) { if (insertArchive == null) { - insertArchive = con.createQuery("insert into archive (archiveId, nameHash, revision) values " - + "(:archiveId, :nameHash, :revision)"); + insertArchive = con.createQuery("insert into archive (archiveId, nameHash, crc, revision, hash) values " + + "(:archiveId, :nameHash, :crc, :revision, :hash)"); } int id = insertArchive .addParameter("archiveId", archiveId) .addParameter("nameHash", nameHash) + .addParameter("crc", crc) .addParameter("revision", revision) + .addParameter("hash", hash) .executeUpdate() .getKey(int.class); @@ -247,7 +267,9 @@ public class CacheDAO entry.setId(id); entry.setArchiveId(archiveId); entry.setNameHash(nameHash); + entry.setCrc(crc); entry.setRevision(revision); + entry.setHash(hash); return entry; } diff --git a/http-service/src/main/java/net/runelite/http/service/cache/CacheService.java b/http-service/src/main/java/net/runelite/http/service/cache/CacheService.java index d1f6251608..5d191a1547 100644 --- a/http-service/src/main/java/net/runelite/http/service/cache/CacheService.java +++ b/http-service/src/main/java/net/runelite/http/service/cache/CacheService.java @@ -24,6 +24,7 @@ */ package net.runelite.http.service.cache; +import com.google.common.io.BaseEncoding; import com.google.common.io.ByteStreams; import io.minio.MinioClient; import io.minio.errors.ErrorResponseException; @@ -106,27 +107,24 @@ public class CacheService /** * retrieve archive from storage - * - * @param indexId - * @param archiveId - * @param revision * @return */ - private byte[] getArchive(int indexId, int archiveId, int revision) + private byte[] getArchive(ArchiveEntry archiveEntry) { + String hashStr = BaseEncoding.base16().encode(archiveEntry.getHash()); String path = new StringBuilder() - .append(indexId) + .append(hashStr.substring(0, 2)) .append('/') - .append(archiveId) - .append('/') - .append(revision) + .append(hashStr.substring(2)) .toString(); try (InputStream in = minioClient.getObject(minioBucket, path)) { return ByteStreams.toByteArray(in); } - catch (InvalidBucketNameException | NoSuchAlgorithmException | InsufficientDataException | IOException | InvalidKeyException | NoResponseException | XmlPullParserException | ErrorResponseException | InternalException | InvalidArgumentException ex) + catch (InvalidBucketNameException | NoSuchAlgorithmException | InsufficientDataException + | IOException | InvalidKeyException | NoResponseException | XmlPullParserException + | ErrorResponseException | InternalException | InvalidArgumentException ex) { logger.warn(null, ex); return null; @@ -144,7 +142,7 @@ public class CacheService files = cacheDao.findFilesForArchive(con, archiveEntry); } - byte[] archiveData = getArchive(index.getNumber(), config.getId(), archiveEntry.getRevision()); + byte[] archiveData = getArchive(archiveEntry); if (archiveData == null) { @@ -289,7 +287,7 @@ public class CacheService archiveEntry = cacheDao.findArchiveForIndex(con, indexEntry, archiveId); } - return getArchive(indexId, archiveId, archiveEntry.getRevision()); + return getArchive(archiveEntry); } @RequestMapping("item/{itemId}") diff --git a/http-service/src/main/java/net/runelite/http/service/cache/CacheStorage.java b/http-service/src/main/java/net/runelite/http/service/cache/CacheStorage.java index 12908d4095..02e1445263 100644 --- a/http-service/src/main/java/net/runelite/http/service/cache/CacheStorage.java +++ b/http-service/src/main/java/net/runelite/http/service/cache/CacheStorage.java @@ -24,6 +24,7 @@ */ package net.runelite.http.service.cache; +import com.google.common.hash.Hashing; import java.io.IOException; import java.util.List; import net.runelite.cache.fs.Archive; @@ -77,14 +78,16 @@ public class CacheStorage implements Storage for (IndexEntry indexEntry : indexes) { Index index = store.addIndex(indexEntry.getIndexId()); + index.setCrc(indexEntry.getCrc()); index.setRevision(indexEntry.getRevision()); List archives = cacheDao.findArchivesForIndex(con, indexEntry); for (ArchiveEntry archiveEntry : archives) { Archive archive = index.addArchive(archiveEntry.getArchiveId()); - archive.setRevision(archiveEntry.getRevision()); archive.setNameHash(archiveEntry.getNameHash()); + archive.setCrc(archiveEntry.getCrc()); + archive.setRevision(archiveEntry.getRevision()); List files = cacheDao.findFilesForArchive(con, archiveEntry); for (FileEntry fileEntry : files) @@ -102,17 +105,19 @@ public class CacheStorage implements Storage { for (Index index : store.getIndexes()) { - IndexEntry entry = cacheDao.findOrCreateIndex(con, cacheEntry, index.getId(), index.getRevision()); + IndexEntry entry = cacheDao.findOrCreateIndex(con, cacheEntry, index.getId(), index.getCrc(), index.getRevision()); // this assumes nothing is associated to the cache yet cacheDao.associateIndexToCache(con, cacheEntry, entry); for (Archive archive : index.getArchives()) { ArchiveEntry archiveEntry = cacheDao.findArchive(con, entry, archive.getArchiveId(), - archive.getNameHash(), archive.getRevision()); + archive.getNameHash(), archive.getCrc(), archive.getRevision()); if (archiveEntry == null) { - archiveEntry = cacheDao.createArchive(con, entry, archive.getArchiveId(), archive.getNameHash(), archive.getRevision()); + byte[] hash = Hashing.sha256().hashBytes(archive.getData()).asBytes(); + archiveEntry = cacheDao.createArchive(con, entry, archive.getArchiveId(), + archive.getNameHash(), archive.getCrc(), archive.getRevision(), hash); for (FSFile file : archive.getFiles()) { diff --git a/http-service/src/main/java/net/runelite/http/service/cache/CacheUpdater.java b/http-service/src/main/java/net/runelite/http/service/cache/CacheUpdater.java index 90223ad7d3..56e5e59739 100644 --- a/http-service/src/main/java/net/runelite/http/service/cache/CacheUpdater.java +++ b/http-service/src/main/java/net/runelite/http/service/cache/CacheUpdater.java @@ -147,7 +147,8 @@ public class CacheUpdater IndexEntry ie = dbIndexes.get(i); if (ii.getId() != ie.getIndexId() - || ii.getRevision() != ie.getRevision()) + || ii.getRevision() != ie.getRevision() + || ii.getCrc() != ie.getCrc()) { return true; } diff --git a/http-service/src/main/java/net/runelite/http/service/cache/CacheUploader.java b/http-service/src/main/java/net/runelite/http/service/cache/CacheUploader.java index 4ede3c4cf8..beef4468a0 100644 --- a/http-service/src/main/java/net/runelite/http/service/cache/CacheUploader.java +++ b/http-service/src/main/java/net/runelite/http/service/cache/CacheUploader.java @@ -24,6 +24,8 @@ */ package net.runelite.http.service.cache; +import com.google.common.hash.Hashing; +import com.google.common.io.BaseEncoding; import io.minio.MinioClient; import io.minio.errors.ErrorResponseException; import io.minio.errors.InsufficientDataException; @@ -60,12 +62,13 @@ public class CacheUploader implements Runnable public void run() { byte[] data = archive.getData(); + byte[] hash = Hashing.sha256().hashBytes(data).asBytes(); + String hashStr = BaseEncoding.base16().encode(hash); + String path = new StringBuilder() - .append(archive.getIndex().getId()) + .append(hashStr.substring(0, 2)) .append('/') - .append(archive.getArchiveId()) - .append('/') - .append(archive.getRevision()) + .append(hashStr.substring(2)) .toString(); try diff --git a/http-service/src/main/java/net/runelite/http/service/cache/beans/ArchiveEntry.java b/http-service/src/main/java/net/runelite/http/service/cache/beans/ArchiveEntry.java index 1a9d0191b2..7b246eaa95 100644 --- a/http-service/src/main/java/net/runelite/http/service/cache/beans/ArchiveEntry.java +++ b/http-service/src/main/java/net/runelite/http/service/cache/beans/ArchiveEntry.java @@ -24,27 +24,33 @@ */ package net.runelite.http.service.cache.beans; +import java.util.Arrays; + public class ArchiveEntry { private int id; private int archiveId; private int nameHash; + private int crc; private int revision; + private byte[] hash; @Override public String toString() { - return "ArchiveEntry{" + "id=" + id + ", archiveId=" + archiveId + ", nameHash=" + nameHash + ", revision=" + revision + '}'; + return "ArchiveEntry{" + "id=" + id + ", archiveId=" + archiveId + ", nameHash=" + nameHash + ", crc=" + crc + ", revision=" + revision + '}'; } @Override public int hashCode() { - int hash = 7; - hash = 89 * hash + this.id; - hash = 89 * hash + this.archiveId; - hash = 89 * hash + this.nameHash; - hash = 89 * hash + this.revision; + int hash = 3; + hash = 53 * hash + this.id; + hash = 53 * hash + this.archiveId; + hash = 53 * hash + this.nameHash; + hash = 53 * hash + this.crc; + hash = 53 * hash + this.revision; + hash = 53 * hash + Arrays.hashCode(this.hash); return hash; } @@ -76,10 +82,18 @@ public class ArchiveEntry { return false; } + if (this.crc != other.crc) + { + return false; + } if (this.revision != other.revision) { return false; } + if (!Arrays.equals(this.hash, other.hash)) + { + return false; + } return true; } @@ -113,6 +127,16 @@ public class ArchiveEntry this.nameHash = nameHash; } + public int getCrc() + { + return crc; + } + + public void setCrc(int crc) + { + this.crc = crc; + } + public int getRevision() { return revision; @@ -122,5 +146,15 @@ public class ArchiveEntry { this.revision = revision; } - + + public byte[] getHash() + { + return hash; + } + + public void setHash(byte[] hash) + { + this.hash = hash; + } + } diff --git a/http-service/src/main/java/net/runelite/http/service/cache/beans/IndexEntry.java b/http-service/src/main/java/net/runelite/http/service/cache/beans/IndexEntry.java index be67820b01..7f89eff79c 100644 --- a/http-service/src/main/java/net/runelite/http/service/cache/beans/IndexEntry.java +++ b/http-service/src/main/java/net/runelite/http/service/cache/beans/IndexEntry.java @@ -28,21 +28,23 @@ public class IndexEntry { private int id; private int indexId; + private int crc; private int revision; @Override public String toString() { - return "IndexEntry{" + "id=" + id + ", indexId=" + indexId + ", revision=" + revision + '}'; + return "IndexEntry{" + "id=" + id + ", indexId=" + indexId + ", crc=" + crc + ", revision=" + revision + '}'; } @Override public int hashCode() { int hash = 5; - hash = 83 * hash + this.id; - hash = 83 * hash + this.indexId; - hash = 83 * hash + this.revision; + hash = 17 * hash + this.id; + hash = 17 * hash + this.indexId; + hash = 17 * hash + this.crc; + hash = 17 * hash + this.revision; return hash; } @@ -70,6 +72,10 @@ public class IndexEntry { return false; } + if (this.crc != other.crc) + { + return false; + } if (this.revision != other.revision) { return false; @@ -97,6 +103,16 @@ public class IndexEntry this.indexId = indexId; } + public int getCrc() + { + return crc; + } + + public void setCrc(int crc) + { + this.crc = crc; + } + public int getRevision() { return revision; diff --git a/http-service/src/main/java/net/runelite/http/service/cache/schema.sql b/http-service/src/main/java/net/runelite/http/service/cache/schema.sql index 3032134734..2f1b106bb3 100644 --- a/http-service/src/main/java/net/runelite/http/service/cache/schema.sql +++ b/http-service/src/main/java/net/runelite/http/service/cache/schema.sql @@ -1,8 +1,8 @@ --- MySQL dump 10.16 Distrib 10.2.8-MariaDB, for osx10.12 (x86_64) +-- MySQL dump 10.16 Distrib 10.1.20-MariaDB, for Linux (x86_64) -- --- Host: localhost Database: cache +-- Host: localhost Database: localhost -- ------------------------------------------------------ --- Server version 10.2.8-MariaDB +-- Server version 10.1.20-MariaDB /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; @@ -26,7 +26,9 @@ CREATE TABLE `archive` ( `id` int(11) NOT NULL AUTO_INCREMENT, `archiveId` int(11) NOT NULL, `nameHash` int(11) NOT NULL, + `crc` int(11) NOT NULL, `revision` int(11) NOT NULL, + `hash` binary(32) NOT NULL, PRIMARY KEY (`id`), KEY `archive_revision` (`archiveId`,`revision`) USING BTREE ) ENGINE=InnoDB DEFAULT CHARSET=latin1; @@ -42,7 +44,7 @@ DROP TABLE IF EXISTS `cache`; CREATE TABLE `cache` ( `id` int(11) NOT NULL AUTO_INCREMENT, `revision` int(11) NOT NULL, - `date` timestamp NOT NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(), + `date` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY `revision_date` (`revision`,`date`) ) ENGINE=InnoDB DEFAULT CHARSET=latin1; @@ -95,9 +97,10 @@ DROP TABLE IF EXISTS `index`; CREATE TABLE `index` ( `id` int(11) NOT NULL AUTO_INCREMENT, `indexId` int(11) NOT NULL, + `crc` int(11) NOT NULL, `revision` int(11) NOT NULL, PRIMARY KEY (`id`), - UNIQUE KEY `indexId` (`indexId`,`revision`) + UNIQUE KEY `indexId` (`indexId`,`revision`,`crc`) USING BTREE ) ENGINE=InnoDB DEFAULT CHARSET=latin1; /*!40101 SET character_set_client = @saved_cs_client */; @@ -129,4 +132,4 @@ CREATE TABLE `index_archive` ( /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; --- Dump completed on 2017-09-10 13:11:23 +-- Dump completed on 2017-09-24 15:38:31