cache: store index and archive crcs, and check them for updates too

Switch archive storage to CAS as sometimes archives change content without changing revision

cache/src/main/java/net/runelite/cache/client/CacheClient.java

@@ -58,6 +58,7 @@ import net.runelite.cache.protocol.packets.ArchiveRequestPacket;
 import net.runelite.cache.protocol.packets.HandshakePacket;
 import net.runelite.cache.protocol.packets.HandshakeResponseType;
 import net.runelite.cache.protocol.packets.HandshakeType;
+import net.runelite.cache.util.Crc32;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -268,7 +269,9 @@ public class CacheClient implements AutoCloseable
 			{
 				Archive existing = index.getArchive(ad.getId());
 
-				if (existing != null && existing.getRevision() == ad.getRevision())
+				if (existing != null && existing.getRevision() == ad.getRevision()
+					&& existing.getCrc() == ad.getCrc()
+					&& existing.getNameHash() == ad.getNameHash())
 				{
 					logger.info("Archive {}/{} in index {} is up to date",
 						ad.getId(), indexData.getArchives().length, index.getId());
@@ -288,9 +291,12 @@ public class CacheClient implements AutoCloseable
 				}
 				else
 				{
-					logger.info("Archive {}/{} in index {} is out of date ({} != {}), downloading",
+					logger.info("Archive {}/{} in index {} is out of date, downloading. " +
+						"revision: ours: {} theirs: {}, crc: ours: {} theirs {}, name: ours {} theirs {}",
 						ad.getId(), indexData.getArchives().length, index.getId(),
-						existing.getRevision(), ad.getRevision());
+						existing.getRevision(), ad.getRevision(),
+						existing.getCrc(), ad.getCrc(),
+						existing.getNameHash(), ad.getNameHash());
 				}
 
 				final Archive archive = existing == null
@@ -313,7 +319,20 @@ public class CacheClient implements AutoCloseable
 				CompletableFuture<FileResult> future = requestFile(index.getId(), ad.getId(), false);
 				future.handle((fr, ex) ->
 				{
-					archive.setData(fr.getCompressedData());
+					byte[] data = fr.getCompressedData();
+
+					Crc32 crc32 = new Crc32();
+					crc32.update(data, 0, data.length);
+					int hash = crc32.getHash();
+
+					if (hash != archive.getCrc())
+					{
+						logger.warn("crc mismatch on downloaded archive {}/{}: {} != {}",
+							archive.getIndex().getId(), archive.getArchiveId(),
+							hash, archive.getCrc());
+					}
+
+					archive.setData(data);
 
 					if (watcher != null)
 					{

cache/src/main/java/net/runelite/cache/protocol/decoders/ArchiveResponseDecoder.java

@@ -115,6 +115,8 @@ public class ArchiveResponseDecoder extends ByteToMessageDecoder
 		archiveResponse.setArchive(file);
 		archiveResponse.setData(compressedData.array());
 		out.add(archiveResponse);
+
+		compressedData.release();
 	}
 
 	/**

http-service/src/main/java/net/runelite/http/service/cache/CacheDAO.java

@@ -55,7 +55,7 @@ public class CacheDAO
 
 	public List<IndexEntry> findIndexesForCache(Connection con, CacheEntry cache)
 	{
-		return con.createQuery("select index.id, index.indexId, index.revision from cache "
+		return con.createQuery("select index.id, index.indexId, index.crc, index.revision from cache "
 			+ "join cache_index on cache_index.cache = cache.id "
 			+ "join `index` on cache_index.index = index.id "
 			+ "where cache.id = :id "
@@ -66,7 +66,7 @@ public class CacheDAO
 
 	public IndexEntry findIndexForCache(Connection con, CacheEntry cache, int indexId)
 	{
-		return con.createQuery("select index.id, index.indexId, index.revision from cache "
+		return con.createQuery("select index.id, index.indexId, index.crc, index.revision from cache "
 			+ "join cache_index on cache_index.cache = cache.id "
 			+ "join `index` on cache_index.index = index.id "
 			+ "where cache.id = :id "
@@ -78,7 +78,8 @@ public class CacheDAO
 
 	public List<ArchiveEntry> findArchivesForIndex(Connection con, IndexEntry indexEntry)
 	{
-		return con.createQuery("select archive.id, archive.archiveId, archive.nameHash, archive.revision from index_archive "
+		return con.createQuery("select archive.id, archive.archiveId, archive.nameHash," +
+			" archive.crc, archive.revision, archive.hash from index_archive "
 			+ "join archive on index_archive.archive = archive.id "
 			+ "where index_archive.index = :id")
 			.addParameter("id", indexEntry.getId())
@@ -87,7 +88,8 @@ public class CacheDAO
 	
 	public ArchiveEntry findArchiveForIndex(Connection con, IndexEntry indexEntry, int archiveId)
 	{
-		return con.createQuery("select archive.id, archive.archiveId, archive.nameHash, archive.revision from index_archive "
+		return con.createQuery("select archive.id, archive.archiveId, archive.nameHash," +
+			" archive.crc, archive.revision, archive.hash from index_archive "
 			+ "join archive on index_archive.archive = archive.id "
 			+ "where index_archive.index = :id "
 			+ "and archive.archiveId = :archiveId")
@@ -105,7 +107,8 @@ public class CacheDAO
 	 */
 	public ArchiveEntry findMostRecentArchive(Connection con, int indexId, int archiveId)
 	{
-		return con.createQuery("select archive.id, archive.archiveId, archive.nameHash, archive.revision from archive "
+		return con.createQuery("select archive.id, archive.archiveId, archive.nameHash," +
+			" archive.crc, archive.revision, archive.hash from archive "
 			+ "join index_archive on index_archive.archive = archive.id "
 			+ "join `index` on index.id = index_archive.index "
 			+ "where index.indexId = :indexId and archive.archiveId = :archiveId "
@@ -153,12 +156,14 @@ public class CacheDAO
 			.executeAndFetchFirst(CacheEntry.class);
 	}
 
-	public IndexEntry findIndex(Connection con, int indexId, int revision)
+	public IndexEntry findIndex(Connection con, int indexId, int crc, int revision)
 	{
-		return con.createQuery("select id, indexId, revision from `index` "
+		return con.createQuery("select id, indexId, crc, revision from `index` "
 			+ "where indexId = :indexId "
+			+ "and crc = :crc "
 			+ "and revision = :revision")
 			.addParameter("indexId", indexId)
+			.addParameter("crc", crc)
 			.addParameter("revision", revision)
 			.executeAndFetchFirst(IndexEntry.class);
 	}
@@ -171,10 +176,14 @@ public class CacheDAO
 			.executeUpdate();
 	}
 
-	public IndexEntry findOrCreateIndex(Connection con, CacheEntry cache, int indexId, int revision)
+	public IndexEntry findOrCreateIndex(Connection con, CacheEntry cache, int indexId, int crc, int revision)
 	{
-		IndexEntry entry = con.createQuery("select id, indexId, revision from `index` where indexId = :indexId and revision = :revision")
+		IndexEntry entry = con.createQuery("select id, indexId, crc, revision from `index`"
+			+ "where indexId = :indexId "
+			+ "and crc = :crc "
+			+ "and revision = :revision")
 			.addParameter("indexId", indexId)
+			.addParameter("crc", crc)
 			.addParameter("revision", revision)
 			.executeAndFetchFirst(IndexEntry.class);
 
@@ -183,8 +192,9 @@ public class CacheDAO
 			return entry;
 		}
 
-		int id = con.createQuery("insert into `index` (indexId, revision) values (:indexId, :revision)")
+		int id = con.createQuery("insert into `index` (indexId, crc, revision) values (:indexId, :crc, :revision)")
 			.addParameter("indexId", indexId)
+			.addParameter("crc", crc)
 			.addParameter("revision", revision)
 			.executeUpdate()
 			.getKey(int.class);
@@ -192,6 +202,7 @@ public class CacheDAO
 		entry = new IndexEntry();
 		entry.setId(id);
 		entry.setIndexId(indexId);
+		entry.setCrc(crc);
 		entry.setRevision(revision);
 		return entry;
 	}
@@ -209,18 +220,25 @@ public class CacheDAO
 	}
 
 	public ArchiveEntry findArchive(Connection con, IndexEntry index,
-		int archiveId, int nameHash, int revision)
+		int archiveId, int nameHash, int crc, int revision)
 	{
 		if (findArchive == null)
 		{
-			findArchive = con.createQuery("select archive.id, archive.archiveId, archive.nameHash, archive.revision from archive "
+			findArchive = con.createQuery("select archive.id, archive.archiveId, archive.nameHash,"
+				+ " archive.crc, archive.revision, archive.hash from archive "
 				+ " join index_archive on index_archive.archive = archive.id"
 				+ " join `index` on index.id = index_archive.index"
-				+ " where archive.archiveId = :archiveId and archive.revision = :revision and index.indexId = :indexId");
+				+ " where archive.archiveId = :archiveId"
+				+ " and archive.nameHash = :nameHash"
+				+ " and archive.crc = :crc"
+				+ " and archive.revision = :revision"
+				+ " and index.indexId = :indexId");
 		}
 
 		ArchiveEntry entry = findArchive
 			.addParameter("archiveId", archiveId)
+			.addParameter("nameHash", nameHash)
+			.addParameter("crc", crc)
 			.addParameter("revision", revision)
 			.addParameter("indexId", index.getIndexId())
 			.executeAndFetchFirst(ArchiveEntry.class);
@@ -228,18 +246,20 @@ public class CacheDAO
 	}
 
 	public ArchiveEntry createArchive(Connection con, IndexEntry index,
-		int archiveId, int nameHash, int revision)
+		int archiveId, int nameHash, int crc, int revision, byte[] hash)
 	{
 		if (insertArchive == null)
 		{
-			insertArchive = con.createQuery("insert into archive (archiveId, nameHash, revision) values "
+			insertArchive = con.createQuery("insert into archive (archiveId, nameHash, crc, revision, hash) values "
-				+ "(:archiveId, :nameHash, :revision)");
+				+ "(:archiveId, :nameHash, :crc, :revision, :hash)");
 		}
 
 		int id = insertArchive
 			.addParameter("archiveId", archiveId)
 			.addParameter("nameHash", nameHash)
+			.addParameter("crc", crc)
 			.addParameter("revision", revision)
+			.addParameter("hash", hash)
 			.executeUpdate()
 			.getKey(int.class);
 
@@ -247,7 +267,9 @@ public class CacheDAO
 		entry.setId(id);
 		entry.setArchiveId(archiveId);
 		entry.setNameHash(nameHash);
+		entry.setCrc(crc);
 		entry.setRevision(revision);
+		entry.setHash(hash);
 		return entry;
 	}
 

http-service/src/main/java/net/runelite/http/service/cache/CacheService.java

@@ -24,6 +24,7 @@
  */
 package net.runelite.http.service.cache;
 
+import com.google.common.io.BaseEncoding;
 import com.google.common.io.ByteStreams;
 import io.minio.MinioClient;
 import io.minio.errors.ErrorResponseException;
@@ -106,27 +107,24 @@ public class CacheService
 
 	/**
 	 * retrieve archive from storage
-	 *
-	 * @param indexId
-	 * @param archiveId
-	 * @param revision
 	 * @return
 	 */
-	private byte[] getArchive(int indexId, int archiveId, int revision)
+	private byte[] getArchive(ArchiveEntry archiveEntry)
 	{
+		String hashStr = BaseEncoding.base16().encode(archiveEntry.getHash());
 		String path = new StringBuilder()
-			.append(indexId)
+			.append(hashStr.substring(0, 2))
 			.append('/')
-			.append(archiveId)
+			.append(hashStr.substring(2))
-			.append('/')
-			.append(revision)
 			.toString();
 
 		try (InputStream in = minioClient.getObject(minioBucket, path))
 		{
 			return ByteStreams.toByteArray(in);
 		}
-		catch (InvalidBucketNameException | NoSuchAlgorithmException | InsufficientDataException | IOException | InvalidKeyException | NoResponseException | XmlPullParserException | ErrorResponseException | InternalException | InvalidArgumentException ex)
+		catch (InvalidBucketNameException | NoSuchAlgorithmException | InsufficientDataException
+			| IOException | InvalidKeyException | NoResponseException | XmlPullParserException
+			| ErrorResponseException | InternalException | InvalidArgumentException ex)
 		{
 			logger.warn(null, ex);
 			return null;
@@ -144,7 +142,7 @@ public class CacheService
 			files = cacheDao.findFilesForArchive(con, archiveEntry);
 		}
 
-		byte[] archiveData = getArchive(index.getNumber(), config.getId(), archiveEntry.getRevision());
+		byte[] archiveData = getArchive(archiveEntry);
 
 		if (archiveData == null)
 		{
@@ -289,7 +287,7 @@ public class CacheService
 			archiveEntry = cacheDao.findArchiveForIndex(con, indexEntry, archiveId);
 		}
 
-		return getArchive(indexId, archiveId, archiveEntry.getRevision());
+		return getArchive(archiveEntry);
 	}
 
 	@RequestMapping("item/{itemId}")

http-service/src/main/java/net/runelite/http/service/cache/CacheStorage.java

@@ -24,6 +24,7 @@
  */
 package net.runelite.http.service.cache;
 
+import com.google.common.hash.Hashing;
 import java.io.IOException;
 import java.util.List;
 import net.runelite.cache.fs.Archive;
@@ -77,14 +78,16 @@ public class CacheStorage implements Storage
 		for (IndexEntry indexEntry : indexes)
 		{
 			Index index = store.addIndex(indexEntry.getIndexId());
+			index.setCrc(indexEntry.getCrc());
 			index.setRevision(indexEntry.getRevision());
 
 			List<ArchiveEntry> archives = cacheDao.findArchivesForIndex(con, indexEntry);
 			for (ArchiveEntry archiveEntry : archives)
 			{
 				Archive archive = index.addArchive(archiveEntry.getArchiveId());
-				archive.setRevision(archiveEntry.getRevision());
 				archive.setNameHash(archiveEntry.getNameHash());
+				archive.setCrc(archiveEntry.getCrc());
+				archive.setRevision(archiveEntry.getRevision());
 
 				List<FileEntry> files = cacheDao.findFilesForArchive(con, archiveEntry);
 				for (FileEntry fileEntry : files)
@@ -102,17 +105,19 @@ public class CacheStorage implements Storage
 	{
 		for (Index index : store.getIndexes())
 		{
-			IndexEntry entry = cacheDao.findOrCreateIndex(con, cacheEntry, index.getId(), index.getRevision());
+			IndexEntry entry = cacheDao.findOrCreateIndex(con, cacheEntry, index.getId(), index.getCrc(), index.getRevision());
 			// this assumes nothing is associated to the cache yet
 			cacheDao.associateIndexToCache(con, cacheEntry, entry);
 
 			for (Archive archive : index.getArchives())
 			{
 				ArchiveEntry archiveEntry = cacheDao.findArchive(con, entry, archive.getArchiveId(),
-					archive.getNameHash(), archive.getRevision());
+					archive.getNameHash(), archive.getCrc(), archive.getRevision());
 				if (archiveEntry == null)
 				{
-					archiveEntry = cacheDao.createArchive(con, entry, archive.getArchiveId(), archive.getNameHash(), archive.getRevision());
+					byte[] hash = Hashing.sha256().hashBytes(archive.getData()).asBytes();
+					archiveEntry = cacheDao.createArchive(con, entry, archive.getArchiveId(),
+						archive.getNameHash(), archive.getCrc(), archive.getRevision(), hash);
 
 					for (FSFile file : archive.getFiles())
 					{

http-service/src/main/java/net/runelite/http/service/cache/CacheUpdater.java

@@ -147,7 +147,8 @@ public class CacheUpdater
 			IndexEntry ie = dbIndexes.get(i);
 
 			if (ii.getId() != ie.getIndexId()
-				|| ii.getRevision() != ie.getRevision())
+				|| ii.getRevision() != ie.getRevision()
+				|| ii.getCrc() != ie.getCrc())
 			{
 				return true;
 			}

http-service/src/main/java/net/runelite/http/service/cache/CacheUploader.java

@@ -24,6 +24,8 @@
  */
 package net.runelite.http.service.cache;
 
+import com.google.common.hash.Hashing;
+import com.google.common.io.BaseEncoding;
 import io.minio.MinioClient;
 import io.minio.errors.ErrorResponseException;
 import io.minio.errors.InsufficientDataException;
@@ -60,12 +62,13 @@ public class CacheUploader implements Runnable
 	public void run()
 	{
 		byte[] data = archive.getData();
+		byte[] hash = Hashing.sha256().hashBytes(data).asBytes();
+		String hashStr = BaseEncoding.base16().encode(hash);
+
 		String path = new StringBuilder()
-			.append(archive.getIndex().getId())
+			.append(hashStr.substring(0, 2))
 			.append('/')
-			.append(archive.getArchiveId())
+			.append(hashStr.substring(2))
-			.append('/')
-			.append(archive.getRevision())
 			.toString();
 
 		try

http-service/src/main/java/net/runelite/http/service/cache/beans/ArchiveEntry.java

@@ -24,27 +24,33 @@
  */
 package net.runelite.http.service.cache.beans;
 
+import java.util.Arrays;
+
 public class ArchiveEntry
 {
 	private int id;
 	private int archiveId;
 	private int nameHash;
+	private int crc;
 	private int revision;
+	private byte[] hash;
 
 	@Override
 	public String toString()
 	{
-		return "ArchiveEntry{" + "id=" + id + ", archiveId=" + archiveId + ", nameHash=" + nameHash + ", revision=" + revision + '}';
+		return "ArchiveEntry{" + "id=" + id + ", archiveId=" + archiveId + ", nameHash=" + nameHash + ", crc=" + crc + ", revision=" + revision + '}';
 	}
 
 	@Override
 	public int hashCode()
 	{
-		int hash = 7;
+		int hash = 3;
-		hash = 89 * hash + this.id;
+		hash = 53 * hash + this.id;
-		hash = 89 * hash + this.archiveId;
+		hash = 53 * hash + this.archiveId;
-		hash = 89 * hash + this.nameHash;
+		hash = 53 * hash + this.nameHash;
-		hash = 89 * hash + this.revision;
+		hash = 53 * hash + this.crc;
+		hash = 53 * hash + this.revision;
+		hash = 53 * hash + Arrays.hashCode(this.hash);
 		return hash;
 	}
 
@@ -76,10 +82,18 @@ public class ArchiveEntry
 		{
 			return false;
 		}
+		if (this.crc != other.crc)
+		{
+			return false;
+		}
 		if (this.revision != other.revision)
 		{
 			return false;
 		}
+		if (!Arrays.equals(this.hash, other.hash))
+		{
+			return false;
+		}
 		return true;
 	}
 
@@ -113,6 +127,16 @@ public class ArchiveEntry
 		this.nameHash = nameHash;
 	}
 
+	public int getCrc()
+	{
+		return crc;
+	}
+
+	public void setCrc(int crc)
+	{
+		this.crc = crc;
+	}
+
 	public int getRevision()
 	{
 		return revision;
@@ -122,5 +146,15 @@ public class ArchiveEntry
 	{
 		this.revision = revision;
 	}
-	
+
+	public byte[] getHash()
+	{
+		return hash;
+	}
+
+	public void setHash(byte[] hash)
+	{
+		this.hash = hash;
+	}
+
 }

http-service/src/main/java/net/runelite/http/service/cache/beans/IndexEntry.java

@@ -28,21 +28,23 @@ public class IndexEntry
 {
 	private int id;
 	private int indexId;
+	private int crc;
 	private int revision;
 
 	@Override
 	public String toString()
 	{
-		return "IndexEntry{" + "id=" + id + ", indexId=" + indexId + ", revision=" + revision + '}';
+		return "IndexEntry{" + "id=" + id + ", indexId=" + indexId + ", crc=" + crc + ", revision=" + revision + '}';
 	}
 
 	@Override
 	public int hashCode()
 	{
 		int hash = 5;
-		hash = 83 * hash + this.id;
+		hash = 17 * hash + this.id;
-		hash = 83 * hash + this.indexId;
+		hash = 17 * hash + this.indexId;
-		hash = 83 * hash + this.revision;
+		hash = 17 * hash + this.crc;
+		hash = 17 * hash + this.revision;
 		return hash;
 	}
 
@@ -70,6 +72,10 @@ public class IndexEntry
 		{
 			return false;
 		}
+		if (this.crc != other.crc)
+		{
+			return false;
+		}
 		if (this.revision != other.revision)
 		{
 			return false;
@@ -97,6 +103,16 @@ public class IndexEntry
 		this.indexId = indexId;
 	}
 
+	public int getCrc()
+	{
+		return crc;
+	}
+
+	public void setCrc(int crc)
+	{
+		this.crc = crc;
+	}
+
 	public int getRevision()
 	{
 		return revision;

http-service/src/main/java/net/runelite/http/service/cache/schema.sql

@@ -1,8 +1,8 @@
--- MySQL dump 10.16  Distrib 10.2.8-MariaDB, for osx10.12 (x86_64)
+-- MySQL dump 10.16  Distrib 10.1.20-MariaDB, for Linux (x86_64)
 --
--- Host: localhost    Database: cache
+-- Host: localhost    Database: localhost
 -- ------------------------------------------------------
--- Server version	10.2.8-MariaDB
+-- Server version	10.1.20-MariaDB
 
 /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
 /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
@@ -26,7 +26,9 @@ CREATE TABLE `archive` (
   `id` int(11) NOT NULL AUTO_INCREMENT,
   `archiveId` int(11) NOT NULL,
   `nameHash` int(11) NOT NULL,
+  `crc` int(11) NOT NULL,
   `revision` int(11) NOT NULL,
+  `hash` binary(32) NOT NULL,
   PRIMARY KEY (`id`),
   KEY `archive_revision` (`archiveId`,`revision`) USING BTREE
 ) ENGINE=InnoDB DEFAULT CHARSET=latin1;
@@ -42,7 +44,7 @@ DROP TABLE IF EXISTS `cache`;
 CREATE TABLE `cache` (
   `id` int(11) NOT NULL AUTO_INCREMENT,
   `revision` int(11) NOT NULL,
-  `date` timestamp NOT NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
+  `date` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
   PRIMARY KEY (`id`),
   UNIQUE KEY `revision_date` (`revision`,`date`)
 ) ENGINE=InnoDB DEFAULT CHARSET=latin1;
@@ -95,9 +97,10 @@ DROP TABLE IF EXISTS `index`;
 CREATE TABLE `index` (
   `id` int(11) NOT NULL AUTO_INCREMENT,
   `indexId` int(11) NOT NULL,
+  `crc` int(11) NOT NULL,
   `revision` int(11) NOT NULL,
   PRIMARY KEY (`id`),
-  UNIQUE KEY `indexId` (`indexId`,`revision`)
+  UNIQUE KEY `indexId` (`indexId`,`revision`,`crc`) USING BTREE
 ) ENGINE=InnoDB DEFAULT CHARSET=latin1;
 /*!40101 SET character_set_client = @saved_cs_client */;
 
@@ -129,4 +132,4 @@ CREATE TABLE `index_archive` (
 /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
 /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
 
--- Dump completed on 2017-09-10 13:11:23
+-- Dump completed on 2017-09-24 15:38:31