Split out cache updater into own project

cache-updater/src/main/java/net/runelite/cache/updater/CacheConfiguration.java

@@ -0,0 +1,88 @@
+/*
+ * Copyright (c) 2018, Adam <Adam@sigterm.info>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package net.runelite.cache.updater;
+
+import io.minio.MinioClient;
+import io.minio.errors.InvalidEndpointException;
+import io.minio.errors.InvalidPortException;
+import java.time.Instant;
+import java.util.HashMap;
+import java.util.Map;
+import javax.sql.DataSource;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.jdbc.datasource.DriverManagerDataSource;
+import org.sql2o.Sql2o;
+import org.sql2o.converters.Converter;
+import org.sql2o.quirks.NoQuirks;
+
+@Configuration
+public class CacheConfiguration
+{
+	@Value("${jdbc.url}")
+	private String jdbcUrl;
+
+	@Value("${jdbc.username}")
+	private String jdbcUsername;
+
+	@Value("${jdbc.password}")
+	private String jdbcPassword;
+
+	@Value("${minio.url}")
+	private String minioUrl;
+
+	@Value("${minio.accesskey}")
+	private String minioAccessKey;
+
+	@Value("${minio.secretkey}")
+	private String minioSecretKey;
+
+	@Bean
+	public DataSource dataSource()
+	{
+		DriverManagerDataSource dataSource = new DriverManagerDataSource();
+		dataSource.setUrl(jdbcUrl);
+		dataSource.setUsername(jdbcUsername);
+		dataSource.setPassword(jdbcPassword);
+		return dataSource;
+	}
+
+	@Bean
+	@Qualifier("Runelite Cache SQL2O")
+	public Sql2o sql2o(DataSource dataSource)
+	{
+		Map<Class, Converter> converters = new HashMap<>();
+		converters.put(Instant.class, new InstantConverter());
+		return new Sql2o(dataSource, new NoQuirks(converters));
+	}
+
+	@Bean
+	public MinioClient minioClient() throws InvalidEndpointException, InvalidPortException
+	{
+		return new MinioClient(minioUrl, minioAccessKey, minioSecretKey);
+	}
+}

cache-updater/src/main/java/net/runelite/cache/updater/CacheDAO.java

@@ -0,0 +1,177 @@
+/*
+ * Copyright (c) 2017, Adam <Adam@sigterm.info>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package net.runelite.cache.updater;
+
+import java.time.Instant;
+import java.util.List;
+import net.runelite.cache.updater.beans.ArchiveEntry;
+import net.runelite.cache.updater.beans.CacheEntry;
+import net.runelite.cache.updater.beans.IndexEntry;
+import org.sql2o.Connection;
+import org.sql2o.Query;
+import org.sql2o.ResultSetIterable;
+
+class CacheDAO
+{
+	// cache prepared statements for high volume queries
+	private Query associateArchive;
+	private Query findArchive, insertArchive;
+	private Query associateFile;
+
+	public CacheEntry findMostRecent(Connection con)
+	{
+		return con.createQuery("select id, revision, date from cache order by revision desc, date desc limit 1")
+			.executeAndFetchFirst(CacheEntry.class);
+	}
+
+	public List<IndexEntry> findIndexesForCache(Connection con, CacheEntry cache)
+	{
+		return con.createQuery("select id, indexId, crc, revision from `index` where cache = :cache")
+			.addParameter("cache", cache.getId())
+			.executeAndFetch(IndexEntry.class);
+	}
+
+	public ResultSetIterable<ArchiveEntry> findArchivesForIndex(Connection con, IndexEntry indexEntry)
+	{
+		return con.createQuery("select archive.id, archive.archiveId, archive.nameHash,"
+			+ " archive.crc, archive.revision, archive.hash from index_archive "
+			+ "join archive on index_archive.archive = archive.id "
+			+ "where index_archive.index = :id")
+			.addParameter("id", indexEntry.getId())
+			.executeAndFetchLazy(ArchiveEntry.class);
+	}
+
+	public CacheEntry createCache(Connection con, int revision, Instant date)
+	{
+		int cacheId = con.createQuery("insert into cache (revision, date) values (:revision, :date)")
+			.addParameter("revision", revision)
+			.addParameter("date", date)
+			.executeUpdate()
+			.getKey(int.class);
+
+		CacheEntry entry = new CacheEntry();
+		entry.setId(cacheId);
+		entry.setRevision(revision);
+		entry.setDate(date);
+		return entry;
+	}
+
+	public IndexEntry createIndex(Connection con, CacheEntry cache, int indexId, int crc, int revision)
+	{
+		int id = con.createQuery("insert into `index` (cache, indexId, crc, revision) values (:cache, :indexId, :crc, :revision)")
+			.addParameter("cache", cache.getId())
+			.addParameter("indexId", indexId)
+			.addParameter("crc", crc)
+			.addParameter("revision", revision)
+			.executeUpdate()
+			.getKey(int.class);
+
+		IndexEntry entry = new IndexEntry();
+		entry.setId(id);
+		entry.setIndexId(indexId);
+		entry.setCrc(crc);
+		entry.setRevision(revision);
+		return entry;
+	}
+
+	public void associateArchiveToIndex(Connection con, ArchiveEntry archive, IndexEntry index)
+	{
+		if (associateArchive == null)
+		{
+			associateArchive = con.createQuery("insert into index_archive (`index`, archive) values (:index, :archive)");
+		}
+		associateArchive
+			.addParameter("index", index.getId())
+			.addParameter("archive", archive.getId())
+			.executeUpdate();
+	}
+
+	public ArchiveEntry findArchive(Connection con, IndexEntry index,
+		int archiveId, int nameHash, int crc, int revision)
+	{
+		if (findArchive == null)
+		{
+			findArchive = con.createQuery("select distinct archive.id, archive.archiveId, archive.nameHash,"
+				+ " archive.crc, archive.revision, archive.hash from archive "
+				+ " join index_archive on index_archive.archive = archive.id"
+				+ " join `index` on index.id = index_archive.index"
+				+ " where archive.archiveId = :archiveId"
+				+ " and archive.nameHash = :nameHash"
+				+ " and archive.crc = :crc"
+				+ " and archive.revision = :revision"
+				+ " and index.indexId = :indexId");
+		}
+
+		ArchiveEntry entry = findArchive
+			.addParameter("archiveId", archiveId)
+			.addParameter("nameHash", nameHash)
+			.addParameter("crc", crc)
+			.addParameter("revision", revision)
+			.addParameter("indexId", index.getIndexId())
+			.executeAndFetchFirst(ArchiveEntry.class);
+		return entry;
+	}
+
+	public ArchiveEntry createArchive(Connection con, IndexEntry index,
+		int archiveId, int nameHash, int crc, int revision, byte[] hash)
+	{
+		if (insertArchive == null)
+		{
+			insertArchive = con.createQuery("insert into archive (archiveId, nameHash, crc, revision, hash) values "
+				+ "(:archiveId, :nameHash, :crc, :revision, :hash)");
+		}
+
+		int id = insertArchive
+			.addParameter("archiveId", archiveId)
+			.addParameter("nameHash", nameHash)
+			.addParameter("crc", crc)
+			.addParameter("revision", revision)
+			.addParameter("hash", hash)
+			.executeUpdate()
+			.getKey(int.class);
+
+		ArchiveEntry entry = new ArchiveEntry();
+		entry.setId(id);
+		entry.setArchiveId(archiveId);
+		entry.setNameHash(nameHash);
+		entry.setCrc(crc);
+		entry.setRevision(revision);
+		entry.setHash(hash);
+		return entry;
+	}
+
+	public void associateFileToArchive(Connection con, ArchiveEntry archive, int fileId, int nameHash)
+	{
+		if (associateFile == null)
+		{
+			associateFile = con.createQuery("insert into file (archive, fileId, nameHash) values (:archive, :fileId, :nameHash)");
+		}
+		associateFile
+			.addParameter("archive", archive.getId())
+			.addParameter("fileId", fileId)
+			.addParameter("nameHash", nameHash)
+			.executeUpdate();
+	}
+}

cache-updater/src/main/java/net/runelite/cache/updater/CacheStorage.java

@@ -0,0 +1,146 @@
+/*
+ * Copyright (c) 2017, Adam <Adam@sigterm.info>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package net.runelite.cache.updater;
+
+import java.io.IOException;
+import java.util.List;
+import lombok.extern.slf4j.Slf4j;
+import net.runelite.cache.fs.Archive;
+import net.runelite.cache.fs.Index;
+import net.runelite.cache.fs.Storage;
+import net.runelite.cache.fs.Store;
+import net.runelite.cache.index.FileData;
+import net.runelite.cache.updater.beans.ArchiveEntry;
+import net.runelite.cache.updater.beans.CacheEntry;
+import net.runelite.cache.updater.beans.IndexEntry;
+import org.sql2o.Connection;
+import org.sql2o.ResultSetIterable;
+
+@Slf4j
+public class CacheStorage implements Storage
+{
+	private CacheEntry cacheEntry;
+	private final CacheDAO cacheDao;
+	private final Connection con;
+
+	public CacheStorage(CacheEntry cacheEntry, CacheDAO cacheDao, Connection con)
+	{
+		this.cacheEntry = cacheEntry;
+		this.cacheDao = cacheDao;
+		this.con = con;
+	}
+
+	public CacheEntry getCacheEntry()
+	{
+		return cacheEntry;
+	}
+
+	public void setCacheEntry(CacheEntry cacheEntry)
+	{
+		this.cacheEntry = cacheEntry;
+	}
+
+	@Override
+	public void init(Store store) throws IOException
+	{
+	}
+
+	@Override
+	public void close() throws IOException
+	{
+	}
+
+	@Override
+	public void load(Store store) throws IOException
+	{
+		List<IndexEntry> indexes = cacheDao.findIndexesForCache(con, cacheEntry);
+		for (IndexEntry indexEntry : indexes)
+		{
+			Index index = store.addIndex(indexEntry.getIndexId());
+			index.setCrc(indexEntry.getCrc());
+			index.setRevision(indexEntry.getRevision());
+
+			try (ResultSetIterable<ArchiveEntry> archives = cacheDao.findArchivesForIndex(con, indexEntry))
+			{
+				for (ArchiveEntry archiveEntry : archives)
+				{
+					if (index.getArchive(archiveEntry.getArchiveId()) != null)
+					{
+						throw new IOException("Duplicate archive " + archiveEntry + " on " + indexEntry);
+					}
+
+					Archive archive = index.addArchive(archiveEntry.getArchiveId());
+					archive.setNameHash(archiveEntry.getNameHash());
+					archive.setCrc(archiveEntry.getCrc());
+					archive.setRevision(archiveEntry.getRevision());
+					archive.setHash(archiveEntry.getHash());
+
+					// File data is not necessary for cache updating
+				}
+			}
+		}
+	}
+
+	@Override
+	public void save(Store store) throws IOException
+	{
+		for (Index index : store.getIndexes())
+		{
+			IndexEntry entry = cacheDao.createIndex(con, cacheEntry, index.getId(), index.getCrc(), index.getRevision());
+
+			for (Archive archive : index.getArchives())
+			{
+				ArchiveEntry archiveEntry = cacheDao.findArchive(con, entry, archive.getArchiveId(),
+					archive.getNameHash(), archive.getCrc(), archive.getRevision());
+				if (archiveEntry == null)
+				{
+					byte[] hash = archive.getHash();
+					archiveEntry = cacheDao.createArchive(con, entry, archive.getArchiveId(),
+						archive.getNameHash(), archive.getCrc(), archive.getRevision(), hash);
+
+					for (FileData file : archive.getFileData())
+					{
+						cacheDao.associateFileToArchive(con, archiveEntry, file.getId(), file.getNameHash());
+					}
+				}
+
+				cacheDao.associateArchiveToIndex(con, archiveEntry, entry);
+			}
+		}
+	}
+
+	@Override
+	public byte[] loadArchive(Archive archive) throws IOException
+	{
+		throw new UnsupportedOperationException();
+	}
+
+	@Override
+	public void saveArchive(Archive archive, byte[] data) throws IOException
+	{
+		throw new UnsupportedOperationException();
+	}
+
+}

cache-updater/src/main/java/net/runelite/cache/updater/CacheUpdater.java

@@ -0,0 +1,173 @@
+/*
+ * Copyright (c) 2017, Adam <Adam@sigterm.info>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package net.runelite.cache.updater;
+
+import io.minio.MinioClient;
+import io.minio.errors.InvalidEndpointException;
+import io.minio.errors.InvalidPortException;
+import java.io.IOException;
+import java.time.Instant;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import net.runelite.cache.client.CacheClient;
+import net.runelite.cache.client.IndexInfo;
+import net.runelite.cache.fs.Archive;
+import net.runelite.cache.fs.Store;
+import net.runelite.cache.updater.beans.CacheEntry;
+import net.runelite.cache.updater.beans.IndexEntry;
+import net.runelite.http.api.RuneLiteAPI;
+import net.runelite.protocol.api.login.HandshakeResponseType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.CommandLineRunner;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.sql2o.Connection;
+import org.sql2o.Sql2o;
+
+@SpringBootApplication
+public class CacheUpdater implements CommandLineRunner
+{
+	private static final Logger logger = LoggerFactory.getLogger(CacheUpdater.class);
+
+	private final Sql2o sql2o;
+	private final MinioClient minioClient;
+
+	@Value("${minio.bucket}")
+	private String minioBucket;
+
+	@Autowired
+	public CacheUpdater(
+		@Qualifier("Runelite Cache SQL2O") Sql2o sql2o,
+		MinioClient minioClient
+	)
+	{
+		this.sql2o = sql2o;
+		this.minioClient = minioClient;
+	}
+
+	public void update() throws IOException, InvalidEndpointException, InvalidPortException, InterruptedException
+	{
+		int rsVersion = RuneLiteAPI.getRsVersion();
+
+		try (Connection con = sql2o.beginTransaction())
+		{
+			CacheDAO cacheDao = new CacheDAO();
+			CacheEntry cache = cacheDao.findMostRecent(con);
+			boolean created = false;
+			if (cache == null)
+			{
+				created = true;
+				cache = cacheDao.createCache(con, rsVersion, Instant.now());
+			}
+
+			CacheStorage storage = new CacheStorage(cache, cacheDao, con);
+			Store store = new Store(storage);
+			store.load();
+
+			ExecutorService executor = Executors.newSingleThreadExecutor();
+
+			CacheClient client = new CacheClient(store, rsVersion,
+				(Archive archive, byte[] data) -> executor.submit(new CacheUploader(minioClient, minioBucket, archive, data)));
+
+			client.connect();
+			HandshakeResponseType result = client.handshake().join();
+
+			if (result != HandshakeResponseType.RESPONSE_OK)
+			{
+				logger.warn("Out of date!");
+				return;
+			}
+
+			List<IndexInfo> indexes = client.requestIndexes();
+			List<IndexEntry> entries = cacheDao.findIndexesForCache(con, cache);
+
+			if (!checkOutOfDate(indexes, entries))
+			{
+				logger.info("All up to date.");
+				return;
+			}
+
+			client.download();
+
+			CacheEntry newCache = created ? cache : cacheDao.createCache(con, rsVersion, Instant.now());
+
+			storage.setCacheEntry(newCache);
+			store.save();
+
+			// ensure objects are added to the store before they become
+			// visible in the database
+			executor.shutdown();
+			while (!executor.awaitTermination(1, TimeUnit.SECONDS))
+			{
+				logger.debug("Waiting for termination of executor...");
+			}
+
+			// commit database
+			con.commit();
+		}
+	}
+
+	private boolean checkOutOfDate(List<IndexInfo> indexes, List<IndexEntry> dbIndexes)
+	{
+		if (indexes.size() != dbIndexes.size())
+		{
+			return true;
+		}
+
+		for (int i = 0; i < indexes.size(); ++i)
+		{
+			IndexInfo ii = indexes.get(i);
+			IndexEntry ie = dbIndexes.get(i);
+
+			if (ii.getId() != ie.getIndexId()
+				|| ii.getRevision() != ie.getRevision()
+				|| ii.getCrc() != ie.getCrc())
+			{
+				return true;
+			}
+		}
+
+		return false;
+	}
+
+	@Override
+	public void run(String... args) throws Exception
+	{
+		update();
+	}
+
+	public static void main(String[] args)
+	{
+		SpringApplication.run(CacheUpdater.class, args).close();
+		System.exit(0);
+	}
+
+}

cache-updater/src/main/java/net/runelite/cache/updater/CacheUploader.java

@@ -0,0 +1,96 @@
+/*
+ * Copyright (c) 2017, Adam <Adam@sigterm.info>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package net.runelite.cache.updater;
+
+import com.google.common.hash.Hashing;
+import com.google.common.io.BaseEncoding;
+import io.minio.MinioClient;
+import io.minio.errors.ErrorResponseException;
+import io.minio.errors.InsufficientDataException;
+import io.minio.errors.InternalException;
+import io.minio.errors.InvalidArgumentException;
+import io.minio.errors.InvalidBucketNameException;
+import io.minio.errors.NoResponseException;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.security.InvalidKeyException;
+import java.security.NoSuchAlgorithmException;
+import net.runelite.cache.fs.Archive;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.xmlpull.v1.XmlPullParserException;
+
+public class CacheUploader implements Runnable
+{
+	private static final Logger logger = LoggerFactory.getLogger(CacheUploader.class);
+
+	private final MinioClient minioClient;
+	private final String minioBucket;
+	private final Archive archive;
+	private final byte[] data;
+
+	public CacheUploader(MinioClient minioClient, String minioBucket, Archive archive, byte[] data)
+	{
+		this.minioClient = minioClient;
+		this.minioBucket = minioBucket;
+		this.archive = archive;
+		this.data = data;
+	}
+
+	@Override
+	public void run()
+	{
+		byte[] hash = Hashing.sha256().hashBytes(data).asBytes();
+		String hashStr = BaseEncoding.base16().encode(hash);
+
+		archive.setHash(hash);
+
+		String path = new StringBuilder()
+			.append(hashStr.substring(0, 2))
+			.append('/')
+			.append(hashStr.substring(2))
+			.toString();
+
+		try
+		{
+			try (InputStream in = minioClient.getObject(minioBucket, path))
+			{
+				return; // already exists
+			}
+			catch (ErrorResponseException ex)
+			{
+				// doesn't exist
+			}
+
+			minioClient.putObject(minioBucket, path, new ByteArrayInputStream(data), data.length, "binary/octet-stream");
+		}
+		catch (ErrorResponseException | InsufficientDataException | InternalException | InvalidArgumentException | InvalidBucketNameException | NoResponseException | IOException | InvalidKeyException | NoSuchAlgorithmException | XmlPullParserException ex)
+		{
+			logger.warn("unable to upload data to store", ex);
+		}
+	}
+
+}

cache-updater/src/main/java/net/runelite/cache/updater/InstantConverter.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2017, Adam <Adam@sigterm.info>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  Redistributions of source code must retain the above copyright notice, this
+ *   list of conditions and the following disclaimer.
+ *
+ *  Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package net.runelite.cache.updater;
+
+import java.sql.Timestamp;
+import java.time.Instant;
+import org.sql2o.converters.Converter;
+import org.sql2o.converters.ConverterException;
+
+public class InstantConverter implements Converter<Instant>
+{
+	@Override
+	public Instant convert(Object val) throws ConverterException
+	{
+		Timestamp ts = (Timestamp) val;
+		return ts.toInstant();
+	}
+
+	@Override
+	public Object toDatabaseParam(Instant val)
+	{
+		return Timestamp.from(val);
+	}
+
+}

cache-updater/src/main/java/net/runelite/cache/updater/beans/ArchiveEntry.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2017, Adam <Adam@sigterm.info>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package net.runelite.cache.updater.beans;
+
+import lombok.Data;
+
+@Data
+public class ArchiveEntry
+{
+	private int id;
+	private int archiveId;
+	private int nameHash;
+	private int crc;
+	private int revision;
+	private byte[] hash;
+}

cache-updater/src/main/java/net/runelite/cache/updater/beans/CacheEntry.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2017, Adam <Adam@sigterm.info>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package net.runelite.cache.updater.beans;
+
+import java.time.Instant;
+import lombok.Data;
+
+@Data
+public class CacheEntry
+{
+	private int id;
+	private int revision;
+	private Instant date;
+}

cache-updater/src/main/java/net/runelite/cache/updater/beans/FileEntry.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2017, Adam <Adam@sigterm.info>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package net.runelite.cache.updater.beans;
+
+import lombok.Data;
+
+@Data
+public class FileEntry
+{
+	private int id;
+	private int archiveId;
+	private int fileId;
+	private int nameHash;
+}

cache-updater/src/main/java/net/runelite/cache/updater/beans/IndexEntry.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2017, Adam <Adam@sigterm.info>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package net.runelite.cache.updater.beans;
+
+import lombok.Data;
+
+@Data
+public class IndexEntry
+{
+	private int id;
+	private int indexId;
+	private int crc;
+	private int revision;
+}