cache: split loading archive files from archive loading

Also no longer store archive contents in memory and instead read it from
storage on demand.

http-service/src/main/java/net/runelite/http/service/cache/CacheService.java

@@ -50,9 +50,8 @@ import net.runelite.cache.definitions.loaders.ItemLoader;
 import net.runelite.cache.definitions.loaders.NpcLoader;
 import net.runelite.cache.definitions.loaders.ObjectLoader;
 import net.runelite.cache.fs.ArchiveFiles;
+import net.runelite.cache.fs.Container;
 import net.runelite.cache.fs.FSFile;
-import net.runelite.cache.fs.jagex.DataFile;
-import net.runelite.cache.fs.jagex.DataFileReadResult;
 import net.runelite.http.api.cache.Cache;
 import net.runelite.http.api.cache.CacheArchive;
 import net.runelite.http.api.cache.CacheIndex;
@@ -150,7 +149,7 @@ public class CacheService
 			return null;
 		}
 
-		DataFileReadResult result = DataFile.decompress(archiveData, null);
+		Container result = Container.decompress(archiveData, null);
 		if (result == null)
 		{
 			return null;

http-service/src/main/java/net/runelite/http/service/cache/CacheStorage.java

@@ -24,14 +24,13 @@
  */
 package net.runelite.http.service.cache;
 
-import com.google.common.hash.Hashing;
 import java.io.IOException;
 import java.util.List;
 import net.runelite.cache.fs.Archive;
-import net.runelite.cache.fs.FSFile;
 import net.runelite.cache.fs.Index;
 import net.runelite.cache.fs.Storage;
 import net.runelite.cache.fs.Store;
+import net.runelite.cache.index.FileData;
 import net.runelite.http.service.cache.beans.ArchiveEntry;
 import net.runelite.http.service.cache.beans.CacheEntry;
 import net.runelite.http.service.cache.beans.FileEntry;
@@ -88,13 +87,17 @@ public class CacheStorage implements Storage
 				archive.setNameHash(archiveEntry.getNameHash());
 				archive.setCrc(archiveEntry.getCrc());
 				archive.setRevision(archiveEntry.getRevision());
+				archive.setHash(archiveEntry.getHash());
 
 				List<FileEntry> files = cacheDao.findFilesForArchive(con, archiveEntry);
+				FileData[] fileData = new FileData[files.size()];
+				archive.setFileData(fileData);
+				int idx = 0;
 				for (FileEntry fileEntry : files)
 				{
-					FSFile file = new FSFile(fileEntry.getFileId());
+					FileData file = fileData[idx++] = new FileData();
+					file.setId(fileEntry.getFileId());
 					file.setNameHash(fileEntry.getNameHash());
-					archive.addFile(file);
 				}
 			}
 		}
@@ -115,13 +118,13 @@ public class CacheStorage implements Storage
 					archive.getNameHash(), archive.getCrc(), archive.getRevision());
 				if (archiveEntry == null)
 				{
-					byte[] hash = Hashing.sha256().hashBytes(archive.getData()).asBytes();
+					byte[] hash = archive.getHash();
 					archiveEntry = cacheDao.createArchive(con, entry, archive.getArchiveId(),
 						archive.getNameHash(), archive.getCrc(), archive.getRevision(), hash);
 
-					for (FSFile file : archive.getFiles())
+					for (FileData file : archive.getFileData())
 					{
-						cacheDao.associateFileToArchive(con, archiveEntry, file.getFileId(), file.getNameHash());
+						cacheDao.associateFileToArchive(con, archiveEntry, file.getId(), file.getNameHash());
 					}
 				}
 				cacheDao.associateArchiveToIndex(con, archiveEntry, entry);
@@ -129,4 +132,16 @@ public class CacheStorage implements Storage
 		}
 	}
 
+	@Override
+	public byte[] loadArchive(Archive archive) throws IOException
+	{
+		throw new UnsupportedOperationException();
+	}
+
+	@Override
+	public void saveArchive(Archive archive, byte[] data) throws IOException
+	{
+		throw new UnsupportedOperationException();
+	}
+
 }

http-service/src/main/java/net/runelite/http/service/cache/CacheUpdater.java

@@ -96,7 +96,7 @@ public class CacheUpdater
 			ExecutorService executor = Executors.newSingleThreadExecutor();
 
 			CacheClient client = new CacheClient(store, rsVersion,
-				(Archive archive) -> executor.submit(new CacheUploader(minioClient, minioBucket, archive)));
+				(Archive archive, byte[] data) -> executor.submit(new CacheUploader(minioClient, minioBucket, archive, data)));
 
 			client.connect();
 			HandshakeResponseType result = client.handshake().join();
@@ -111,6 +111,7 @@ public class CacheUpdater
 
 			if (!checkOutOfDate(indexes, entries))
 			{
+				logger.info("All up to date.");
 				return;
 			}
 

http-service/src/main/java/net/runelite/http/service/cache/CacheUploader.java

@@ -50,21 +50,24 @@ public class CacheUploader implements Runnable
 	private final MinioClient minioClient;
 	private final String minioBucket;
 	private final Archive archive;
+	private final byte[] data;
 
-	public CacheUploader(MinioClient minioClient, String minioBucket, Archive archive)
+	public CacheUploader(MinioClient minioClient, String minioBucket, Archive archive, byte[] data)
 	{
 		this.minioClient = minioClient;
 		this.minioBucket = minioBucket;
 		this.archive = archive;
+		this.data = data;
 	}
 
 	@Override
 	public void run()
 	{
-		byte[] data = archive.getData();
 		byte[] hash = Hashing.sha256().hashBytes(data).asBytes();
 		String hashStr = BaseEncoding.base16().encode(hash);
 
+		archive.setHash(hash);
+
 		String path = new StringBuilder()
 			.append(hashStr.substring(0, 2))
 			.append('/')

http-service/src/main/java/net/runelite/http/service/xtea/XteaService.java

@@ -28,7 +28,7 @@ import java.io.IOException;
 import java.util.List;
 import java.util.stream.Collectors;
 import net.runelite.cache.IndexType;
-import net.runelite.cache.fs.jagex.DataFile;
+import net.runelite.cache.fs.Container;
 import net.runelite.cache.util.Djb2;
 import net.runelite.http.api.xtea.XteaKey;
 import net.runelite.http.api.xtea.XteaRequest;
@@ -226,7 +226,7 @@ public class XteaService
 
 		try
 		{
-			DataFile.decompress(data, keys);
+			Container.decompress(data, keys);
 			return true;
 		}
 		catch (IOException ex)