cache: split loading archive files from archive loading

Also no longer store archive contents in memory and instead read it from
storage on demand.

cache/pom.xml

@@ -37,7 +37,7 @@
 	<name>Cache</name>
 
 	<properties>
-		<cache.version>154</cache.version>
+		<cache.version>160</cache.version>
 		
 		<antlr4.version>4.6</antlr4.version>
 	</properties>

cache/src/main/java/net/runelite/cache/AreaManager.java

@@ -24,6 +24,7 @@
  */
 package net.runelite.cache;
 
+import java.io.IOException;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
@@ -31,8 +32,10 @@ import java.util.Map;
 import net.runelite.cache.definitions.AreaDefinition;
 import net.runelite.cache.definitions.loaders.AreaLoader;
 import net.runelite.cache.fs.Archive;
+import net.runelite.cache.fs.ArchiveFiles;
 import net.runelite.cache.fs.FSFile;
 import net.runelite.cache.fs.Index;
+import net.runelite.cache.fs.Storage;
 import net.runelite.cache.fs.Store;
 
 public class AreaManager
@@ -45,12 +48,16 @@ public class AreaManager
 		this.store = store;
 	}
 
-	public void load()
+	public void load() throws IOException
 	{
+		Storage storage = store.getStorage();
 		Index index = store.getIndex(IndexType.CONFIGS);
 		Archive archive = index.getArchive(ConfigType.AREA.getId());
 
-		for (FSFile file : archive.getFiles())
+		byte[] archiveData = storage.loadArchive(archive);
+		ArchiveFiles files = archive.getFiles(archiveData);
+
+		for (FSFile file : files.getFiles())
 		{
 			AreaLoader loader = new AreaLoader();
 			AreaDefinition area = loader.load(file.getContents(), file.getFileId());

cache/src/main/java/net/runelite/cache/InterfaceManager.java

@@ -33,8 +33,10 @@ import net.runelite.cache.definitions.InterfaceDefinition;
 import net.runelite.cache.definitions.exporters.InterfaceExporter;
 import net.runelite.cache.definitions.loaders.InterfaceLoader;
 import net.runelite.cache.fs.Archive;
+import net.runelite.cache.fs.ArchiveFiles;
 import net.runelite.cache.fs.FSFile;
 import net.runelite.cache.fs.Index;
+import net.runelite.cache.fs.Storage;
 import net.runelite.cache.fs.Store;
 import net.runelite.cache.util.Namer;
 
@@ -49,17 +51,20 @@ public class InterfaceManager
 		this.store = store;
 	}
 
-	public void load()
+	public void load() throws IOException
 	{
 		InterfaceLoader loader = new InterfaceLoader();
 
+		Storage storage = store.getStorage();
 		Index index = store.getIndex(IndexType.INTERFACES);
 
 		for (Archive archive : index.getArchives())
 		{
 			int archiveId = archive.getArchiveId();
+			byte[] archiveData = storage.loadArchive(archive);
+			ArchiveFiles files = archive.getFiles(archiveData);
 
-			for (FSFile file : archive.getFiles())
+			for (FSFile file : files.getFiles())
 			{
 				int fileId = file.getFileId();
 

cache/src/main/java/net/runelite/cache/ItemManager.java

@@ -33,8 +33,10 @@ import net.runelite.cache.definitions.ItemDefinition;
 import net.runelite.cache.definitions.exporters.ItemExporter;
 import net.runelite.cache.definitions.loaders.ItemLoader;
 import net.runelite.cache.fs.Archive;
+import net.runelite.cache.fs.ArchiveFiles;
 import net.runelite.cache.fs.FSFile;
 import net.runelite.cache.fs.Index;
+import net.runelite.cache.fs.Storage;
 import net.runelite.cache.fs.Store;
 import net.runelite.cache.util.Namer;
 
@@ -49,14 +51,18 @@ public class ItemManager
 		this.store = store;
 	}
 
-	public void load()
+	public void load() throws IOException
 	{
 		ItemLoader loader = new ItemLoader();
 
+		Storage storage = store.getStorage();
 		Index index = store.getIndex(IndexType.CONFIGS);
 		Archive archive = index.getArchive(ConfigType.ITEM.getId());
 
-		for (FSFile f : archive.getFiles())
+		byte[] archiveData = storage.loadArchive(archive);
+		ArchiveFiles files = archive.getFiles(archiveData);
+
+		for (FSFile f : files.getFiles())
 		{
 			ItemDefinition def = loader.load(f.getFileId(), f.getContents());
 			items.add(def);
@@ -75,7 +81,7 @@ public class ItemManager
 		for (ItemDefinition def : items)
 		{
 			ItemExporter exporter = new ItemExporter(def);
-			
+
 			File targ = new File(out, def.id + ".json");
 			exporter.exportTo(targ);
 		}

cache/src/main/java/net/runelite/cache/MapImageDumper.java

@@ -39,14 +39,15 @@ import net.runelite.cache.definitions.OverlayDefinition;
 import net.runelite.cache.definitions.SpriteDefinition;
 import net.runelite.cache.definitions.TextureDefinition;
 import net.runelite.cache.definitions.UnderlayDefinition;
-import net.runelite.cache.definitions.loaders.ObjectLoader;
 import net.runelite.cache.definitions.loaders.OverlayLoader;
 import net.runelite.cache.definitions.loaders.SpriteLoader;
 import net.runelite.cache.definitions.loaders.TextureLoader;
 import net.runelite.cache.definitions.loaders.UnderlayLoader;
 import net.runelite.cache.fs.Archive;
+import net.runelite.cache.fs.ArchiveFiles;
 import net.runelite.cache.fs.FSFile;
 import net.runelite.cache.fs.Index;
+import net.runelite.cache.fs.Storage;
 import net.runelite.cache.fs.Store;
 import net.runelite.cache.region.Location;
 import net.runelite.cache.region.Region;
@@ -367,12 +368,16 @@ public class MapImageDumper
 		logger.info("East most region:  {}", regionLoader.getHighestX().getBaseX());
 	}
 
-	private void loadUnderlays(Store store)
+	private void loadUnderlays(Store store) throws IOException
 	{
+		Storage storage = store.getStorage();
 		Index index = store.getIndex(IndexType.CONFIGS);
 		Archive archive = index.getArchive(ConfigType.UNDERLAY.getId());
 
-		for (FSFile file : archive.getFiles())
+		byte[] archiveData = storage.loadArchive(archive);
+		ArchiveFiles files = archive.getFiles(archiveData);
+
+		for (FSFile file : files.getFiles())
 		{
 			UnderlayLoader loader = new UnderlayLoader();
 			UnderlayDefinition underlay = loader.load(file.getFileId(), file.getContents());
@@ -393,12 +398,16 @@ public class MapImageDumper
 		return null;
 	}
 
-	private void loadOverlays(Store store)
+	private void loadOverlays(Store store) throws IOException
 	{
+		Storage storage = store.getStorage();
 		Index index = store.getIndex(IndexType.CONFIGS);
 		Archive archive = index.getArchive(ConfigType.OVERLAY.getId());
 
-		for (FSFile file : archive.getFiles())
+		byte[] archiveData = storage.loadArchive(archive);
+		ArchiveFiles files = archive.getFiles(archiveData);
+
+		for (FSFile file : files.getFiles())
 		{
 			OverlayLoader loader = new OverlayLoader();
 			OverlayDefinition underlay = loader.load(file.getFileId(), file.getContents());
@@ -419,12 +428,16 @@ public class MapImageDumper
 		return null;
 	}
 
-	private void loadTextures(Store store)
+	private void loadTextures(Store store) throws IOException
 	{
+		Storage storage = store.getStorage();
 		Index index = store.getIndex(IndexType.TEXTURES);
 		Archive archive = index.getArchive(0);
 
-		for (FSFile file : archive.getFiles())
+		byte[] archiveData = storage.loadArchive(archive);
+		ArchiveFiles files = archive.getFiles(archiveData);
+
+		for (FSFile file : files.getFiles())
 		{
 			TextureLoader loader = new TextureLoader();
 			TextureDefinition texture = loader.load(file.getFileId(), file.getContents());
@@ -446,19 +459,15 @@ public class MapImageDumper
 		return null;
 	}
 
-	private void loadSprites()
+	private void loadSprites() throws IOException
 	{
+		Storage storage = store.getStorage();
 		Index index = store.getIndex(IndexType.SPRITES);
 		final int mapsceneHash = Djb2.hash("mapscene");
 
 		for (Archive a : index.getArchives())
 		{
-			List<FSFile> files = a.getFiles();
+			byte[] contents = a.decompress(storage.loadArchive(a));
-
-			assert files.size() == 1;
-
-			FSFile file = files.get(0);
-			byte[] contents = file.getContents();
 
 			SpriteLoader loader = new SpriteLoader();
 			SpriteDefinition[] sprites = loader.load(a.getArchiveId(), contents);
@@ -515,16 +524,12 @@ public class MapImageDumper
 		return color.getRGB();
 	}
 
-	private void loadObjects(Store store)
+	private void loadObjects(Store store) throws IOException
 	{
-		Index index = store.getIndex(IndexType.CONFIGS);
+		ObjectManager manager = new ObjectManager(store);
-		Archive archive = index.getArchive(ConfigType.OBJECT.getId());
+		manager.load();
-
+		for (ObjectDefinition def : manager.getObjects())
-		ObjectLoader loader = new ObjectLoader();
-
-		for (FSFile f : archive.getFiles())
 		{
-			ObjectDefinition def = loader.load(f.getFileId(), f.getContents());
 			objects.put(def.getId(), def);
 		}
 	}

cache/src/main/java/net/runelite/cache/NpcManager.java

@@ -33,8 +33,10 @@ import net.runelite.cache.definitions.NpcDefinition;
 import net.runelite.cache.definitions.exporters.NpcExporter;
 import net.runelite.cache.definitions.loaders.NpcLoader;
 import net.runelite.cache.fs.Archive;
+import net.runelite.cache.fs.ArchiveFiles;
 import net.runelite.cache.fs.FSFile;
 import net.runelite.cache.fs.Index;
+import net.runelite.cache.fs.Storage;
 import net.runelite.cache.fs.Store;
 import net.runelite.cache.util.Namer;
 
@@ -53,10 +55,14 @@ public class NpcManager
 	{
 		NpcLoader loader = new NpcLoader();
 
+		Storage storage = store.getStorage();
 		Index index = store.getIndex(IndexType.CONFIGS);
 		Archive archive = index.getArchive(ConfigType.NPC.getId());
 
-		for (FSFile f : archive.getFiles())
+		byte[] archiveData = storage.loadArchive(archive);
+		ArchiveFiles files = archive.getFiles(archiveData);
+
+		for (FSFile f : files.getFiles())
 		{
 			NpcDefinition npc = loader.load(f.getFileId(), f.getContents());
 			npcs.add(npc);

cache/src/main/java/net/runelite/cache/ObjectManager.java

@@ -33,8 +33,10 @@ import net.runelite.cache.definitions.ObjectDefinition;
 import net.runelite.cache.definitions.exporters.ObjectExporter;
 import net.runelite.cache.definitions.loaders.ObjectLoader;
 import net.runelite.cache.fs.Archive;
+import net.runelite.cache.fs.ArchiveFiles;
 import net.runelite.cache.fs.FSFile;
 import net.runelite.cache.fs.Index;
+import net.runelite.cache.fs.Storage;
 import net.runelite.cache.fs.Store;
 import net.runelite.cache.util.Namer;
 
@@ -53,10 +55,14 @@ public class ObjectManager
 	{
 		ObjectLoader loader = new ObjectLoader();
 
+		Storage storage = store.getStorage();
 		Index index = store.getIndex(IndexType.CONFIGS);
 		Archive archive = index.getArchive(ConfigType.OBJECT.getId());
 
-		for (FSFile f : archive.getFiles())
+		byte[] archiveData = storage.loadArchive(archive);
+		ArchiveFiles files = archive.getFiles(archiveData);
+
+		for (FSFile f : files.getFiles())
 		{
 			ObjectDefinition def = loader.load(f.getFileId(), f.getContents());
 			objects.add(def);

cache/src/main/java/net/runelite/cache/SpriteManager.java

@@ -31,13 +31,12 @@ import java.io.File;
 import java.io.IOException;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.List;
 import net.runelite.cache.definitions.SpriteDefinition;
 import net.runelite.cache.definitions.exporters.SpriteExporter;
 import net.runelite.cache.definitions.loaders.SpriteLoader;
 import net.runelite.cache.fs.Archive;
-import net.runelite.cache.fs.FSFile;
 import net.runelite.cache.fs.Index;
+import net.runelite.cache.fs.Storage;
 import net.runelite.cache.fs.Store;
 
 public class SpriteManager
@@ -50,18 +49,14 @@ public class SpriteManager
 		this.store = store;
 	}
 
-	public void load()
+	public void load() throws IOException
 	{
+		Storage storage = store.getStorage();
 		Index index = store.getIndex(IndexType.SPRITES);
 
 		for (Archive a : index.getArchives())
 		{
-			List<FSFile> files = a.getFiles();
+			byte[] contents = a.decompress(storage.loadArchive(a));
-
-			assert files.size() == 1;
-
-			FSFile file = files.get(0);
-			byte[] contents = file.getContents();
 
 			SpriteLoader loader = new SpriteLoader();
 			SpriteDefinition[] defs = loader.load(a.getArchiveId(), contents);

cache/src/main/java/net/runelite/cache/TextureManager.java

@@ -24,13 +24,16 @@
  */
 package net.runelite.cache;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import net.runelite.cache.definitions.TextureDefinition;
 import net.runelite.cache.definitions.loaders.TextureLoader;
 import net.runelite.cache.fs.Archive;
+import net.runelite.cache.fs.ArchiveFiles;
 import net.runelite.cache.fs.FSFile;
 import net.runelite.cache.fs.Index;
+import net.runelite.cache.fs.Storage;
 import net.runelite.cache.fs.Store;
 
 public class TextureManager
@@ -43,14 +46,18 @@ public class TextureManager
 		this.store = store;
 	}
 
-	public void load()
+	public void load() throws IOException
 	{
+		Storage storage = store.getStorage();
 		Index index = store.getIndex(IndexType.TEXTURES);
 		Archive archive = index.getArchive(0);
 
+		byte[] archiveData = storage.loadArchive(archive);
+		ArchiveFiles files = archive.getFiles(archiveData);
+
 		TextureLoader loader = new TextureLoader();
 
-		for (FSFile file : archive.getFiles())
+		for (FSFile file : files.getFiles())
 		{
 			TextureDefinition texture = loader.load(file.getFileId(), file.getContents());
 			textures.add(texture);

cache/src/main/java/net/runelite/cache/client/CacheClient.java

@@ -44,11 +44,10 @@ import java.util.List;
 import java.util.Queue;
 import java.util.concurrent.CompletableFuture;
 import net.runelite.cache.fs.Archive;
-import net.runelite.cache.fs.FSFile;
 import net.runelite.cache.fs.Index;
+import net.runelite.cache.fs.Storage;
 import net.runelite.cache.fs.Store;
 import net.runelite.cache.index.ArchiveData;
-import net.runelite.cache.index.FileData;
 import net.runelite.cache.index.IndexData;
 import net.runelite.cache.protocol.decoders.HandshakeResponseDecoder;
 import net.runelite.cache.protocol.encoders.ArchiveRequestEncoder;
@@ -275,7 +274,7 @@ public class CacheClient implements AutoCloseable
 					&& existing.getCrc() == ad.getCrc()
 					&& existing.getNameHash() == ad.getNameHash())
 				{
-					logger.info("Archive {}/{} in index {} is up to date",
+					logger.debug("Archive {}/{} in index {} is up to date",
 						ad.getId(), indexData.getArchives().length, index.getId());
 					continue;
 				}
@@ -310,13 +309,7 @@ public class CacheClient implements AutoCloseable
 				archive.setNameHash(ad.getNameHash());
 
 				// Add files
-				archive.clearFiles();
+				archive.setFileData(ad.getFiles());
-				for (FileData fd : ad.getFiles())
-				{
-					FSFile file = new FSFile(fd.getId());
-					file.setNameHash(fd.getNameHash());
-					archive.addFile(file);
-				}
 
 				CompletableFuture<FileResult> future = requestFile(index.getId(), ad.getId(), false);
 				future.handle((fr, ex) ->
@@ -332,13 +325,24 @@ public class CacheClient implements AutoCloseable
 						logger.warn("crc mismatch on downloaded archive {}/{}: {} != {}",
 							archive.getIndex().getId(), archive.getArchiveId(),
 							hash, archive.getCrc());
+						throw new RuntimeException("crc mismatch");
 					}
 
-					archive.setData(data);
-
 					if (watcher != null)
 					{
-						watcher.downloadComplete(archive);
+						watcher.downloadComplete(archive, data);
+					}
+					else
+					{
+						try
+						{
+							Storage storage = store.getStorage();
+							storage.saveArchive(archive, data);
+						}
+						catch (IOException ex1)
+						{
+							logger.warn("unable to save archive data", ex1);
+						}
 					}
 					return null;
 				});

cache/src/main/java/net/runelite/cache/client/DownloadWatcher.java

@@ -29,5 +29,5 @@ import net.runelite.cache.fs.Archive;
 @FunctionalInterface
 public interface DownloadWatcher
 {
-	void downloadComplete(Archive archive);
+	void downloadComplete(Archive archive, byte[] data);
 }

cache/src/main/java/net/runelite/cache/client/FileResult.java

@@ -25,8 +25,7 @@
 package net.runelite.cache.client;
 
 import java.io.IOException;
-import net.runelite.cache.fs.jagex.DataFile;
+import net.runelite.cache.fs.Container;
-import net.runelite.cache.fs.jagex.DataFileReadResult;
 
 public class FileResult
 {
@@ -63,7 +62,7 @@ public class FileResult
 
 	public void decompress(int[] keys) throws IOException
 	{
-		DataFileReadResult res = DataFile.decompress(compressedData, keys);
+		Container res = Container.decompress(compressedData, keys);
 
 		contents = res.data;
 		revision = res.revision;

cache/src/main/java/net/runelite/cache/fs/Archive.java

@@ -24,11 +24,8 @@
  */
 package net.runelite.cache.fs;
 
-import net.runelite.cache.fs.jagex.DataFile;
-import net.runelite.cache.fs.jagex.DataFileReadResult;
 import java.io.IOException;
-import java.util.List;
+import net.runelite.cache.index.FileData;
-import java.util.Objects;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -38,15 +35,13 @@ public class Archive
 
 	private final Index index; // member of this index
 
-	private byte[] data; // raw data from the datafile, compressed/encrypted
-
 	private final int archiveId;
 	private int nameHash;
 	private int crc;
 	private int revision;
 	private int compression;
-
+	private FileData[] fileData;
-	private final ArchiveFiles files = new ArchiveFiles();
+	private byte[] hash; // used by webservice, sha256 hash of content
 
 	public Archive(Index index, int id)
 	{
@@ -61,7 +56,6 @@ public class Archive
 		hash = 47 * hash + this.archiveId;
 		hash = 47 * hash + this.nameHash;
 		hash = 47 * hash + this.revision;
-		hash = 47 * hash + Objects.hashCode(this.files);
 		return hash;
 	}
 
@@ -89,10 +83,6 @@ public class Archive
 		{
 			return false;
 		}
-		if (!Objects.equals(this.files, other.files))
-		{
-			return false;
-		}
 		return true;
 	}
 
@@ -101,68 +91,70 @@ public class Archive
 		return index;
 	}
 
-	public byte[] getData()
+	public byte[] decompress(byte[] data) throws IOException
 	{
-		return data;
+		return decompress(data, null);
 	}
 
-	public void setData(byte[] data)
+	public byte[] decompress(byte[] data, int[] keys) throws IOException
 	{
-		this.data = data;
+		if (data == null)
-	}
+		{
+			return null;
+		}
 
-	public FSFile addFile(FSFile file)
+		byte[] encryptedData = data;
-	{
-		this.files.addFile(file);
-		return file;
-	}
 
-	public FSFile findFile(int id)
+		Container container = Container.decompress(encryptedData, keys);
-	{
+		if (container == null)
-		return this.files.findFile(id);
-	}
-
-	public void decompressAndLoad(int[] keys) throws IOException
-	{
-		byte[] encryptedData = this.getData();
-
-		DataFileReadResult res = DataFile.decompress(encryptedData, keys);
-		if (res == null)
 		{
 			logger.warn("Unable to decrypt archive {}", this);
-			return;
+			return null;
 		}
 
-		byte[] decompressedData = res.data;
+		byte[] decompressedData = container.data;
 
-		if (this.crc != res.crc)
+		if (this.crc != container.crc)
 		{
 			logger.warn("crc mismatch for archive {}/{}", index.getId(), this.getArchiveId());
-			this.setCrc(res.crc);
+			throw new IOException("CRC mismatch for " + index.getId() + "/" + this.getArchiveId());
 		}
 
-		if (res.revision != -1 && this.getRevision() != res.revision)
+		if (container.revision != -1 && this.getRevision() != container.revision)
 		{
 			// compressed data doesn't always include a revision, but check it if it does
 			logger.warn("revision mismatch for archive {}/{}, expected {} was {}",
 				index.getId(), this.getArchiveId(),
-				this.getRevision(), res.revision);
+				this.getRevision(), container.revision);
 			// I've seen this happen with vanilla caches where the
 			// revision in the index data differs from the revision
 			// stored for the archive data on disk... I assume this
 			// is more correct
-			this.setRevision(res.revision);
+			this.setRevision(container.revision);
 		}
 
-		setCompression(res.compression);
+		setCompression(container.compression);
-
+		return decompressedData;
-		files.loadContents(decompressedData);
-		this.setData(null); // now that we've loaded it, clean it so it doesn't get written back
 	}
 
-	public byte[] saveContents()
+	public ArchiveFiles getFiles(byte[] data) throws IOException
 	{
-		return files.saveContents();
+		return getFiles(data, null);
+	}
+
+	public ArchiveFiles getFiles(byte[] data, int[] keys) throws IOException
+	{
+		byte[] decompressedData = decompress(data, keys);
+
+		ArchiveFiles files = new ArchiveFiles();
+		for (FileData fileEntry : fileData)
+		{
+			FSFile file = new FSFile(fileEntry.getId());
+			file.setNameHash(fileEntry.getNameHash());
+			files.addFile(file);
+		}
+		files.loadContents(decompressedData);
+		return files;
 	}
 
 	public int getArchiveId()
@@ -210,13 +202,23 @@ public class Archive
 		this.compression = compression;
 	}
 
-	public List<FSFile> getFiles()
+	public FileData[] getFileData()
 	{
-		return files.getFiles();
+		return fileData;
 	}
 
-	public void clearFiles()
+	public void setFileData(FileData[] fileData)
 	{
-		files.clear();
+		this.fileData = fileData;
+	}
+
+	public byte[] getHash()
+	{
+		return hash;
+	}
+
+	public void setHash(byte[] hash)
+	{
+		this.hash = hash;
 	}
 }

cache/src/main/java/net/runelite/cache/fs/Container.java

@@ -0,0 +1,216 @@
+/*
+ * Copyright (c) 2016-2017, Adam <Adam@sigterm.info>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package net.runelite.cache.fs;
+
+import static com.google.common.primitives.Bytes.concat;
+import com.google.common.primitives.Ints;
+import java.io.IOException;
+import net.runelite.cache.fs.jagex.CompressionType;
+import net.runelite.cache.io.InputStream;
+import net.runelite.cache.io.OutputStream;
+import net.runelite.cache.util.BZip2;
+import net.runelite.cache.util.Crc32;
+import net.runelite.cache.util.GZip;
+import net.runelite.cache.util.Xtea;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class Container
+{
+	private static final Logger logger = LoggerFactory.getLogger(Container.class);
+
+	public byte[] data;
+	public int compression; // compression
+	public int revision;
+	public int crc; // crc of compressed data
+
+	public Container(int compression, int revision)
+	{
+		this.compression = compression;
+		this.revision = revision;
+	}
+
+	public void compress(byte[] data, int[] keys) throws IOException
+	{
+		OutputStream stream = new OutputStream();
+
+		byte[] compressedData;
+		int length;
+		switch (compression)
+		{
+			case CompressionType.NONE:
+				compressedData = data;
+				length = compressedData.length;
+				break;
+			case CompressionType.BZ2:
+				compressedData = concat(Ints.toByteArray(data.length), BZip2.compress(data));
+				length = compressedData.length - 4;
+				break;
+			case CompressionType.GZ:
+				compressedData = concat(Ints.toByteArray(data.length), GZip.compress(data));
+				length = compressedData.length - 4;
+				break;
+			default:
+				throw new RuntimeException("Unknown compression type");
+		}
+
+		compressedData = encrypt(compressedData, compressedData.length, keys);
+
+		stream.writeByte(compression);
+		stream.writeInt(length);
+
+		stream.writeBytes(compressedData);
+		if (revision != -1)
+		{
+			stream.writeShort(revision);
+		}
+
+		this.data = stream.flip();
+	}
+
+	public static Container decompress(byte[] b, int[] keys) throws IOException
+	{
+		InputStream stream = new InputStream(b);
+
+		int compression = stream.readUnsignedByte();
+		int compressedLength = stream.readInt();
+		if (compressedLength < 0 || compressedLength > 1000000)
+		{
+			throw new RuntimeException("Invalid data");
+		}
+
+		Crc32 crc32 = new Crc32();
+		crc32.update(b, 0, 5); // compression + length
+
+		byte[] data;
+		int revision = -1;
+		switch (compression)
+		{
+			case CompressionType.NONE:
+			{
+				byte[] encryptedData = new byte[compressedLength];
+				stream.readBytes(encryptedData, 0, compressedLength);
+
+				crc32.update(encryptedData, 0, compressedLength);
+				byte[] decryptedData = decrypt(encryptedData, encryptedData.length, keys);
+
+				if (stream.remaining() >= 2)
+				{
+					revision = stream.readUnsignedShort();
+					assert revision != -1;
+				}
+
+				data = decryptedData;
+
+				break;
+			}
+			case CompressionType.BZ2:
+			{
+				byte[] encryptedData = new byte[compressedLength + 4];
+				stream.readBytes(encryptedData);
+
+				crc32.update(encryptedData, 0, encryptedData.length);
+				byte[] decryptedData = decrypt(encryptedData, encryptedData.length, keys);
+
+				if (stream.remaining() >= 2)
+				{
+					revision = stream.readUnsignedShort();
+					assert revision != -1;
+				}
+
+				stream = new InputStream(decryptedData);
+
+				int decompressedLength = stream.readInt();
+				data = BZip2.decompress(stream.getRemaining(), compressedLength);
+
+				if (data == null)
+				{
+					return null;
+				}
+
+				assert data.length == decompressedLength;
+
+				break;
+			}
+			case CompressionType.GZ:
+			{
+				byte[] encryptedData = new byte[compressedLength + 4];
+				stream.readBytes(encryptedData);
+
+				crc32.update(encryptedData, 0, encryptedData.length);
+				byte[] decryptedData = decrypt(encryptedData, encryptedData.length, keys);
+
+				if (stream.remaining() >= 2)
+				{
+					revision = stream.readUnsignedShort();
+					assert revision != -1;
+				}
+
+				stream = new InputStream(decryptedData);
+
+				int decompressedLength = stream.readInt();
+				data = GZip.decompress(stream.getRemaining(), compressedLength);
+
+				if (data == null)
+				{
+					return null;
+				}
+
+				assert data.length == decompressedLength;
+
+				break;
+			}
+			default:
+				throw new RuntimeException("Unknown decompression type");
+		}
+
+		Container container = new Container(compression, revision);
+		container.data = data;
+		container.crc = crc32.getHash();
+		return container;
+	}
+
+	private static byte[] decrypt(byte[] data, int length, int[] keys)
+	{
+		if (keys == null)
+		{
+			return data;
+		}
+
+		Xtea xtea = new Xtea(keys);
+		return xtea.decrypt(data, length);
+	}
+
+	private static byte[] encrypt(byte[] data, int length, int[] keys)
+	{
+		if (keys == null)
+		{
+			return data;
+		}
+
+		Xtea xtea = new Xtea(keys);
+		return xtea.encrypt(data, length);
+	}
+}

cache/src/main/java/net/runelite/cache/fs/Index.java

@@ -24,20 +24,13 @@
  */
 package net.runelite.cache.fs;
 
-import net.runelite.cache.fs.jagex.DataFile;
-import net.runelite.cache.fs.jagex.CompressionType;
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.Unpooled;
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
 import net.runelite.cache.index.ArchiveData;
 import net.runelite.cache.index.FileData;
 import net.runelite.cache.index.IndexData;
-import net.runelite.cache.util.Crc32;
 import net.runelite.cache.util.Djb2;
-import net.runelite.cache.util.XteaKeyManager;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -45,11 +38,8 @@ public class Index
 {
 	private static final Logger logger = LoggerFactory.getLogger(Index.class);
 
-	private final Store store;
 	private final int id;
 
-	private XteaKeyManager xteaManager;
-
 	private int protocol = 7;
 	private boolean named = true;
 	private int revision;
@@ -58,9 +48,8 @@ public class Index
 
 	private final List<Archive> archives = new ArrayList<>();
 
-	public Index(Store store, int id)
+	public Index(int id)
 	{
-		this.store = store;
 		this.id = id;
 	}
 
@@ -101,16 +90,6 @@ public class Index
 		return true;
 	}
 
-	public XteaKeyManager getXteaManager()
-	{
-		return xteaManager;
-	}
-
-	public void setXteaManager(XteaKeyManager xteaManager)
-	{
-		this.xteaManager = xteaManager;
-	}
-
 	public int getId()
 	{
 		return id;
@@ -203,48 +182,6 @@ public class Index
 		return null;
 	}
 
-	public void rebuildCrc() throws IOException
-	{
-		for (Archive a : archives)
-		{
-			int rev; // used for determining what part of compressedData to crc
-			byte[] compressedData;
-
-			if (a.getData() != null)
-			{
-				compressedData = a.getData(); // data was never decompressed or loaded
-				rev = -1; // assume that this data has no revision?
-			}
-			else
-			{
-				byte[] fileData = a.saveContents();
-				rev = a.getRevision();
-				compressedData = DataFile.compress(fileData, a.getCompression(), a.getRevision(), null);
-			}
-
-			int length = rev != -1 ? compressedData.length - 2 : compressedData.length;
-			Crc32 crc32 = new Crc32();
-			crc32.update(compressedData, 0, length);
-
-			int crc = crc32.getHash();
-
-			a.setCrc(crc);
-		}
-
-		Crc32 crc = new Crc32();
-		byte[] indexData = toIndexData().writeIndexData();
-
-		ByteBuf b = Unpooled.buffer(5, 5);
-		b.writeByte((byte) CompressionType.NONE);
-		b.writeInt(indexData.length);
-
-		crc.update(b.array(), 0, 5); // crc includes compression type and length
-		crc.update(indexData, 0, indexData.length);
-
-		int hash = crc.getHash();
-		this.setCrc(hash);
-	}
-
 	public IndexData toIndexData()
 	{
 		IndexData data = new IndexData();
@@ -264,16 +201,8 @@ public class Index
 			ad.setCrc(archive.getCrc());
 			ad.setRevision(archive.getRevision());
 
-			FileData[] files = new FileData[archive.getFiles().size()];
+			FileData[] files = archive.getFileData();
 			ad.setFiles(files);
-
-			int idx2 = 0;
-			for (FSFile file : archive.getFiles())
-			{
-				FileData fd = files[idx2++] = new FileData();
-				fd.setId(file.getFileId());
-				fd.setNameHash(file.getNameHash());
-			}
 		}
 		return data;
 	}

cache/src/main/java/net/runelite/cache/fs/Storage.java

@@ -36,4 +36,8 @@ public interface Storage extends AutoCloseable
 	void load(Store store) throws IOException;
 
 	void save(Store store) throws IOException;
+
+	byte[] loadArchive(Archive archive) throws IOException;
+
+	void saveArchive(Archive archive, byte[] data) throws IOException;
 }

cache/src/main/java/net/runelite/cache/fs/Store.java

@@ -33,7 +33,6 @@ import java.util.List;
 import java.util.Objects;
 import net.runelite.cache.IndexType;
 import net.runelite.cache.fs.jagex.DiskStorage;
-import net.runelite.cache.util.XteaKeyManager;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -48,14 +47,6 @@ public final class Store implements Closeable
 	{
 		storage = new DiskStorage(folder);
 		storage.init(this);
-
-		Index maps = this.findIndex(IndexType.MAPS.getNumber());
-		if (maps != null)
-		{
-			XteaKeyManager mapKeys = new XteaKeyManager();
-			mapKeys.loadKeys();
-			maps.setXteaManager(mapKeys);
-		}
 	}
 
 	public Store(Storage storage) throws IOException
@@ -65,6 +56,11 @@ public final class Store implements Closeable
 		storage.init(this);
 	}
 
+	public Storage getStorage()
+	{
+		return storage;
+	}
+
 	@Override
 	public void close() throws IOException
 	{
@@ -108,7 +104,7 @@ public final class Store implements Closeable
 			}
 		}
 
-		Index index = new Index(this, id);
+		Index index = new Index(id);
 		this.indexes.add(index);
 
 		return index;
@@ -120,18 +116,6 @@ public final class Store implements Closeable
 		indexes.remove(index);
 	}
 
-	/*
-	 * we rebuild data differently, so the CRCs aren't right.
-	 * rebuild them.
-	 */
-	public void rebuildCrc() throws IOException
-	{
-		for (Index i : indexes)
-		{
-			i.rebuildCrc();
-		}
-	}
-
 	public void load() throws IOException
 	{
 		storage.load(this);

cache/src/main/java/net/runelite/cache/fs/jagex/DataFile.java

@@ -24,20 +24,12 @@
  */
 package net.runelite.cache.fs.jagex;
 
-import static com.google.common.primitives.Bytes.concat;
-import com.google.common.primitives.Ints;
 import java.io.Closeable;
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.RandomAccessFile;
 import java.nio.ByteBuffer;
-import net.runelite.cache.util.BZip2;
-import net.runelite.cache.io.InputStream;
-import net.runelite.cache.io.OutputStream;
-import net.runelite.cache.util.Crc32;
-import net.runelite.cache.util.GZip;
-import net.runelite.cache.util.Xtea;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -177,7 +169,7 @@ public class DataFile implements Closeable
 		return buffer.array();
 	}
 
-	public DataFileWriteResult write(int indexId, int archiveId, byte[] compressedData, int revision) throws IOException
+	public DataFileWriteResult write(int indexId, int archiveId, byte[] compressedData) throws IOException
 	{
 		int sector;
 		int startSector;
@@ -270,174 +262,6 @@ public class DataFile implements Closeable
 		DataFileWriteResult res = new DataFileWriteResult();
 		res.sector = startSector;
 		res.compressedLength = compressedData.length;
-
-		int length = revision != -1 ? compressedData.length - 2 : compressedData.length;
-		Crc32 crc32 = new Crc32();
-		crc32.update(compressedData, 0, length);
-		res.crc = crc32.getHash();
 		return res;
 	}
-
-	public static DataFileReadResult decompress(byte[] b, int[] keys) throws IOException
-	{
-		InputStream stream = new InputStream(b);
-
-		int compression = stream.readUnsignedByte();
-		int compressedLength = stream.readInt();
-		if (compressedLength < 0 || compressedLength > 1000000)
-		{
-			throw new RuntimeException("Invalid data");
-		}
-
-		Crc32 crc32 = new Crc32();
-		crc32.update(b, 0, 5); // compression + length
-
-		byte[] data;
-		int revision = -1;
-		switch (compression)
-		{
-			case CompressionType.NONE:
-			{
-				byte[] encryptedData = new byte[compressedLength];
-				stream.readBytes(encryptedData, 0, compressedLength);
-
-				crc32.update(encryptedData, 0, compressedLength);
-				byte[] decryptedData = decrypt(encryptedData, encryptedData.length, keys);
-
-				if (stream.remaining() >= 2)
-				{
-					revision = stream.readUnsignedShort();
-					assert revision != -1;
-				}
-
-				data = decryptedData;
-
-				break;
-			}
-			case CompressionType.BZ2:
-			{
-				byte[] encryptedData = new byte[compressedLength + 4];
-				stream.readBytes(encryptedData);
-
-				crc32.update(encryptedData, 0, encryptedData.length);
-				byte[] decryptedData = decrypt(encryptedData, encryptedData.length, keys);
-
-				if (stream.remaining() >= 2)
-				{
-					revision = stream.readUnsignedShort();
-					assert revision != -1;
-				}
-
-				stream = new InputStream(decryptedData);
-
-				int decompressedLength = stream.readInt();
-				data = BZip2.decompress(stream.getRemaining(), compressedLength);
-
-				if (data == null)
-				{
-					return null;
-				}
-
-				assert data.length == decompressedLength;
-
-				break;
-			}
-			case CompressionType.GZ:
-			{
-				byte[] encryptedData = new byte[compressedLength + 4];
-				stream.readBytes(encryptedData);
-
-				crc32.update(encryptedData, 0, encryptedData.length);
-				byte[] decryptedData = decrypt(encryptedData, encryptedData.length, keys);
-
-				if (stream.remaining() >= 2)
-				{
-					revision = stream.readUnsignedShort();
-					assert revision != -1;
-				}
-
-				stream = new InputStream(decryptedData);
-
-				int decompressedLength = stream.readInt();
-				data = GZip.decompress(stream.getRemaining(), compressedLength);
-
-				if (data == null)
-				{
-					return null;
-				}
-
-				assert data.length == decompressedLength;
-
-				break;
-			}
-			default:
-				throw new RuntimeException("Unknown decompression type");
-		}
-
-		DataFileReadResult res = new DataFileReadResult();
-		res.data = data;
-		res.revision = revision;
-		res.crc = crc32.getHash();
-		res.compression = compression;
-		return res;
-	}
-
-	public static byte[] compress(byte[] data, int compression, int revision, int[] keys) throws IOException
-	{
-		OutputStream stream = new OutputStream();
-		byte[] compressedData;
-		int length;
-		switch (compression)
-		{
-			case CompressionType.NONE:
-				compressedData = data;
-				length = compressedData.length;
-				break;
-			case CompressionType.BZ2:
-				compressedData = concat(Ints.toByteArray(data.length), BZip2.compress(data));
-				length = compressedData.length - 4;
-				break;
-			case CompressionType.GZ:
-				compressedData = concat(Ints.toByteArray(data.length), GZip.compress(data));
-				length = compressedData.length - 4;
-				break;
-			default:
-				throw new RuntimeException("Unknown compression type");
-		}
-
-		compressedData = encrypt(compressedData, compressedData.length, keys);
-
-		stream.writeByte(compression);
-		stream.writeInt(length);
-
-		stream.writeBytes(compressedData);
-		if (revision != -1)
-		{
-			stream.writeShort(revision);
-		}
-
-		return stream.flip();
-	}
-
-	private static byte[] decrypt(byte[] data, int length, int[] keys)
-	{
-		if (keys == null)
-		{
-			return data;
-		}
-
-		Xtea xtea = new Xtea(keys);
-		return xtea.decrypt(data, length);
-	}
-
-	private static byte[] encrypt(byte[] data, int length, int[] keys)
-	{
-		if (keys == null)
-		{
-			return data;
-		}
-
-		Xtea xtea = new Xtea(keys);
-		return xtea.encrypt(data, length);
-	}
 }

cache/src/main/java/net/runelite/cache/fs/jagex/DataFileWriteResult.java

@@ -28,5 +28,4 @@ package net.runelite.cache.fs.jagex;
 public class DataFileWriteResult
 {
 	public int sector, compressedLength;
-	public int crc; // crc of compressed data
 }

cache/src/main/java/net/runelite/cache/fs/jagex/DiskStorage.java

@@ -24,19 +24,20 @@
  */
 package net.runelite.cache.fs.jagex;
 
+import com.google.common.primitives.Ints;
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import net.runelite.cache.fs.Archive;
-import net.runelite.cache.fs.FSFile;
+import net.runelite.cache.fs.Container;
 import net.runelite.cache.fs.Index;
 import net.runelite.cache.fs.Storage;
 import net.runelite.cache.fs.Store;
 import net.runelite.cache.index.ArchiveData;
-import net.runelite.cache.index.FileData;
 import net.runelite.cache.index.IndexData;
+import net.runelite.cache.util.Crc32;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -108,13 +109,19 @@ public class DiskStorage implements Storage
 		}
 	}
 
+	public byte[] readIndex(int indexId) throws IOException
+	{
+		IndexEntry entry = index255.read(indexId);
+		byte[] indexData = data.read(index255.getIndexFileId(), entry.getId(), entry.getSector(), entry.getLength());
+		return indexData;
+	}
+
 	private void loadIndex(Index index) throws IOException
 	{
 		logger.trace("Loading index {}", index.getId());
 
-		IndexEntry entry = index255.read(index.getId());
+		byte[] indexData = readIndex(index.getId());
-		byte[] indexData = data.read(index255.getIndexFileId(), entry.getId(), entry.getSector(), entry.getLength());
+		Container res = Container.decompress(indexData, null);
-		DataFileReadResult res = DataFile.decompress(indexData, null);
 		byte[] data = res.data;
 
 		IndexData id = new IndexData();
@@ -130,28 +137,18 @@ public class DiskStorage implements Storage
 			archive.setNameHash(ad.getNameHash());
 			archive.setCrc(ad.getCrc());
 			archive.setRevision(ad.getRevision());
+			archive.setFileData(ad.getFiles());
 
 			assert ad.getFiles().length > 0;
-
-			for (FileData fd : ad.getFiles())
-			{
-				FSFile file = new FSFile(fd.getId());
-				file.setNameHash(fd.getNameHash());
-				archive.addFile(file);
-			}
 		}
 
 		index.setCrc(res.crc);
 		index.setCompression(res.compression);
 		assert res.revision == -1;
-
-		for (Archive archive : new ArrayList<>(index.getArchives()))
-		{
-			loadArchive(archive);
-		}
 	}
 
-	private void loadArchive(Archive archive) throws IOException
+	@Override
+	public byte[] loadArchive(Archive archive) throws IOException
 	{
 		Index index = archive.getIndex();
 		IndexFile indexFile = getIndex(index.getId());
@@ -162,8 +159,7 @@ public class DiskStorage implements Storage
 		if (entry == null)
 		{
 			logger.debug("can't read archive " + archive.getArchiveId() + " from index " + index.getId());
-			index.getArchives().remove(archive); // is this correct?
+			return null;
-			return;
 		}
 
 		assert entry.getId() == archive.getArchiveId();
@@ -172,28 +168,12 @@ public class DiskStorage implements Storage
 			archive.getArchiveId(), index.getId(), entry.getSector(), entry.getLength());
 
 		byte[] archiveData = data.read(index.getId(), entry.getId(), entry.getSector(), entry.getLength());
-		archive.setData(archiveData);
+		return archiveData;
-
-		if (index.getXteaManager() != null)
-		{
-			return; // can't decrypt this yet
-		}
-
-		archive.decompressAndLoad(null);
 	}
 
 	@Override
 	public void save(Store store) throws IOException
 	{
-		logger.debug("Clearing data and indexes in preparation for store save");
-
-		data.clear();
-
-		for (IndexFile indexFile : indexFiles)
-		{
-			indexFile.clear();
-		}
-
 		logger.debug("Saving store");
 
 		for (Index i : store.getIndexes())
@@ -204,49 +184,46 @@ public class DiskStorage implements Storage
 
 	private void saveIndex(Index index) throws IOException
 	{
-		// This updates archive CRCs for writeIndexData
-		for (Archive archive : index.getArchives())
-		{
-			saveArchive(archive);
-		}
-
 		IndexData indexData = index.toIndexData();
 		byte[] data = indexData.writeIndexData();
 
-		byte[] compressedData = DataFile.compress(data, index.getCompression(), -1, null); // index data revision is always -1
+		Container container = new Container(index.getCompression(), -1); // index data revision is always -1
-		DataFileWriteResult res = this.data.write(index255.getIndexFileId(), index.getId(), compressedData, index.getRevision());
+		container.compress(data, null);
+		byte[] compressedData = container.data;
+		DataFileWriteResult res = this.data.write(index255.getIndexFileId(), index.getId(), compressedData);
 
 		index255.write(new IndexEntry(index255, index.getId(), res.sector, res.compressedLength));
 
-		index.setCrc(res.crc);
+		Crc32 crc = new Crc32();
+		crc.update(compressedData, 0, compressedData.length);
+		index.setCrc(crc.getHash());
 	}
 
-	private void saveArchive(Archive a) throws IOException
+	@Override
+	public void saveArchive(Archive a, byte[] archiveData) throws IOException
 	{
 		Index index = a.getIndex();
 		IndexFile indexFile = getIndex(index.getId());
 		assert indexFile.getIndexFileId() == index.getId();
 
-		int rev; // used for determining what part of compressedData to crc
+		DataFileWriteResult res = data.write(index.getId(), a.getArchiveId(), archiveData);
-		byte[] compressedData;
-
-		if (a.getData() != null)
-		{
-			compressedData = a.getData(); // data was never decompressed or loaded
-			rev = -1; // assume that this data has no revision?
-		}
-		else
-		{
-			byte[] fileData = a.saveContents();
-			rev = a.getRevision();
-			compressedData = DataFile.compress(fileData, a.getCompression(), a.getRevision(), null);
-		}
-
-		DataFileWriteResult res = data.write(index.getId(), a.getArchiveId(), compressedData, rev);
 		indexFile.write(new IndexEntry(indexFile, a.getArchiveId(), res.sector, res.compressedLength));
 
-		logger.trace("Saved archive {}/{} at sector {}, compressed length {}", index.getId(), a.getArchiveId(), res.sector, res.compressedLength);
+		byte compression = archiveData[0];
+		int compressedSize = Ints.fromBytes(archiveData[1], archiveData[2],
+			archiveData[3], archiveData[4]);
 
-		a.setCrc(res.crc);
+		// don't crc the appended revision, if it is there
+		int length = 1 // compression type
+			+ 4 // compressed size
+			+ compressedSize
+			+ (compression != CompressionType.NONE ? 4 : 0);
+
+		Crc32 crc = new Crc32();
+		crc.update(archiveData, 0, length);
+		a.setCrc(crc.getHash());
+
+		logger.trace("Saved archive {}/{} at sector {}, compressed length {}",
+			index.getId(), a.getArchiveId(), res.sector, res.compressedLength);
 	}
 }

cache/src/main/java/net/runelite/cache/region/RegionLoader.java

@@ -30,6 +30,7 @@ import java.util.List;
 import net.runelite.cache.IndexType;
 import net.runelite.cache.fs.Archive;
 import net.runelite.cache.fs.Index;
+import net.runelite.cache.fs.Storage;
 import net.runelite.cache.fs.Store;
 import net.runelite.cache.util.XteaKeyManager;
 import org.slf4j.Logger;
@@ -53,7 +54,8 @@ public class RegionLoader
 	{
 		this.store = store;
 		index = store.getIndex(IndexType.MAPS);
-		keyManager = index.getXteaManager();
+		keyManager = new XteaKeyManager();
+		keyManager.loadKeys();
 	}
 
 	public void loadRegions() throws IOException
@@ -71,6 +73,7 @@ public class RegionLoader
 		int x = i >> 8;
 		int y = i & 0xFF;
 
+		Storage storage = store.getStorage();
 		Archive map = index.findArchiveByName("m" + x + "_" + y);
 		Archive land = index.findArchiveByName("l" + x + "_" + y);
 
@@ -81,12 +84,7 @@ public class RegionLoader
 			return null;
 		}
 
-		assert map.getFiles().size() == 1;
+		byte[] data = map.decompress(storage.loadArchive(map));
-		assert land.getFiles().size() == 1;
-
-		map.decompressAndLoad(null);
-
-		byte[] data = map.getFiles().get(0).getContents();
 
 		Region region = new Region(i);
 		region.loadTerrain(data);
@@ -96,9 +94,7 @@ public class RegionLoader
 		{
 			try
 			{
-				land.decompressAndLoad(keys);
+				data = land.decompress(storage.loadArchive(land), keys);
-
-				data = land.getFiles().get(0).getContents();
 				region.loadLocations(data);
 			}
 			catch (IOException ex)

cache/src/main/java/net/runelite/cache/server/ArchiveRequestHandler.java

@@ -32,10 +32,12 @@ import io.netty.channel.SimpleChannelInboundHandler;
 import java.io.IOException;
 import java.util.Arrays;
 import net.runelite.cache.fs.Archive;
+import net.runelite.cache.fs.Container;
 import net.runelite.cache.fs.Index;
+import net.runelite.cache.fs.Storage;
 import net.runelite.cache.fs.Store;
 import net.runelite.cache.fs.jagex.CompressionType;
-import net.runelite.cache.fs.jagex.DataFile;
+import net.runelite.cache.fs.jagex.DiskStorage;
 import net.runelite.cache.protocol.packets.ArchiveRequestPacket;
 import net.runelite.cache.protocol.packets.ArchiveResponsePacket;
 import org.slf4j.Logger;
@@ -88,12 +90,10 @@ public class ArchiveRequestHandler extends SimpleChannelInboundHandler<ArchiveRe
 		}
 		else
 		{
-			Index i = store.findIndex(archiveId);
+			// Requires disk storage. Use packed index data from
-			assert i != null;
+			// store as its crc matches
-
+			DiskStorage storage = (DiskStorage) store.getStorage();
-			byte[] indexData = i.toIndexData().writeIndexData();
+			compressed = storage.readIndex(archiveId);
-
-			compressed = compress(CompressionType.NONE, indexData);
 		}
 
 		ArchiveResponsePacket response = new ArchiveResponsePacket();
@@ -114,33 +114,31 @@ public class ArchiveRequestHandler extends SimpleChannelInboundHandler<ArchiveRe
 		Archive archive = i.getArchive(archiveId);
 		assert archive != null;
 
-		byte[] packed;
+		Storage storage = store.getStorage();
-		if (archive.getData() != null)
+		byte[] packed = storage.loadArchive(archive); // is compressed, includes length and type
+
+		if (packed == null)
 		{
-			packed = archive.getData(); // is compressed, includes length and type
+			logger.warn("Missing archive {}/{}", index, archiveId);
-
+			return; // is it possible to notify the client of an error with this?
-			byte compression = packed[0];
-			int compressedSize = Ints.fromBytes(packed[1], packed[2],
-				packed[3], packed[4]);
-
-			// size the client expects the data to be
-			int expectedSize = 1 // compression
-				+ 4 // compressed size
-				+ compressedSize
-				+ (compression != CompressionType.NONE ? 4 : 0);
-			if (packed.length != expectedSize)
-			{
-				// It may have the archive revision appended at the end.
-				// The data the client writes will have it, but the data fetched from
-				// the update server will never have it
-				assert packed.length - expectedSize == 2 : "packed length != expected size";
-				packed = Arrays.copyOf(packed, packed.length - 2);
-			}
 		}
-		else
+
+		byte compression = packed[0];
+		int compressedSize = Ints.fromBytes(packed[1], packed[2],
+			packed[3], packed[4]);
+
+		// size the client expects the data to be
+		int expectedSize = 1 // compression type
+			+ 4 // compressed size
+			+ compressedSize
+			+ (compression != CompressionType.NONE ? 4 : 0);
+		if (packed.length != expectedSize)
 		{
-			byte[] data = archive.saveContents();
+			// It may have the archive revision appended at the end.
-			packed = compress(archive.getCompression(), data);
+			// The data the client writes will have it, but the data fetched from
+			// the update server will never have it
+			assert packed.length - expectedSize == 2 : "packed length != expected size";
+			packed = Arrays.copyOf(packed, packed.length - 2);
 		}
 
 		ArchiveResponsePacket response = new ArchiveResponsePacket();
@@ -153,6 +151,8 @@ public class ArchiveRequestHandler extends SimpleChannelInboundHandler<ArchiveRe
 
 	private byte[] compress(int compression, byte[] data) throws IOException
 	{
-		return DataFile.compress(data, compression, -1, null);
+		Container container = new Container(compression, -1);
+		container.compress(data, null);
+		return container.data;
 	}
 }

cache/src/test/java/net/runelite/cache/EnumDumperTest.java

@@ -22,7 +22,6 @@
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
-
 package net.runelite.cache;
 
 import com.google.common.io.Files;
@@ -34,8 +33,10 @@ import java.nio.charset.Charset;
 import net.runelite.cache.definitions.EnumDefinition;
 import net.runelite.cache.definitions.loaders.EnumLoader;
 import net.runelite.cache.fs.Archive;
+import net.runelite.cache.fs.ArchiveFiles;
 import net.runelite.cache.fs.FSFile;
 import net.runelite.cache.fs.Index;
+import net.runelite.cache.fs.Storage;
 import net.runelite.cache.fs.Store;
 import org.junit.Rule;
 import org.junit.Test;
@@ -62,12 +63,16 @@ public class EnumDumperTest
 		{
 			store.load();
 
+			Storage storage = store.getStorage();
 			Index index = store.getIndex(IndexType.CONFIGS);
 			Archive archive = index.getArchive(ConfigType.ENUM.getId());
 
+			byte[] archiveData = storage.loadArchive(archive);
+			ArchiveFiles files = archive.getFiles(archiveData);
+
 			EnumLoader loader = new EnumLoader();
 
-			for (FSFile file : archive.getFiles())
+			for (FSFile file : files.getFiles())
 			{
 				byte[] b = file.getContents();
 

cache/src/test/java/net/runelite/cache/FrameDumper.java

@@ -37,11 +37,10 @@ import net.runelite.cache.definitions.FramemapDefinition;
 import net.runelite.cache.definitions.loaders.FrameLoader;
 import net.runelite.cache.definitions.loaders.FramemapLoader;
 import net.runelite.cache.fs.Archive;
-import net.runelite.cache.fs.FSFile;
 import net.runelite.cache.fs.Index;
+import net.runelite.cache.fs.Storage;
 import net.runelite.cache.fs.Store;
 import org.junit.Rule;
-import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -67,6 +66,7 @@ public class FrameDumper
 		{
 			store.load();
 
+			Storage storage = store.getStorage();
 			Index frameIndex = store.getIndex(IndexType.FRAMES);
 			Index framemapIndex = store.getIndex(IndexType.FRAMEMAPS);
 
@@ -74,24 +74,22 @@ public class FrameDumper
 			{
 				List<FrameDefinition> frames = new ArrayList<>();
 
-				for (FSFile file : archive.getFiles())
+				byte[] archiveData = storage.loadArchive(archive);
-				{
+				byte[] contents = archive.decompress(archiveData);
-					byte[] contents = file.getContents();
 
-					int framemapArchiveId = (contents[0] & 0xff) << 8 | contents[1] & 0xff;
+				int framemapArchiveId = (contents[0] & 0xff) << 8 | contents[1] & 0xff;
 
-					Archive framemapArchive = framemapIndex.getArchives().get(framemapArchiveId);
+				Archive framemapArchive = framemapIndex.getArchives().get(framemapArchiveId);
-					assert framemapArchive.getFiles().size() == 1;
+				archiveData = storage.loadArchive(framemapArchive);
-					FSFile framemapFile = framemapArchive.getFiles().get(0);
+				byte[] framemapContents = framemapArchive.decompress(archiveData);
 
-					FramemapLoader fmloader = new FramemapLoader();
+				FramemapLoader fmloader = new FramemapLoader();
-					FramemapDefinition framemap = fmloader.load(framemapFile.getFileId(), framemapFile.getContents());
+				FramemapDefinition framemap = fmloader.load(0, framemapContents);
 
-					FrameLoader frameLoader = new FrameLoader();
+				FrameLoader frameLoader = new FrameLoader();
-					FrameDefinition frame = frameLoader.load(framemap, contents);
+				FrameDefinition frame = frameLoader.load(framemap, contents);
 
-					frames.add(frame);
+				frames.add(frame);
-				}
 
 				Files.write(gson.toJson(frames), new File(outDir, archive.getArchiveId() + ".json"), Charset.defaultCharset());
 				++count;

cache/src/test/java/net/runelite/cache/FramemapDumper.java

@@ -33,8 +33,8 @@ import java.nio.charset.Charset;
 import net.runelite.cache.definitions.FramemapDefinition;
 import net.runelite.cache.definitions.loaders.FramemapLoader;
 import net.runelite.cache.fs.Archive;
-import net.runelite.cache.fs.FSFile;
 import net.runelite.cache.fs.Index;
+import net.runelite.cache.fs.Storage;
 import net.runelite.cache.fs.Store;
 import org.junit.Rule;
 import org.junit.Test;
@@ -63,14 +63,16 @@ public class FramemapDumper
 		{
 			store.load();
 
+			Storage storage = store.getStorage();
 			Index index = store.getIndex(IndexType.FRAMEMAPS);
+
 			for (Archive archive : index.getArchives())
 			{
-				assert archive.getFiles().size() == 1;
+				byte[] archiveData = storage.loadArchive(archive);
-				FSFile file = archive.getFiles().get(0);
+				byte[] contents = archive.decompress(archiveData);
 
 				FramemapLoader loader = new FramemapLoader();
-				FramemapDefinition framemap = loader.load(file.getFileId(), file.getContents());
+				FramemapDefinition framemap = loader.load(0, contents);
 
 				Files.write(gson.toJson(framemap), new File(outDir, archive.getArchiveId() + ".json"), Charset.defaultCharset());
 				++count;

cache/src/test/java/net/runelite/cache/InventoryDumper.java

@@ -33,8 +33,10 @@ import java.nio.charset.Charset;
 import net.runelite.cache.definitions.InventoryDefinition;
 import net.runelite.cache.definitions.loaders.InventoryLoader;
 import net.runelite.cache.fs.Archive;
+import net.runelite.cache.fs.ArchiveFiles;
 import net.runelite.cache.fs.FSFile;
 import net.runelite.cache.fs.Index;
+import net.runelite.cache.fs.Storage;
 import net.runelite.cache.fs.Store;
 import org.junit.Rule;
 import org.junit.Test;
@@ -63,10 +65,14 @@ public class InventoryDumper
 		{
 			store.load();
 
+			Storage storage = store.getStorage();
 			Index index = store.getIndex(IndexType.CONFIGS);
 			Archive archive = index.getArchive(ConfigType.INV.getId());
 
-			for (FSFile file : archive.getFiles())
+			byte[] archiveData = storage.loadArchive(archive);
+			ArchiveFiles files = archive.getFiles(archiveData);
+
+			for (FSFile file : files.getFiles())
 			{
 				InventoryLoader loader = new InventoryLoader();
 				InventoryDefinition inv = loader.load(file.getContents());

cache/src/test/java/net/runelite/cache/KitDumperTest.java

@@ -33,8 +33,10 @@ import java.nio.charset.Charset;
 import net.runelite.cache.definitions.KitDefinition;
 import net.runelite.cache.definitions.loaders.KitLoader;
 import net.runelite.cache.fs.Archive;
+import net.runelite.cache.fs.ArchiveFiles;
 import net.runelite.cache.fs.FSFile;
 import net.runelite.cache.fs.Index;
+import net.runelite.cache.fs.Storage;
 import net.runelite.cache.fs.Store;
 import org.junit.Rule;
 import org.junit.Test;
@@ -61,12 +63,16 @@ public class KitDumperTest
 		{
 			store.load();
 
+			Storage storage = store.getStorage();
 			Index index = store.getIndex(IndexType.CONFIGS);
 			Archive archive = index.getArchive(ConfigType.IDENTKIT.getId());
 
 			KitLoader loader = new KitLoader();
 
-			for (FSFile file : archive.getFiles())
+			byte[] archiveData = storage.loadArchive(archive);
+			ArchiveFiles files = archive.getFiles(archiveData);
+
+			for (FSFile file : files.getFiles())
 			{
 				byte[] b = file.getContents();
 

cache/src/test/java/net/runelite/cache/MapDumperTest.java

@@ -33,6 +33,7 @@ import java.util.ArrayList;
 import java.util.List;
 import net.runelite.cache.fs.Archive;
 import net.runelite.cache.fs.Index;
+import net.runelite.cache.fs.Storage;
 import net.runelite.cache.fs.Store;
 import net.runelite.cache.region.Region;
 import net.runelite.cache.util.XteaKeyManager;
@@ -59,13 +60,15 @@ public class MapDumperTest
 	{
 		File base = StoreLocation.LOCATION,
 			outDir = folder.newFolder();
+		XteaKeyManager keyManager = new XteaKeyManager();
+		keyManager.loadKeys();
 
 		try (Store store = new Store(base))
 		{
 			store.load();
 
+			Storage storage = store.getStorage();
 			Index index = store.getIndex(IndexType.MAPS);
-			XteaKeyManager keyManager = index.getXteaManager();
 
 			for (int i = 0; i < MAX_REGIONS; i++)
 			{
@@ -84,14 +87,7 @@ public class MapDumperTest
 					continue;
 				}
 
-				assert map.getFiles().size() == 1;
+				byte[] data = map.decompress(storage.loadArchive(map));
-				assert land.getFiles().size() == 1;
-
-				// maps aren't encrypted, but we don't load archive data of any archive in
-				// the maps index, so load it
-				map.decompressAndLoad(null);
-
-				byte[] data = map.getFiles().get(0).getContents();
 
 				Files.write(data, new File(outDir, "m" + x + "_" + y + ".dat"));
 
@@ -99,7 +95,7 @@ public class MapDumperTest
 				{
 					try
 					{
-						land.decompressAndLoad(keys);
+						data = land.decompress(storage.loadArchive(land), keys);
 					}
 					catch (IOException ex)
 					{
@@ -109,8 +105,6 @@ public class MapDumperTest
 
 					logger.info("Decrypted region {} coords {},{}", i, x, y);
 
-					data = land.getFiles().get(0).getContents();
-
 					Files.write(data, new File(outDir, "l" + x + "_" + y + ".dat"));
 				}
 			}
@@ -119,8 +113,10 @@ public class MapDumperTest
 
 	private void loadRegions(Store store) throws IOException
 	{
+		Storage storage = store.getStorage();
 		Index index = store.getIndex(IndexType.MAPS);
-		XteaKeyManager keyManager = index.getXteaManager();
+		XteaKeyManager keyManager = new XteaKeyManager();
+		keyManager.loadKeys();
 
 		for (int i = 0; i < MAX_REGIONS; ++i)
 		{
@@ -137,12 +133,7 @@ public class MapDumperTest
 				continue;
 			}
 
-			assert map.getFiles().size() == 1;
+			byte[] data = map.decompress(storage.loadArchive(map));
-			assert land.getFiles().size() == 1;
-
-			map.decompressAndLoad(null);
-
-			byte[] data = map.getFiles().get(0).getContents();
 
 			Region region = new Region(i);
 			region.loadTerrain(data);
@@ -152,14 +143,13 @@ public class MapDumperTest
 			{
 				try
 				{
-					land.decompressAndLoad(keys);
+					data = land.decompress(storage.loadArchive(land), keys);
 				}
 				catch (IOException ex)
 				{
 					continue;
 				}
 
-				data = land.getFiles().get(0).getContents();
 				region.loadLocations(data);
 			}
 

cache/src/test/java/net/runelite/cache/ModelDumperTest.java

@@ -27,13 +27,12 @@ package net.runelite.cache;
 
 
 import com.google.common.io.Files;
-import com.google.gson.Gson;
 import java.io.File;
 import java.io.IOException;
 import net.runelite.cache.definitions.loaders.ModelLoader;
 import net.runelite.cache.fs.Archive;
-import net.runelite.cache.fs.FSFile;
 import net.runelite.cache.fs.Index;
+import net.runelite.cache.fs.Storage;
 import net.runelite.cache.fs.Store;
 import org.junit.Rule;
 import org.junit.Test;
@@ -45,8 +44,6 @@ public class ModelDumperTest
 {
 	private static final Logger logger = LoggerFactory.getLogger(ModelDumperTest.class);
 
-	private Gson gson = new Gson();
-
 	@Rule
 	public TemporaryFolder folder = StoreLocation.getTemporaryFolder();
 
@@ -60,20 +57,17 @@ public class ModelDumperTest
 		{
 			store.load();
 
+			Storage storage = store.getStorage();
 			Index index = store.getIndex(IndexType.MODELS);
 
 			for (Archive archive : index.getArchives())
 			{
-				assert archive.getFiles().size() == 1;
+				byte[] contents = archive.decompress(storage.loadArchive(archive));
-
-				FSFile file = archive.getFiles().get(0);
-				byte[] contents = file.getContents();
 
 				ModelLoader loader = new ModelLoader();
 				loader.load(archive.getArchiveId(), contents);
 
 				Files.write(contents, new File(modelDir, archive.getArchiveId() + ".model"));
-				//Files.write(gson.toJson(loader), new File(modelDir, archive.getArchiveId() + ".json"), Charset.defaultCharset());
 				++count;
 			}
 		}

cache/src/test/java/net/runelite/cache/OverlayDumper.java

@@ -33,8 +33,10 @@ import java.nio.charset.Charset;
 import net.runelite.cache.definitions.OverlayDefinition;
 import net.runelite.cache.definitions.loaders.OverlayLoader;
 import net.runelite.cache.fs.Archive;
+import net.runelite.cache.fs.ArchiveFiles;
 import net.runelite.cache.fs.FSFile;
 import net.runelite.cache.fs.Index;
+import net.runelite.cache.fs.Storage;
 import net.runelite.cache.fs.Store;
 import org.junit.Rule;
 import org.junit.Test;
@@ -63,10 +65,14 @@ public class OverlayDumper
 		{
 			store.load();
 
+			Storage storage = store.getStorage();
 			Index index = store.getIndex(IndexType.CONFIGS);
 			Archive archive = index.getArchive(ConfigType.OVERLAY.getId());
 
-			for (FSFile file : archive.getFiles())
+			byte[] archiveData = storage.loadArchive(archive);
+			ArchiveFiles files = archive.getFiles(archiveData);
+
+			for (FSFile file : files.getFiles())
 			{
 				OverlayLoader loader = new OverlayLoader();
 				OverlayDefinition overlay = loader.load(file.getFileId(), file.getContents());

cache/src/test/java/net/runelite/cache/SequenceDumper.java

@@ -33,10 +33,11 @@ import java.nio.charset.Charset;
 import net.runelite.cache.definitions.SequenceDefinition;
 import net.runelite.cache.definitions.loaders.SequenceLoader;
 import net.runelite.cache.fs.Archive;
+import net.runelite.cache.fs.ArchiveFiles;
 import net.runelite.cache.fs.FSFile;
 import net.runelite.cache.fs.Index;
+import net.runelite.cache.fs.Storage;
 import net.runelite.cache.fs.Store;
-import net.runelite.cache.io.InputStream;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
@@ -64,10 +65,14 @@ public class SequenceDumper
 		{
 			store.load();
 
+			Storage storage = store.getStorage();
 			Index index = store.getIndex(IndexType.CONFIGS);
 			Archive archive = index.getArchive(ConfigType.SEQUENCE.getId());
 
-			for (FSFile file : archive.getFiles())
+			byte[] archiveData = storage.loadArchive(archive);
+			ArchiveFiles files = archive.getFiles(archiveData);
+
+			for (FSFile file : files.getFiles())
 			{
 				SequenceLoader loader = new SequenceLoader();
 				SequenceDefinition seq = loader.load(file.getFileId(), file.getContents());

cache/src/test/java/net/runelite/cache/SoundEffectsDumperTest.java

@@ -33,8 +33,8 @@ import java.nio.charset.Charset;
 import net.runelite.cache.definitions.loaders.sound.SoundEffectLoader;
 import net.runelite.cache.definitions.sound.SoundEffectDefinition;
 import net.runelite.cache.fs.Archive;
-import net.runelite.cache.fs.FSFile;
 import net.runelite.cache.fs.Index;
+import net.runelite.cache.fs.Storage;
 import net.runelite.cache.fs.Store;
 import org.junit.Rule;
 import org.junit.Test;
@@ -61,16 +61,15 @@ public class SoundEffectsDumperTest
 		{
 			store.load();
 
+			Storage storage = store.getStorage();
 			Index index = store.getIndex(IndexType.SOUNDEFFECTS);
 
 			for (Archive archive : index.getArchives())
 			{
-				assert archive.getFiles().size() == 1;
+				byte[] contents = archive.decompress(storage.loadArchive(archive));
-
-				FSFile file = archive.getFiles().get(0);
 
 				SoundEffectLoader soundEffectLoader = new SoundEffectLoader();
-				SoundEffectDefinition soundEffect = soundEffectLoader.load(file.getContents());
+				SoundEffectDefinition soundEffect = soundEffectLoader.load(contents);
 
 				Files.write(gson.toJson(soundEffect), new File(dumpDir, archive.getArchiveId() + ".json"), Charset.defaultCharset());
 				++count;

cache/src/test/java/net/runelite/cache/TitleDumper.java

@@ -22,15 +22,14 @@
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
-
 package net.runelite.cache;
 
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
 import net.runelite.cache.fs.Archive;
-import net.runelite.cache.fs.FSFile;
 import net.runelite.cache.fs.Index;
+import net.runelite.cache.fs.Storage;
 import net.runelite.cache.fs.Store;
 import org.junit.Rule;
 import org.junit.Test;
@@ -55,11 +54,12 @@ public class TitleDumper
 		{
 			store.load();
 
+			Storage storage = store.getStorage();
 			Index index = store.getIndex(IndexType.BINARY);
-			Archive a = index.findArchiveByName("title.jpg");
+			Archive archive = index.findArchiveByName("title.jpg");
-			FSFile file = a.getFiles().get(0);
+			byte[] contents = archive.decompress(storage.loadArchive(archive));
 
-			Files.write(outFile.toPath(), file.getContents());
+			Files.write(outFile.toPath(), contents);
 		}
 
 		logger.info("Dumped to {}", outFile);

cache/src/test/java/net/runelite/cache/TrackDumperTest.java

@@ -33,8 +33,8 @@ import javax.sound.midi.Sequencer;
 import net.runelite.cache.definitions.TrackDefinition;
 import net.runelite.cache.definitions.loaders.TrackLoader;
 import net.runelite.cache.fs.Archive;
-import net.runelite.cache.fs.FSFile;
 import net.runelite.cache.fs.Index;
+import net.runelite.cache.fs.Storage;
 import net.runelite.cache.fs.Store;
 import net.runelite.cache.util.Djb2Manager;
 import org.junit.Ignore;
@@ -66,18 +66,19 @@ public class TrackDumperTest
 		{
 			store.load();
 
+			Storage storage = store.getStorage();
 			Index index = store.getIndex(IndexType.TRACK1);
 			Index index2 = store.getIndex(IndexType.TRACK2);
 
 			for (Archive archive : index.getArchives())
 			{
-				dumpTrackArchive(dumpDir1, archive);
+				dumpTrackArchive(dumpDir1, storage, archive);
 				++idx1;
 			}
 
 			for (Archive archive : index2.getArchives())
 			{
-				dumpTrackArchive(dumpDir2, archive);
+				dumpTrackArchive(dumpDir2, storage, archive);
 				++idx2;
 			}
 		}
@@ -85,14 +86,17 @@ public class TrackDumperTest
 		logger.info("Dumped {} sound tracks ({} idx1, {} idx2) to {} and {}", idx1 + idx2, idx1, idx2, dumpDir1, dumpDir2);
 	}
 
-	private void dumpTrackArchive(File dumpDir, Archive archive) throws IOException
+	private void dumpTrackArchive(File dumpDir, Storage storage, Archive archive) throws IOException
 	{
-		assert archive.getFiles().size() == 1;
+		byte[] contents = archive.decompress(storage.loadArchive(archive));
 
-		FSFile file = archive.getFiles().get(0);
+		if (contents == null)
+		{
+			return;
+		}
 
 		TrackLoader loader = new TrackLoader();
-		TrackDefinition def = loader.load(file.getContents());
+		TrackDefinition def = loader.load(contents);
 
 		String name;
 		if (archive.getNameHash() != 0)

cache/src/test/java/net/runelite/cache/UnderlayDumper.java

@@ -33,8 +33,10 @@ import java.nio.charset.Charset;
 import net.runelite.cache.definitions.UnderlayDefinition;
 import net.runelite.cache.definitions.loaders.UnderlayLoader;
 import net.runelite.cache.fs.Archive;
+import net.runelite.cache.fs.ArchiveFiles;
 import net.runelite.cache.fs.FSFile;
 import net.runelite.cache.fs.Index;
+import net.runelite.cache.fs.Storage;
 import net.runelite.cache.fs.Store;
 import org.junit.Rule;
 import org.junit.Test;
@@ -63,10 +65,14 @@ public class UnderlayDumper
 		{
 			store.load();
 
+			Storage storage = store.getStorage();
 			Index index = store.getIndex(IndexType.CONFIGS);
 			Archive archive = index.getArchive(ConfigType.UNDERLAY.getId());
 
-			for (FSFile file : archive.getFiles())
+			byte[] archiveData = storage.loadArchive(archive);
+			ArchiveFiles files = archive.getFiles(archiveData);
+
+			for (FSFile file : files.getFiles())
 			{
 				UnderlayLoader loader = new UnderlayLoader();
 				UnderlayDefinition underlay = loader.load(file.getFileId(), file.getContents());

cache/src/test/java/net/runelite/cache/VarbitDumper.java

@@ -33,8 +33,10 @@ import java.nio.charset.Charset;
 import net.runelite.cache.definitions.VarbitDefinition;
 import net.runelite.cache.definitions.loaders.VarbitLoader;
 import net.runelite.cache.fs.Archive;
+import net.runelite.cache.fs.ArchiveFiles;
 import net.runelite.cache.fs.FSFile;
 import net.runelite.cache.fs.Index;
+import net.runelite.cache.fs.Storage;
 import net.runelite.cache.fs.Store;
 import org.junit.Rule;
 import org.junit.Test;
@@ -63,10 +65,14 @@ public class VarbitDumper
 		{
 			store.load();
 
+			Storage storage = store.getStorage();
 			Index index = store.getIndex(IndexType.CONFIGS);
 			Archive archive = index.getArchive(ConfigType.VARBIT.getId());
 
-			for (FSFile file : archive.getFiles())
+			byte[] archiveData = storage.loadArchive(archive);
+			ArchiveFiles files = archive.getFiles(archiveData);
+
+			for (FSFile file : files.getFiles())
 			{
 				VarbitLoader loader = new VarbitLoader();
 				VarbitDefinition varbit = loader.load(file.getFileId(), file.getContents());

cache/src/test/java/net/runelite/cache/WorldMapDumperTest.java

@@ -33,8 +33,10 @@ import java.nio.charset.Charset;
 import net.runelite.cache.definitions.WorldMapDefinition;
 import net.runelite.cache.definitions.loaders.WorldMapLoader;
 import net.runelite.cache.fs.Archive;
+import net.runelite.cache.fs.ArchiveFiles;
 import net.runelite.cache.fs.FSFile;
 import net.runelite.cache.fs.Index;
+import net.runelite.cache.fs.Storage;
 import net.runelite.cache.fs.Store;
 import org.junit.Rule;
 import org.junit.Test;
@@ -63,10 +65,14 @@ public class WorldMapDumperTest
 		{
 			store.load();
 
+			Storage storage = store.getStorage();
 			Index index = store.getIndex(IndexType.WORLDMAP);
 			Archive archive = index.getArchive(0); // there is also archive 1/2, but their data format is not this
 
-			for (FSFile file : archive.getFiles())
+			byte[] archiveData = storage.loadArchive(archive);
+			ArchiveFiles files = archive.getFiles(archiveData);
+
+			for (FSFile file : files.getFiles())
 			{
 				WorldMapLoader loader = new WorldMapLoader();
 				WorldMapDefinition def = loader.load(file.getContents(), file.getFileId());

cache/src/test/java/net/runelite/cache/fs/ContainerTest.java

@@ -1,34 +1,55 @@
-/*
+/*
- * Copyright (c) 2016-2017, Adam <Adam@sigterm.info>
+ * Copyright (c) 2016-2017, Adam <Adam@sigterm.info>
- * All rights reserved.
+ * All rights reserved.
- *
+ *
- * Redistribution and use in source and binary forms, with or without
+ * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
+ * modification, are permitted provided that the following conditions are met:
- *
+ *
- * 1. Redistributions of source code must retain the above copyright notice, this
+ * 1. Redistributions of source code must retain the above copyright notice, this
- *    list of conditions and the following disclaimer.
+ *    list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
- *    this list of conditions and the following disclaimer in the documentation
+ *    this list of conditions and the following disclaimer in the documentation
- *    and/or other materials provided with the distribution.
+ *    and/or other materials provided with the distribution.
- *
+ *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
- * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
+ */
-
+package net.runelite.cache.fs;
-package net.runelite.cache.fs.jagex;
+
-
+import java.io.IOException;
-public class DataFileReadResult
+import java.util.Random;
-{
+import static net.runelite.cache.fs.jagex.CompressionType.GZ;
-	public byte[] data;
+import static org.junit.Assert.assertArrayEquals;
-	public int revision;
+import org.junit.Test;
-	public int crc; // crc of compressed data
+
-	public int compression; // compression method data was compressed with
+public class ContainerTest
-}
+{
+
+	@Test
+	public void testCompress() throws IOException
+	{
+		int[] keys = new int[]
+		{
+			4, 8, 15, 16
+		};
+		Random random = new Random(42L);
+		byte[] data = new byte[1024];
+		random.nextBytes(data);
+
+		Container container = new Container(GZ, -1);
+		container.compress(data, keys);
+		byte[] compressedData = container.data;
+
+		container = Container.decompress(compressedData, keys);
+		assertArrayEquals(data, container.data);
+	}
+
+}

cache/src/test/java/net/runelite/cache/fs/StoreLoadTest.java

@@ -44,8 +44,6 @@ public class StoreLoadTest
 		try (Store store = new Store(StoreLocation.LOCATION))
 		{
 			store.load();
-
-			System.out.println(store);
 		}
 	}
 

cache/src/test/java/net/runelite/cache/fs/StoreTest.java

@@ -28,6 +28,7 @@ import java.io.File;
 import java.io.IOException;
 import java.util.Random;
 import net.runelite.cache.StoreLocation;
+import net.runelite.cache.index.FileData;
 import org.junit.Assert;
 import org.junit.Rule;
 import org.junit.Test;
@@ -47,10 +48,10 @@ public class StoreTest
 		{
 			Index index = store.addIndex(0);
 			Archive archive = index.addArchive(0);
-			FSFile file = new FSFile(0);
+			archive.setFileData(new FileData[1]);
-			archive.addFile(file);
+			FileData fileData = archive.getFileData()[0] = new FileData();
-			file.setNameHash(7);
+			fileData.setId(42);
-			file.setContents("test".getBytes());
+			fileData.setNameHash(7);
 
 			store.save();
 
@@ -74,15 +75,14 @@ public class StoreTest
 			Index index = store.addIndex(0);
 			Archive archive = index.addArchive(0);
 			archive.setNameHash(random.nextInt());
+			archive.setFileData(new FileData[NUMBER_OF_FILES]);
 
 			for (int i = 0; i < NUMBER_OF_FILES; ++i)
 			{
-				FSFile file = new FSFile(i);
+				FileData[] fileData = archive.getFileData();
+				FileData file = fileData[i] = new FileData();
+				file.setId(i);
 				file.setNameHash(random.nextInt());
-				archive.addFile(file);
-				byte[] data = new byte[random.nextInt(1024)];
-				random.nextBytes(data);
-				file.setContents(data);
 			}
 
 			store.save();
@@ -109,39 +109,33 @@ public class StoreTest
 
 			Archive archive = index.addArchive(0);
 			archive.setNameHash(random.nextInt(Integer.MAX_VALUE));
+			archive.setFileData(new FileData[NUMBER_OF_FILES]);
 
 			Archive archive2 = index.addArchive(1);
+			archive2.setFileData(new FileData[NUMBER_OF_FILES]);
 
 			Archive archive3 = index2.addArchive(0);
+			archive3.setFileData(new FileData[NUMBER_OF_FILES]);
 
 			for (int i = 0; i < NUMBER_OF_FILES; ++i)
 			{
-				FSFile file = new FSFile(i);
+				FileData[] fileData = archive.getFileData();
+				FileData file = fileData[i] = new FileData();
 				file.setNameHash(random.nextInt(Integer.MAX_VALUE));
-				archive.addFile(file);
-				byte[] data = new byte[random.nextInt(1024)];
-				random.nextBytes(data);
-				file.setContents(data);
 			}
 
 			for (int i = 0; i < NUMBER_OF_FILES; ++i)
 			{
-				FSFile file = new FSFile(i);
+				FileData[] fileData = archive2.getFileData();
+				FileData file = fileData[i] = new FileData();
 				file.setNameHash(random.nextInt(Integer.MAX_VALUE));
-				archive2.addFile(file);
-				byte[] data = new byte[random.nextInt(1024)];
-				random.nextBytes(data);
-				file.setContents(data);
 			}
 
 			for (int i = 0; i < NUMBER_OF_FILES; ++i)
 			{
-				FSFile file = new FSFile(i);
+				FileData[] fileData = archive3.getFileData();
+				FileData file = fileData[i] = new FileData();
 				file.setNameHash(random.nextInt(Integer.MAX_VALUE));
-				archive3.addFile(file);
-				byte[] data = new byte[random.nextInt(1024)];
-				random.nextBytes(data);
-				file.setContents(data);
 			}
 
 			store.save();

cache/src/test/java/net/runelite/cache/fs/jagex/DataFileTest.java

@@ -27,6 +27,7 @@ package net.runelite.cache.fs.jagex;
 import java.io.File;
 import java.io.IOException;
 import net.runelite.cache.StoreLocation;
+import net.runelite.cache.fs.Container;
 import org.junit.Assert;
 import org.junit.Rule;
 import org.junit.Test;
@@ -43,17 +44,17 @@ public class DataFileTest
 		File file = folder.newFile();
 		DataFile df = new DataFile(file);
 
-		byte[] compressedData = DataFile.compress("test".getBytes(), CompressionType.NONE, 0, null);
+		Container container = new Container(CompressionType.NONE, 0);
-		DataFileWriteResult res = df.write(42, 3, compressedData, 0);
+		container.compress("test".getBytes(), null);
+		byte[] compressedData = container.data;
+		DataFileWriteResult res = df.write(42, 3, compressedData);
 
 		compressedData = df.read(42, 3, res.sector, res.compressedLength);
-		DataFileReadResult res2 = DataFile.decompress(compressedData, null);
+		Container res2 = Container.decompress(compressedData, null);
 
 		byte[] buf = res2.data;
 		String str = new String(buf);
 		Assert.assertEquals("test", str);
-
-		Assert.assertEquals(res.crc, res2.crc);
 	}
 
 	@Test
@@ -69,11 +70,13 @@ public class DataFileTest
 
 		DataFile df = new DataFile(file);
 
-		byte[] compressedData = DataFile.compress(b, CompressionType.BZ2, 42, null);
+		Container container = new Container(CompressionType.BZ2, 42);
-		DataFileWriteResult res = df.write(42, 0x1FFFF, compressedData, 42);
+		container.compress(b, null);
+		byte[] compressedData = container.data;
+		DataFileWriteResult res = df.write(42, 0x1FFFF, compressedData);
 
 		compressedData = df.read(42, 0x1FFFF, res.sector, res.compressedLength);
-		DataFileReadResult res2 = DataFile.decompress(compressedData, null);
+		Container res2 = Container.decompress(compressedData, null);
 
 		byte[] buf = res2.data;
 		Assert.assertArrayEquals(b, buf);
@@ -84,11 +87,13 @@ public class DataFileTest
 	{
 		DataFile df = new DataFile(folder.newFile());
 
-		byte[] compressedData = DataFile.compress("test".getBytes(), CompressionType.GZ, 0, null);
+		Container container = new Container(CompressionType.GZ, 0);
-		DataFileWriteResult res = df.write(41, 4, compressedData, 0);
+		container.compress("test".getBytes(), null);
+		byte[] compressedData = container.data;
+		DataFileWriteResult res = df.write(41, 4, compressedData);
 
 		compressedData = df.read(41, 4, res.sector, res.compressedLength);
-		DataFileReadResult res2 = DataFile.decompress(compressedData, null);
+		Container res2 = Container.decompress(compressedData, null);
 
 		byte[] buf = res2.data;
 		String str = new String(buf);
@@ -100,36 +105,19 @@ public class DataFileTest
 	{
 		DataFile df = new DataFile(folder.newFile());
 
-		byte[] compressedData = DataFile.compress("test".getBytes(), CompressionType.BZ2, 5, null);
+		Container container = new Container(CompressionType.BZ2, 5);
-		DataFileWriteResult res = df.write(41, 4, compressedData, 0);
+		container.compress("test".getBytes(), null);
+		byte[] compressedData = container.data;
+		DataFileWriteResult res = df.write(41, 4, compressedData);
 
 		compressedData = df.read(41, 4, res.sector, res.compressedLength);
-		DataFileReadResult res2 = DataFile.decompress(compressedData, null);
+		Container res2 = Container.decompress(compressedData, null);
 
 		byte[] buf = res2.data;
 		String str = new String(buf);
 		Assert.assertEquals("test", str);
 	}
 
-	@Test
-	public void testCrc() throws IOException
-	{
-		File file = folder.newFile();
-		DataFile df = new DataFile(file);
-
-		byte[] compressedData = DataFile.compress("test".getBytes(), CompressionType.NONE, 42, null);
-		DataFileWriteResult res = df.write(42, 3, compressedData, 0);
-
-		compressedData = df.read(42, 3, res.sector, res.compressedLength);
-		DataFileReadResult res2 = DataFile.decompress(compressedData, null);
-
-		byte[] buf = res2.data;
-		String str = new String(buf);
-		Assert.assertEquals("test", str);
-		Assert.assertEquals(res.crc, res2.crc);
-		Assert.assertEquals(42, res2.revision);
-	}
-
 	@Test
 	public void testEnc() throws IOException
 	{
@@ -141,16 +129,17 @@ public class DataFileTest
 
 		DataFile df = new DataFile(file);
 
-		byte[] compressedData = DataFile.compress("testtesttesttest1".getBytes(), CompressionType.NONE, 42, keys);
+		Container container = new Container(CompressionType.NONE, 42);
-		DataFileWriteResult res = df.write(42, 3, compressedData, 0);
+		container.compress("testtesttesttest1".getBytes(), keys);
+		byte[] compressedData = container.data;
+		DataFileWriteResult res = df.write(42, 3, compressedData);
 
 		compressedData = df.read(42, 3, res.sector, res.compressedLength);
-		DataFileReadResult res2 = DataFile.decompress(compressedData, keys);
+		Container res2 = Container.decompress(compressedData, keys);
 
 		byte[] buf = res2.data;
 		String str = new String(buf);
 		Assert.assertEquals("testtesttesttest1", str);
-		Assert.assertEquals(res.crc, res2.crc);
 		Assert.assertEquals(42, res2.revision);
 	}
 
@@ -165,16 +154,17 @@ public class DataFileTest
 
 		DataFile df = new DataFile(file);
 
-		byte[] compressedData = DataFile.compress("testtesttesttest1".getBytes(), CompressionType.GZ, 42, keys);
+		Container container = new Container(CompressionType.GZ, 42);
-		DataFileWriteResult res = df.write(42, 3, compressedData, 0);
+		container.compress("testtesttesttest1".getBytes(), keys);
+		byte[] compressedData = container.data;
+		DataFileWriteResult res = df.write(42, 3, compressedData);
 
 		compressedData = df.read(42, 3, res.sector, res.compressedLength);
-		DataFileReadResult res2 = DataFile.decompress(compressedData, keys);
+		Container res2 = Container.decompress(compressedData, keys);
 
 		byte[] buf = res2.data;
 		String str = new String(buf);
 		Assert.assertEquals("testtesttesttest1", str);
-		Assert.assertEquals(res.crc, res2.crc);
 		Assert.assertEquals(42, res2.revision);
 	}
 }

cache/src/test/java/net/runelite/cache/fs/jagex/DiskStorageTest.java

@@ -0,0 +1,102 @@
+/*
+ * Copyright (c) 2016-2017, Adam <Adam@sigterm.info>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package net.runelite.cache.fs.jagex;
+
+import java.io.File;
+import net.runelite.cache.StoreLocation;
+import net.runelite.cache.fs.Archive;
+import net.runelite.cache.fs.Container;
+import net.runelite.cache.fs.Index;
+import net.runelite.cache.fs.Store;
+import net.runelite.cache.index.FileData;
+import org.junit.Test;
+import static org.junit.Assert.*;
+import org.junit.Rule;
+import org.junit.rules.TemporaryFolder;
+
+public class DiskStorageTest
+{
+	@Rule
+	public TemporaryFolder folder = StoreLocation.getTemporaryFolder();
+
+	@Test
+	public void testSaveArchive() throws Exception
+	{
+		File file = folder.newFolder();
+		DiskStorage storage = new DiskStorage(file);
+		Archive archive;
+		Archive archive2;
+		try (Store store = new Store(storage))
+		{
+			Index index = store.addIndex(0);
+			archive = index.addArchive(0);
+			archive2 = index.addArchive(1);
+
+			FileData[] fileData = new FileData[1];
+			archive.setFileData(fileData);
+			fileData[0] = new FileData();
+
+			FileData[] fileData2 = new FileData[1];
+			archive2.setFileData(fileData2);
+			fileData2[0] = new FileData();
+
+			byte[] data = "test".getBytes();
+			Container container = new Container(archive.getCompression(), -1);
+			container.compress(data, null);
+			byte[] compressedData = container.data;
+			storage.saveArchive(archive, compressedData);
+
+			container = new Container(archive.getCompression(), 42);
+			container.compress(data, null);
+			compressedData = container.data;
+			archive2.setRevision(42);
+			storage.saveArchive(archive2, compressedData);
+
+			store.save();
+		}
+
+		storage = new DiskStorage(file);
+		try (Store store = new Store(storage))
+		{
+			store.load();
+			Index index = store.findIndex(0);
+			Archive archive2_1 = index.getArchive(0);
+			Archive archive2_2 = index.getArchive(1);
+
+			byte[] comprsesedData = storage.loadArchive(archive2_1);
+			byte[] data = archive2_1.decompress(comprsesedData);
+			assertArrayEquals("test".getBytes(), data);
+			assertEquals(archive.getCrc(), archive2_1.getCrc());
+			assertEquals(archive.getRevision(), archive2_1.getRevision());
+
+			comprsesedData = storage.loadArchive(archive2_2);
+			data = archive2_2.decompress(comprsesedData);
+			assertArrayEquals("test".getBytes(), data);
+			assertEquals(archive2.getCrc(), archive2_2.getCrc());
+			assertEquals(archive2.getRevision(), archive2_2.getRevision());
+		}
+	}
+
+}

cache/src/test/java/net/runelite/cache/protocol/encoders/ArchiveResponseEncoderTest.java

@@ -29,6 +29,7 @@ import io.netty.buffer.Unpooled;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Random;
+import net.runelite.cache.fs.Container;
 import net.runelite.cache.fs.jagex.CompressionType;
 import net.runelite.cache.fs.jagex.DataFile;
 import net.runelite.cache.protocol.decoders.ArchiveResponseDecoder;
@@ -45,7 +46,9 @@ public class ArchiveResponseEncoderTest
 		Random random = new Random(42L);
 		random.nextBytes(data);
 
-		byte[] compressedData = DataFile.compress(data, CompressionType.NONE, -1, null);
+		Container container =new Container(CompressionType.NONE, -1);
+		container.compress(data, null);
+		byte[] compressedData = container.data;//DataFile.compress(data, CompressionType.NONE, -1, null);
 
 		ArchiveResponsePacket archiveResponse = new ArchiveResponsePacket();
 		archiveResponse.setIndex(0);
@@ -67,7 +70,7 @@ public class ArchiveResponseEncoderTest
 		Assert.assertEquals(archiveResponse.getArchive(), response.getArchive());
 		Assert.assertArrayEquals(archiveResponse.getData(), response.getData());
 
-		byte[] decompressedData = DataFile.decompress(response.getData(), null).data;
+		byte[] decompressedData = Container.decompress(response.getData(), null).data;
 		Assert.assertArrayEquals(data, decompressedData);
 	}
 

cache/src/test/java/net/runelite/cache/script/disassembler/DisassemblerTest.java

@@ -32,8 +32,8 @@ import net.runelite.cache.StoreLocation;
 import net.runelite.cache.definitions.ScriptDefinition;
 import net.runelite.cache.definitions.loaders.ScriptLoader;
 import net.runelite.cache.fs.Archive;
-import net.runelite.cache.fs.FSFile;
 import net.runelite.cache.fs.Index;
+import net.runelite.cache.fs.Storage;
 import net.runelite.cache.fs.Store;
 import net.runelite.cache.script.Instructions;
 import org.junit.Rule;
@@ -61,17 +61,20 @@ public class DisassemblerTest
 		{
 			store.load();
 
+			Storage storage = store.getStorage();
 			Index index = store.getIndex(IndexType.CLIENTSCRIPT);
 			ScriptLoader loader = new ScriptLoader();
 
 			for (Archive archive : index.getArchives())
 			{
-				assert archive.getFiles().size() == 1;
+				byte[] contents = archive.decompress(storage.loadArchive(archive));
 
-				FSFile file = archive.getFiles().get(0);
+				if (contents == null)
-				byte[] contents = file.getContents();
+				{
+					continue;
+				}
 
-				ScriptDefinition script = loader.load(file.getFileId(), contents);
+				ScriptDefinition script = loader.load(0, contents);
 
 				File outFile = new File(outDir, archive.getArchiveId() + ".rs2asm");
 

cache/src/test/java/net/runelite/cache/server/CacheServerTest.java

@@ -24,15 +24,18 @@
  */
 package net.runelite.cache.server;
 
-import java.io.File;
 import java.io.FileNotFoundException;
+import java.io.IOException;
 import net.runelite.cache.StoreLocation;
 import net.runelite.cache.client.CacheClient;
 import net.runelite.cache.fs.Archive;
-import net.runelite.cache.fs.FSFile;
+import net.runelite.cache.fs.Container;
 import net.runelite.cache.fs.Index;
+import net.runelite.cache.fs.Storage;
 import net.runelite.cache.fs.Store;
-import org.junit.Assert;
+import net.runelite.cache.index.FileData;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
 import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
@@ -54,7 +57,6 @@ public class CacheServerTest
 			CacheServer server = new CacheServer(store, REVISION))
 		{
 			store.load();
-			store.rebuildCrc();
 
 			server.start();
 
@@ -75,7 +77,7 @@ public class CacheServerTest
 		{
 			addInitialFilesToStore(store);
 
-			store.rebuildCrc();
+			store.save();
 
 			server.start();
 
@@ -88,22 +90,36 @@ public class CacheServerTest
 
 				Index index = store2.findIndex(0);
 				Archive archive = index.getArchive(0);
-				archive.decompressAndLoad(null); // cache client doesn't decompress archive
 
-				FSFile file = archive.getFiles().get(0);
+				FileData[] files = archive.getFileData();
-				Assert.assertArrayEquals("test".getBytes(), file.getContents());
+				FileData file = files[0];
+				assertEquals(7, file.getNameHash());
+
+				Storage storage = store2.getStorage();
+				byte[] data = storage.loadArchive(archive);
+				data = archive.decompress(data);
+				assertArrayEquals("test".getBytes(), data);
+				assertEquals(store.getIndexes().get(0).getArchive(0).getCrc(), archive.getCrc());
 			}
 		}
 	}
 
-	private void addInitialFilesToStore(Store store) throws FileNotFoundException
+	private void addInitialFilesToStore(Store store) throws FileNotFoundException, IOException
 	{
+		Storage storage = store.getStorage();
 		Index index = store.addIndex(0);
+
 		Archive archive = index.addArchive(0);
-		FSFile file = new FSFile(0);
+		FileData[] files = new FileData[1];
+		archive.setFileData(files);
+		FileData file = files[0] = new FileData();
 		file.setNameHash(7);
-		file.setContents("test".getBytes());
+		byte[] data = "test".getBytes();
-		archive.addFile(file);
+
+		Container container = new Container(archive.getCompression(), -1);
+		container.compress(data, null);
+		byte[] compressedData = container.data;
+		storage.saveArchive(archive, compressedData);
 	}
 
 }

cache/src/test/java/net/runelite/cache/util/XteaTest.java

@@ -24,18 +24,13 @@
  */
 package net.runelite.cache.util;
 
-import java.security.InvalidKeyException;
-import java.security.NoSuchAlgorithmException;
-import javax.crypto.BadPaddingException;
-import javax.crypto.IllegalBlockSizeException;
-import javax.crypto.NoSuchPaddingException;
 import static org.junit.Assert.assertArrayEquals;
 import org.junit.Test;
 
 public class XteaTest
 {
 	@Test
-	public void test() throws InvalidKeyException, IllegalBlockSizeException, BadPaddingException, NoSuchAlgorithmException, NoSuchPaddingException
+	public void test()
 	{
 		byte[] data = "testtesttest1".getBytes();
 

cache/src/test/resources/simplelogger.properties

@@ -0,0 +1 @@
+org.slf4j.simpleLogger.defaultLogLevel=info

http-service/src/main/java/net/runelite/http/service/cache/CacheService.java

@@ -50,9 +50,8 @@ import net.runelite.cache.definitions.loaders.ItemLoader;
 import net.runelite.cache.definitions.loaders.NpcLoader;
 import net.runelite.cache.definitions.loaders.ObjectLoader;
 import net.runelite.cache.fs.ArchiveFiles;
+import net.runelite.cache.fs.Container;
 import net.runelite.cache.fs.FSFile;
-import net.runelite.cache.fs.jagex.DataFile;
-import net.runelite.cache.fs.jagex.DataFileReadResult;
 import net.runelite.http.api.cache.Cache;
 import net.runelite.http.api.cache.CacheArchive;
 import net.runelite.http.api.cache.CacheIndex;
@@ -150,7 +149,7 @@ public class CacheService
 			return null;
 		}
 
-		DataFileReadResult result = DataFile.decompress(archiveData, null);
+		Container result = Container.decompress(archiveData, null);
 		if (result == null)
 		{
 			return null;

http-service/src/main/java/net/runelite/http/service/cache/CacheStorage.java

@@ -24,14 +24,13 @@
  */
 package net.runelite.http.service.cache;
 
-import com.google.common.hash.Hashing;
 import java.io.IOException;
 import java.util.List;
 import net.runelite.cache.fs.Archive;
-import net.runelite.cache.fs.FSFile;
 import net.runelite.cache.fs.Index;
 import net.runelite.cache.fs.Storage;
 import net.runelite.cache.fs.Store;
+import net.runelite.cache.index.FileData;
 import net.runelite.http.service.cache.beans.ArchiveEntry;
 import net.runelite.http.service.cache.beans.CacheEntry;
 import net.runelite.http.service.cache.beans.FileEntry;
@@ -88,13 +87,17 @@ public class CacheStorage implements Storage
 				archive.setNameHash(archiveEntry.getNameHash());
 				archive.setCrc(archiveEntry.getCrc());
 				archive.setRevision(archiveEntry.getRevision());
+				archive.setHash(archiveEntry.getHash());
 
 				List<FileEntry> files = cacheDao.findFilesForArchive(con, archiveEntry);
+				FileData[] fileData = new FileData[files.size()];
+				archive.setFileData(fileData);
+				int idx = 0;
 				for (FileEntry fileEntry : files)
 				{
-					FSFile file = new FSFile(fileEntry.getFileId());
+					FileData file = fileData[idx++] = new FileData();
+					file.setId(fileEntry.getFileId());
 					file.setNameHash(fileEntry.getNameHash());
-					archive.addFile(file);
 				}
 			}
 		}
@@ -115,13 +118,13 @@ public class CacheStorage implements Storage
 					archive.getNameHash(), archive.getCrc(), archive.getRevision());
 				if (archiveEntry == null)
 				{
-					byte[] hash = Hashing.sha256().hashBytes(archive.getData()).asBytes();
+					byte[] hash = archive.getHash();
 					archiveEntry = cacheDao.createArchive(con, entry, archive.getArchiveId(),
 						archive.getNameHash(), archive.getCrc(), archive.getRevision(), hash);
 
-					for (FSFile file : archive.getFiles())
+					for (FileData file : archive.getFileData())
 					{
-						cacheDao.associateFileToArchive(con, archiveEntry, file.getFileId(), file.getNameHash());
+						cacheDao.associateFileToArchive(con, archiveEntry, file.getId(), file.getNameHash());
 					}
 				}
 				cacheDao.associateArchiveToIndex(con, archiveEntry, entry);
@@ -129,4 +132,16 @@ public class CacheStorage implements Storage
 		}
 	}
 
+	@Override
+	public byte[] loadArchive(Archive archive) throws IOException
+	{
+		throw new UnsupportedOperationException();
+	}
+
+	@Override
+	public void saveArchive(Archive archive, byte[] data) throws IOException
+	{
+		throw new UnsupportedOperationException();
+	}
+
 }

http-service/src/main/java/net/runelite/http/service/cache/CacheUpdater.java

@@ -96,7 +96,7 @@ public class CacheUpdater
 			ExecutorService executor = Executors.newSingleThreadExecutor();
 
 			CacheClient client = new CacheClient(store, rsVersion,
-				(Archive archive) -> executor.submit(new CacheUploader(minioClient, minioBucket, archive)));
+				(Archive archive, byte[] data) -> executor.submit(new CacheUploader(minioClient, minioBucket, archive, data)));
 
 			client.connect();
 			HandshakeResponseType result = client.handshake().join();
@@ -111,6 +111,7 @@ public class CacheUpdater
 
 			if (!checkOutOfDate(indexes, entries))
 			{
+				logger.info("All up to date.");
 				return;
 			}
 

http-service/src/main/java/net/runelite/http/service/cache/CacheUploader.java

@@ -50,21 +50,24 @@ public class CacheUploader implements Runnable
 	private final MinioClient minioClient;
 	private final String minioBucket;
 	private final Archive archive;
+	private final byte[] data;
 
-	public CacheUploader(MinioClient minioClient, String minioBucket, Archive archive)
+	public CacheUploader(MinioClient minioClient, String minioBucket, Archive archive, byte[] data)
 	{
 		this.minioClient = minioClient;
 		this.minioBucket = minioBucket;
 		this.archive = archive;
+		this.data = data;
 	}
 
 	@Override
 	public void run()
 	{
-		byte[] data = archive.getData();
 		byte[] hash = Hashing.sha256().hashBytes(data).asBytes();
 		String hashStr = BaseEncoding.base16().encode(hash);
 
+		archive.setHash(hash);
+
 		String path = new StringBuilder()
 			.append(hashStr.substring(0, 2))
 			.append('/')

http-service/src/main/java/net/runelite/http/service/xtea/XteaService.java

@@ -28,7 +28,7 @@ import java.io.IOException;
 import java.util.List;
 import java.util.stream.Collectors;
 import net.runelite.cache.IndexType;
-import net.runelite.cache.fs.jagex.DataFile;
+import net.runelite.cache.fs.Container;
 import net.runelite.cache.util.Djb2;
 import net.runelite.http.api.xtea.XteaKey;
 import net.runelite.http.api.xtea.XteaRequest;
@@ -226,7 +226,7 @@ public class XteaService
 
 		try
 		{
-			DataFile.decompress(data, keys);
+			Container.decompress(data, keys);
 			return true;
 		}
 		catch (IOException ex)