datafile can handle all of the compression

src/main/java/net/runelite/cache/fs/DataFile.java

@@ -7,6 +7,10 @@ import java.io.IOException;
 import java.io.RandomAccessFile;
 import java.nio.ByteBuffer;
 import java.util.Objects;
+import net.runelite.cache.fs.io.InputStream;
+import net.runelite.cache.fs.io.OutputStream;
+import net.runelite.cache.fs.util.bzip2.BZip2Decompressor;
+import net.runelite.cache.fs.util.gzip.GZipDecompressor;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -70,7 +74,7 @@ public class DataFile implements Closeable
 	 * @return
 	 * @throws IOException 
 	 */
-	public synchronized byte[] read(int indexId, int archiveId, int sector, int size) throws IOException
+	public synchronized DataFileReadResult read(int indexId, int archiveId, int sector, int size) throws IOException
 	{
 		if (sector <= 0L || dat.length() / 520L < (long) sector)
 		{
@@ -154,7 +158,10 @@ public class DataFile implements Closeable
 		}
 
 		buffer.flip();
-		return buffer.array();
+				
+		//XTEA decrypt here?
+		
+		return this.decompress(buffer.array());
 	}
 	
 	/**
@@ -165,11 +172,14 @@ public class DataFile implements Closeable
 	 * @return the sector the data starts at
 	 * @throws IOException 
 	 */
-	public synchronized int write(int indexId, int archiveId, ByteBuffer data) throws IOException
+	public synchronized DataFileWriteResult write(int indexId, int archiveId, ByteBuffer data, int compression, int revision) throws IOException
 	{
 		int sector;
 		int startSector;
 		
+		data = ByteBuffer.wrap(this.compress(data.array(), compression, revision));
+		int dataLen = data.remaining();
+		
 		sector = (int) ((dat.length() + (long) (SECTOR_SIZE - 1)) / (long) SECTOR_SIZE);
 		if (sector == 0)
 		{
@@ -253,6 +263,98 @@ public class DataFile implements Closeable
 			sector = nextSector;
 		}
 		
-		return startSector;
+		DataFileWriteResult res = new DataFileWriteResult();
+		res.sector = startSector;
+		res.compressedLength = dataLen;
+		return res;
+	}
+	
+	private DataFileReadResult decompress(byte[] b)
+	{
+		InputStream stream = new InputStream(b);
+		
+		int compression = stream.readUnsignedByte();
+		int compressedLength = stream.readInt();
+		if (compressedLength < 0 || compressedLength > 1000000)
+			throw new RuntimeException("Invalid data");
+		
+		byte[] data;
+		int revision;
+		switch (compression)
+		{
+			case 0:
+				data = new byte[compressedLength];
+				revision = this.checkRevision(stream, compressedLength);
+				stream.readBytes(data, 0, compressedLength);
+				break;
+			case 1:
+			{
+				int length = stream.readInt();
+				data = new byte[length];
+				revision = this.checkRevision(stream, compressedLength);
+				BZip2Decompressor.decompress(data, b, compressedLength, 9);
+				break;
+			}
+			default:
+			{
+				int length = stream.readInt();
+				data = new byte[length];
+				revision = this.checkRevision(stream, compressedLength);
+				GZipDecompressor.decompress(stream, data);
+			}
+		}
+		
+		DataFileReadResult res = new DataFileReadResult();
+		res.data = data;
+		res.revision = revision;
+		return res;
+	}
+	
+	private byte[] compress(byte[] data, int compression, int revision)
+	{
+		OutputStream stream = new OutputStream();
+		stream.writeByte(compression);
+		byte[] compressedData;
+		switch (compression)
+		{
+			case 0:
+				compressedData = data;
+				stream.writeInt(data.length);
+				break;
+			default:
+				throw new RuntimeException();
+//			case 1:
+//				compressedData = (byte[]) null;
+//				break;
+//			default:
+//				compressedData = GZipCompressor.compress(data);
+//				stream.writeInt(compressedData.length);
+//				stream.writeInt(data.length);
+		}
+
+		stream.writeBytes(compressedData);
+		stream.writeShort(revision);
+		
+		byte[] compressed = new byte[stream.getOffset()];
+		stream.setOffset(0);
+		stream.getBytes(compressed, 0, compressed.length);
+		return compressed;
+	}
+	
+	private int checkRevision(InputStream stream, int compressedLength)
+	{
+		int offset = stream.getOffset();
+		int revision;
+		if (stream.getLength() - (compressedLength + stream.getOffset()) >= 2)
+		{
+			stream.setOffset(stream.getLength() - 2);
+			revision = stream.readUnsignedShort();
+			stream.setOffset(offset);
+		}
+		else
+		{
+			revision = -1;
+		}
+		return revision;
 	}
 }

src/main/java/net/runelite/cache/fs/DataFileReadResult.java

@@ -0,0 +1,7 @@
+package net.runelite.cache.fs;
+
+public class DataFileReadResult
+{
+	public byte[] data;
+	public int revision;
+}

src/main/java/net/runelite/cache/fs/DataFileWriteResult.java

@@ -0,0 +1,6 @@
+package net.runelite.cache.fs;
+
+public class DataFileWriteResult
+{
+	public int sector, compressedLength;
+}

src/main/java/net/runelite/cache/fs/Index.java

@@ -10,7 +10,6 @@ import java.util.Objects;
 import net.runelite.cache.fs.io.InputStream;
 import net.runelite.cache.fs.io.OutputStream;
 import net.runelite.cache.fs.util.bzip2.BZip2Decompressor;
-import net.runelite.cache.fs.util.gzip.GZipCompressor;
 import net.runelite.cache.fs.util.gzip.GZipDecompressor;
 
 public class Index implements Closeable
@@ -113,41 +112,43 @@ public class Index implements Closeable
 		IndexFile index255 = store.getIndex255();
 		
 		IndexEntry entry = index255.read(id);
-		byte[] b = dataFile.read(index255.getIndexFileId(), entry.getId(), entry.getSector(), entry.getLength());
+		DataFileReadResult res = dataFile.read(index255.getIndexFileId(), entry.getId(), entry.getSector(), entry.getLength());
-		
+		byte[] data = res.data;
-		InputStream stream = new InputStream(b);
+//		byte[] b = dataFile.read(index255.getIndexFileId(), entry.getId(), entry.getSector(), entry.getLength());
-		
+//		
-		//XTEA decrypt here
+//		InputStream stream = new InputStream(b);
-		
+//		
-		this.compression = stream.readUnsignedByte();
+//		//XTEA decrypt here
-		int compressedLength = stream.readInt();
+//		
-		if (compressedLength < 0 || compressedLength > 1000000)
+//		this.compression = stream.readUnsignedByte();
-			throw new RuntimeException("Invalid archive header");
+//		int compressedLength = stream.readInt();
-		
+//		if (compressedLength < 0 || compressedLength > 1000000)
-		byte[] data;
+//			throw new RuntimeException("Invalid archive header");
-		switch (compression)
+//		
-		{
+//		byte[] data;
-			case 0:
+//		switch (compression)
-				data = new byte[compressedLength];
+//		{
-				this.checkRevision(stream, compressedLength);
+//			case 0:
-				stream.readBytes(data, 0, compressedLength);
+//				data = new byte[compressedLength];
-				break;
+//				this.checkRevision(stream, compressedLength);
-			case 1:
+//				stream.readBytes(data, 0, compressedLength);
-			{
+//				break;
-				int length = stream.readInt();
+//			case 1:
-				data = new byte[length];
+//			{
-				this.checkRevision(stream, compressedLength);
+//				int length = stream.readInt();
-				BZip2Decompressor.decompress(data, b, compressedLength, 9);
+//				data = new byte[length];
-				break;
+//				this.checkRevision(stream, compressedLength);
-			}
+//				BZip2Decompressor.decompress(data, b, compressedLength, 9);
-			default:
+//				break;
-			{
+//			}
-				int length = stream.readInt();
+//			default:
-				data = new byte[length];
+//			{
-				this.checkRevision(stream, compressedLength);
+//				int length = stream.readInt();
-				GZipDecompressor.decompress(stream, data);
+//				data = new byte[length];
-			}
+//				this.checkRevision(stream, compressedLength);
-		}
+//				GZipDecompressor.decompress(stream, data);
+//			}
+//		}
 		
 		readIndexData(data);
 		
@@ -160,51 +161,53 @@ public class Index implements Closeable
 		
 		byte[] data = this.writeIndexData();
 		
-		OutputStream stream = new OutputStream();
+//		OutputStream stream = new OutputStream();
-		stream.writeByte(this.compression);
+//		stream.writeByte(this.compression);
-		byte[] compressedData;
+//		byte[] compressedData;
-		switch (this.compression)
+//		switch (this.compression)
-		{
+//		{
-			case 0:
+//			case 0:
-				compressedData = data;
+//				compressedData = data;
-				stream.writeInt(data.length);
+//				stream.writeInt(data.length);
-				break;
-			default:
-				throw new RuntimeException();
-//			case 1:
-//				compressedData = (byte[]) null;
 //				break;
 //			default:
-//				compressedData = GZipCompressor.compress(data);
+//				throw new RuntimeException();
-//				stream.writeInt(compressedData.length);
+////			case 1:
-//				stream.writeInt(data.length);
+////				compressedData = (byte[]) null;
-		}
+////				break;
-
+////			default:
-		stream.writeBytes(compressedData);
+////				compressedData = GZipCompressor.compress(data);
-		stream.writeShort(this.revision);
+////				stream.writeInt(compressedData.length);
-		
+////				stream.writeInt(data.length);
-		byte[] compressed = new byte[stream.getOffset()];
+//		}
-		stream.setOffset(0);
+//
-		stream.getBytes(compressed, 0, compressed.length);
+//		stream.writeBytes(compressedData);
-		
+//		stream.writeShort(this.revision);
-		//XTEA encrypt here
+//		
+//		byte[] compressed = new byte[stream.getOffset()];
+//		stream.setOffset(0);
+//		stream.getBytes(compressed, 0, compressed.length);
+//		
+//		//XTEA encrypt here
 
 		DataFile dataFile = store.getData();
 		IndexFile index255 = store.getIndex255();
 		
-		int sector = dataFile.write(index255.getIndexFileId(), this.id, ByteBuffer.wrap(compressed));
+		DataFileWriteResult res = dataFile.write(index255.getIndexFileId(), this.id, ByteBuffer.wrap(data), 0, this.revision);
-		index255.write(new IndexEntry(index255, id, sector, compressed.length));
+		index255.write(new IndexEntry(index255, id, res.sector, res.compressedLength));
 	}
 	
 	private void checkRevision(InputStream stream, int compressedLength)
 	{
 		int offset = stream.getOffset();
-		if (stream.getLength() - (compressedLength + stream.getOffset()) >= 2) {
+		if (stream.getLength() - (compressedLength + stream.getOffset()) >= 2)
+		{
 			stream.setOffset(stream.getLength() - 2);
 			this.revision = stream.readUnsignedShort();
 			stream.setOffset(offset);
 		}
-		else {
+		else
+		{
 			this.revision = -1;
 		}
 
@@ -308,41 +311,42 @@ public class Index implements Closeable
 			IndexEntry entry = this.index.read(a.getArchiveId());
 			assert this.index.getIndexFileId() == this.id;
 			assert entry.getId() == a.getArchiveId();
-			byte[] b = store.getData().read(this.id, entry.getId(), entry.getSector(), entry.getLength()); // needs decompress etc...
+			DataFileReadResult res = store.getData().read(this.id, entry.getId(), entry.getSector(), entry.getLength()); // needs decompress etc...
-			
+			byte[] data = res.data;
-			InputStream stream = new InputStream(b);
+//			
-
+//			InputStream stream = new InputStream(b);
-			this.compression = stream.readUnsignedByte();
+//
-			int compressedLength = stream.readInt();
+//			this.compression = stream.readUnsignedByte();
-			if (compressedLength < 0 || compressedLength > 1000000)
+//			int compressedLength = stream.readInt();
-			{
+//			if (compressedLength < 0 || compressedLength > 1000000)
-				throw new RuntimeException("Invalid archive header");
+//			{
-			}
+//				throw new RuntimeException("Invalid archive header");
-
+//			}
-			byte[] data;
+//
-			switch (compression)
+//			byte[] data;
-			{
+//			switch (compression)
-				case 0:
+//			{
-					data = new byte[compressedLength];
+//				case 0:
-					this.checkRevision(stream, compressedLength);
+//					data = new byte[compressedLength];
-					stream.readBytes(data, 0, compressedLength);
+//					this.checkRevision(stream, compressedLength);
-					break;
+//					stream.readBytes(data, 0, compressedLength);
-				case 1:
+//					break;
-				{
+//				case 1:
-					int length = stream.readInt();
+//				{
-					data = new byte[length];
+//					int length = stream.readInt();
-					this.checkRevision(stream, compressedLength);
+//					data = new byte[length];
-					BZip2Decompressor.decompress(data, b, compressedLength, 9);
+//					this.checkRevision(stream, compressedLength);
-					break;
+//					BZip2Decompressor.decompress(data, b, compressedLength, 9);
-				}
+//					break;
-				default:
+//				}
-				{
+//				default:
-					int length = stream.readInt();
+//				{
-					data = new byte[length];
+//					int length = stream.readInt();
-					this.checkRevision(stream, compressedLength);
+//					data = new byte[length];
-					GZipDecompressor.decompress(stream, data);
+//					this.checkRevision(stream, compressedLength);
-				}
+//					GZipDecompressor.decompress(stream, data);
-			}
+//				}
+//			}
 			
 			if (a.getFiles().size() == 1)
 			{
@@ -356,7 +360,7 @@ public class Index implements Closeable
 			--readPosition;
 			int amtOfLoops = data[readPosition] & 255;
 			readPosition -= amtOfLoops * filesCount * 4;
-			stream = new InputStream(data);
+			InputStream stream = new InputStream(data);
 			stream.setOffset(readPosition);
 			int[] filesSize = new int[filesCount];
 
@@ -444,24 +448,24 @@ public class Index implements Closeable
 			stream.setOffset(0);
 			stream.getBytes(fileData, 0, fileData.length);
 
-			stream = new OutputStream();
+//			stream = new OutputStream();
-			
+//			
-			stream.writeByte(0); // compression
+//			stream.writeByte(0); // compression
-			stream.writeInt(fileData.length);
+//			stream.writeInt(fileData.length);
-			
+//			
-			stream.writeBytes(fileData);
+//			stream.writeBytes(fileData);
-			stream.writeShort(this.revision);
+//			stream.writeShort(this.revision);
-			
+//			
-			byte[] finalFileData = new byte[stream.getOffset()];
+//			byte[] finalFileData = new byte[stream.getOffset()];
-			stream.setOffset(0);
+//			stream.setOffset(0);
-			stream.getBytes(finalFileData, 0, finalFileData.length);
+//			stream.getBytes(finalFileData, 0, finalFileData.length);
 			
 			assert this.index.getIndexFileId() == this.id;
 			DataFile data = store.getData();
 			
 			// XXX old data is just left there in the file?
-			int sector = data.write(this.id, a.getArchiveId(), ByteBuffer.wrap(finalFileData));
+			DataFileWriteResult res = data.write(this.id, a.getArchiveId(), ByteBuffer.wrap(fileData), 0, this.revision);
-			this.index.write(new IndexEntry(this.index, a.getArchiveId(), sector, finalFileData.length));
+			this.index.write(new IndexEntry(this.index, a.getArchiveId(), res.sector, res.compressedLength));
 		}
 	}
 	

src/test/java/net/runelite/cache/fs/DataFileTest.java

@@ -26,8 +26,9 @@ public class DataFileTest
 		File file = folder.newFile();
 		Store store = new Store(folder.getRoot());
 		DataFile df = new DataFile(store, file);
-		int sector = df.write(42, 3, ByteBuffer.wrap("test".getBytes()));
+		DataFileWriteResult res = df.write(42, 3, ByteBuffer.wrap("test".getBytes()), 0, 0);
-		byte[] buf = df.read(42, 3, sector, 4);
+		DataFileReadResult res2 = df.read(42, 3, res.sector, res.compressedLength);
+		byte[] buf = res2.data;
 		String str = new String(buf);
 		Assert.assertEquals("test", str);
 		file.delete();
@@ -37,13 +38,15 @@ public class DataFileTest
 	public void test2() throws IOException
 	{
 		byte[] b = new byte[1024];
-		for (int i = 0; i < 1024; ++i) b[i] = (byte) i;
+		for (int i = 0; i < 1024; ++i)
+			b[i] = (byte) i;
 		
 		File file = folder.newFile();
 		Store store = new Store(folder.getRoot());
 		DataFile df = new DataFile(store, file);
-		int sector = df.write(42, 0x1FFFF, ByteBuffer.wrap(b));
+		DataFileWriteResult res = df.write(42, 0x1FFFF, ByteBuffer.wrap(b), 0, 0);
-		byte[] buf = df.read(42, 0x1FFFF, sector, b.length);
+		DataFileReadResult res2 = df.read(42, 0x1FFFF, res.sector, res.compressedLength);
+		byte[] buf = res2.data;
 		Assert.assertArrayEquals(b, buf);
 		file.delete();
 	}