From a16bf14e08f6e42208c28a9ca0a8ec1f709fb48d Mon Sep 17 00:00:00 2001 From: Adam Date: Sat, 17 Oct 2015 11:47:26 -0400 Subject: [PATCH] datafile can handle all of the compression --- .../java/net/runelite/cache/fs/DataFile.java | 110 +++++++- .../runelite/cache/fs/DataFileReadResult.java | 7 + .../cache/fs/DataFileWriteResult.java | 6 + .../java/net/runelite/cache/fs/Index.java | 234 +++++++++--------- .../net/runelite/cache/fs/DataFileTest.java | 13 +- 5 files changed, 246 insertions(+), 124 deletions(-) create mode 100644 src/main/java/net/runelite/cache/fs/DataFileReadResult.java create mode 100644 src/main/java/net/runelite/cache/fs/DataFileWriteResult.java diff --git a/src/main/java/net/runelite/cache/fs/DataFile.java b/src/main/java/net/runelite/cache/fs/DataFile.java index bee25c46ea..24997d58dc 100644 --- a/src/main/java/net/runelite/cache/fs/DataFile.java +++ b/src/main/java/net/runelite/cache/fs/DataFile.java @@ -7,6 +7,10 @@ import java.io.IOException; import java.io.RandomAccessFile; import java.nio.ByteBuffer; import java.util.Objects; +import net.runelite.cache.fs.io.InputStream; +import net.runelite.cache.fs.io.OutputStream; +import net.runelite.cache.fs.util.bzip2.BZip2Decompressor; +import net.runelite.cache.fs.util.gzip.GZipDecompressor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -70,7 +74,7 @@ public class DataFile implements Closeable * @return * @throws IOException */ - public synchronized byte[] read(int indexId, int archiveId, int sector, int size) throws IOException + public synchronized DataFileReadResult read(int indexId, int archiveId, int sector, int size) throws IOException { if (sector <= 0L || dat.length() / 520L < (long) sector) { @@ -154,7 +158,10 @@ public class DataFile implements Closeable } buffer.flip(); - return buffer.array(); + + //XTEA decrypt here? + + return this.decompress(buffer.array()); } /** @@ -165,11 +172,14 @@ public class DataFile implements Closeable * @return the sector the data starts at * @throws IOException */ - public synchronized int write(int indexId, int archiveId, ByteBuffer data) throws IOException + public synchronized DataFileWriteResult write(int indexId, int archiveId, ByteBuffer data, int compression, int revision) throws IOException { int sector; int startSector; + data = ByteBuffer.wrap(this.compress(data.array(), compression, revision)); + int dataLen = data.remaining(); + sector = (int) ((dat.length() + (long) (SECTOR_SIZE - 1)) / (long) SECTOR_SIZE); if (sector == 0) { @@ -253,6 +263,98 @@ public class DataFile implements Closeable sector = nextSector; } - return startSector; + DataFileWriteResult res = new DataFileWriteResult(); + res.sector = startSector; + res.compressedLength = dataLen; + return res; + } + + private DataFileReadResult decompress(byte[] b) + { + InputStream stream = new InputStream(b); + + int compression = stream.readUnsignedByte(); + int compressedLength = stream.readInt(); + if (compressedLength < 0 || compressedLength > 1000000) + throw new RuntimeException("Invalid data"); + + byte[] data; + int revision; + switch (compression) + { + case 0: + data = new byte[compressedLength]; + revision = this.checkRevision(stream, compressedLength); + stream.readBytes(data, 0, compressedLength); + break; + case 1: + { + int length = stream.readInt(); + data = new byte[length]; + revision = this.checkRevision(stream, compressedLength); + BZip2Decompressor.decompress(data, b, compressedLength, 9); + break; + } + default: + { + int length = stream.readInt(); + data = new byte[length]; + revision = this.checkRevision(stream, compressedLength); + GZipDecompressor.decompress(stream, data); + } + } + + DataFileReadResult res = new DataFileReadResult(); + res.data = data; + res.revision = revision; + return res; + } + + private byte[] compress(byte[] data, int compression, int revision) + { + OutputStream stream = new OutputStream(); + stream.writeByte(compression); + byte[] compressedData; + switch (compression) + { + case 0: + compressedData = data; + stream.writeInt(data.length); + break; + default: + throw new RuntimeException(); +// case 1: +// compressedData = (byte[]) null; +// break; +// default: +// compressedData = GZipCompressor.compress(data); +// stream.writeInt(compressedData.length); +// stream.writeInt(data.length); + } + + stream.writeBytes(compressedData); + stream.writeShort(revision); + + byte[] compressed = new byte[stream.getOffset()]; + stream.setOffset(0); + stream.getBytes(compressed, 0, compressed.length); + return compressed; + } + + private int checkRevision(InputStream stream, int compressedLength) + { + int offset = stream.getOffset(); + int revision; + if (stream.getLength() - (compressedLength + stream.getOffset()) >= 2) + { + stream.setOffset(stream.getLength() - 2); + revision = stream.readUnsignedShort(); + stream.setOffset(offset); + } + else + { + revision = -1; + } + return revision; } } diff --git a/src/main/java/net/runelite/cache/fs/DataFileReadResult.java b/src/main/java/net/runelite/cache/fs/DataFileReadResult.java new file mode 100644 index 0000000000..28842ec242 --- /dev/null +++ b/src/main/java/net/runelite/cache/fs/DataFileReadResult.java @@ -0,0 +1,7 @@ +package net.runelite.cache.fs; + +public class DataFileReadResult +{ + public byte[] data; + public int revision; +} diff --git a/src/main/java/net/runelite/cache/fs/DataFileWriteResult.java b/src/main/java/net/runelite/cache/fs/DataFileWriteResult.java new file mode 100644 index 0000000000..6fa07179ef --- /dev/null +++ b/src/main/java/net/runelite/cache/fs/DataFileWriteResult.java @@ -0,0 +1,6 @@ +package net.runelite.cache.fs; + +public class DataFileWriteResult +{ + public int sector, compressedLength; +} diff --git a/src/main/java/net/runelite/cache/fs/Index.java b/src/main/java/net/runelite/cache/fs/Index.java index 412bf0ed9d..9d3a14a116 100644 --- a/src/main/java/net/runelite/cache/fs/Index.java +++ b/src/main/java/net/runelite/cache/fs/Index.java @@ -10,7 +10,6 @@ import java.util.Objects; import net.runelite.cache.fs.io.InputStream; import net.runelite.cache.fs.io.OutputStream; import net.runelite.cache.fs.util.bzip2.BZip2Decompressor; -import net.runelite.cache.fs.util.gzip.GZipCompressor; import net.runelite.cache.fs.util.gzip.GZipDecompressor; public class Index implements Closeable @@ -113,41 +112,43 @@ public class Index implements Closeable IndexFile index255 = store.getIndex255(); IndexEntry entry = index255.read(id); - byte[] b = dataFile.read(index255.getIndexFileId(), entry.getId(), entry.getSector(), entry.getLength()); - - InputStream stream = new InputStream(b); - - //XTEA decrypt here - - this.compression = stream.readUnsignedByte(); - int compressedLength = stream.readInt(); - if (compressedLength < 0 || compressedLength > 1000000) - throw new RuntimeException("Invalid archive header"); - - byte[] data; - switch (compression) - { - case 0: - data = new byte[compressedLength]; - this.checkRevision(stream, compressedLength); - stream.readBytes(data, 0, compressedLength); - break; - case 1: - { - int length = stream.readInt(); - data = new byte[length]; - this.checkRevision(stream, compressedLength); - BZip2Decompressor.decompress(data, b, compressedLength, 9); - break; - } - default: - { - int length = stream.readInt(); - data = new byte[length]; - this.checkRevision(stream, compressedLength); - GZipDecompressor.decompress(stream, data); - } - } + DataFileReadResult res = dataFile.read(index255.getIndexFileId(), entry.getId(), entry.getSector(), entry.getLength()); + byte[] data = res.data; +// byte[] b = dataFile.read(index255.getIndexFileId(), entry.getId(), entry.getSector(), entry.getLength()); +// +// InputStream stream = new InputStream(b); +// +// //XTEA decrypt here +// +// this.compression = stream.readUnsignedByte(); +// int compressedLength = stream.readInt(); +// if (compressedLength < 0 || compressedLength > 1000000) +// throw new RuntimeException("Invalid archive header"); +// +// byte[] data; +// switch (compression) +// { +// case 0: +// data = new byte[compressedLength]; +// this.checkRevision(stream, compressedLength); +// stream.readBytes(data, 0, compressedLength); +// break; +// case 1: +// { +// int length = stream.readInt(); +// data = new byte[length]; +// this.checkRevision(stream, compressedLength); +// BZip2Decompressor.decompress(data, b, compressedLength, 9); +// break; +// } +// default: +// { +// int length = stream.readInt(); +// data = new byte[length]; +// this.checkRevision(stream, compressedLength); +// GZipDecompressor.decompress(stream, data); +// } +// } readIndexData(data); @@ -160,51 +161,53 @@ public class Index implements Closeable byte[] data = this.writeIndexData(); - OutputStream stream = new OutputStream(); - stream.writeByte(this.compression); - byte[] compressedData; - switch (this.compression) - { - case 0: - compressedData = data; - stream.writeInt(data.length); - break; - default: - throw new RuntimeException(); -// case 1: -// compressedData = (byte[]) null; +// OutputStream stream = new OutputStream(); +// stream.writeByte(this.compression); +// byte[] compressedData; +// switch (this.compression) +// { +// case 0: +// compressedData = data; +// stream.writeInt(data.length); // break; // default: -// compressedData = GZipCompressor.compress(data); -// stream.writeInt(compressedData.length); -// stream.writeInt(data.length); - } - - stream.writeBytes(compressedData); - stream.writeShort(this.revision); - - byte[] compressed = new byte[stream.getOffset()]; - stream.setOffset(0); - stream.getBytes(compressed, 0, compressed.length); - - //XTEA encrypt here +// throw new RuntimeException(); +//// case 1: +//// compressedData = (byte[]) null; +//// break; +//// default: +//// compressedData = GZipCompressor.compress(data); +//// stream.writeInt(compressedData.length); +//// stream.writeInt(data.length); +// } +// +// stream.writeBytes(compressedData); +// stream.writeShort(this.revision); +// +// byte[] compressed = new byte[stream.getOffset()]; +// stream.setOffset(0); +// stream.getBytes(compressed, 0, compressed.length); +// +// //XTEA encrypt here DataFile dataFile = store.getData(); IndexFile index255 = store.getIndex255(); - int sector = dataFile.write(index255.getIndexFileId(), this.id, ByteBuffer.wrap(compressed)); - index255.write(new IndexEntry(index255, id, sector, compressed.length)); + DataFileWriteResult res = dataFile.write(index255.getIndexFileId(), this.id, ByteBuffer.wrap(data), 0, this.revision); + index255.write(new IndexEntry(index255, id, res.sector, res.compressedLength)); } private void checkRevision(InputStream stream, int compressedLength) { int offset = stream.getOffset(); - if (stream.getLength() - (compressedLength + stream.getOffset()) >= 2) { + if (stream.getLength() - (compressedLength + stream.getOffset()) >= 2) + { stream.setOffset(stream.getLength() - 2); this.revision = stream.readUnsignedShort(); stream.setOffset(offset); } - else { + else + { this.revision = -1; } @@ -308,41 +311,42 @@ public class Index implements Closeable IndexEntry entry = this.index.read(a.getArchiveId()); assert this.index.getIndexFileId() == this.id; assert entry.getId() == a.getArchiveId(); - byte[] b = store.getData().read(this.id, entry.getId(), entry.getSector(), entry.getLength()); // needs decompress etc... - - InputStream stream = new InputStream(b); - - this.compression = stream.readUnsignedByte(); - int compressedLength = stream.readInt(); - if (compressedLength < 0 || compressedLength > 1000000) - { - throw new RuntimeException("Invalid archive header"); - } - - byte[] data; - switch (compression) - { - case 0: - data = new byte[compressedLength]; - this.checkRevision(stream, compressedLength); - stream.readBytes(data, 0, compressedLength); - break; - case 1: - { - int length = stream.readInt(); - data = new byte[length]; - this.checkRevision(stream, compressedLength); - BZip2Decompressor.decompress(data, b, compressedLength, 9); - break; - } - default: - { - int length = stream.readInt(); - data = new byte[length]; - this.checkRevision(stream, compressedLength); - GZipDecompressor.decompress(stream, data); - } - } + DataFileReadResult res = store.getData().read(this.id, entry.getId(), entry.getSector(), entry.getLength()); // needs decompress etc... + byte[] data = res.data; +// +// InputStream stream = new InputStream(b); +// +// this.compression = stream.readUnsignedByte(); +// int compressedLength = stream.readInt(); +// if (compressedLength < 0 || compressedLength > 1000000) +// { +// throw new RuntimeException("Invalid archive header"); +// } +// +// byte[] data; +// switch (compression) +// { +// case 0: +// data = new byte[compressedLength]; +// this.checkRevision(stream, compressedLength); +// stream.readBytes(data, 0, compressedLength); +// break; +// case 1: +// { +// int length = stream.readInt(); +// data = new byte[length]; +// this.checkRevision(stream, compressedLength); +// BZip2Decompressor.decompress(data, b, compressedLength, 9); +// break; +// } +// default: +// { +// int length = stream.readInt(); +// data = new byte[length]; +// this.checkRevision(stream, compressedLength); +// GZipDecompressor.decompress(stream, data); +// } +// } if (a.getFiles().size() == 1) { @@ -356,7 +360,7 @@ public class Index implements Closeable --readPosition; int amtOfLoops = data[readPosition] & 255; readPosition -= amtOfLoops * filesCount * 4; - stream = new InputStream(data); + InputStream stream = new InputStream(data); stream.setOffset(readPosition); int[] filesSize = new int[filesCount]; @@ -444,24 +448,24 @@ public class Index implements Closeable stream.setOffset(0); stream.getBytes(fileData, 0, fileData.length); - stream = new OutputStream(); - - stream.writeByte(0); // compression - stream.writeInt(fileData.length); - - stream.writeBytes(fileData); - stream.writeShort(this.revision); - - byte[] finalFileData = new byte[stream.getOffset()]; - stream.setOffset(0); - stream.getBytes(finalFileData, 0, finalFileData.length); +// stream = new OutputStream(); +// +// stream.writeByte(0); // compression +// stream.writeInt(fileData.length); +// +// stream.writeBytes(fileData); +// stream.writeShort(this.revision); +// +// byte[] finalFileData = new byte[stream.getOffset()]; +// stream.setOffset(0); +// stream.getBytes(finalFileData, 0, finalFileData.length); assert this.index.getIndexFileId() == this.id; DataFile data = store.getData(); // XXX old data is just left there in the file? - int sector = data.write(this.id, a.getArchiveId(), ByteBuffer.wrap(finalFileData)); - this.index.write(new IndexEntry(this.index, a.getArchiveId(), sector, finalFileData.length)); + DataFileWriteResult res = data.write(this.id, a.getArchiveId(), ByteBuffer.wrap(fileData), 0, this.revision); + this.index.write(new IndexEntry(this.index, a.getArchiveId(), res.sector, res.compressedLength)); } } diff --git a/src/test/java/net/runelite/cache/fs/DataFileTest.java b/src/test/java/net/runelite/cache/fs/DataFileTest.java index 42d2e89ec0..7fca5fb158 100644 --- a/src/test/java/net/runelite/cache/fs/DataFileTest.java +++ b/src/test/java/net/runelite/cache/fs/DataFileTest.java @@ -26,8 +26,9 @@ public class DataFileTest File file = folder.newFile(); Store store = new Store(folder.getRoot()); DataFile df = new DataFile(store, file); - int sector = df.write(42, 3, ByteBuffer.wrap("test".getBytes())); - byte[] buf = df.read(42, 3, sector, 4); + DataFileWriteResult res = df.write(42, 3, ByteBuffer.wrap("test".getBytes()), 0, 0); + DataFileReadResult res2 = df.read(42, 3, res.sector, res.compressedLength); + byte[] buf = res2.data; String str = new String(buf); Assert.assertEquals("test", str); file.delete(); @@ -37,13 +38,15 @@ public class DataFileTest public void test2() throws IOException { byte[] b = new byte[1024]; - for (int i = 0; i < 1024; ++i) b[i] = (byte) i; + for (int i = 0; i < 1024; ++i) + b[i] = (byte) i; File file = folder.newFile(); Store store = new Store(folder.getRoot()); DataFile df = new DataFile(store, file); - int sector = df.write(42, 0x1FFFF, ByteBuffer.wrap(b)); - byte[] buf = df.read(42, 0x1FFFF, sector, b.length); + DataFileWriteResult res = df.write(42, 0x1FFFF, ByteBuffer.wrap(b), 0, 0); + DataFileReadResult res2 = df.read(42, 0x1FFFF, res.sector, res.compressedLength); + byte[] buf = res2.data; Assert.assertArrayEquals(b, buf); file.delete(); }