diff --git a/cache/src/main/java/net/runelite/cache/fs/Archive.java b/cache/src/main/java/net/runelite/cache/fs/Archive.java index ec0853dd92..57c016bc22 100644 --- a/cache/src/main/java/net/runelite/cache/fs/Archive.java +++ b/cache/src/main/java/net/runelite/cache/fs/Archive.java @@ -22,12 +22,14 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ - package net.runelite.cache.fs; +import com.google.common.io.Files; import java.io.IOException; +import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Objects; import net.runelite.cache.io.InputStream; @@ -38,9 +40,9 @@ import org.slf4j.LoggerFactory; public class Archive { private static final Logger logger = LoggerFactory.getLogger(Archive.class); - + private Index index; // member of this index - + private byte[] data; // raw data from the datafile, compressed/encrypted private int archiveId; @@ -51,7 +53,7 @@ public class Archive private int compression; private List files = new ArrayList<>(); - + public Archive(Index index, int id) { this.index = index; @@ -109,18 +111,18 @@ public class Archive { this.data = data; } - + public File addFile(int id) { File file = new File(this, id); this.files.add(file); return file; } - - public void load(InputStream stream, int numberOfFiles, int protocol) + + public void loadFiles(InputStream stream, int numberOfFiles, int protocol) { int archive = 0; - + for (int i = 0; i < numberOfFiles; ++i) { int fileId = archive += protocol >= 7 ? stream.readBigSmart() : stream.readUnsignedShort(); @@ -140,7 +142,7 @@ public class Archive logger.warn("Unable to decrypt archive {}", this); return; } - + byte[] decompressedData = res.data; if (this.crc != res.crc) @@ -164,7 +166,7 @@ public class Archive loadContents(decompressedData); this.setData(null); // now that we've loaded it, clean it so it doesn't get written back } - + public void loadContents(byte[] data) { logger.trace("Loading contents of archive {} ({} files)", archiveId, files.size()); @@ -217,9 +219,9 @@ public class Archive for (int id = 0; id < filesCount; ++id) { int chunkSize = chunkSizes[id][chunk]; - + stream.readBytes(fileContents[id], fileOffsets[id], chunkSize); - + fileOffsets[id] += chunkSize; } } @@ -238,7 +240,7 @@ public class Archive logger.trace("Saving contents of archive {}/{} using cached data", index.getId(), archiveId); return data; } - + OutputStream stream = new OutputStream(); int filesCount = this.getFiles().size(); @@ -276,7 +278,143 @@ public class Archive return fileData; } - + + public void saveTree(java.io.File to) throws IOException + { + if (data != null) + { + assert files.size() == 1; // this is the maps + + File file = files.get(0); + + java.io.File archiveFile = new java.io.File(to, this.getArchiveId() + "-" + file.getFileId() + "-" + file.getNameHash() + ".datc"); + Files.write(data, archiveFile); + + archiveFile = new java.io.File(to, this.getArchiveId() + ".rev"); + Files.write("" + this.getRevision(), archiveFile, Charset.defaultCharset()); + + archiveFile = new java.io.File(to, this.getArchiveId() + ".name"); + Files.write("" + this.getNameHash(), archiveFile, Charset.defaultCharset()); + return; + } + + if (files.size() == 1) + { + File file = this.getFiles().get(0); + + java.io.File archiveFile = new java.io.File(to, this.getArchiveId() + "-" + file.getFileId() + "-" + file.getNameHash() + ".dat"); + byte[] contents = file.getContents(); + + Files.write(contents, archiveFile); + + archiveFile = new java.io.File(to, this.getArchiveId() + ".rev"); + Files.write("" + this.getRevision(), archiveFile, Charset.defaultCharset()); + + archiveFile = new java.io.File(to, this.getArchiveId() + ".name"); + Files.write("" + this.getNameHash(), archiveFile, Charset.defaultCharset()); + return; + } + + java.io.File archiveFile = new java.io.File(to, this.getArchiveId() + ".rev"); + Files.write("" + this.getRevision(), archiveFile, Charset.defaultCharset()); + + archiveFile = new java.io.File(to, this.getArchiveId() + ".name"); + Files.write("" + this.getNameHash(), archiveFile, Charset.defaultCharset()); + + java.io.File archiveFolder = new java.io.File(to, "" + this.getArchiveId()); + archiveFolder.mkdirs(); + + for (File file : files) + { + archiveFile = new java.io.File(archiveFolder, file.getFileId() + "-" + file.getNameHash() + ".dat"); + byte[] contents = file.getContents(); + Files.write(contents, archiveFile); + } + } + + public void loadTreeData(java.io.File parent, java.io.File from) throws IOException + { + //archiveId-fileId-fileName - assumes name isn't negative + String[] parts = Files.getNameWithoutExtension(from.getName()).split("-"); + int archiveId = Integer.parseInt(parts[0]); + int fileId = Integer.parseInt(parts[1]); + int nameHash = Integer.parseInt(parts[2]); + + assert archiveId == this.getArchiveId(); + + data = Files.toByteArray(from); + + File file = new File(this, fileId); + file.setNameHash(nameHash); + + files.add(file); + + java.io.File archiveFile = new java.io.File(parent, this.getArchiveId() + ".rev"); + int rev = Integer.parseInt(Files.readFirstLine(archiveFile, Charset.defaultCharset())); + this.setRevision(rev); + + archiveFile = new java.io.File(parent, this.getArchiveId() + ".name"); + int name = Integer.parseInt(Files.readFirstLine(archiveFile, Charset.defaultCharset())); + this.setNameHash(name); + } + + public void loadTreeSingleFile(java.io.File parent, java.io.File from) throws IOException + { + //archiveId-fileId-fileName + String[] parts = Files.getNameWithoutExtension(from.getName()).split("-"); + int archiveId = Integer.parseInt(parts[0]); + int fileId = Integer.parseInt(parts[1]); + int nameHash = Integer.parseInt(parts[2]); + + assert archiveId == this.getArchiveId(); + + File file = new File(this, fileId); + file.setNameHash(nameHash); + + byte[] contents = Files.toByteArray(from); + file.setContents(contents); + + files.add(file); + + java.io.File archiveFile = new java.io.File(parent, this.getArchiveId() + ".rev"); + int rev = Integer.parseInt(Files.readFirstLine(archiveFile, Charset.defaultCharset())); + this.setRevision(rev); + + archiveFile = new java.io.File(parent, this.getArchiveId() + ".name"); + int name = Integer.parseInt(Files.readFirstLine(archiveFile, Charset.defaultCharset())); + this.setNameHash(name); + } + + public void loadTree(java.io.File parent, java.io.File from) throws IOException + { + for (java.io.File file : from.listFiles()) + { + //fileId-fileName.dat + String[] split = Files.getNameWithoutExtension(file.getName()).split("-"); + int fileId = Integer.parseInt(split[0]); + int fileName = Integer.parseInt(split[1]); + + File f = new File(this, fileId); + f.setNameHash(fileName); + + byte[] contents = Files.toByteArray(file); + f.setContents(contents); + + files.add(f); + } + + java.io.File archiveFile = new java.io.File(parent, this.getArchiveId() + ".rev"); + int rev = Integer.parseInt(Files.readFirstLine(archiveFile, Charset.defaultCharset())); + this.setRevision(rev); + + archiveFile = new java.io.File(parent, this.getArchiveId() + ".name"); + int name = Integer.parseInt(Files.readFirstLine(archiveFile, Charset.defaultCharset())); + this.setNameHash(name); + + // the filesystem may order these differently (eg, 1, 10, 2) + Collections.sort(files, (f1, f2) -> Integer.compare(f1.getFileId(), f2.getFileId())); + } + public void loadNames(InputStream stream, int numberOfFiles) { for (int i = 0; i < numberOfFiles; ++i) diff --git a/cache/src/main/java/net/runelite/cache/fs/Index.java b/cache/src/main/java/net/runelite/cache/fs/Index.java index 47390cd6a0..b0aee52de8 100644 --- a/cache/src/main/java/net/runelite/cache/fs/Index.java +++ b/cache/src/main/java/net/runelite/cache/fs/Index.java @@ -22,12 +22,14 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ - package net.runelite.cache.fs; +import com.google.common.io.Files; import java.io.Closeable; import java.io.IOException; +import java.nio.charset.Charset; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Objects; import net.runelite.cache.util.Djb2; @@ -40,11 +42,11 @@ import org.slf4j.LoggerFactory; public class Index implements Closeable { private static final Logger logger = LoggerFactory.getLogger(Index.class); - + private final Store store; private final IndexFile index; private final int id; - + private XteaKeyManager xteaManager; private int protocol = 7; @@ -55,14 +57,14 @@ public class Index implements Closeable private int compression; // compression method of this index's data in 255 private final List archives = new ArrayList<>(); - + public Index(Store store, IndexFile index, int id) { this.store = store; this.index = index; this.id = id; } - + @Override public void close() throws IOException { @@ -160,19 +162,23 @@ public class Index implements Closeable { return archives; } - + public Archive addArchive(int id) { Archive archive = new Archive(this, id); this.archives.add(archive); return archive; } - + public Archive getArchive(int id) { for (Archive a : archives) + { if (a.getArchiveId() == id) + { return a; + } + } return null; } @@ -180,25 +186,29 @@ public class Index implements Closeable { int hash = Djb2.hash(name); for (Archive a : archives) + { if (a.getNameHash() == hash) + { return a; + } + } return null; } - + public void load() throws IOException { logger.trace("Loading index {}", id); DataFile dataFile = store.getData(); IndexFile index255 = store.getIndex255(); - + IndexEntry entry = index255.read(id); byte[] indexData = dataFile.read(index255.getIndexFileId(), entry.getId(), entry.getSector(), entry.getLength()); DataFileReadResult res = DataFile.decompress(indexData, null); byte[] data = res.data; archives.clear(); - + readIndexData(data); this.crc = res.crc; @@ -208,11 +218,11 @@ public class Index implements Closeable this.loadArchives(); } - + public void save() throws IOException { saveArchives(); - + byte[] data = this.writeIndexData(); DataFile dataFile = store.getData(); @@ -226,7 +236,63 @@ public class Index implements Closeable this.crc = res.crc; this.whirlpool = res.whirlpool; } - + + public void saveTree(java.io.File to) throws IOException + { + java.io.File idx = new java.io.File(to, "" + this.getId()); + idx.mkdirs(); + + for (Archive a : archives) + { + a.saveTree(idx); + } + + java.io.File rev = new java.io.File(to, this.getId() + ".rev"); + Files.write("" + this.getRevision(), rev, Charset.defaultCharset()); + } + + public void loadTree(java.io.File parent, java.io.File to) throws IOException + { + for (java.io.File f : to.listFiles()) + { + if (f.isDirectory()) + { + int id = Integer.parseInt(f.getName()); + + Archive archive = new Archive(this, id); + archive.loadTree(to, f); + archives.add(archive); + } + else if (f.getName().endsWith(".dat")) + { + // one file. archiveId-fileId-name + String[] parts = Files.getNameWithoutExtension(f.getName()).split("-"); + + int id = Integer.parseInt(parts[0]); + + Archive archive = new Archive(this, id); + archive.loadTreeSingleFile(to, f); + archives.add(archive); + } + else if (f.getName().endsWith(".datc")) + { + // packed data + String[] parts = Files.getNameWithoutExtension(f.getName()).split("-"); + + int id = Integer.parseInt(parts[0]); + + Archive archive = new Archive(this, id); + archive.loadTreeData(to, f); + archives.add(archive); + } + } + + String str = Files.readFirstLine(new java.io.File(parent, this.getId() + ".rev"), Charset.defaultCharset()); + revision = Integer.parseInt(str); + + Collections.sort(archives, (ar1, ar2) -> Integer.compare(ar1.getArchiveId(), ar2.getArchiveId())); + } + public void readIndexData(byte[] data) { InputStream stream = new InputStream(data); @@ -304,7 +370,7 @@ public class Index implements Closeable archive = 0; Archive a = this.archives.get(index); - a.load(stream, numberOfFiles[index], protocol); + a.loadFiles(stream, numberOfFiles[index], protocol); } if (named) @@ -317,7 +383,7 @@ public class Index implements Closeable } } } - + private void loadArchives() throws IOException { // get data from index file @@ -347,7 +413,7 @@ public class Index implements Closeable a.decompressAndLoad(null); } } - + public void saveArchives() throws IOException { for (Archive a : archives) @@ -372,14 +438,14 @@ public class Index implements Closeable DataFileWriteResult res = data.write(this.id, a.getArchiveId(), compressedData, rev); this.index.write(new IndexEntry(this.index, a.getArchiveId(), res.sector, res.compressedLength)); - + logger.trace("Saved archive {}/{} at sector {}, compressed length {}", this.getId(), a.getArchiveId(), res.sector, res.compressedLength); a.setCrc(res.crc); a.setWhirlpool(res.whirlpool); } } - + public byte[] writeIndexData() { OutputStream stream = new OutputStream(); @@ -454,7 +520,7 @@ public class Index implements Closeable for (data = 0; data < this.archives.size(); ++data) { Archive a = this.archives.get(data); - + int len = a.getFiles().size(); if (protocol >= 7) diff --git a/cache/src/main/java/net/runelite/cache/fs/Store.java b/cache/src/main/java/net/runelite/cache/fs/Store.java index ac2a0b3f13..1976856117 100644 --- a/cache/src/main/java/net/runelite/cache/fs/Store.java +++ b/cache/src/main/java/net/runelite/cache/fs/Store.java @@ -22,7 +22,6 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ - package net.runelite.cache.fs; import java.io.Closeable; @@ -30,6 +29,7 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Objects; import net.runelite.cache.IndexType; @@ -43,19 +43,19 @@ public class Store implements Closeable private static final String MAIN_FILE_CACHE_DAT = "main_file_cache.dat2"; private static final String MAIN_FILE_CACHE_IDX = "main_file_cache.idx"; - + private final File folder; private final DataFile data; private final IndexFile index255; private final List indexes = new ArrayList<>(); - + public Store(File folder) throws IOException { this.folder = folder; - + data = new DataFile(this, new File(folder, MAIN_FILE_CACHE_DAT)); index255 = new IndexFile(this, 255, new File(folder, MAIN_FILE_CACHE_IDX + "255")); - + for (int i = 0; i < index255.getIndexCount(); ++i) { this.addIndex(i); @@ -76,7 +76,9 @@ public class Store implements Closeable data.close(); index255.close(); for (Index i : indexes) + { i.close(); + } } @Override @@ -105,18 +107,22 @@ public class Store implements Closeable } return true; } - + public final Index addIndex(int id) throws FileNotFoundException { for (Index i : indexes) + { if (i.getIndex().getIndexFileId() == id) + { throw new IllegalArgumentException("index " + id + " already exists"); - + } + } + IndexFile indexFile = new IndexFile(this, id, new File(folder, MAIN_FILE_CACHE_IDX + id)); Index index = new Index(this, indexFile, id); - + this.indexes.add(index); - + return index; } @@ -125,13 +131,15 @@ public class Store implements Closeable assert indexes.contains(index); indexes.remove(index); } - + public void load() throws IOException { for (Index i : indexes) + { i.load(); + } } - + public void save() throws IOException { logger.debug("Clearing data and indexes in preparation for store save"); @@ -139,10 +147,41 @@ public class Store implements Closeable data.clear(); for (Index i : indexes) + { i.clear(); + } for (Index i : indexes) + { i.save(); + } + } + + public void saveTree(java.io.File to) throws IOException + { + for (Index i : indexes) + { + i.saveTree(to); + } + } + + public void loadTree(java.io.File from) throws IOException + { + for (java.io.File idx : from.listFiles()) + { + if (!idx.isDirectory()) + { + continue; + } + + int id = Integer.parseInt(idx.getName()); + IndexFile indexFile = new IndexFile(this, id, new File(folder, MAIN_FILE_CACHE_IDX + id)); + Index index = new Index(this, indexFile, id); + index.loadTree(from, idx); + indexes.add(index); + } + + Collections.sort(indexes, (idx1, idx2) -> Integer.compare(idx1.getId(), idx2.getId())); } public DataFile getData() @@ -159,7 +198,7 @@ public class Store implements Closeable { return indexes; } - + public Index getIndex(IndexType type) { return indexes.get(type.getNumber()); @@ -168,8 +207,12 @@ public class Store implements Closeable public final Index findIndex(int id) { for (Index i : indexes) + { if (i.getId() == id) + { return i; + } + } return null; } } diff --git a/cache/src/test/java/net/runelite/cache/fs/StoreLoadTest.java b/cache/src/test/java/net/runelite/cache/fs/StoreLoadTest.java index f9033efe38..c8052e9975 100644 --- a/cache/src/test/java/net/runelite/cache/fs/StoreLoadTest.java +++ b/cache/src/test/java/net/runelite/cache/fs/StoreLoadTest.java @@ -22,14 +22,13 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ - package net.runelite.cache.fs; import com.google.common.io.Files; -import java.io.FileOutputStream; import java.io.IOException; import net.runelite.cache.StoreLocation; import org.junit.Assert; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -55,7 +54,9 @@ public class StoreLoadTest java.io.File testStoreFile = folder.newFolder(); for (java.io.File f : StoreLocation.LOCATION.listFiles()) + { Files.copy(f, new java.io.File(testStoreFile, f.getName())); + } Store testStore = new Store(testStoreFile); testStore.load(); @@ -68,7 +69,8 @@ public class StoreLoadTest Assert.assertTrue(store.equals(testStore)); } - //@Test + @Test + @Ignore public void unpackStore() throws IOException { java.io.File base = StoreLocation.LOCATION; @@ -76,29 +78,22 @@ public class StoreLoadTest { store.load(); - for (Index i : store.getIndexes()) - { - java.io.File ifile = new java.io.File(folder.newFolder(), "" + i.getId()); - ifile.mkdir(); + store.saveTree(folder.newFolder()); + } + } - for (Archive a : i.getArchives()) - { - java.io.File afile = new java.io.File(ifile, "" + a.getArchiveId()); - afile.mkdir(); + @Test + @Ignore + public void loadTree() throws IOException + { + Store store = new Store(folder.newFolder()); + store.loadTree(new java.io.File("C:\\rs\\temp\\tree")); - for (File f : a.getFiles()) - { - java.io.File ffile = new java.io.File(afile, "" + f.getFileId()); - try (FileOutputStream fout = new FileOutputStream(ffile)) - { - if (f.getContents() != null) - { - fout.write(f.getContents()); - } - } - } - } - } + try (Store store2 = new Store(StoreLocation.LOCATION)) + { + store2.load(); + + Assert.assertEquals(store, store2); } } } diff --git a/cache/src/test/java/net/runelite/cache/fs/StoreTest.java b/cache/src/test/java/net/runelite/cache/fs/StoreTest.java index cea72809cd..bb5ec930a7 100644 --- a/cache/src/test/java/net/runelite/cache/fs/StoreTest.java +++ b/cache/src/test/java/net/runelite/cache/fs/StoreTest.java @@ -22,7 +22,6 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ - package net.runelite.cache.fs; import java.io.IOException; @@ -35,9 +34,11 @@ import org.junit.rules.TemporaryFolder; public class StoreTest { + private static final int NUMBER_OF_FILES = 1024; + @Rule public TemporaryFolder folder = StoreLocation.getTemporaryFolder(); - + @Test public void testOneFile() throws IOException { @@ -50,29 +51,28 @@ public class StoreTest file.setContents("test".getBytes()); store.save(); - + try (Store store2 = new Store(folder.getRoot())) { store2.load(); - + Assert.assertEquals(store, store2); } } } - - private static final int NUMBER_OF_FILES = 1024; - + @Test public void testManyFiles() throws IOException { Random random = new Random(42L); + java.io.File root = folder.newFolder(); - try (Store store = new Store(folder.getRoot())) + try (Store store = new Store(root)) { Index index = store.addIndex(0); Archive archive = index.addArchive(0); archive.setNameHash(random.nextInt()); - + for (int i = 0; i < NUMBER_OF_FILES; ++i) { File file = archive.addFile(i); @@ -81,70 +81,80 @@ public class StoreTest random.nextBytes(data); file.setContents(data); } - + store.save(); - - try (Store store2 = new Store(folder.getRoot())) + + try (Store store2 = new Store(root)) { store2.load(); - + Assert.assertEquals(store, store2); } } } - + @Test public void testMultipleArchives() throws IOException { Random random = new Random(43L); + java.io.File root = folder.newFolder(); - try (Store store = new Store(folder.getRoot())) + try (Store store = new Store(root)) { Index index = store.addIndex(0); Index index2 = store.addIndex(1); - + Archive archive = index.addArchive(0); - archive.setNameHash(random.nextInt()); - + archive.setNameHash(random.nextInt(Integer.MAX_VALUE)); + Archive archive2 = index.addArchive(1); - + Archive archive3 = index2.addArchive(0); - + for (int i = 0; i < NUMBER_OF_FILES; ++i) { File file = archive.addFile(i); - file.setNameHash(random.nextInt()); + file.setNameHash(random.nextInt(Integer.MAX_VALUE)); byte[] data = new byte[random.nextInt(1024)]; random.nextBytes(data); file.setContents(data); } - + for (int i = 0; i < NUMBER_OF_FILES; ++i) { File file = archive2.addFile(i); - file.setNameHash(random.nextInt()); + file.setNameHash(random.nextInt(Integer.MAX_VALUE)); byte[] data = new byte[random.nextInt(1024)]; random.nextBytes(data); file.setContents(data); } - + for (int i = 0; i < NUMBER_OF_FILES; ++i) { File file = archive3.addFile(i); - file.setNameHash(random.nextInt()); + file.setNameHash(random.nextInt(Integer.MAX_VALUE)); byte[] data = new byte[random.nextInt(1024)]; random.nextBytes(data); file.setContents(data); } - + store.save(); - - try (Store store2 = new Store(folder.getRoot())) + + try (Store store2 = new Store(root)) { store2.load(); - + Assert.assertEquals(store, store2); } + + // Test tree save/load + java.io.File tree = folder.newFolder(); + store.saveTree(tree); + + try (Store store2 = new Store(folder.newFolder())) + { + store2.loadTree(tree); + } } } }