diff --git a/cache/pom.xml b/cache/pom.xml index f4091957fa..6436e19ac4 100644 --- a/cache/pom.xml +++ b/cache/pom.xml @@ -37,7 +37,7 @@ Cache - 154 + 160 4.6 diff --git a/cache/src/main/java/net/runelite/cache/AreaManager.java b/cache/src/main/java/net/runelite/cache/AreaManager.java index 3f45072fec..96967b84b6 100644 --- a/cache/src/main/java/net/runelite/cache/AreaManager.java +++ b/cache/src/main/java/net/runelite/cache/AreaManager.java @@ -24,6 +24,7 @@ */ package net.runelite.cache; +import java.io.IOException; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -31,8 +32,10 @@ import java.util.Map; import net.runelite.cache.definitions.AreaDefinition; import net.runelite.cache.definitions.loaders.AreaLoader; import net.runelite.cache.fs.Archive; +import net.runelite.cache.fs.ArchiveFiles; import net.runelite.cache.fs.FSFile; import net.runelite.cache.fs.Index; +import net.runelite.cache.fs.Storage; import net.runelite.cache.fs.Store; public class AreaManager @@ -45,12 +48,16 @@ public class AreaManager this.store = store; } - public void load() + public void load() throws IOException { + Storage storage = store.getStorage(); Index index = store.getIndex(IndexType.CONFIGS); Archive archive = index.getArchive(ConfigType.AREA.getId()); - for (FSFile file : archive.getFiles()) + byte[] archiveData = storage.loadArchive(archive); + ArchiveFiles files = archive.getFiles(archiveData); + + for (FSFile file : files.getFiles()) { AreaLoader loader = new AreaLoader(); AreaDefinition area = loader.load(file.getContents(), file.getFileId()); diff --git a/cache/src/main/java/net/runelite/cache/InterfaceManager.java b/cache/src/main/java/net/runelite/cache/InterfaceManager.java index 2856bc3381..d9e22139ec 100644 --- a/cache/src/main/java/net/runelite/cache/InterfaceManager.java +++ b/cache/src/main/java/net/runelite/cache/InterfaceManager.java @@ -33,8 +33,10 @@ import net.runelite.cache.definitions.InterfaceDefinition; import net.runelite.cache.definitions.exporters.InterfaceExporter; import net.runelite.cache.definitions.loaders.InterfaceLoader; import net.runelite.cache.fs.Archive; +import net.runelite.cache.fs.ArchiveFiles; import net.runelite.cache.fs.FSFile; import net.runelite.cache.fs.Index; +import net.runelite.cache.fs.Storage; import net.runelite.cache.fs.Store; import net.runelite.cache.util.Namer; @@ -49,17 +51,20 @@ public class InterfaceManager this.store = store; } - public void load() + public void load() throws IOException { InterfaceLoader loader = new InterfaceLoader(); + Storage storage = store.getStorage(); Index index = store.getIndex(IndexType.INTERFACES); for (Archive archive : index.getArchives()) { int archiveId = archive.getArchiveId(); + byte[] archiveData = storage.loadArchive(archive); + ArchiveFiles files = archive.getFiles(archiveData); - for (FSFile file : archive.getFiles()) + for (FSFile file : files.getFiles()) { int fileId = file.getFileId(); diff --git a/cache/src/main/java/net/runelite/cache/ItemManager.java b/cache/src/main/java/net/runelite/cache/ItemManager.java index 7ab3b4aea3..9d11ec1668 100644 --- a/cache/src/main/java/net/runelite/cache/ItemManager.java +++ b/cache/src/main/java/net/runelite/cache/ItemManager.java @@ -33,8 +33,10 @@ import net.runelite.cache.definitions.ItemDefinition; import net.runelite.cache.definitions.exporters.ItemExporter; import net.runelite.cache.definitions.loaders.ItemLoader; import net.runelite.cache.fs.Archive; +import net.runelite.cache.fs.ArchiveFiles; import net.runelite.cache.fs.FSFile; import net.runelite.cache.fs.Index; +import net.runelite.cache.fs.Storage; import net.runelite.cache.fs.Store; import net.runelite.cache.util.Namer; @@ -49,14 +51,18 @@ public class ItemManager this.store = store; } - public void load() + public void load() throws IOException { ItemLoader loader = new ItemLoader(); + Storage storage = store.getStorage(); Index index = store.getIndex(IndexType.CONFIGS); Archive archive = index.getArchive(ConfigType.ITEM.getId()); - for (FSFile f : archive.getFiles()) + byte[] archiveData = storage.loadArchive(archive); + ArchiveFiles files = archive.getFiles(archiveData); + + for (FSFile f : files.getFiles()) { ItemDefinition def = loader.load(f.getFileId(), f.getContents()); items.add(def); @@ -75,7 +81,7 @@ public class ItemManager for (ItemDefinition def : items) { ItemExporter exporter = new ItemExporter(def); - + File targ = new File(out, def.id + ".json"); exporter.exportTo(targ); } diff --git a/cache/src/main/java/net/runelite/cache/MapImageDumper.java b/cache/src/main/java/net/runelite/cache/MapImageDumper.java index 04d73c43d4..089ec5e062 100644 --- a/cache/src/main/java/net/runelite/cache/MapImageDumper.java +++ b/cache/src/main/java/net/runelite/cache/MapImageDumper.java @@ -39,14 +39,15 @@ import net.runelite.cache.definitions.OverlayDefinition; import net.runelite.cache.definitions.SpriteDefinition; import net.runelite.cache.definitions.TextureDefinition; import net.runelite.cache.definitions.UnderlayDefinition; -import net.runelite.cache.definitions.loaders.ObjectLoader; import net.runelite.cache.definitions.loaders.OverlayLoader; import net.runelite.cache.definitions.loaders.SpriteLoader; import net.runelite.cache.definitions.loaders.TextureLoader; import net.runelite.cache.definitions.loaders.UnderlayLoader; import net.runelite.cache.fs.Archive; +import net.runelite.cache.fs.ArchiveFiles; import net.runelite.cache.fs.FSFile; import net.runelite.cache.fs.Index; +import net.runelite.cache.fs.Storage; import net.runelite.cache.fs.Store; import net.runelite.cache.region.Location; import net.runelite.cache.region.Region; @@ -367,12 +368,16 @@ public class MapImageDumper logger.info("East most region: {}", regionLoader.getHighestX().getBaseX()); } - private void loadUnderlays(Store store) + private void loadUnderlays(Store store) throws IOException { + Storage storage = store.getStorage(); Index index = store.getIndex(IndexType.CONFIGS); Archive archive = index.getArchive(ConfigType.UNDERLAY.getId()); - for (FSFile file : archive.getFiles()) + byte[] archiveData = storage.loadArchive(archive); + ArchiveFiles files = archive.getFiles(archiveData); + + for (FSFile file : files.getFiles()) { UnderlayLoader loader = new UnderlayLoader(); UnderlayDefinition underlay = loader.load(file.getFileId(), file.getContents()); @@ -393,12 +398,16 @@ public class MapImageDumper return null; } - private void loadOverlays(Store store) + private void loadOverlays(Store store) throws IOException { + Storage storage = store.getStorage(); Index index = store.getIndex(IndexType.CONFIGS); Archive archive = index.getArchive(ConfigType.OVERLAY.getId()); - for (FSFile file : archive.getFiles()) + byte[] archiveData = storage.loadArchive(archive); + ArchiveFiles files = archive.getFiles(archiveData); + + for (FSFile file : files.getFiles()) { OverlayLoader loader = new OverlayLoader(); OverlayDefinition underlay = loader.load(file.getFileId(), file.getContents()); @@ -419,12 +428,16 @@ public class MapImageDumper return null; } - private void loadTextures(Store store) + private void loadTextures(Store store) throws IOException { + Storage storage = store.getStorage(); Index index = store.getIndex(IndexType.TEXTURES); Archive archive = index.getArchive(0); - for (FSFile file : archive.getFiles()) + byte[] archiveData = storage.loadArchive(archive); + ArchiveFiles files = archive.getFiles(archiveData); + + for (FSFile file : files.getFiles()) { TextureLoader loader = new TextureLoader(); TextureDefinition texture = loader.load(file.getFileId(), file.getContents()); @@ -446,19 +459,15 @@ public class MapImageDumper return null; } - private void loadSprites() + private void loadSprites() throws IOException { + Storage storage = store.getStorage(); Index index = store.getIndex(IndexType.SPRITES); final int mapsceneHash = Djb2.hash("mapscene"); for (Archive a : index.getArchives()) { - List files = a.getFiles(); - - assert files.size() == 1; - - FSFile file = files.get(0); - byte[] contents = file.getContents(); + byte[] contents = a.decompress(storage.loadArchive(a)); SpriteLoader loader = new SpriteLoader(); SpriteDefinition[] sprites = loader.load(a.getArchiveId(), contents); @@ -515,16 +524,12 @@ public class MapImageDumper return color.getRGB(); } - private void loadObjects(Store store) + private void loadObjects(Store store) throws IOException { - Index index = store.getIndex(IndexType.CONFIGS); - Archive archive = index.getArchive(ConfigType.OBJECT.getId()); - - ObjectLoader loader = new ObjectLoader(); - - for (FSFile f : archive.getFiles()) + ObjectManager manager = new ObjectManager(store); + manager.load(); + for (ObjectDefinition def : manager.getObjects()) { - ObjectDefinition def = loader.load(f.getFileId(), f.getContents()); objects.put(def.getId(), def); } } diff --git a/cache/src/main/java/net/runelite/cache/NpcManager.java b/cache/src/main/java/net/runelite/cache/NpcManager.java index b9f590257b..c881c776de 100644 --- a/cache/src/main/java/net/runelite/cache/NpcManager.java +++ b/cache/src/main/java/net/runelite/cache/NpcManager.java @@ -33,8 +33,10 @@ import net.runelite.cache.definitions.NpcDefinition; import net.runelite.cache.definitions.exporters.NpcExporter; import net.runelite.cache.definitions.loaders.NpcLoader; import net.runelite.cache.fs.Archive; +import net.runelite.cache.fs.ArchiveFiles; import net.runelite.cache.fs.FSFile; import net.runelite.cache.fs.Index; +import net.runelite.cache.fs.Storage; import net.runelite.cache.fs.Store; import net.runelite.cache.util.Namer; @@ -53,10 +55,14 @@ public class NpcManager { NpcLoader loader = new NpcLoader(); + Storage storage = store.getStorage(); Index index = store.getIndex(IndexType.CONFIGS); Archive archive = index.getArchive(ConfigType.NPC.getId()); - for (FSFile f : archive.getFiles()) + byte[] archiveData = storage.loadArchive(archive); + ArchiveFiles files = archive.getFiles(archiveData); + + for (FSFile f : files.getFiles()) { NpcDefinition npc = loader.load(f.getFileId(), f.getContents()); npcs.add(npc); diff --git a/cache/src/main/java/net/runelite/cache/ObjectManager.java b/cache/src/main/java/net/runelite/cache/ObjectManager.java index b7c6240cc4..5edfdfd718 100644 --- a/cache/src/main/java/net/runelite/cache/ObjectManager.java +++ b/cache/src/main/java/net/runelite/cache/ObjectManager.java @@ -33,8 +33,10 @@ import net.runelite.cache.definitions.ObjectDefinition; import net.runelite.cache.definitions.exporters.ObjectExporter; import net.runelite.cache.definitions.loaders.ObjectLoader; import net.runelite.cache.fs.Archive; +import net.runelite.cache.fs.ArchiveFiles; import net.runelite.cache.fs.FSFile; import net.runelite.cache.fs.Index; +import net.runelite.cache.fs.Storage; import net.runelite.cache.fs.Store; import net.runelite.cache.util.Namer; @@ -53,10 +55,14 @@ public class ObjectManager { ObjectLoader loader = new ObjectLoader(); + Storage storage = store.getStorage(); Index index = store.getIndex(IndexType.CONFIGS); Archive archive = index.getArchive(ConfigType.OBJECT.getId()); - for (FSFile f : archive.getFiles()) + byte[] archiveData = storage.loadArchive(archive); + ArchiveFiles files = archive.getFiles(archiveData); + + for (FSFile f : files.getFiles()) { ObjectDefinition def = loader.load(f.getFileId(), f.getContents()); objects.add(def); diff --git a/cache/src/main/java/net/runelite/cache/SpriteManager.java b/cache/src/main/java/net/runelite/cache/SpriteManager.java index 653a84b748..b622ae366a 100644 --- a/cache/src/main/java/net/runelite/cache/SpriteManager.java +++ b/cache/src/main/java/net/runelite/cache/SpriteManager.java @@ -31,13 +31,12 @@ import java.io.File; import java.io.IOException; import java.util.Collection; import java.util.Collections; -import java.util.List; import net.runelite.cache.definitions.SpriteDefinition; import net.runelite.cache.definitions.exporters.SpriteExporter; import net.runelite.cache.definitions.loaders.SpriteLoader; import net.runelite.cache.fs.Archive; -import net.runelite.cache.fs.FSFile; import net.runelite.cache.fs.Index; +import net.runelite.cache.fs.Storage; import net.runelite.cache.fs.Store; public class SpriteManager @@ -50,18 +49,14 @@ public class SpriteManager this.store = store; } - public void load() + public void load() throws IOException { + Storage storage = store.getStorage(); Index index = store.getIndex(IndexType.SPRITES); for (Archive a : index.getArchives()) { - List files = a.getFiles(); - - assert files.size() == 1; - - FSFile file = files.get(0); - byte[] contents = file.getContents(); + byte[] contents = a.decompress(storage.loadArchive(a)); SpriteLoader loader = new SpriteLoader(); SpriteDefinition[] defs = loader.load(a.getArchiveId(), contents); diff --git a/cache/src/main/java/net/runelite/cache/TextureManager.java b/cache/src/main/java/net/runelite/cache/TextureManager.java index 05c22f7b44..3014624b3b 100644 --- a/cache/src/main/java/net/runelite/cache/TextureManager.java +++ b/cache/src/main/java/net/runelite/cache/TextureManager.java @@ -24,13 +24,16 @@ */ package net.runelite.cache; +import java.io.IOException; import java.util.ArrayList; import java.util.List; import net.runelite.cache.definitions.TextureDefinition; import net.runelite.cache.definitions.loaders.TextureLoader; import net.runelite.cache.fs.Archive; +import net.runelite.cache.fs.ArchiveFiles; import net.runelite.cache.fs.FSFile; import net.runelite.cache.fs.Index; +import net.runelite.cache.fs.Storage; import net.runelite.cache.fs.Store; public class TextureManager @@ -43,14 +46,18 @@ public class TextureManager this.store = store; } - public void load() + public void load() throws IOException { + Storage storage = store.getStorage(); Index index = store.getIndex(IndexType.TEXTURES); Archive archive = index.getArchive(0); + byte[] archiveData = storage.loadArchive(archive); + ArchiveFiles files = archive.getFiles(archiveData); + TextureLoader loader = new TextureLoader(); - for (FSFile file : archive.getFiles()) + for (FSFile file : files.getFiles()) { TextureDefinition texture = loader.load(file.getFileId(), file.getContents()); textures.add(texture); diff --git a/cache/src/main/java/net/runelite/cache/client/CacheClient.java b/cache/src/main/java/net/runelite/cache/client/CacheClient.java index 59c126aff9..723c029696 100644 --- a/cache/src/main/java/net/runelite/cache/client/CacheClient.java +++ b/cache/src/main/java/net/runelite/cache/client/CacheClient.java @@ -44,11 +44,10 @@ import java.util.List; import java.util.Queue; import java.util.concurrent.CompletableFuture; import net.runelite.cache.fs.Archive; -import net.runelite.cache.fs.FSFile; import net.runelite.cache.fs.Index; +import net.runelite.cache.fs.Storage; import net.runelite.cache.fs.Store; import net.runelite.cache.index.ArchiveData; -import net.runelite.cache.index.FileData; import net.runelite.cache.index.IndexData; import net.runelite.cache.protocol.decoders.HandshakeResponseDecoder; import net.runelite.cache.protocol.encoders.ArchiveRequestEncoder; @@ -275,7 +274,7 @@ public class CacheClient implements AutoCloseable && existing.getCrc() == ad.getCrc() && existing.getNameHash() == ad.getNameHash()) { - logger.info("Archive {}/{} in index {} is up to date", + logger.debug("Archive {}/{} in index {} is up to date", ad.getId(), indexData.getArchives().length, index.getId()); continue; } @@ -310,13 +309,7 @@ public class CacheClient implements AutoCloseable archive.setNameHash(ad.getNameHash()); // Add files - archive.clearFiles(); - for (FileData fd : ad.getFiles()) - { - FSFile file = new FSFile(fd.getId()); - file.setNameHash(fd.getNameHash()); - archive.addFile(file); - } + archive.setFileData(ad.getFiles()); CompletableFuture future = requestFile(index.getId(), ad.getId(), false); future.handle((fr, ex) -> @@ -332,13 +325,24 @@ public class CacheClient implements AutoCloseable logger.warn("crc mismatch on downloaded archive {}/{}: {} != {}", archive.getIndex().getId(), archive.getArchiveId(), hash, archive.getCrc()); + throw new RuntimeException("crc mismatch"); } - archive.setData(data); - if (watcher != null) { - watcher.downloadComplete(archive); + watcher.downloadComplete(archive, data); + } + else + { + try + { + Storage storage = store.getStorage(); + storage.saveArchive(archive, data); + } + catch (IOException ex1) + { + logger.warn("unable to save archive data", ex1); + } } return null; }); diff --git a/cache/src/main/java/net/runelite/cache/client/DownloadWatcher.java b/cache/src/main/java/net/runelite/cache/client/DownloadWatcher.java index cd3cd0369b..3924c74541 100644 --- a/cache/src/main/java/net/runelite/cache/client/DownloadWatcher.java +++ b/cache/src/main/java/net/runelite/cache/client/DownloadWatcher.java @@ -29,5 +29,5 @@ import net.runelite.cache.fs.Archive; @FunctionalInterface public interface DownloadWatcher { - void downloadComplete(Archive archive); + void downloadComplete(Archive archive, byte[] data); } diff --git a/cache/src/main/java/net/runelite/cache/client/FileResult.java b/cache/src/main/java/net/runelite/cache/client/FileResult.java index 6bcefba514..2c5046af62 100644 --- a/cache/src/main/java/net/runelite/cache/client/FileResult.java +++ b/cache/src/main/java/net/runelite/cache/client/FileResult.java @@ -25,8 +25,7 @@ package net.runelite.cache.client; import java.io.IOException; -import net.runelite.cache.fs.jagex.DataFile; -import net.runelite.cache.fs.jagex.DataFileReadResult; +import net.runelite.cache.fs.Container; public class FileResult { @@ -63,7 +62,7 @@ public class FileResult public void decompress(int[] keys) throws IOException { - DataFileReadResult res = DataFile.decompress(compressedData, keys); + Container res = Container.decompress(compressedData, keys); contents = res.data; revision = res.revision; diff --git a/cache/src/main/java/net/runelite/cache/fs/Archive.java b/cache/src/main/java/net/runelite/cache/fs/Archive.java index 99b5ef3861..d7b8435f3c 100644 --- a/cache/src/main/java/net/runelite/cache/fs/Archive.java +++ b/cache/src/main/java/net/runelite/cache/fs/Archive.java @@ -24,11 +24,8 @@ */ package net.runelite.cache.fs; -import net.runelite.cache.fs.jagex.DataFile; -import net.runelite.cache.fs.jagex.DataFileReadResult; import java.io.IOException; -import java.util.List; -import java.util.Objects; +import net.runelite.cache.index.FileData; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,15 +35,13 @@ public class Archive private final Index index; // member of this index - private byte[] data; // raw data from the datafile, compressed/encrypted - private final int archiveId; private int nameHash; private int crc; private int revision; private int compression; - - private final ArchiveFiles files = new ArchiveFiles(); + private FileData[] fileData; + private byte[] hash; // used by webservice, sha256 hash of content public Archive(Index index, int id) { @@ -61,7 +56,6 @@ public class Archive hash = 47 * hash + this.archiveId; hash = 47 * hash + this.nameHash; hash = 47 * hash + this.revision; - hash = 47 * hash + Objects.hashCode(this.files); return hash; } @@ -89,10 +83,6 @@ public class Archive { return false; } - if (!Objects.equals(this.files, other.files)) - { - return false; - } return true; } @@ -101,68 +91,70 @@ public class Archive return index; } - public byte[] getData() + public byte[] decompress(byte[] data) throws IOException { - return data; + return decompress(data, null); } - public void setData(byte[] data) + public byte[] decompress(byte[] data, int[] keys) throws IOException { - this.data = data; - } + if (data == null) + { + return null; + } - public FSFile addFile(FSFile file) - { - this.files.addFile(file); - return file; - } + byte[] encryptedData = data; - public FSFile findFile(int id) - { - return this.files.findFile(id); - } - - public void decompressAndLoad(int[] keys) throws IOException - { - byte[] encryptedData = this.getData(); - - DataFileReadResult res = DataFile.decompress(encryptedData, keys); - if (res == null) + Container container = Container.decompress(encryptedData, keys); + if (container == null) { logger.warn("Unable to decrypt archive {}", this); - return; + return null; } - byte[] decompressedData = res.data; + byte[] decompressedData = container.data; - if (this.crc != res.crc) + if (this.crc != container.crc) { logger.warn("crc mismatch for archive {}/{}", index.getId(), this.getArchiveId()); - this.setCrc(res.crc); + throw new IOException("CRC mismatch for " + index.getId() + "/" + this.getArchiveId()); } - if (res.revision != -1 && this.getRevision() != res.revision) + if (container.revision != -1 && this.getRevision() != container.revision) { // compressed data doesn't always include a revision, but check it if it does logger.warn("revision mismatch for archive {}/{}, expected {} was {}", index.getId(), this.getArchiveId(), - this.getRevision(), res.revision); + this.getRevision(), container.revision); // I've seen this happen with vanilla caches where the // revision in the index data differs from the revision // stored for the archive data on disk... I assume this // is more correct - this.setRevision(res.revision); + this.setRevision(container.revision); } - setCompression(res.compression); - - files.loadContents(decompressedData); - this.setData(null); // now that we've loaded it, clean it so it doesn't get written back + setCompression(container.compression); + return decompressedData; } - public byte[] saveContents() + public ArchiveFiles getFiles(byte[] data) throws IOException { - return files.saveContents(); + return getFiles(data, null); + } + + public ArchiveFiles getFiles(byte[] data, int[] keys) throws IOException + { + byte[] decompressedData = decompress(data, keys); + + ArchiveFiles files = new ArchiveFiles(); + for (FileData fileEntry : fileData) + { + FSFile file = new FSFile(fileEntry.getId()); + file.setNameHash(fileEntry.getNameHash()); + files.addFile(file); + } + files.loadContents(decompressedData); + return files; } public int getArchiveId() @@ -210,13 +202,23 @@ public class Archive this.compression = compression; } - public List getFiles() + public FileData[] getFileData() { - return files.getFiles(); + return fileData; } - public void clearFiles() + public void setFileData(FileData[] fileData) { - files.clear(); + this.fileData = fileData; + } + + public byte[] getHash() + { + return hash; + } + + public void setHash(byte[] hash) + { + this.hash = hash; } } diff --git a/cache/src/main/java/net/runelite/cache/fs/Container.java b/cache/src/main/java/net/runelite/cache/fs/Container.java new file mode 100644 index 0000000000..9d57264bc7 --- /dev/null +++ b/cache/src/main/java/net/runelite/cache/fs/Container.java @@ -0,0 +1,216 @@ +/* + * Copyright (c) 2016-2017, Adam + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package net.runelite.cache.fs; + +import static com.google.common.primitives.Bytes.concat; +import com.google.common.primitives.Ints; +import java.io.IOException; +import net.runelite.cache.fs.jagex.CompressionType; +import net.runelite.cache.io.InputStream; +import net.runelite.cache.io.OutputStream; +import net.runelite.cache.util.BZip2; +import net.runelite.cache.util.Crc32; +import net.runelite.cache.util.GZip; +import net.runelite.cache.util.Xtea; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class Container +{ + private static final Logger logger = LoggerFactory.getLogger(Container.class); + + public byte[] data; + public int compression; // compression + public int revision; + public int crc; // crc of compressed data + + public Container(int compression, int revision) + { + this.compression = compression; + this.revision = revision; + } + + public void compress(byte[] data, int[] keys) throws IOException + { + OutputStream stream = new OutputStream(); + + byte[] compressedData; + int length; + switch (compression) + { + case CompressionType.NONE: + compressedData = data; + length = compressedData.length; + break; + case CompressionType.BZ2: + compressedData = concat(Ints.toByteArray(data.length), BZip2.compress(data)); + length = compressedData.length - 4; + break; + case CompressionType.GZ: + compressedData = concat(Ints.toByteArray(data.length), GZip.compress(data)); + length = compressedData.length - 4; + break; + default: + throw new RuntimeException("Unknown compression type"); + } + + compressedData = encrypt(compressedData, compressedData.length, keys); + + stream.writeByte(compression); + stream.writeInt(length); + + stream.writeBytes(compressedData); + if (revision != -1) + { + stream.writeShort(revision); + } + + this.data = stream.flip(); + } + + public static Container decompress(byte[] b, int[] keys) throws IOException + { + InputStream stream = new InputStream(b); + + int compression = stream.readUnsignedByte(); + int compressedLength = stream.readInt(); + if (compressedLength < 0 || compressedLength > 1000000) + { + throw new RuntimeException("Invalid data"); + } + + Crc32 crc32 = new Crc32(); + crc32.update(b, 0, 5); // compression + length + + byte[] data; + int revision = -1; + switch (compression) + { + case CompressionType.NONE: + { + byte[] encryptedData = new byte[compressedLength]; + stream.readBytes(encryptedData, 0, compressedLength); + + crc32.update(encryptedData, 0, compressedLength); + byte[] decryptedData = decrypt(encryptedData, encryptedData.length, keys); + + if (stream.remaining() >= 2) + { + revision = stream.readUnsignedShort(); + assert revision != -1; + } + + data = decryptedData; + + break; + } + case CompressionType.BZ2: + { + byte[] encryptedData = new byte[compressedLength + 4]; + stream.readBytes(encryptedData); + + crc32.update(encryptedData, 0, encryptedData.length); + byte[] decryptedData = decrypt(encryptedData, encryptedData.length, keys); + + if (stream.remaining() >= 2) + { + revision = stream.readUnsignedShort(); + assert revision != -1; + } + + stream = new InputStream(decryptedData); + + int decompressedLength = stream.readInt(); + data = BZip2.decompress(stream.getRemaining(), compressedLength); + + if (data == null) + { + return null; + } + + assert data.length == decompressedLength; + + break; + } + case CompressionType.GZ: + { + byte[] encryptedData = new byte[compressedLength + 4]; + stream.readBytes(encryptedData); + + crc32.update(encryptedData, 0, encryptedData.length); + byte[] decryptedData = decrypt(encryptedData, encryptedData.length, keys); + + if (stream.remaining() >= 2) + { + revision = stream.readUnsignedShort(); + assert revision != -1; + } + + stream = new InputStream(decryptedData); + + int decompressedLength = stream.readInt(); + data = GZip.decompress(stream.getRemaining(), compressedLength); + + if (data == null) + { + return null; + } + + assert data.length == decompressedLength; + + break; + } + default: + throw new RuntimeException("Unknown decompression type"); + } + + Container container = new Container(compression, revision); + container.data = data; + container.crc = crc32.getHash(); + return container; + } + + private static byte[] decrypt(byte[] data, int length, int[] keys) + { + if (keys == null) + { + return data; + } + + Xtea xtea = new Xtea(keys); + return xtea.decrypt(data, length); + } + + private static byte[] encrypt(byte[] data, int length, int[] keys) + { + if (keys == null) + { + return data; + } + + Xtea xtea = new Xtea(keys); + return xtea.encrypt(data, length); + } +} diff --git a/cache/src/main/java/net/runelite/cache/fs/Index.java b/cache/src/main/java/net/runelite/cache/fs/Index.java index 5ccf5e6e5d..c9331dbeda 100644 --- a/cache/src/main/java/net/runelite/cache/fs/Index.java +++ b/cache/src/main/java/net/runelite/cache/fs/Index.java @@ -24,20 +24,13 @@ */ package net.runelite.cache.fs; -import net.runelite.cache.fs.jagex.DataFile; -import net.runelite.cache.fs.jagex.CompressionType; -import io.netty.buffer.ByteBuf; -import io.netty.buffer.Unpooled; -import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Objects; import net.runelite.cache.index.ArchiveData; import net.runelite.cache.index.FileData; import net.runelite.cache.index.IndexData; -import net.runelite.cache.util.Crc32; import net.runelite.cache.util.Djb2; -import net.runelite.cache.util.XteaKeyManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,11 +38,8 @@ public class Index { private static final Logger logger = LoggerFactory.getLogger(Index.class); - private final Store store; private final int id; - private XteaKeyManager xteaManager; - private int protocol = 7; private boolean named = true; private int revision; @@ -58,9 +48,8 @@ public class Index private final List archives = new ArrayList<>(); - public Index(Store store, int id) + public Index(int id) { - this.store = store; this.id = id; } @@ -101,16 +90,6 @@ public class Index return true; } - public XteaKeyManager getXteaManager() - { - return xteaManager; - } - - public void setXteaManager(XteaKeyManager xteaManager) - { - this.xteaManager = xteaManager; - } - public int getId() { return id; @@ -203,48 +182,6 @@ public class Index return null; } - public void rebuildCrc() throws IOException - { - for (Archive a : archives) - { - int rev; // used for determining what part of compressedData to crc - byte[] compressedData; - - if (a.getData() != null) - { - compressedData = a.getData(); // data was never decompressed or loaded - rev = -1; // assume that this data has no revision? - } - else - { - byte[] fileData = a.saveContents(); - rev = a.getRevision(); - compressedData = DataFile.compress(fileData, a.getCompression(), a.getRevision(), null); - } - - int length = rev != -1 ? compressedData.length - 2 : compressedData.length; - Crc32 crc32 = new Crc32(); - crc32.update(compressedData, 0, length); - - int crc = crc32.getHash(); - - a.setCrc(crc); - } - - Crc32 crc = new Crc32(); - byte[] indexData = toIndexData().writeIndexData(); - - ByteBuf b = Unpooled.buffer(5, 5); - b.writeByte((byte) CompressionType.NONE); - b.writeInt(indexData.length); - - crc.update(b.array(), 0, 5); // crc includes compression type and length - crc.update(indexData, 0, indexData.length); - - int hash = crc.getHash(); - this.setCrc(hash); - } - public IndexData toIndexData() { IndexData data = new IndexData(); @@ -264,16 +201,8 @@ public class Index ad.setCrc(archive.getCrc()); ad.setRevision(archive.getRevision()); - FileData[] files = new FileData[archive.getFiles().size()]; + FileData[] files = archive.getFileData(); ad.setFiles(files); - - int idx2 = 0; - for (FSFile file : archive.getFiles()) - { - FileData fd = files[idx2++] = new FileData(); - fd.setId(file.getFileId()); - fd.setNameHash(file.getNameHash()); - } } return data; } diff --git a/cache/src/main/java/net/runelite/cache/fs/Storage.java b/cache/src/main/java/net/runelite/cache/fs/Storage.java index 0b10b075c9..6bcdb88819 100644 --- a/cache/src/main/java/net/runelite/cache/fs/Storage.java +++ b/cache/src/main/java/net/runelite/cache/fs/Storage.java @@ -36,4 +36,8 @@ public interface Storage extends AutoCloseable void load(Store store) throws IOException; void save(Store store) throws IOException; + + byte[] loadArchive(Archive archive) throws IOException; + + void saveArchive(Archive archive, byte[] data) throws IOException; } diff --git a/cache/src/main/java/net/runelite/cache/fs/Store.java b/cache/src/main/java/net/runelite/cache/fs/Store.java index ad3be228f5..97e741e61c 100644 --- a/cache/src/main/java/net/runelite/cache/fs/Store.java +++ b/cache/src/main/java/net/runelite/cache/fs/Store.java @@ -33,7 +33,6 @@ import java.util.List; import java.util.Objects; import net.runelite.cache.IndexType; import net.runelite.cache.fs.jagex.DiskStorage; -import net.runelite.cache.util.XteaKeyManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,14 +47,6 @@ public final class Store implements Closeable { storage = new DiskStorage(folder); storage.init(this); - - Index maps = this.findIndex(IndexType.MAPS.getNumber()); - if (maps != null) - { - XteaKeyManager mapKeys = new XteaKeyManager(); - mapKeys.loadKeys(); - maps.setXteaManager(mapKeys); - } } public Store(Storage storage) throws IOException @@ -65,6 +56,11 @@ public final class Store implements Closeable storage.init(this); } + public Storage getStorage() + { + return storage; + } + @Override public void close() throws IOException { @@ -108,7 +104,7 @@ public final class Store implements Closeable } } - Index index = new Index(this, id); + Index index = new Index(id); this.indexes.add(index); return index; @@ -120,18 +116,6 @@ public final class Store implements Closeable indexes.remove(index); } - /* - * we rebuild data differently, so the CRCs aren't right. - * rebuild them. - */ - public void rebuildCrc() throws IOException - { - for (Index i : indexes) - { - i.rebuildCrc(); - } - } - public void load() throws IOException { storage.load(this); diff --git a/cache/src/main/java/net/runelite/cache/fs/jagex/DataFile.java b/cache/src/main/java/net/runelite/cache/fs/jagex/DataFile.java index cd770999ff..c589685017 100644 --- a/cache/src/main/java/net/runelite/cache/fs/jagex/DataFile.java +++ b/cache/src/main/java/net/runelite/cache/fs/jagex/DataFile.java @@ -24,20 +24,12 @@ */ package net.runelite.cache.fs.jagex; -import static com.google.common.primitives.Bytes.concat; -import com.google.common.primitives.Ints; import java.io.Closeable; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.RandomAccessFile; import java.nio.ByteBuffer; -import net.runelite.cache.util.BZip2; -import net.runelite.cache.io.InputStream; -import net.runelite.cache.io.OutputStream; -import net.runelite.cache.util.Crc32; -import net.runelite.cache.util.GZip; -import net.runelite.cache.util.Xtea; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -177,7 +169,7 @@ public class DataFile implements Closeable return buffer.array(); } - public DataFileWriteResult write(int indexId, int archiveId, byte[] compressedData, int revision) throws IOException + public DataFileWriteResult write(int indexId, int archiveId, byte[] compressedData) throws IOException { int sector; int startSector; @@ -270,174 +262,6 @@ public class DataFile implements Closeable DataFileWriteResult res = new DataFileWriteResult(); res.sector = startSector; res.compressedLength = compressedData.length; - - int length = revision != -1 ? compressedData.length - 2 : compressedData.length; - Crc32 crc32 = new Crc32(); - crc32.update(compressedData, 0, length); - res.crc = crc32.getHash(); return res; } - - public static DataFileReadResult decompress(byte[] b, int[] keys) throws IOException - { - InputStream stream = new InputStream(b); - - int compression = stream.readUnsignedByte(); - int compressedLength = stream.readInt(); - if (compressedLength < 0 || compressedLength > 1000000) - { - throw new RuntimeException("Invalid data"); - } - - Crc32 crc32 = new Crc32(); - crc32.update(b, 0, 5); // compression + length - - byte[] data; - int revision = -1; - switch (compression) - { - case CompressionType.NONE: - { - byte[] encryptedData = new byte[compressedLength]; - stream.readBytes(encryptedData, 0, compressedLength); - - crc32.update(encryptedData, 0, compressedLength); - byte[] decryptedData = decrypt(encryptedData, encryptedData.length, keys); - - if (stream.remaining() >= 2) - { - revision = stream.readUnsignedShort(); - assert revision != -1; - } - - data = decryptedData; - - break; - } - case CompressionType.BZ2: - { - byte[] encryptedData = new byte[compressedLength + 4]; - stream.readBytes(encryptedData); - - crc32.update(encryptedData, 0, encryptedData.length); - byte[] decryptedData = decrypt(encryptedData, encryptedData.length, keys); - - if (stream.remaining() >= 2) - { - revision = stream.readUnsignedShort(); - assert revision != -1; - } - - stream = new InputStream(decryptedData); - - int decompressedLength = stream.readInt(); - data = BZip2.decompress(stream.getRemaining(), compressedLength); - - if (data == null) - { - return null; - } - - assert data.length == decompressedLength; - - break; - } - case CompressionType.GZ: - { - byte[] encryptedData = new byte[compressedLength + 4]; - stream.readBytes(encryptedData); - - crc32.update(encryptedData, 0, encryptedData.length); - byte[] decryptedData = decrypt(encryptedData, encryptedData.length, keys); - - if (stream.remaining() >= 2) - { - revision = stream.readUnsignedShort(); - assert revision != -1; - } - - stream = new InputStream(decryptedData); - - int decompressedLength = stream.readInt(); - data = GZip.decompress(stream.getRemaining(), compressedLength); - - if (data == null) - { - return null; - } - - assert data.length == decompressedLength; - - break; - } - default: - throw new RuntimeException("Unknown decompression type"); - } - - DataFileReadResult res = new DataFileReadResult(); - res.data = data; - res.revision = revision; - res.crc = crc32.getHash(); - res.compression = compression; - return res; - } - - public static byte[] compress(byte[] data, int compression, int revision, int[] keys) throws IOException - { - OutputStream stream = new OutputStream(); - byte[] compressedData; - int length; - switch (compression) - { - case CompressionType.NONE: - compressedData = data; - length = compressedData.length; - break; - case CompressionType.BZ2: - compressedData = concat(Ints.toByteArray(data.length), BZip2.compress(data)); - length = compressedData.length - 4; - break; - case CompressionType.GZ: - compressedData = concat(Ints.toByteArray(data.length), GZip.compress(data)); - length = compressedData.length - 4; - break; - default: - throw new RuntimeException("Unknown compression type"); - } - - compressedData = encrypt(compressedData, compressedData.length, keys); - - stream.writeByte(compression); - stream.writeInt(length); - - stream.writeBytes(compressedData); - if (revision != -1) - { - stream.writeShort(revision); - } - - return stream.flip(); - } - - private static byte[] decrypt(byte[] data, int length, int[] keys) - { - if (keys == null) - { - return data; - } - - Xtea xtea = new Xtea(keys); - return xtea.decrypt(data, length); - } - - private static byte[] encrypt(byte[] data, int length, int[] keys) - { - if (keys == null) - { - return data; - } - - Xtea xtea = new Xtea(keys); - return xtea.encrypt(data, length); - } } diff --git a/cache/src/main/java/net/runelite/cache/fs/jagex/DataFileWriteResult.java b/cache/src/main/java/net/runelite/cache/fs/jagex/DataFileWriteResult.java index 23481c60bc..7a7d651d74 100644 --- a/cache/src/main/java/net/runelite/cache/fs/jagex/DataFileWriteResult.java +++ b/cache/src/main/java/net/runelite/cache/fs/jagex/DataFileWriteResult.java @@ -28,5 +28,4 @@ package net.runelite.cache.fs.jagex; public class DataFileWriteResult { public int sector, compressedLength; - public int crc; // crc of compressed data } diff --git a/cache/src/main/java/net/runelite/cache/fs/jagex/DiskStorage.java b/cache/src/main/java/net/runelite/cache/fs/jagex/DiskStorage.java index a430dee37b..bebcc9711f 100644 --- a/cache/src/main/java/net/runelite/cache/fs/jagex/DiskStorage.java +++ b/cache/src/main/java/net/runelite/cache/fs/jagex/DiskStorage.java @@ -24,19 +24,20 @@ */ package net.runelite.cache.fs.jagex; +import com.google.common.primitives.Ints; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; import java.util.List; import net.runelite.cache.fs.Archive; -import net.runelite.cache.fs.FSFile; +import net.runelite.cache.fs.Container; import net.runelite.cache.fs.Index; import net.runelite.cache.fs.Storage; import net.runelite.cache.fs.Store; import net.runelite.cache.index.ArchiveData; -import net.runelite.cache.index.FileData; import net.runelite.cache.index.IndexData; +import net.runelite.cache.util.Crc32; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -108,13 +109,19 @@ public class DiskStorage implements Storage } } + public byte[] readIndex(int indexId) throws IOException + { + IndexEntry entry = index255.read(indexId); + byte[] indexData = data.read(index255.getIndexFileId(), entry.getId(), entry.getSector(), entry.getLength()); + return indexData; + } + private void loadIndex(Index index) throws IOException { logger.trace("Loading index {}", index.getId()); - IndexEntry entry = index255.read(index.getId()); - byte[] indexData = data.read(index255.getIndexFileId(), entry.getId(), entry.getSector(), entry.getLength()); - DataFileReadResult res = DataFile.decompress(indexData, null); + byte[] indexData = readIndex(index.getId()); + Container res = Container.decompress(indexData, null); byte[] data = res.data; IndexData id = new IndexData(); @@ -130,28 +137,18 @@ public class DiskStorage implements Storage archive.setNameHash(ad.getNameHash()); archive.setCrc(ad.getCrc()); archive.setRevision(ad.getRevision()); + archive.setFileData(ad.getFiles()); assert ad.getFiles().length > 0; - - for (FileData fd : ad.getFiles()) - { - FSFile file = new FSFile(fd.getId()); - file.setNameHash(fd.getNameHash()); - archive.addFile(file); - } } index.setCrc(res.crc); index.setCompression(res.compression); assert res.revision == -1; - - for (Archive archive : new ArrayList<>(index.getArchives())) - { - loadArchive(archive); - } } - private void loadArchive(Archive archive) throws IOException + @Override + public byte[] loadArchive(Archive archive) throws IOException { Index index = archive.getIndex(); IndexFile indexFile = getIndex(index.getId()); @@ -162,8 +159,7 @@ public class DiskStorage implements Storage if (entry == null) { logger.debug("can't read archive " + archive.getArchiveId() + " from index " + index.getId()); - index.getArchives().remove(archive); // is this correct? - return; + return null; } assert entry.getId() == archive.getArchiveId(); @@ -172,28 +168,12 @@ public class DiskStorage implements Storage archive.getArchiveId(), index.getId(), entry.getSector(), entry.getLength()); byte[] archiveData = data.read(index.getId(), entry.getId(), entry.getSector(), entry.getLength()); - archive.setData(archiveData); - - if (index.getXteaManager() != null) - { - return; // can't decrypt this yet - } - - archive.decompressAndLoad(null); + return archiveData; } @Override public void save(Store store) throws IOException { - logger.debug("Clearing data and indexes in preparation for store save"); - - data.clear(); - - for (IndexFile indexFile : indexFiles) - { - indexFile.clear(); - } - logger.debug("Saving store"); for (Index i : store.getIndexes()) @@ -204,49 +184,46 @@ public class DiskStorage implements Storage private void saveIndex(Index index) throws IOException { - // This updates archive CRCs for writeIndexData - for (Archive archive : index.getArchives()) - { - saveArchive(archive); - } - IndexData indexData = index.toIndexData(); byte[] data = indexData.writeIndexData(); - byte[] compressedData = DataFile.compress(data, index.getCompression(), -1, null); // index data revision is always -1 - DataFileWriteResult res = this.data.write(index255.getIndexFileId(), index.getId(), compressedData, index.getRevision()); + Container container = new Container(index.getCompression(), -1); // index data revision is always -1 + container.compress(data, null); + byte[] compressedData = container.data; + DataFileWriteResult res = this.data.write(index255.getIndexFileId(), index.getId(), compressedData); index255.write(new IndexEntry(index255, index.getId(), res.sector, res.compressedLength)); - index.setCrc(res.crc); + Crc32 crc = new Crc32(); + crc.update(compressedData, 0, compressedData.length); + index.setCrc(crc.getHash()); } - private void saveArchive(Archive a) throws IOException + @Override + public void saveArchive(Archive a, byte[] archiveData) throws IOException { Index index = a.getIndex(); IndexFile indexFile = getIndex(index.getId()); assert indexFile.getIndexFileId() == index.getId(); - int rev; // used for determining what part of compressedData to crc - byte[] compressedData; - - if (a.getData() != null) - { - compressedData = a.getData(); // data was never decompressed or loaded - rev = -1; // assume that this data has no revision? - } - else - { - byte[] fileData = a.saveContents(); - rev = a.getRevision(); - compressedData = DataFile.compress(fileData, a.getCompression(), a.getRevision(), null); - } - - DataFileWriteResult res = data.write(index.getId(), a.getArchiveId(), compressedData, rev); + DataFileWriteResult res = data.write(index.getId(), a.getArchiveId(), archiveData); indexFile.write(new IndexEntry(indexFile, a.getArchiveId(), res.sector, res.compressedLength)); - logger.trace("Saved archive {}/{} at sector {}, compressed length {}", index.getId(), a.getArchiveId(), res.sector, res.compressedLength); + byte compression = archiveData[0]; + int compressedSize = Ints.fromBytes(archiveData[1], archiveData[2], + archiveData[3], archiveData[4]); - a.setCrc(res.crc); + // don't crc the appended revision, if it is there + int length = 1 // compression type + + 4 // compressed size + + compressedSize + + (compression != CompressionType.NONE ? 4 : 0); + + Crc32 crc = new Crc32(); + crc.update(archiveData, 0, length); + a.setCrc(crc.getHash()); + + logger.trace("Saved archive {}/{} at sector {}, compressed length {}", + index.getId(), a.getArchiveId(), res.sector, res.compressedLength); } } diff --git a/cache/src/main/java/net/runelite/cache/region/RegionLoader.java b/cache/src/main/java/net/runelite/cache/region/RegionLoader.java index c247003210..6395ccc707 100644 --- a/cache/src/main/java/net/runelite/cache/region/RegionLoader.java +++ b/cache/src/main/java/net/runelite/cache/region/RegionLoader.java @@ -30,6 +30,7 @@ import java.util.List; import net.runelite.cache.IndexType; import net.runelite.cache.fs.Archive; import net.runelite.cache.fs.Index; +import net.runelite.cache.fs.Storage; import net.runelite.cache.fs.Store; import net.runelite.cache.util.XteaKeyManager; import org.slf4j.Logger; @@ -53,7 +54,8 @@ public class RegionLoader { this.store = store; index = store.getIndex(IndexType.MAPS); - keyManager = index.getXteaManager(); + keyManager = new XteaKeyManager(); + keyManager.loadKeys(); } public void loadRegions() throws IOException @@ -71,6 +73,7 @@ public class RegionLoader int x = i >> 8; int y = i & 0xFF; + Storage storage = store.getStorage(); Archive map = index.findArchiveByName("m" + x + "_" + y); Archive land = index.findArchiveByName("l" + x + "_" + y); @@ -81,12 +84,7 @@ public class RegionLoader return null; } - assert map.getFiles().size() == 1; - assert land.getFiles().size() == 1; - - map.decompressAndLoad(null); - - byte[] data = map.getFiles().get(0).getContents(); + byte[] data = map.decompress(storage.loadArchive(map)); Region region = new Region(i); region.loadTerrain(data); @@ -96,9 +94,7 @@ public class RegionLoader { try { - land.decompressAndLoad(keys); - - data = land.getFiles().get(0).getContents(); + data = land.decompress(storage.loadArchive(land), keys); region.loadLocations(data); } catch (IOException ex) diff --git a/cache/src/main/java/net/runelite/cache/server/ArchiveRequestHandler.java b/cache/src/main/java/net/runelite/cache/server/ArchiveRequestHandler.java index a79ba8c663..e4ced6da04 100644 --- a/cache/src/main/java/net/runelite/cache/server/ArchiveRequestHandler.java +++ b/cache/src/main/java/net/runelite/cache/server/ArchiveRequestHandler.java @@ -32,10 +32,12 @@ import io.netty.channel.SimpleChannelInboundHandler; import java.io.IOException; import java.util.Arrays; import net.runelite.cache.fs.Archive; +import net.runelite.cache.fs.Container; import net.runelite.cache.fs.Index; +import net.runelite.cache.fs.Storage; import net.runelite.cache.fs.Store; import net.runelite.cache.fs.jagex.CompressionType; -import net.runelite.cache.fs.jagex.DataFile; +import net.runelite.cache.fs.jagex.DiskStorage; import net.runelite.cache.protocol.packets.ArchiveRequestPacket; import net.runelite.cache.protocol.packets.ArchiveResponsePacket; import org.slf4j.Logger; @@ -88,12 +90,10 @@ public class ArchiveRequestHandler extends SimpleChannelInboundHandler frames = new ArrayList<>(); - for (FSFile file : archive.getFiles()) - { - byte[] contents = file.getContents(); + byte[] archiveData = storage.loadArchive(archive); + byte[] contents = archive.decompress(archiveData); - int framemapArchiveId = (contents[0] & 0xff) << 8 | contents[1] & 0xff; + int framemapArchiveId = (contents[0] & 0xff) << 8 | contents[1] & 0xff; - Archive framemapArchive = framemapIndex.getArchives().get(framemapArchiveId); - assert framemapArchive.getFiles().size() == 1; - FSFile framemapFile = framemapArchive.getFiles().get(0); + Archive framemapArchive = framemapIndex.getArchives().get(framemapArchiveId); + archiveData = storage.loadArchive(framemapArchive); + byte[] framemapContents = framemapArchive.decompress(archiveData); - FramemapLoader fmloader = new FramemapLoader(); - FramemapDefinition framemap = fmloader.load(framemapFile.getFileId(), framemapFile.getContents()); + FramemapLoader fmloader = new FramemapLoader(); + FramemapDefinition framemap = fmloader.load(0, framemapContents); - FrameLoader frameLoader = new FrameLoader(); - FrameDefinition frame = frameLoader.load(framemap, contents); + FrameLoader frameLoader = new FrameLoader(); + FrameDefinition frame = frameLoader.load(framemap, contents); - frames.add(frame); - } + frames.add(frame); Files.write(gson.toJson(frames), new File(outDir, archive.getArchiveId() + ".json"), Charset.defaultCharset()); ++count; diff --git a/cache/src/test/java/net/runelite/cache/FramemapDumper.java b/cache/src/test/java/net/runelite/cache/FramemapDumper.java index 55e428a38c..4513bf3dfb 100644 --- a/cache/src/test/java/net/runelite/cache/FramemapDumper.java +++ b/cache/src/test/java/net/runelite/cache/FramemapDumper.java @@ -33,8 +33,8 @@ import java.nio.charset.Charset; import net.runelite.cache.definitions.FramemapDefinition; import net.runelite.cache.definitions.loaders.FramemapLoader; import net.runelite.cache.fs.Archive; -import net.runelite.cache.fs.FSFile; import net.runelite.cache.fs.Index; +import net.runelite.cache.fs.Storage; import net.runelite.cache.fs.Store; import org.junit.Rule; import org.junit.Test; @@ -63,14 +63,16 @@ public class FramemapDumper { store.load(); + Storage storage = store.getStorage(); Index index = store.getIndex(IndexType.FRAMEMAPS); + for (Archive archive : index.getArchives()) { - assert archive.getFiles().size() == 1; - FSFile file = archive.getFiles().get(0); + byte[] archiveData = storage.loadArchive(archive); + byte[] contents = archive.decompress(archiveData); FramemapLoader loader = new FramemapLoader(); - FramemapDefinition framemap = loader.load(file.getFileId(), file.getContents()); + FramemapDefinition framemap = loader.load(0, contents); Files.write(gson.toJson(framemap), new File(outDir, archive.getArchiveId() + ".json"), Charset.defaultCharset()); ++count; diff --git a/cache/src/test/java/net/runelite/cache/InventoryDumper.java b/cache/src/test/java/net/runelite/cache/InventoryDumper.java index 5ff860fd03..5a46b79b8d 100644 --- a/cache/src/test/java/net/runelite/cache/InventoryDumper.java +++ b/cache/src/test/java/net/runelite/cache/InventoryDumper.java @@ -33,8 +33,10 @@ import java.nio.charset.Charset; import net.runelite.cache.definitions.InventoryDefinition; import net.runelite.cache.definitions.loaders.InventoryLoader; import net.runelite.cache.fs.Archive; +import net.runelite.cache.fs.ArchiveFiles; import net.runelite.cache.fs.FSFile; import net.runelite.cache.fs.Index; +import net.runelite.cache.fs.Storage; import net.runelite.cache.fs.Store; import org.junit.Rule; import org.junit.Test; @@ -63,10 +65,14 @@ public class InventoryDumper { store.load(); + Storage storage = store.getStorage(); Index index = store.getIndex(IndexType.CONFIGS); Archive archive = index.getArchive(ConfigType.INV.getId()); - for (FSFile file : archive.getFiles()) + byte[] archiveData = storage.loadArchive(archive); + ArchiveFiles files = archive.getFiles(archiveData); + + for (FSFile file : files.getFiles()) { InventoryLoader loader = new InventoryLoader(); InventoryDefinition inv = loader.load(file.getContents()); diff --git a/cache/src/test/java/net/runelite/cache/KitDumperTest.java b/cache/src/test/java/net/runelite/cache/KitDumperTest.java index edca618821..286ca6ea8e 100644 --- a/cache/src/test/java/net/runelite/cache/KitDumperTest.java +++ b/cache/src/test/java/net/runelite/cache/KitDumperTest.java @@ -33,8 +33,10 @@ import java.nio.charset.Charset; import net.runelite.cache.definitions.KitDefinition; import net.runelite.cache.definitions.loaders.KitLoader; import net.runelite.cache.fs.Archive; +import net.runelite.cache.fs.ArchiveFiles; import net.runelite.cache.fs.FSFile; import net.runelite.cache.fs.Index; +import net.runelite.cache.fs.Storage; import net.runelite.cache.fs.Store; import org.junit.Rule; import org.junit.Test; @@ -61,12 +63,16 @@ public class KitDumperTest { store.load(); + Storage storage = store.getStorage(); Index index = store.getIndex(IndexType.CONFIGS); Archive archive = index.getArchive(ConfigType.IDENTKIT.getId()); KitLoader loader = new KitLoader(); - for (FSFile file : archive.getFiles()) + byte[] archiveData = storage.loadArchive(archive); + ArchiveFiles files = archive.getFiles(archiveData); + + for (FSFile file : files.getFiles()) { byte[] b = file.getContents(); diff --git a/cache/src/test/java/net/runelite/cache/MapDumperTest.java b/cache/src/test/java/net/runelite/cache/MapDumperTest.java index 791eb27a24..91395b0bcd 100644 --- a/cache/src/test/java/net/runelite/cache/MapDumperTest.java +++ b/cache/src/test/java/net/runelite/cache/MapDumperTest.java @@ -33,6 +33,7 @@ import java.util.ArrayList; import java.util.List; import net.runelite.cache.fs.Archive; import net.runelite.cache.fs.Index; +import net.runelite.cache.fs.Storage; import net.runelite.cache.fs.Store; import net.runelite.cache.region.Region; import net.runelite.cache.util.XteaKeyManager; @@ -59,13 +60,15 @@ public class MapDumperTest { File base = StoreLocation.LOCATION, outDir = folder.newFolder(); + XteaKeyManager keyManager = new XteaKeyManager(); + keyManager.loadKeys(); try (Store store = new Store(base)) { store.load(); + Storage storage = store.getStorage(); Index index = store.getIndex(IndexType.MAPS); - XteaKeyManager keyManager = index.getXteaManager(); for (int i = 0; i < MAX_REGIONS; i++) { @@ -84,14 +87,7 @@ public class MapDumperTest continue; } - assert map.getFiles().size() == 1; - assert land.getFiles().size() == 1; - - // maps aren't encrypted, but we don't load archive data of any archive in - // the maps index, so load it - map.decompressAndLoad(null); - - byte[] data = map.getFiles().get(0).getContents(); + byte[] data = map.decompress(storage.loadArchive(map)); Files.write(data, new File(outDir, "m" + x + "_" + y + ".dat")); @@ -99,7 +95,7 @@ public class MapDumperTest { try { - land.decompressAndLoad(keys); + data = land.decompress(storage.loadArchive(land), keys); } catch (IOException ex) { @@ -109,8 +105,6 @@ public class MapDumperTest logger.info("Decrypted region {} coords {},{}", i, x, y); - data = land.getFiles().get(0).getContents(); - Files.write(data, new File(outDir, "l" + x + "_" + y + ".dat")); } } @@ -119,8 +113,10 @@ public class MapDumperTest private void loadRegions(Store store) throws IOException { + Storage storage = store.getStorage(); Index index = store.getIndex(IndexType.MAPS); - XteaKeyManager keyManager = index.getXteaManager(); + XteaKeyManager keyManager = new XteaKeyManager(); + keyManager.loadKeys(); for (int i = 0; i < MAX_REGIONS; ++i) { @@ -137,12 +133,7 @@ public class MapDumperTest continue; } - assert map.getFiles().size() == 1; - assert land.getFiles().size() == 1; - - map.decompressAndLoad(null); - - byte[] data = map.getFiles().get(0).getContents(); + byte[] data = map.decompress(storage.loadArchive(map)); Region region = new Region(i); region.loadTerrain(data); @@ -152,14 +143,13 @@ public class MapDumperTest { try { - land.decompressAndLoad(keys); + data = land.decompress(storage.loadArchive(land), keys); } catch (IOException ex) { continue; } - data = land.getFiles().get(0).getContents(); region.loadLocations(data); } diff --git a/cache/src/test/java/net/runelite/cache/ModelDumperTest.java b/cache/src/test/java/net/runelite/cache/ModelDumperTest.java index a481b38eb0..32f5879ac1 100644 --- a/cache/src/test/java/net/runelite/cache/ModelDumperTest.java +++ b/cache/src/test/java/net/runelite/cache/ModelDumperTest.java @@ -27,13 +27,12 @@ package net.runelite.cache; import com.google.common.io.Files; -import com.google.gson.Gson; import java.io.File; import java.io.IOException; import net.runelite.cache.definitions.loaders.ModelLoader; import net.runelite.cache.fs.Archive; -import net.runelite.cache.fs.FSFile; import net.runelite.cache.fs.Index; +import net.runelite.cache.fs.Storage; import net.runelite.cache.fs.Store; import org.junit.Rule; import org.junit.Test; @@ -45,8 +44,6 @@ public class ModelDumperTest { private static final Logger logger = LoggerFactory.getLogger(ModelDumperTest.class); - private Gson gson = new Gson(); - @Rule public TemporaryFolder folder = StoreLocation.getTemporaryFolder(); @@ -60,20 +57,17 @@ public class ModelDumperTest { store.load(); + Storage storage = store.getStorage(); Index index = store.getIndex(IndexType.MODELS); for (Archive archive : index.getArchives()) { - assert archive.getFiles().size() == 1; - - FSFile file = archive.getFiles().get(0); - byte[] contents = file.getContents(); + byte[] contents = archive.decompress(storage.loadArchive(archive)); ModelLoader loader = new ModelLoader(); loader.load(archive.getArchiveId(), contents); Files.write(contents, new File(modelDir, archive.getArchiveId() + ".model")); - //Files.write(gson.toJson(loader), new File(modelDir, archive.getArchiveId() + ".json"), Charset.defaultCharset()); ++count; } } diff --git a/cache/src/test/java/net/runelite/cache/OverlayDumper.java b/cache/src/test/java/net/runelite/cache/OverlayDumper.java index 6d4c86cafa..4ac4b53a7c 100644 --- a/cache/src/test/java/net/runelite/cache/OverlayDumper.java +++ b/cache/src/test/java/net/runelite/cache/OverlayDumper.java @@ -33,8 +33,10 @@ import java.nio.charset.Charset; import net.runelite.cache.definitions.OverlayDefinition; import net.runelite.cache.definitions.loaders.OverlayLoader; import net.runelite.cache.fs.Archive; +import net.runelite.cache.fs.ArchiveFiles; import net.runelite.cache.fs.FSFile; import net.runelite.cache.fs.Index; +import net.runelite.cache.fs.Storage; import net.runelite.cache.fs.Store; import org.junit.Rule; import org.junit.Test; @@ -63,10 +65,14 @@ public class OverlayDumper { store.load(); + Storage storage = store.getStorage(); Index index = store.getIndex(IndexType.CONFIGS); Archive archive = index.getArchive(ConfigType.OVERLAY.getId()); - for (FSFile file : archive.getFiles()) + byte[] archiveData = storage.loadArchive(archive); + ArchiveFiles files = archive.getFiles(archiveData); + + for (FSFile file : files.getFiles()) { OverlayLoader loader = new OverlayLoader(); OverlayDefinition overlay = loader.load(file.getFileId(), file.getContents()); diff --git a/cache/src/test/java/net/runelite/cache/SequenceDumper.java b/cache/src/test/java/net/runelite/cache/SequenceDumper.java index 7f5c56de4a..94dd175bbb 100644 --- a/cache/src/test/java/net/runelite/cache/SequenceDumper.java +++ b/cache/src/test/java/net/runelite/cache/SequenceDumper.java @@ -33,10 +33,11 @@ import java.nio.charset.Charset; import net.runelite.cache.definitions.SequenceDefinition; import net.runelite.cache.definitions.loaders.SequenceLoader; import net.runelite.cache.fs.Archive; +import net.runelite.cache.fs.ArchiveFiles; import net.runelite.cache.fs.FSFile; import net.runelite.cache.fs.Index; +import net.runelite.cache.fs.Storage; import net.runelite.cache.fs.Store; -import net.runelite.cache.io.InputStream; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -64,10 +65,14 @@ public class SequenceDumper { store.load(); + Storage storage = store.getStorage(); Index index = store.getIndex(IndexType.CONFIGS); Archive archive = index.getArchive(ConfigType.SEQUENCE.getId()); - for (FSFile file : archive.getFiles()) + byte[] archiveData = storage.loadArchive(archive); + ArchiveFiles files = archive.getFiles(archiveData); + + for (FSFile file : files.getFiles()) { SequenceLoader loader = new SequenceLoader(); SequenceDefinition seq = loader.load(file.getFileId(), file.getContents()); diff --git a/cache/src/test/java/net/runelite/cache/SoundEffectsDumperTest.java b/cache/src/test/java/net/runelite/cache/SoundEffectsDumperTest.java index 2bb3006f79..c2a420c0de 100644 --- a/cache/src/test/java/net/runelite/cache/SoundEffectsDumperTest.java +++ b/cache/src/test/java/net/runelite/cache/SoundEffectsDumperTest.java @@ -33,8 +33,8 @@ import java.nio.charset.Charset; import net.runelite.cache.definitions.loaders.sound.SoundEffectLoader; import net.runelite.cache.definitions.sound.SoundEffectDefinition; import net.runelite.cache.fs.Archive; -import net.runelite.cache.fs.FSFile; import net.runelite.cache.fs.Index; +import net.runelite.cache.fs.Storage; import net.runelite.cache.fs.Store; import org.junit.Rule; import org.junit.Test; @@ -61,16 +61,15 @@ public class SoundEffectsDumperTest { store.load(); + Storage storage = store.getStorage(); Index index = store.getIndex(IndexType.SOUNDEFFECTS); for (Archive archive : index.getArchives()) { - assert archive.getFiles().size() == 1; - - FSFile file = archive.getFiles().get(0); + byte[] contents = archive.decompress(storage.loadArchive(archive)); SoundEffectLoader soundEffectLoader = new SoundEffectLoader(); - SoundEffectDefinition soundEffect = soundEffectLoader.load(file.getContents()); + SoundEffectDefinition soundEffect = soundEffectLoader.load(contents); Files.write(gson.toJson(soundEffect), new File(dumpDir, archive.getArchiveId() + ".json"), Charset.defaultCharset()); ++count; diff --git a/cache/src/test/java/net/runelite/cache/TitleDumper.java b/cache/src/test/java/net/runelite/cache/TitleDumper.java index eeba647616..c611ddf04a 100644 --- a/cache/src/test/java/net/runelite/cache/TitleDumper.java +++ b/cache/src/test/java/net/runelite/cache/TitleDumper.java @@ -22,15 +22,14 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ - package net.runelite.cache; import java.io.File; import java.io.IOException; import java.nio.file.Files; import net.runelite.cache.fs.Archive; -import net.runelite.cache.fs.FSFile; import net.runelite.cache.fs.Index; +import net.runelite.cache.fs.Storage; import net.runelite.cache.fs.Store; import org.junit.Rule; import org.junit.Test; @@ -55,11 +54,12 @@ public class TitleDumper { store.load(); + Storage storage = store.getStorage(); Index index = store.getIndex(IndexType.BINARY); - Archive a = index.findArchiveByName("title.jpg"); - FSFile file = a.getFiles().get(0); + Archive archive = index.findArchiveByName("title.jpg"); + byte[] contents = archive.decompress(storage.loadArchive(archive)); - Files.write(outFile.toPath(), file.getContents()); + Files.write(outFile.toPath(), contents); } logger.info("Dumped to {}", outFile); diff --git a/cache/src/test/java/net/runelite/cache/TrackDumperTest.java b/cache/src/test/java/net/runelite/cache/TrackDumperTest.java index b06757d62c..dc70a798ef 100644 --- a/cache/src/test/java/net/runelite/cache/TrackDumperTest.java +++ b/cache/src/test/java/net/runelite/cache/TrackDumperTest.java @@ -33,8 +33,8 @@ import javax.sound.midi.Sequencer; import net.runelite.cache.definitions.TrackDefinition; import net.runelite.cache.definitions.loaders.TrackLoader; import net.runelite.cache.fs.Archive; -import net.runelite.cache.fs.FSFile; import net.runelite.cache.fs.Index; +import net.runelite.cache.fs.Storage; import net.runelite.cache.fs.Store; import net.runelite.cache.util.Djb2Manager; import org.junit.Ignore; @@ -66,18 +66,19 @@ public class TrackDumperTest { store.load(); + Storage storage = store.getStorage(); Index index = store.getIndex(IndexType.TRACK1); Index index2 = store.getIndex(IndexType.TRACK2); for (Archive archive : index.getArchives()) { - dumpTrackArchive(dumpDir1, archive); + dumpTrackArchive(dumpDir1, storage, archive); ++idx1; } for (Archive archive : index2.getArchives()) { - dumpTrackArchive(dumpDir2, archive); + dumpTrackArchive(dumpDir2, storage, archive); ++idx2; } } @@ -85,14 +86,17 @@ public class TrackDumperTest logger.info("Dumped {} sound tracks ({} idx1, {} idx2) to {} and {}", idx1 + idx2, idx1, idx2, dumpDir1, dumpDir2); } - private void dumpTrackArchive(File dumpDir, Archive archive) throws IOException + private void dumpTrackArchive(File dumpDir, Storage storage, Archive archive) throws IOException { - assert archive.getFiles().size() == 1; + byte[] contents = archive.decompress(storage.loadArchive(archive)); - FSFile file = archive.getFiles().get(0); + if (contents == null) + { + return; + } TrackLoader loader = new TrackLoader(); - TrackDefinition def = loader.load(file.getContents()); + TrackDefinition def = loader.load(contents); String name; if (archive.getNameHash() != 0) diff --git a/cache/src/test/java/net/runelite/cache/UnderlayDumper.java b/cache/src/test/java/net/runelite/cache/UnderlayDumper.java index cf540be771..db735d40cf 100644 --- a/cache/src/test/java/net/runelite/cache/UnderlayDumper.java +++ b/cache/src/test/java/net/runelite/cache/UnderlayDumper.java @@ -33,8 +33,10 @@ import java.nio.charset.Charset; import net.runelite.cache.definitions.UnderlayDefinition; import net.runelite.cache.definitions.loaders.UnderlayLoader; import net.runelite.cache.fs.Archive; +import net.runelite.cache.fs.ArchiveFiles; import net.runelite.cache.fs.FSFile; import net.runelite.cache.fs.Index; +import net.runelite.cache.fs.Storage; import net.runelite.cache.fs.Store; import org.junit.Rule; import org.junit.Test; @@ -63,10 +65,14 @@ public class UnderlayDumper { store.load(); + Storage storage = store.getStorage(); Index index = store.getIndex(IndexType.CONFIGS); Archive archive = index.getArchive(ConfigType.UNDERLAY.getId()); - for (FSFile file : archive.getFiles()) + byte[] archiveData = storage.loadArchive(archive); + ArchiveFiles files = archive.getFiles(archiveData); + + for (FSFile file : files.getFiles()) { UnderlayLoader loader = new UnderlayLoader(); UnderlayDefinition underlay = loader.load(file.getFileId(), file.getContents()); diff --git a/cache/src/test/java/net/runelite/cache/VarbitDumper.java b/cache/src/test/java/net/runelite/cache/VarbitDumper.java index fe775bde3f..5f75c7a157 100644 --- a/cache/src/test/java/net/runelite/cache/VarbitDumper.java +++ b/cache/src/test/java/net/runelite/cache/VarbitDumper.java @@ -33,8 +33,10 @@ import java.nio.charset.Charset; import net.runelite.cache.definitions.VarbitDefinition; import net.runelite.cache.definitions.loaders.VarbitLoader; import net.runelite.cache.fs.Archive; +import net.runelite.cache.fs.ArchiveFiles; import net.runelite.cache.fs.FSFile; import net.runelite.cache.fs.Index; +import net.runelite.cache.fs.Storage; import net.runelite.cache.fs.Store; import org.junit.Rule; import org.junit.Test; @@ -63,10 +65,14 @@ public class VarbitDumper { store.load(); + Storage storage = store.getStorage(); Index index = store.getIndex(IndexType.CONFIGS); Archive archive = index.getArchive(ConfigType.VARBIT.getId()); - for (FSFile file : archive.getFiles()) + byte[] archiveData = storage.loadArchive(archive); + ArchiveFiles files = archive.getFiles(archiveData); + + for (FSFile file : files.getFiles()) { VarbitLoader loader = new VarbitLoader(); VarbitDefinition varbit = loader.load(file.getFileId(), file.getContents()); diff --git a/cache/src/test/java/net/runelite/cache/WorldMapDumperTest.java b/cache/src/test/java/net/runelite/cache/WorldMapDumperTest.java index 43cc7429a9..8d8dd956c4 100644 --- a/cache/src/test/java/net/runelite/cache/WorldMapDumperTest.java +++ b/cache/src/test/java/net/runelite/cache/WorldMapDumperTest.java @@ -33,8 +33,10 @@ import java.nio.charset.Charset; import net.runelite.cache.definitions.WorldMapDefinition; import net.runelite.cache.definitions.loaders.WorldMapLoader; import net.runelite.cache.fs.Archive; +import net.runelite.cache.fs.ArchiveFiles; import net.runelite.cache.fs.FSFile; import net.runelite.cache.fs.Index; +import net.runelite.cache.fs.Storage; import net.runelite.cache.fs.Store; import org.junit.Rule; import org.junit.Test; @@ -63,10 +65,14 @@ public class WorldMapDumperTest { store.load(); + Storage storage = store.getStorage(); Index index = store.getIndex(IndexType.WORLDMAP); Archive archive = index.getArchive(0); // there is also archive 1/2, but their data format is not this - for (FSFile file : archive.getFiles()) + byte[] archiveData = storage.loadArchive(archive); + ArchiveFiles files = archive.getFiles(archiveData); + + for (FSFile file : files.getFiles()) { WorldMapLoader loader = new WorldMapLoader(); WorldMapDefinition def = loader.load(file.getContents(), file.getFileId()); diff --git a/cache/src/main/java/net/runelite/cache/fs/jagex/DataFileReadResult.java b/cache/src/test/java/net/runelite/cache/fs/ContainerTest.java similarity index 65% rename from cache/src/main/java/net/runelite/cache/fs/jagex/DataFileReadResult.java rename to cache/src/test/java/net/runelite/cache/fs/ContainerTest.java index af8cb7aff5..1aca0df4ba 100644 --- a/cache/src/main/java/net/runelite/cache/fs/jagex/DataFileReadResult.java +++ b/cache/src/test/java/net/runelite/cache/fs/ContainerTest.java @@ -1,34 +1,55 @@ -/* - * Copyright (c) 2016-2017, Adam - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -package net.runelite.cache.fs.jagex; - -public class DataFileReadResult -{ - public byte[] data; - public int revision; - public int crc; // crc of compressed data - public int compression; // compression method data was compressed with -} +/* + * Copyright (c) 2016-2017, Adam + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package net.runelite.cache.fs; + +import java.io.IOException; +import java.util.Random; +import static net.runelite.cache.fs.jagex.CompressionType.GZ; +import static org.junit.Assert.assertArrayEquals; +import org.junit.Test; + +public class ContainerTest +{ + + @Test + public void testCompress() throws IOException + { + int[] keys = new int[] + { + 4, 8, 15, 16 + }; + Random random = new Random(42L); + byte[] data = new byte[1024]; + random.nextBytes(data); + + Container container = new Container(GZ, -1); + container.compress(data, keys); + byte[] compressedData = container.data; + + container = Container.decompress(compressedData, keys); + assertArrayEquals(data, container.data); + } + +} diff --git a/cache/src/test/java/net/runelite/cache/fs/StoreLoadTest.java b/cache/src/test/java/net/runelite/cache/fs/StoreLoadTest.java index ab7a782cf1..91bbc3c3f0 100644 --- a/cache/src/test/java/net/runelite/cache/fs/StoreLoadTest.java +++ b/cache/src/test/java/net/runelite/cache/fs/StoreLoadTest.java @@ -44,8 +44,6 @@ public class StoreLoadTest try (Store store = new Store(StoreLocation.LOCATION)) { store.load(); - - System.out.println(store); } } diff --git a/cache/src/test/java/net/runelite/cache/fs/StoreTest.java b/cache/src/test/java/net/runelite/cache/fs/StoreTest.java index a827aee9b8..8953d481cf 100644 --- a/cache/src/test/java/net/runelite/cache/fs/StoreTest.java +++ b/cache/src/test/java/net/runelite/cache/fs/StoreTest.java @@ -28,6 +28,7 @@ import java.io.File; import java.io.IOException; import java.util.Random; import net.runelite.cache.StoreLocation; +import net.runelite.cache.index.FileData; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; @@ -47,10 +48,10 @@ public class StoreTest { Index index = store.addIndex(0); Archive archive = index.addArchive(0); - FSFile file = new FSFile(0); - archive.addFile(file); - file.setNameHash(7); - file.setContents("test".getBytes()); + archive.setFileData(new FileData[1]); + FileData fileData = archive.getFileData()[0] = new FileData(); + fileData.setId(42); + fileData.setNameHash(7); store.save(); @@ -74,15 +75,14 @@ public class StoreTest Index index = store.addIndex(0); Archive archive = index.addArchive(0); archive.setNameHash(random.nextInt()); + archive.setFileData(new FileData[NUMBER_OF_FILES]); for (int i = 0; i < NUMBER_OF_FILES; ++i) { - FSFile file = new FSFile(i); + FileData[] fileData = archive.getFileData(); + FileData file = fileData[i] = new FileData(); + file.setId(i); file.setNameHash(random.nextInt()); - archive.addFile(file); - byte[] data = new byte[random.nextInt(1024)]; - random.nextBytes(data); - file.setContents(data); } store.save(); @@ -109,39 +109,33 @@ public class StoreTest Archive archive = index.addArchive(0); archive.setNameHash(random.nextInt(Integer.MAX_VALUE)); + archive.setFileData(new FileData[NUMBER_OF_FILES]); Archive archive2 = index.addArchive(1); + archive2.setFileData(new FileData[NUMBER_OF_FILES]); Archive archive3 = index2.addArchive(0); + archive3.setFileData(new FileData[NUMBER_OF_FILES]); for (int i = 0; i < NUMBER_OF_FILES; ++i) { - FSFile file = new FSFile(i); + FileData[] fileData = archive.getFileData(); + FileData file = fileData[i] = new FileData(); file.setNameHash(random.nextInt(Integer.MAX_VALUE)); - archive.addFile(file); - byte[] data = new byte[random.nextInt(1024)]; - random.nextBytes(data); - file.setContents(data); } for (int i = 0; i < NUMBER_OF_FILES; ++i) { - FSFile file = new FSFile(i); + FileData[] fileData = archive2.getFileData(); + FileData file = fileData[i] = new FileData(); file.setNameHash(random.nextInt(Integer.MAX_VALUE)); - archive2.addFile(file); - byte[] data = new byte[random.nextInt(1024)]; - random.nextBytes(data); - file.setContents(data); } for (int i = 0; i < NUMBER_OF_FILES; ++i) { - FSFile file = new FSFile(i); + FileData[] fileData = archive3.getFileData(); + FileData file = fileData[i] = new FileData(); file.setNameHash(random.nextInt(Integer.MAX_VALUE)); - archive3.addFile(file); - byte[] data = new byte[random.nextInt(1024)]; - random.nextBytes(data); - file.setContents(data); } store.save(); diff --git a/cache/src/test/java/net/runelite/cache/fs/jagex/DataFileTest.java b/cache/src/test/java/net/runelite/cache/fs/jagex/DataFileTest.java index bc7b8ea962..7e66d0b1e9 100644 --- a/cache/src/test/java/net/runelite/cache/fs/jagex/DataFileTest.java +++ b/cache/src/test/java/net/runelite/cache/fs/jagex/DataFileTest.java @@ -27,6 +27,7 @@ package net.runelite.cache.fs.jagex; import java.io.File; import java.io.IOException; import net.runelite.cache.StoreLocation; +import net.runelite.cache.fs.Container; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; @@ -43,17 +44,17 @@ public class DataFileTest File file = folder.newFile(); DataFile df = new DataFile(file); - byte[] compressedData = DataFile.compress("test".getBytes(), CompressionType.NONE, 0, null); - DataFileWriteResult res = df.write(42, 3, compressedData, 0); + Container container = new Container(CompressionType.NONE, 0); + container.compress("test".getBytes(), null); + byte[] compressedData = container.data; + DataFileWriteResult res = df.write(42, 3, compressedData); compressedData = df.read(42, 3, res.sector, res.compressedLength); - DataFileReadResult res2 = DataFile.decompress(compressedData, null); + Container res2 = Container.decompress(compressedData, null); byte[] buf = res2.data; String str = new String(buf); Assert.assertEquals("test", str); - - Assert.assertEquals(res.crc, res2.crc); } @Test @@ -69,11 +70,13 @@ public class DataFileTest DataFile df = new DataFile(file); - byte[] compressedData = DataFile.compress(b, CompressionType.BZ2, 42, null); - DataFileWriteResult res = df.write(42, 0x1FFFF, compressedData, 42); + Container container = new Container(CompressionType.BZ2, 42); + container.compress(b, null); + byte[] compressedData = container.data; + DataFileWriteResult res = df.write(42, 0x1FFFF, compressedData); compressedData = df.read(42, 0x1FFFF, res.sector, res.compressedLength); - DataFileReadResult res2 = DataFile.decompress(compressedData, null); + Container res2 = Container.decompress(compressedData, null); byte[] buf = res2.data; Assert.assertArrayEquals(b, buf); @@ -84,11 +87,13 @@ public class DataFileTest { DataFile df = new DataFile(folder.newFile()); - byte[] compressedData = DataFile.compress("test".getBytes(), CompressionType.GZ, 0, null); - DataFileWriteResult res = df.write(41, 4, compressedData, 0); + Container container = new Container(CompressionType.GZ, 0); + container.compress("test".getBytes(), null); + byte[] compressedData = container.data; + DataFileWriteResult res = df.write(41, 4, compressedData); compressedData = df.read(41, 4, res.sector, res.compressedLength); - DataFileReadResult res2 = DataFile.decompress(compressedData, null); + Container res2 = Container.decompress(compressedData, null); byte[] buf = res2.data; String str = new String(buf); @@ -100,36 +105,19 @@ public class DataFileTest { DataFile df = new DataFile(folder.newFile()); - byte[] compressedData = DataFile.compress("test".getBytes(), CompressionType.BZ2, 5, null); - DataFileWriteResult res = df.write(41, 4, compressedData, 0); + Container container = new Container(CompressionType.BZ2, 5); + container.compress("test".getBytes(), null); + byte[] compressedData = container.data; + DataFileWriteResult res = df.write(41, 4, compressedData); compressedData = df.read(41, 4, res.sector, res.compressedLength); - DataFileReadResult res2 = DataFile.decompress(compressedData, null); + Container res2 = Container.decompress(compressedData, null); byte[] buf = res2.data; String str = new String(buf); Assert.assertEquals("test", str); } - @Test - public void testCrc() throws IOException - { - File file = folder.newFile(); - DataFile df = new DataFile(file); - - byte[] compressedData = DataFile.compress("test".getBytes(), CompressionType.NONE, 42, null); - DataFileWriteResult res = df.write(42, 3, compressedData, 0); - - compressedData = df.read(42, 3, res.sector, res.compressedLength); - DataFileReadResult res2 = DataFile.decompress(compressedData, null); - - byte[] buf = res2.data; - String str = new String(buf); - Assert.assertEquals("test", str); - Assert.assertEquals(res.crc, res2.crc); - Assert.assertEquals(42, res2.revision); - } - @Test public void testEnc() throws IOException { @@ -141,16 +129,17 @@ public class DataFileTest DataFile df = new DataFile(file); - byte[] compressedData = DataFile.compress("testtesttesttest1".getBytes(), CompressionType.NONE, 42, keys); - DataFileWriteResult res = df.write(42, 3, compressedData, 0); + Container container = new Container(CompressionType.NONE, 42); + container.compress("testtesttesttest1".getBytes(), keys); + byte[] compressedData = container.data; + DataFileWriteResult res = df.write(42, 3, compressedData); compressedData = df.read(42, 3, res.sector, res.compressedLength); - DataFileReadResult res2 = DataFile.decompress(compressedData, keys); + Container res2 = Container.decompress(compressedData, keys); byte[] buf = res2.data; String str = new String(buf); Assert.assertEquals("testtesttesttest1", str); - Assert.assertEquals(res.crc, res2.crc); Assert.assertEquals(42, res2.revision); } @@ -165,16 +154,17 @@ public class DataFileTest DataFile df = new DataFile(file); - byte[] compressedData = DataFile.compress("testtesttesttest1".getBytes(), CompressionType.GZ, 42, keys); - DataFileWriteResult res = df.write(42, 3, compressedData, 0); + Container container = new Container(CompressionType.GZ, 42); + container.compress("testtesttesttest1".getBytes(), keys); + byte[] compressedData = container.data; + DataFileWriteResult res = df.write(42, 3, compressedData); compressedData = df.read(42, 3, res.sector, res.compressedLength); - DataFileReadResult res2 = DataFile.decompress(compressedData, keys); + Container res2 = Container.decompress(compressedData, keys); byte[] buf = res2.data; String str = new String(buf); Assert.assertEquals("testtesttesttest1", str); - Assert.assertEquals(res.crc, res2.crc); Assert.assertEquals(42, res2.revision); } } diff --git a/cache/src/test/java/net/runelite/cache/fs/jagex/DiskStorageTest.java b/cache/src/test/java/net/runelite/cache/fs/jagex/DiskStorageTest.java new file mode 100644 index 0000000000..6d6035a61e --- /dev/null +++ b/cache/src/test/java/net/runelite/cache/fs/jagex/DiskStorageTest.java @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2016-2017, Adam + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package net.runelite.cache.fs.jagex; + +import java.io.File; +import net.runelite.cache.StoreLocation; +import net.runelite.cache.fs.Archive; +import net.runelite.cache.fs.Container; +import net.runelite.cache.fs.Index; +import net.runelite.cache.fs.Store; +import net.runelite.cache.index.FileData; +import org.junit.Test; +import static org.junit.Assert.*; +import org.junit.Rule; +import org.junit.rules.TemporaryFolder; + +public class DiskStorageTest +{ + @Rule + public TemporaryFolder folder = StoreLocation.getTemporaryFolder(); + + @Test + public void testSaveArchive() throws Exception + { + File file = folder.newFolder(); + DiskStorage storage = new DiskStorage(file); + Archive archive; + Archive archive2; + try (Store store = new Store(storage)) + { + Index index = store.addIndex(0); + archive = index.addArchive(0); + archive2 = index.addArchive(1); + + FileData[] fileData = new FileData[1]; + archive.setFileData(fileData); + fileData[0] = new FileData(); + + FileData[] fileData2 = new FileData[1]; + archive2.setFileData(fileData2); + fileData2[0] = new FileData(); + + byte[] data = "test".getBytes(); + Container container = new Container(archive.getCompression(), -1); + container.compress(data, null); + byte[] compressedData = container.data; + storage.saveArchive(archive, compressedData); + + container = new Container(archive.getCompression(), 42); + container.compress(data, null); + compressedData = container.data; + archive2.setRevision(42); + storage.saveArchive(archive2, compressedData); + + store.save(); + } + + storage = new DiskStorage(file); + try (Store store = new Store(storage)) + { + store.load(); + Index index = store.findIndex(0); + Archive archive2_1 = index.getArchive(0); + Archive archive2_2 = index.getArchive(1); + + byte[] comprsesedData = storage.loadArchive(archive2_1); + byte[] data = archive2_1.decompress(comprsesedData); + assertArrayEquals("test".getBytes(), data); + assertEquals(archive.getCrc(), archive2_1.getCrc()); + assertEquals(archive.getRevision(), archive2_1.getRevision()); + + comprsesedData = storage.loadArchive(archive2_2); + data = archive2_2.decompress(comprsesedData); + assertArrayEquals("test".getBytes(), data); + assertEquals(archive2.getCrc(), archive2_2.getCrc()); + assertEquals(archive2.getRevision(), archive2_2.getRevision()); + } + } + +} diff --git a/cache/src/test/java/net/runelite/cache/protocol/encoders/ArchiveResponseEncoderTest.java b/cache/src/test/java/net/runelite/cache/protocol/encoders/ArchiveResponseEncoderTest.java index 3e3f8f80e5..290ed2511b 100644 --- a/cache/src/test/java/net/runelite/cache/protocol/encoders/ArchiveResponseEncoderTest.java +++ b/cache/src/test/java/net/runelite/cache/protocol/encoders/ArchiveResponseEncoderTest.java @@ -29,6 +29,7 @@ import io.netty.buffer.Unpooled; import java.util.ArrayList; import java.util.List; import java.util.Random; +import net.runelite.cache.fs.Container; import net.runelite.cache.fs.jagex.CompressionType; import net.runelite.cache.fs.jagex.DataFile; import net.runelite.cache.protocol.decoders.ArchiveResponseDecoder; @@ -45,7 +46,9 @@ public class ArchiveResponseEncoderTest Random random = new Random(42L); random.nextBytes(data); - byte[] compressedData = DataFile.compress(data, CompressionType.NONE, -1, null); + Container container =new Container(CompressionType.NONE, -1); + container.compress(data, null); + byte[] compressedData = container.data;//DataFile.compress(data, CompressionType.NONE, -1, null); ArchiveResponsePacket archiveResponse = new ArchiveResponsePacket(); archiveResponse.setIndex(0); @@ -67,7 +70,7 @@ public class ArchiveResponseEncoderTest Assert.assertEquals(archiveResponse.getArchive(), response.getArchive()); Assert.assertArrayEquals(archiveResponse.getData(), response.getData()); - byte[] decompressedData = DataFile.decompress(response.getData(), null).data; + byte[] decompressedData = Container.decompress(response.getData(), null).data; Assert.assertArrayEquals(data, decompressedData); } diff --git a/cache/src/test/java/net/runelite/cache/script/disassembler/DisassemblerTest.java b/cache/src/test/java/net/runelite/cache/script/disassembler/DisassemblerTest.java index ae740f734b..e883a26f52 100644 --- a/cache/src/test/java/net/runelite/cache/script/disassembler/DisassemblerTest.java +++ b/cache/src/test/java/net/runelite/cache/script/disassembler/DisassemblerTest.java @@ -32,8 +32,8 @@ import net.runelite.cache.StoreLocation; import net.runelite.cache.definitions.ScriptDefinition; import net.runelite.cache.definitions.loaders.ScriptLoader; import net.runelite.cache.fs.Archive; -import net.runelite.cache.fs.FSFile; import net.runelite.cache.fs.Index; +import net.runelite.cache.fs.Storage; import net.runelite.cache.fs.Store; import net.runelite.cache.script.Instructions; import org.junit.Rule; @@ -61,17 +61,20 @@ public class DisassemblerTest { store.load(); + Storage storage = store.getStorage(); Index index = store.getIndex(IndexType.CLIENTSCRIPT); ScriptLoader loader = new ScriptLoader(); for (Archive archive : index.getArchives()) { - assert archive.getFiles().size() == 1; + byte[] contents = archive.decompress(storage.loadArchive(archive)); - FSFile file = archive.getFiles().get(0); - byte[] contents = file.getContents(); + if (contents == null) + { + continue; + } - ScriptDefinition script = loader.load(file.getFileId(), contents); + ScriptDefinition script = loader.load(0, contents); File outFile = new File(outDir, archive.getArchiveId() + ".rs2asm"); diff --git a/cache/src/test/java/net/runelite/cache/server/CacheServerTest.java b/cache/src/test/java/net/runelite/cache/server/CacheServerTest.java index dfa0d445c0..a9d3e1f64d 100644 --- a/cache/src/test/java/net/runelite/cache/server/CacheServerTest.java +++ b/cache/src/test/java/net/runelite/cache/server/CacheServerTest.java @@ -24,15 +24,18 @@ */ package net.runelite.cache.server; -import java.io.File; import java.io.FileNotFoundException; +import java.io.IOException; import net.runelite.cache.StoreLocation; import net.runelite.cache.client.CacheClient; import net.runelite.cache.fs.Archive; -import net.runelite.cache.fs.FSFile; +import net.runelite.cache.fs.Container; import net.runelite.cache.fs.Index; +import net.runelite.cache.fs.Storage; import net.runelite.cache.fs.Store; -import org.junit.Assert; +import net.runelite.cache.index.FileData; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; @@ -54,7 +57,6 @@ public class CacheServerTest CacheServer server = new CacheServer(store, REVISION)) { store.load(); - store.rebuildCrc(); server.start(); @@ -75,7 +77,7 @@ public class CacheServerTest { addInitialFilesToStore(store); - store.rebuildCrc(); + store.save(); server.start(); @@ -88,22 +90,36 @@ public class CacheServerTest Index index = store2.findIndex(0); Archive archive = index.getArchive(0); - archive.decompressAndLoad(null); // cache client doesn't decompress archive - FSFile file = archive.getFiles().get(0); - Assert.assertArrayEquals("test".getBytes(), file.getContents()); + FileData[] files = archive.getFileData(); + FileData file = files[0]; + assertEquals(7, file.getNameHash()); + + Storage storage = store2.getStorage(); + byte[] data = storage.loadArchive(archive); + data = archive.decompress(data); + assertArrayEquals("test".getBytes(), data); + assertEquals(store.getIndexes().get(0).getArchive(0).getCrc(), archive.getCrc()); } } } - private void addInitialFilesToStore(Store store) throws FileNotFoundException + private void addInitialFilesToStore(Store store) throws FileNotFoundException, IOException { + Storage storage = store.getStorage(); Index index = store.addIndex(0); + Archive archive = index.addArchive(0); - FSFile file = new FSFile(0); + FileData[] files = new FileData[1]; + archive.setFileData(files); + FileData file = files[0] = new FileData(); file.setNameHash(7); - file.setContents("test".getBytes()); - archive.addFile(file); + byte[] data = "test".getBytes(); + + Container container = new Container(archive.getCompression(), -1); + container.compress(data, null); + byte[] compressedData = container.data; + storage.saveArchive(archive, compressedData); } } diff --git a/cache/src/test/java/net/runelite/cache/util/XteaTest.java b/cache/src/test/java/net/runelite/cache/util/XteaTest.java index ef381817ef..8d6d703989 100644 --- a/cache/src/test/java/net/runelite/cache/util/XteaTest.java +++ b/cache/src/test/java/net/runelite/cache/util/XteaTest.java @@ -24,18 +24,13 @@ */ package net.runelite.cache.util; -import java.security.InvalidKeyException; -import java.security.NoSuchAlgorithmException; -import javax.crypto.BadPaddingException; -import javax.crypto.IllegalBlockSizeException; -import javax.crypto.NoSuchPaddingException; import static org.junit.Assert.assertArrayEquals; import org.junit.Test; public class XteaTest { @Test - public void test() throws InvalidKeyException, IllegalBlockSizeException, BadPaddingException, NoSuchAlgorithmException, NoSuchPaddingException + public void test() { byte[] data = "testtesttest1".getBytes(); diff --git a/cache/src/test/resources/simplelogger.properties b/cache/src/test/resources/simplelogger.properties new file mode 100644 index 0000000000..40cfffe8a7 --- /dev/null +++ b/cache/src/test/resources/simplelogger.properties @@ -0,0 +1 @@ +org.slf4j.simpleLogger.defaultLogLevel=info \ No newline at end of file diff --git a/http-service/src/main/java/net/runelite/http/service/cache/CacheService.java b/http-service/src/main/java/net/runelite/http/service/cache/CacheService.java index b2b31604f5..362948f724 100644 --- a/http-service/src/main/java/net/runelite/http/service/cache/CacheService.java +++ b/http-service/src/main/java/net/runelite/http/service/cache/CacheService.java @@ -50,9 +50,8 @@ import net.runelite.cache.definitions.loaders.ItemLoader; import net.runelite.cache.definitions.loaders.NpcLoader; import net.runelite.cache.definitions.loaders.ObjectLoader; import net.runelite.cache.fs.ArchiveFiles; +import net.runelite.cache.fs.Container; import net.runelite.cache.fs.FSFile; -import net.runelite.cache.fs.jagex.DataFile; -import net.runelite.cache.fs.jagex.DataFileReadResult; import net.runelite.http.api.cache.Cache; import net.runelite.http.api.cache.CacheArchive; import net.runelite.http.api.cache.CacheIndex; @@ -150,7 +149,7 @@ public class CacheService return null; } - DataFileReadResult result = DataFile.decompress(archiveData, null); + Container result = Container.decompress(archiveData, null); if (result == null) { return null; diff --git a/http-service/src/main/java/net/runelite/http/service/cache/CacheStorage.java b/http-service/src/main/java/net/runelite/http/service/cache/CacheStorage.java index 02e1445263..83b0eaffa1 100644 --- a/http-service/src/main/java/net/runelite/http/service/cache/CacheStorage.java +++ b/http-service/src/main/java/net/runelite/http/service/cache/CacheStorage.java @@ -24,14 +24,13 @@ */ package net.runelite.http.service.cache; -import com.google.common.hash.Hashing; import java.io.IOException; import java.util.List; import net.runelite.cache.fs.Archive; -import net.runelite.cache.fs.FSFile; import net.runelite.cache.fs.Index; import net.runelite.cache.fs.Storage; import net.runelite.cache.fs.Store; +import net.runelite.cache.index.FileData; import net.runelite.http.service.cache.beans.ArchiveEntry; import net.runelite.http.service.cache.beans.CacheEntry; import net.runelite.http.service.cache.beans.FileEntry; @@ -88,13 +87,17 @@ public class CacheStorage implements Storage archive.setNameHash(archiveEntry.getNameHash()); archive.setCrc(archiveEntry.getCrc()); archive.setRevision(archiveEntry.getRevision()); + archive.setHash(archiveEntry.getHash()); List files = cacheDao.findFilesForArchive(con, archiveEntry); + FileData[] fileData = new FileData[files.size()]; + archive.setFileData(fileData); + int idx = 0; for (FileEntry fileEntry : files) { - FSFile file = new FSFile(fileEntry.getFileId()); + FileData file = fileData[idx++] = new FileData(); + file.setId(fileEntry.getFileId()); file.setNameHash(fileEntry.getNameHash()); - archive.addFile(file); } } } @@ -115,13 +118,13 @@ public class CacheStorage implements Storage archive.getNameHash(), archive.getCrc(), archive.getRevision()); if (archiveEntry == null) { - byte[] hash = Hashing.sha256().hashBytes(archive.getData()).asBytes(); + byte[] hash = archive.getHash(); archiveEntry = cacheDao.createArchive(con, entry, archive.getArchiveId(), archive.getNameHash(), archive.getCrc(), archive.getRevision(), hash); - for (FSFile file : archive.getFiles()) + for (FileData file : archive.getFileData()) { - cacheDao.associateFileToArchive(con, archiveEntry, file.getFileId(), file.getNameHash()); + cacheDao.associateFileToArchive(con, archiveEntry, file.getId(), file.getNameHash()); } } cacheDao.associateArchiveToIndex(con, archiveEntry, entry); @@ -129,4 +132,16 @@ public class CacheStorage implements Storage } } + @Override + public byte[] loadArchive(Archive archive) throws IOException + { + throw new UnsupportedOperationException(); + } + + @Override + public void saveArchive(Archive archive, byte[] data) throws IOException + { + throw new UnsupportedOperationException(); + } + } diff --git a/http-service/src/main/java/net/runelite/http/service/cache/CacheUpdater.java b/http-service/src/main/java/net/runelite/http/service/cache/CacheUpdater.java index 56e5e59739..78b7cdb0e1 100644 --- a/http-service/src/main/java/net/runelite/http/service/cache/CacheUpdater.java +++ b/http-service/src/main/java/net/runelite/http/service/cache/CacheUpdater.java @@ -96,7 +96,7 @@ public class CacheUpdater ExecutorService executor = Executors.newSingleThreadExecutor(); CacheClient client = new CacheClient(store, rsVersion, - (Archive archive) -> executor.submit(new CacheUploader(minioClient, minioBucket, archive))); + (Archive archive, byte[] data) -> executor.submit(new CacheUploader(minioClient, minioBucket, archive, data))); client.connect(); HandshakeResponseType result = client.handshake().join(); @@ -111,6 +111,7 @@ public class CacheUpdater if (!checkOutOfDate(indexes, entries)) { + logger.info("All up to date."); return; } diff --git a/http-service/src/main/java/net/runelite/http/service/cache/CacheUploader.java b/http-service/src/main/java/net/runelite/http/service/cache/CacheUploader.java index beef4468a0..cea25cfd08 100644 --- a/http-service/src/main/java/net/runelite/http/service/cache/CacheUploader.java +++ b/http-service/src/main/java/net/runelite/http/service/cache/CacheUploader.java @@ -50,21 +50,24 @@ public class CacheUploader implements Runnable private final MinioClient minioClient; private final String minioBucket; private final Archive archive; + private final byte[] data; - public CacheUploader(MinioClient minioClient, String minioBucket, Archive archive) + public CacheUploader(MinioClient minioClient, String minioBucket, Archive archive, byte[] data) { this.minioClient = minioClient; this.minioBucket = minioBucket; this.archive = archive; + this.data = data; } @Override public void run() { - byte[] data = archive.getData(); byte[] hash = Hashing.sha256().hashBytes(data).asBytes(); String hashStr = BaseEncoding.base16().encode(hash); + archive.setHash(hash); + String path = new StringBuilder() .append(hashStr.substring(0, 2)) .append('/') diff --git a/http-service/src/main/java/net/runelite/http/service/xtea/XteaService.java b/http-service/src/main/java/net/runelite/http/service/xtea/XteaService.java index 355ef2246a..c6db5f47b1 100644 --- a/http-service/src/main/java/net/runelite/http/service/xtea/XteaService.java +++ b/http-service/src/main/java/net/runelite/http/service/xtea/XteaService.java @@ -28,7 +28,7 @@ import java.io.IOException; import java.util.List; import java.util.stream.Collectors; import net.runelite.cache.IndexType; -import net.runelite.cache.fs.jagex.DataFile; +import net.runelite.cache.fs.Container; import net.runelite.cache.util.Djb2; import net.runelite.http.api.xtea.XteaKey; import net.runelite.http.api.xtea.XteaRequest; @@ -226,7 +226,7 @@ public class XteaService try { - DataFile.decompress(data, keys); + Container.decompress(data, keys); return true; } catch (IOException ex) diff --git a/http-service/src/main/java/net/runelite/http/service/cache/schema.sql b/http-service/src/main/resources/net/runelite/http/service/cache/schema.sql similarity index 100% rename from http-service/src/main/java/net/runelite/http/service/cache/schema.sql rename to http-service/src/main/resources/net/runelite/http/service/cache/schema.sql