From 33e76436d33fa04a0bbcc2b24209cb118cb5de17 Mon Sep 17 00:00:00 2001 From: Max Weber Date: Sun, 6 Jan 2019 21:10:03 -0700 Subject: [PATCH 1/2] cache: Add FlatStorage --- .../runelite/cache/fs/flat/FlatStorage.java | 268 ++++++++++++++++++ .../cache/fs/flat/FlatStorageTest.java | 101 +++++++ 2 files changed, 369 insertions(+) create mode 100644 cache/src/main/java/net/runelite/cache/fs/flat/FlatStorage.java create mode 100644 cache/src/test/java/net/runelite/cache/fs/flat/FlatStorageTest.java diff --git a/cache/src/main/java/net/runelite/cache/fs/flat/FlatStorage.java b/cache/src/main/java/net/runelite/cache/fs/flat/FlatStorage.java new file mode 100644 index 0000000000..fdcdadeb75 --- /dev/null +++ b/cache/src/main/java/net/runelite/cache/fs/flat/FlatStorage.java @@ -0,0 +1,268 @@ +/* + * Copyright (c) 2018 Abex + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package net.runelite.cache.fs.flat; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import net.runelite.cache.fs.Archive; +import net.runelite.cache.fs.Index; +import net.runelite.cache.fs.Storage; +import net.runelite.cache.fs.Store; +import net.runelite.cache.index.FileData; + +/** + * A Storage that stores the cache as a series of flat files, designed + * to be git revisioned. + */ +public class FlatStorage implements Storage +{ + protected static final String EXTENSION = ".flatcache"; + + private final File directory; + private final Map data = new HashMap<>(); + + public FlatStorage(File directory) throws IOException + { + this.directory = directory; + } + + protected FlatStorage() + { + this.directory = null; + } + + protected InputStream openReader(String filename) throws IOException + { + return new FileInputStream(new File(directory, filename)); + } + + protected OutputStream openWriter(String filename) throws IOException + { + return new FileOutputStream(new File(directory, filename)); + } + + protected String[] listFlatcacheFiles() throws IOException + { + return directory.list((dir, name) -> name.endsWith(EXTENSION)); + } + + @Override + public void init(Store store) throws IOException + { + String[] idxs = listFlatcacheFiles(); + for (String idx : idxs) + { + int id = Integer.parseInt(idx.substring(0, idx.length() - EXTENSION.length())); + store.addIndex(id); + } + } + + @Override + public void close() throws IOException + { + } + + @Override + public void load(Store store) throws IOException + { + for (Index idx : store.getIndexes()) + { + String file = idx.getId() + EXTENSION; + try (BufferedReader br = new BufferedReader(new InputStreamReader(openReader(file)))) + { + int lineNo = 0; + Archive archive = null; + List fileData = null; + for (String line = br.readLine(); line != null; line = br.readLine()) + { + lineNo++; + + try + { + int lidx = line.indexOf('='); + String key = line.substring(0, lidx); + String value = line.substring(lidx + 1, line.length()); + + if ("file".equals(key)) + { + if (fileData == null) + { + fileData = new ArrayList<>(); + } + + int vidx = value.indexOf('='); + FileData fd = new FileData(); + fd.setId(Integer.parseInt(value.substring(0, vidx))); + fd.setNameHash(Integer.parseInt(value.substring(vidx + 1, value.length()))); + fileData.add(fd); + continue; + } + else if (fileData != null) + { + archive.setFileData(fileData.toArray(new FileData[0])); + fileData = null; + } + + if ("id".equals(key)) + { + archive = idx.addArchive(Integer.parseInt(value)); + continue; + } + + if (archive == null) + { + switch (key) + { + case "protocol": + idx.setProtocol(Integer.parseInt(value)); + continue; + case "revision": + idx.setRevision(Integer.parseInt(value)); + continue; + case "compression": + idx.setCompression(Integer.parseInt(value)); + continue; + case "crc": + idx.setCrc(Integer.parseInt(value)); + continue; + case "named": + idx.setNamed(Boolean.parseBoolean(value)); + continue; + } + } + else + { + switch (key) + { + case "namehash": + archive.setNameHash(Integer.parseInt(value)); + continue; + case "revision": + archive.setRevision(Integer.parseInt(value)); + continue; + case "crc": + archive.setCrc(Integer.parseInt(value)); + continue; + case "hash": + archive.setHash(Base64.getDecoder().decode(value)); + continue; + case "compression": + archive.setCompression(Integer.parseInt(value)); + continue; + case "contents": + data.put((long) idx.getId() << 32 | archive.getArchiveId(), Base64.getDecoder().decode(value)); + continue; + } + } + throw new IOException("unknown key: \"" + key + "\""); + } + catch (Exception e) + { + throw new IOException("error reading flatcache at " + file + ":" + lineNo, e); + } + } + + if (fileData != null) + { + archive.setFileData(fileData.toArray(new FileData[0])); + fileData = null; + } + } + } + } + + @Override + public void save(Store store) throws IOException + { + store.getIndexes().sort(Comparator.comparing(Index::getId)); + for (Index idx : store.getIndexes()) + { + String file = idx.getId() + EXTENSION; + try (PrintStream br = new PrintStream(openWriter(file))) + { + br.printf("protocol=%d\n", idx.getProtocol()); + br.printf("revision=%d\n", idx.getRevision()); + br.printf("compression=%d\n", idx.getCompression()); + br.printf("crc=%d\n", idx.getCrc()); + br.printf("named=%b\n", idx.getCompression()); + + idx.getArchives().sort(Comparator.comparing(Archive::getArchiveId)); + for (Archive archive : idx.getArchives()) + { + br.printf("id=%d\n", archive.getArchiveId()); + br.printf("namehash=%d\n", archive.getNameHash()); + br.printf("revision=%d\n", archive.getRevision()); + br.printf("crc=%d\n", archive.getCrc()); + + if (archive.getHash() != null) + { + br.append("hash="); + br.write(Base64.getEncoder().encode(archive.getHash())); + br.append("\n"); + } + + byte[] contents = store.getStorage().loadArchive(archive); + if (contents != null) + { + br.append("contents="); + br.write(Base64.getEncoder().encode(contents)); + br.append("\n"); + } + + br.printf("compression=%d\n", archive.getCompression()); + for (FileData fd : archive.getFileData()) + { + br.printf("file=%d=%d\n", fd.getId(), fd.getNameHash()); + } + } + } + } + } + + @Override + public byte[] loadArchive(Archive archive) throws IOException + { + return data.get((long) archive.getIndex().getId() << 32 | archive.getArchiveId()); + } + + @Override + public void saveArchive(Archive archive, byte[] bytes) throws IOException + { + data.put((long) archive.getIndex().getId() << 32 | archive.getArchiveId(), bytes); + } +} diff --git a/cache/src/test/java/net/runelite/cache/fs/flat/FlatStorageTest.java b/cache/src/test/java/net/runelite/cache/fs/flat/FlatStorageTest.java new file mode 100644 index 0000000000..37eb72c8c4 --- /dev/null +++ b/cache/src/test/java/net/runelite/cache/fs/flat/FlatStorageTest.java @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2016-2017, Adam + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package net.runelite.cache.fs.flat; + +import java.io.File; +import net.runelite.cache.fs.Archive; +import net.runelite.cache.fs.Container; +import net.runelite.cache.fs.Index; +import net.runelite.cache.fs.Store; +import net.runelite.cache.fs.jagex.DiskStorage; +import net.runelite.cache.index.FileData; +import org.junit.Test; +import static org.junit.Assert.*; +import org.junit.Rule; +import org.junit.rules.TemporaryFolder; + +public class FlatStorageTest +{ + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + @Test + public void testSaveArchive() throws Exception + { + File file = folder.newFolder(); + DiskStorage storage = new DiskStorage(file); + Archive archive; + Archive archive2; + try (Store store = new Store(storage)) + { + Index index = store.addIndex(0); + archive = index.addArchive(0); + archive2 = index.addArchive(1); + + FileData[] fileData = new FileData[1]; + archive.setFileData(fileData); + fileData[0] = new FileData(); + + FileData[] fileData2 = new FileData[1]; + archive2.setFileData(fileData2); + fileData2[0] = new FileData(); + + byte[] data = "test".getBytes(); + Container container = new Container(archive.getCompression(), -1); + container.compress(data, null); + byte[] compressedData = container.data; + storage.saveArchive(archive, compressedData); + + container = new Container(archive.getCompression(), 42); + container.compress(data, null); + compressedData = container.data; + archive2.setRevision(42); + storage.saveArchive(archive2, compressedData); + + store.save(); + } + + storage = new DiskStorage(file); + try (Store store = new Store(storage)) + { + store.load(); + Index index = store.findIndex(0); + Archive archive2_1 = index.getArchive(0); + Archive archive2_2 = index.getArchive(1); + + byte[] comprsesedData = storage.loadArchive(archive2_1); + byte[] data = archive2_1.decompress(comprsesedData); + assertArrayEquals("test".getBytes(), data); + assertEquals(archive.getCrc(), archive2_1.getCrc()); + assertEquals(archive.getRevision(), archive2_1.getRevision()); + + comprsesedData = storage.loadArchive(archive2_2); + data = archive2_2.decompress(comprsesedData); + assertArrayEquals("test".getBytes(), data); + assertEquals(archive2.getCrc(), archive2_2.getCrc()); + assertEquals(archive2.getRevision(), archive2_2.getRevision()); + } + } +} From 330d47b5ecf10a0911cfa9fb27e17ad7ba5cc6f2 Mon Sep 17 00:00:00 2001 From: Max Weber Date: Thu, 10 Jan 2019 03:19:55 -0700 Subject: [PATCH 2/2] cache: Add equals and hashCode to all Definitions Lombok is updated due to lombok issue 1724 breaking ModelDefinition --- .../java/net/runelite/cache/definitions/AreaDefinition.java | 3 +++ .../runelite/cache/definitions/ClientScript1Instruction.java | 2 ++ .../java/net/runelite/cache/definitions/FrameDefinition.java | 3 +++ .../net/runelite/cache/definitions/FramemapDefinition.java | 3 +++ .../net/runelite/cache/definitions/InterfaceDefinition.java | 3 +++ .../net/runelite/cache/definitions/InventoryDefinition.java | 3 +++ .../java/net/runelite/cache/definitions/ItemDefinition.java | 2 ++ .../java/net/runelite/cache/definitions/KitDefinition.java | 2 ++ .../java/net/runelite/cache/definitions/ModelDefinition.java | 2 ++ .../java/net/runelite/cache/definitions/NpcDefinition.java | 2 ++ .../net/runelite/cache/definitions/SequenceDefinition.java | 2 ++ .../net/runelite/cache/definitions/SpotAnimDefinition.java | 3 +++ .../java/net/runelite/cache/definitions/StructDefinition.java | 2 ++ .../java/net/runelite/cache/definitions/TrackDefinition.java | 3 +++ .../java/net/runelite/cache/definitions/WorldMapType0.java | 3 +++ pom.xml | 2 +- 16 files changed, 39 insertions(+), 1 deletion(-) diff --git a/cache/src/main/java/net/runelite/cache/definitions/AreaDefinition.java b/cache/src/main/java/net/runelite/cache/definitions/AreaDefinition.java index fe615b4250..27a722014b 100644 --- a/cache/src/main/java/net/runelite/cache/definitions/AreaDefinition.java +++ b/cache/src/main/java/net/runelite/cache/definitions/AreaDefinition.java @@ -24,6 +24,9 @@ */ package net.runelite.cache.definitions; +import lombok.EqualsAndHashCode; + +@EqualsAndHashCode public class AreaDefinition { public int id; diff --git a/cache/src/main/java/net/runelite/cache/definitions/ClientScript1Instruction.java b/cache/src/main/java/net/runelite/cache/definitions/ClientScript1Instruction.java index f0fe7d4683..2b80b8c1dc 100644 --- a/cache/src/main/java/net/runelite/cache/definitions/ClientScript1Instruction.java +++ b/cache/src/main/java/net/runelite/cache/definitions/ClientScript1Instruction.java @@ -24,8 +24,10 @@ */ package net.runelite.cache.definitions; +import lombok.EqualsAndHashCode; import lombok.RequiredArgsConstructor; +@EqualsAndHashCode public class ClientScript1Instruction { @RequiredArgsConstructor diff --git a/cache/src/main/java/net/runelite/cache/definitions/FrameDefinition.java b/cache/src/main/java/net/runelite/cache/definitions/FrameDefinition.java index 07fb2c7e31..fde09dfb87 100644 --- a/cache/src/main/java/net/runelite/cache/definitions/FrameDefinition.java +++ b/cache/src/main/java/net/runelite/cache/definitions/FrameDefinition.java @@ -24,6 +24,9 @@ */ package net.runelite.cache.definitions; +import lombok.EqualsAndHashCode; + +@EqualsAndHashCode public class FrameDefinition { public int id; // file id diff --git a/cache/src/main/java/net/runelite/cache/definitions/FramemapDefinition.java b/cache/src/main/java/net/runelite/cache/definitions/FramemapDefinition.java index 250204d24b..7e958ff78b 100644 --- a/cache/src/main/java/net/runelite/cache/definitions/FramemapDefinition.java +++ b/cache/src/main/java/net/runelite/cache/definitions/FramemapDefinition.java @@ -24,6 +24,9 @@ */ package net.runelite.cache.definitions; +import lombok.EqualsAndHashCode; + +@EqualsAndHashCode public class FramemapDefinition { public int id; diff --git a/cache/src/main/java/net/runelite/cache/definitions/InterfaceDefinition.java b/cache/src/main/java/net/runelite/cache/definitions/InterfaceDefinition.java index 4bbee579e2..d314a7a347 100644 --- a/cache/src/main/java/net/runelite/cache/definitions/InterfaceDefinition.java +++ b/cache/src/main/java/net/runelite/cache/definitions/InterfaceDefinition.java @@ -24,6 +24,9 @@ */ package net.runelite.cache.definitions; +import lombok.EqualsAndHashCode; + +@EqualsAndHashCode public class InterfaceDefinition { public int id = -1; diff --git a/cache/src/main/java/net/runelite/cache/definitions/InventoryDefinition.java b/cache/src/main/java/net/runelite/cache/definitions/InventoryDefinition.java index d8f383ac58..95c683e7d8 100644 --- a/cache/src/main/java/net/runelite/cache/definitions/InventoryDefinition.java +++ b/cache/src/main/java/net/runelite/cache/definitions/InventoryDefinition.java @@ -24,6 +24,9 @@ */ package net.runelite.cache.definitions; +import lombok.EqualsAndHashCode; + +@EqualsAndHashCode public class InventoryDefinition { public int id; diff --git a/cache/src/main/java/net/runelite/cache/definitions/ItemDefinition.java b/cache/src/main/java/net/runelite/cache/definitions/ItemDefinition.java index 3856b0bb61..18961a38ce 100644 --- a/cache/src/main/java/net/runelite/cache/definitions/ItemDefinition.java +++ b/cache/src/main/java/net/runelite/cache/definitions/ItemDefinition.java @@ -25,9 +25,11 @@ package net.runelite.cache.definitions; import java.util.Map; +import lombok.EqualsAndHashCode; import lombok.RequiredArgsConstructor; @RequiredArgsConstructor +@EqualsAndHashCode public class ItemDefinition { public final int id; diff --git a/cache/src/main/java/net/runelite/cache/definitions/KitDefinition.java b/cache/src/main/java/net/runelite/cache/definitions/KitDefinition.java index 8db2095eb3..9b355b80e2 100644 --- a/cache/src/main/java/net/runelite/cache/definitions/KitDefinition.java +++ b/cache/src/main/java/net/runelite/cache/definitions/KitDefinition.java @@ -24,9 +24,11 @@ */ package net.runelite.cache.definitions; +import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.RequiredArgsConstructor; +@EqualsAndHashCode @RequiredArgsConstructor public class KitDefinition { diff --git a/cache/src/main/java/net/runelite/cache/definitions/ModelDefinition.java b/cache/src/main/java/net/runelite/cache/definitions/ModelDefinition.java index 2ec675d0d0..9229c731c7 100644 --- a/cache/src/main/java/net/runelite/cache/definitions/ModelDefinition.java +++ b/cache/src/main/java/net/runelite/cache/definitions/ModelDefinition.java @@ -1,10 +1,12 @@ package net.runelite.cache.definitions; import java.util.Arrays; +import lombok.EqualsAndHashCode; import net.runelite.cache.models.CircularAngle; import net.runelite.cache.models.FaceNormal; import net.runelite.cache.models.VertexNormal; +@EqualsAndHashCode public class ModelDefinition { public int id; diff --git a/cache/src/main/java/net/runelite/cache/definitions/NpcDefinition.java b/cache/src/main/java/net/runelite/cache/definitions/NpcDefinition.java index ffb8e1888a..00e94938ab 100644 --- a/cache/src/main/java/net/runelite/cache/definitions/NpcDefinition.java +++ b/cache/src/main/java/net/runelite/cache/definitions/NpcDefinition.java @@ -26,8 +26,10 @@ package net.runelite.cache.definitions; import java.util.Map; +import lombok.EqualsAndHashCode; import lombok.RequiredArgsConstructor; +@EqualsAndHashCode @RequiredArgsConstructor public class NpcDefinition { diff --git a/cache/src/main/java/net/runelite/cache/definitions/SequenceDefinition.java b/cache/src/main/java/net/runelite/cache/definitions/SequenceDefinition.java index 257be8398a..5d9632656d 100644 --- a/cache/src/main/java/net/runelite/cache/definitions/SequenceDefinition.java +++ b/cache/src/main/java/net/runelite/cache/definitions/SequenceDefinition.java @@ -24,9 +24,11 @@ */ package net.runelite.cache.definitions; +import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.RequiredArgsConstructor; +@EqualsAndHashCode @RequiredArgsConstructor public class SequenceDefinition { diff --git a/cache/src/main/java/net/runelite/cache/definitions/SpotAnimDefinition.java b/cache/src/main/java/net/runelite/cache/definitions/SpotAnimDefinition.java index c0107e0848..6a155ba354 100644 --- a/cache/src/main/java/net/runelite/cache/definitions/SpotAnimDefinition.java +++ b/cache/src/main/java/net/runelite/cache/definitions/SpotAnimDefinition.java @@ -24,6 +24,9 @@ */ package net.runelite.cache.definitions; +import lombok.EqualsAndHashCode; + +@EqualsAndHashCode public class SpotAnimDefinition { public int rotaton = 0; diff --git a/cache/src/main/java/net/runelite/cache/definitions/StructDefinition.java b/cache/src/main/java/net/runelite/cache/definitions/StructDefinition.java index cd1d2de749..265fcb1bf4 100644 --- a/cache/src/main/java/net/runelite/cache/definitions/StructDefinition.java +++ b/cache/src/main/java/net/runelite/cache/definitions/StructDefinition.java @@ -25,8 +25,10 @@ package net.runelite.cache.definitions; import java.util.Map; +import lombok.EqualsAndHashCode; import lombok.RequiredArgsConstructor; +@EqualsAndHashCode @RequiredArgsConstructor public class StructDefinition { diff --git a/cache/src/main/java/net/runelite/cache/definitions/TrackDefinition.java b/cache/src/main/java/net/runelite/cache/definitions/TrackDefinition.java index 33007ce073..5e5cb1af9d 100644 --- a/cache/src/main/java/net/runelite/cache/definitions/TrackDefinition.java +++ b/cache/src/main/java/net/runelite/cache/definitions/TrackDefinition.java @@ -24,6 +24,9 @@ */ package net.runelite.cache.definitions; +import lombok.EqualsAndHashCode; + +@EqualsAndHashCode public class TrackDefinition { public byte[] midi; // midi file contents diff --git a/cache/src/main/java/net/runelite/cache/definitions/WorldMapType0.java b/cache/src/main/java/net/runelite/cache/definitions/WorldMapType0.java index c69c0d873a..1294d0eb4d 100644 --- a/cache/src/main/java/net/runelite/cache/definitions/WorldMapType0.java +++ b/cache/src/main/java/net/runelite/cache/definitions/WorldMapType0.java @@ -24,6 +24,9 @@ */ package net.runelite.cache.definitions; +import lombok.Data; + +@Data public class WorldMapType0 implements WorldMapTypeBase { public int field600; diff --git a/pom.xml b/pom.xml index e643be5913..a441fb3372 100644 --- a/pom.xml +++ b/pom.xml @@ -38,7 +38,7 @@ UTF-8 1.8 - 1.16.22 + 1.18.4 true true