From 33e76436d33fa04a0bbcc2b24209cb118cb5de17 Mon Sep 17 00:00:00 2001 From: Max Weber Date: Sun, 6 Jan 2019 21:10:03 -0700 Subject: [PATCH] cache: Add FlatStorage --- .../runelite/cache/fs/flat/FlatStorage.java | 268 ++++++++++++++++++ .../cache/fs/flat/FlatStorageTest.java | 101 +++++++ 2 files changed, 369 insertions(+) create mode 100644 cache/src/main/java/net/runelite/cache/fs/flat/FlatStorage.java create mode 100644 cache/src/test/java/net/runelite/cache/fs/flat/FlatStorageTest.java diff --git a/cache/src/main/java/net/runelite/cache/fs/flat/FlatStorage.java b/cache/src/main/java/net/runelite/cache/fs/flat/FlatStorage.java new file mode 100644 index 0000000000..fdcdadeb75 --- /dev/null +++ b/cache/src/main/java/net/runelite/cache/fs/flat/FlatStorage.java @@ -0,0 +1,268 @@ +/* + * Copyright (c) 2018 Abex + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package net.runelite.cache.fs.flat; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import net.runelite.cache.fs.Archive; +import net.runelite.cache.fs.Index; +import net.runelite.cache.fs.Storage; +import net.runelite.cache.fs.Store; +import net.runelite.cache.index.FileData; + +/** + * A Storage that stores the cache as a series of flat files, designed + * to be git revisioned. + */ +public class FlatStorage implements Storage +{ + protected static final String EXTENSION = ".flatcache"; + + private final File directory; + private final Map data = new HashMap<>(); + + public FlatStorage(File directory) throws IOException + { + this.directory = directory; + } + + protected FlatStorage() + { + this.directory = null; + } + + protected InputStream openReader(String filename) throws IOException + { + return new FileInputStream(new File(directory, filename)); + } + + protected OutputStream openWriter(String filename) throws IOException + { + return new FileOutputStream(new File(directory, filename)); + } + + protected String[] listFlatcacheFiles() throws IOException + { + return directory.list((dir, name) -> name.endsWith(EXTENSION)); + } + + @Override + public void init(Store store) throws IOException + { + String[] idxs = listFlatcacheFiles(); + for (String idx : idxs) + { + int id = Integer.parseInt(idx.substring(0, idx.length() - EXTENSION.length())); + store.addIndex(id); + } + } + + @Override + public void close() throws IOException + { + } + + @Override + public void load(Store store) throws IOException + { + for (Index idx : store.getIndexes()) + { + String file = idx.getId() + EXTENSION; + try (BufferedReader br = new BufferedReader(new InputStreamReader(openReader(file)))) + { + int lineNo = 0; + Archive archive = null; + List fileData = null; + for (String line = br.readLine(); line != null; line = br.readLine()) + { + lineNo++; + + try + { + int lidx = line.indexOf('='); + String key = line.substring(0, lidx); + String value = line.substring(lidx + 1, line.length()); + + if ("file".equals(key)) + { + if (fileData == null) + { + fileData = new ArrayList<>(); + } + + int vidx = value.indexOf('='); + FileData fd = new FileData(); + fd.setId(Integer.parseInt(value.substring(0, vidx))); + fd.setNameHash(Integer.parseInt(value.substring(vidx + 1, value.length()))); + fileData.add(fd); + continue; + } + else if (fileData != null) + { + archive.setFileData(fileData.toArray(new FileData[0])); + fileData = null; + } + + if ("id".equals(key)) + { + archive = idx.addArchive(Integer.parseInt(value)); + continue; + } + + if (archive == null) + { + switch (key) + { + case "protocol": + idx.setProtocol(Integer.parseInt(value)); + continue; + case "revision": + idx.setRevision(Integer.parseInt(value)); + continue; + case "compression": + idx.setCompression(Integer.parseInt(value)); + continue; + case "crc": + idx.setCrc(Integer.parseInt(value)); + continue; + case "named": + idx.setNamed(Boolean.parseBoolean(value)); + continue; + } + } + else + { + switch (key) + { + case "namehash": + archive.setNameHash(Integer.parseInt(value)); + continue; + case "revision": + archive.setRevision(Integer.parseInt(value)); + continue; + case "crc": + archive.setCrc(Integer.parseInt(value)); + continue; + case "hash": + archive.setHash(Base64.getDecoder().decode(value)); + continue; + case "compression": + archive.setCompression(Integer.parseInt(value)); + continue; + case "contents": + data.put((long) idx.getId() << 32 | archive.getArchiveId(), Base64.getDecoder().decode(value)); + continue; + } + } + throw new IOException("unknown key: \"" + key + "\""); + } + catch (Exception e) + { + throw new IOException("error reading flatcache at " + file + ":" + lineNo, e); + } + } + + if (fileData != null) + { + archive.setFileData(fileData.toArray(new FileData[0])); + fileData = null; + } + } + } + } + + @Override + public void save(Store store) throws IOException + { + store.getIndexes().sort(Comparator.comparing(Index::getId)); + for (Index idx : store.getIndexes()) + { + String file = idx.getId() + EXTENSION; + try (PrintStream br = new PrintStream(openWriter(file))) + { + br.printf("protocol=%d\n", idx.getProtocol()); + br.printf("revision=%d\n", idx.getRevision()); + br.printf("compression=%d\n", idx.getCompression()); + br.printf("crc=%d\n", idx.getCrc()); + br.printf("named=%b\n", idx.getCompression()); + + idx.getArchives().sort(Comparator.comparing(Archive::getArchiveId)); + for (Archive archive : idx.getArchives()) + { + br.printf("id=%d\n", archive.getArchiveId()); + br.printf("namehash=%d\n", archive.getNameHash()); + br.printf("revision=%d\n", archive.getRevision()); + br.printf("crc=%d\n", archive.getCrc()); + + if (archive.getHash() != null) + { + br.append("hash="); + br.write(Base64.getEncoder().encode(archive.getHash())); + br.append("\n"); + } + + byte[] contents = store.getStorage().loadArchive(archive); + if (contents != null) + { + br.append("contents="); + br.write(Base64.getEncoder().encode(contents)); + br.append("\n"); + } + + br.printf("compression=%d\n", archive.getCompression()); + for (FileData fd : archive.getFileData()) + { + br.printf("file=%d=%d\n", fd.getId(), fd.getNameHash()); + } + } + } + } + } + + @Override + public byte[] loadArchive(Archive archive) throws IOException + { + return data.get((long) archive.getIndex().getId() << 32 | archive.getArchiveId()); + } + + @Override + public void saveArchive(Archive archive, byte[] bytes) throws IOException + { + data.put((long) archive.getIndex().getId() << 32 | archive.getArchiveId(), bytes); + } +} diff --git a/cache/src/test/java/net/runelite/cache/fs/flat/FlatStorageTest.java b/cache/src/test/java/net/runelite/cache/fs/flat/FlatStorageTest.java new file mode 100644 index 0000000000..37eb72c8c4 --- /dev/null +++ b/cache/src/test/java/net/runelite/cache/fs/flat/FlatStorageTest.java @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2016-2017, Adam + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package net.runelite.cache.fs.flat; + +import java.io.File; +import net.runelite.cache.fs.Archive; +import net.runelite.cache.fs.Container; +import net.runelite.cache.fs.Index; +import net.runelite.cache.fs.Store; +import net.runelite.cache.fs.jagex.DiskStorage; +import net.runelite.cache.index.FileData; +import org.junit.Test; +import static org.junit.Assert.*; +import org.junit.Rule; +import org.junit.rules.TemporaryFolder; + +public class FlatStorageTest +{ + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + @Test + public void testSaveArchive() throws Exception + { + File file = folder.newFolder(); + DiskStorage storage = new DiskStorage(file); + Archive archive; + Archive archive2; + try (Store store = new Store(storage)) + { + Index index = store.addIndex(0); + archive = index.addArchive(0); + archive2 = index.addArchive(1); + + FileData[] fileData = new FileData[1]; + archive.setFileData(fileData); + fileData[0] = new FileData(); + + FileData[] fileData2 = new FileData[1]; + archive2.setFileData(fileData2); + fileData2[0] = new FileData(); + + byte[] data = "test".getBytes(); + Container container = new Container(archive.getCompression(), -1); + container.compress(data, null); + byte[] compressedData = container.data; + storage.saveArchive(archive, compressedData); + + container = new Container(archive.getCompression(), 42); + container.compress(data, null); + compressedData = container.data; + archive2.setRevision(42); + storage.saveArchive(archive2, compressedData); + + store.save(); + } + + storage = new DiskStorage(file); + try (Store store = new Store(storage)) + { + store.load(); + Index index = store.findIndex(0); + Archive archive2_1 = index.getArchive(0); + Archive archive2_2 = index.getArchive(1); + + byte[] comprsesedData = storage.loadArchive(archive2_1); + byte[] data = archive2_1.decompress(comprsesedData); + assertArrayEquals("test".getBytes(), data); + assertEquals(archive.getCrc(), archive2_1.getCrc()); + assertEquals(archive.getRevision(), archive2_1.getRevision()); + + comprsesedData = storage.loadArchive(archive2_2); + data = archive2_2.decompress(comprsesedData); + assertArrayEquals("test".getBytes(), data); + assertEquals(archive2.getCrc(), archive2_2.getCrc()); + assertEquals(archive2.getRevision(), archive2_2.getRevision()); + } + } +}