ra/runelite
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

cache: remove tree storage

Browse Source
This commit is contained in:
Adam
2017-12-16 17:50:47 -05:00
parent 440ecd6c7c
commit cba9866304
5 changed files with 5 additions and 450 deletions
Download Patch File Download Diff File Expand all files Collapse all files

79
cache/src/main/java/net/runelite/cache/Cache.java vendored
View File

@@ -26,10 +26,7 @@ package net.runelite.cache;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import net.runelite.cache.fs.Storage;
import net.runelite.cache.fs.Store; import net.runelite.cache.fs.Store;
import net.runelite.cache.fs.jagex.DiskStorage;
import net.runelite.cache.fs.tree.TreeStorage;
import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.DefaultParser; import org.apache.commons.cli.DefaultParser;
@@ -43,10 +40,6 @@ public class Cache
Options options = new Options(); Options options = new Options();
options.addOption("c", "cache", true, "cache base"); options.addOption("c", "cache", true, "cache base");
options.addOption("t", "tree", true, "tree base");
options.addOption("u", "unpack", false, "unpack cache");
options.addOption("p", "pack", false, "pack cache");
options.addOption(null, "items", true, "directory to dump items to"); options.addOption(null, "items", true, "directory to dump items to");
options.addOption(null, "npcs", true, "directory to dump npcs to"); options.addOption(null, "npcs", true, "directory to dump npcs to");
@@ -67,59 +60,8 @@ public class Cache
} }
String cache = cmd.getOptionValue("cache"); String cache = cmd.getOptionValue("cache");
String tree = cmd.getOptionValue("tree");
if (cmd.hasOption("p")) Store store = loadStore(cache);
{
if (cache == null || tree == null)
{
System.err.println("Cache and tree base must be specified to pack");
System.exit(-1);
}
System.out.print("Packing tree from " + tree + " to " + cache + "...");
File cacheDir = new File(cache),
treeDir = new File(tree);
Storage from = new TreeStorage(treeDir);
Storage to = new DiskStorage(cacheDir);
Store store = new Store(from);
store.load();
to.save(store);
System.out.println(" done!");
return;
}
else if (cmd.hasOption("u"))
{
if (cache == null || tree == null)
{
System.err.println("Cache and tree base must be specified to unpack");
System.exit(-1);
}
System.out.print("Unpacking cache from " + cache + " to " + tree + "...");
Store treeBase = new Store(new File(cache));
treeBase.load();
TreeStorage storage = new TreeStorage(new File(tree));
storage.save(treeBase);
System.out.println(" done!");
return;
}
if (cache != null && tree != null)
{
System.err.println("Cannot specify both cache and tree");
System.exit(-1);
return;
}
Store store = loadStore(cache, tree);
if (cmd.hasOption("items")) if (cmd.hasOption("items"))
{ {
@@ -179,22 +121,11 @@ public class Cache
} }
} }
private static Store loadStore(String cache, String tree) throws IOException private static Store loadStore(String cache) throws IOException
{ {
if (cache == null) Store store = new Store(new File(cache));
{ store.load();
Storage storage = new TreeStorage(new File(tree)); return store;
Store store = new Store(storage);
store.load();
return store;
}
else
{
Store store = new Store(new File(cache));
store.load();
return store;
}
} }
private static void dumpItems(Store store, File itemdir) throws IOException private static void dumpItems(Store store, File itemdir) throws IOException

289
cache/src/main/java/net/runelite/cache/fs/tree/TreeStorage.java vendored
View File

@@ -1,289 +0,0 @@
/*
* Copyright (c) 2016-2017, Adam <Adam@sigterm.info>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package net.runelite.cache.fs.tree;
import com.google.common.io.Files;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import net.runelite.cache.fs.Archive;
import net.runelite.cache.fs.FSFile;
import net.runelite.cache.fs.Index;
import net.runelite.cache.fs.Storage;
import net.runelite.cache.fs.Store;
public class TreeStorage implements Storage
{
private final File folder;
public TreeStorage(File folder)
{
this.folder = folder;
}
@Override
public void init(Store store) throws IOException
{
}
@Override
public void close() throws IOException
{
}
@Override
public void load(Store store) throws IOException
{
for (File idx : folder.listFiles())
{
if (!idx.isDirectory())
{
continue;
}
int id = Integer.parseInt(idx.getName());
Index index = store.addIndex(id);
loadIndex(index, folder, idx);
}
Collections.sort(store.getIndexes(), (idx1, idx2) -> Integer.compare(idx1.getId(), idx2.getId()));
}
private void loadIndex(Index index, File parent, File to) throws IOException
{
for (File f : to.listFiles())
{
if (f.isDirectory())
{
int id = Integer.parseInt(f.getName());
Archive archive = index.addArchive(id);
loadTree(archive, to, f);
}
else if (f.getName().endsWith(".dat"))
{
// one file. archiveId-fileId-name
String[] parts = Files.getNameWithoutExtension(f.getName()).split("-");
int id = Integer.parseInt(parts[0]);
Archive archive = index.addArchive(id);
loadTreeSingleFile(archive, to, f);
}
else if (f.getName().endsWith(".datc"))
{
// packed data
String[] parts = Files.getNameWithoutExtension(f.getName()).split("-");
int id = Integer.parseInt(parts[0]);
Archive archive = index.addArchive(id);
loadTreeData(archive, to, f);
}
}
String str = Files.readFirstLine(new File(parent, index.getId() + ".rev"), Charset.defaultCharset());
int revision = Integer.parseInt(str);
index.setRevision(revision);
Collections.sort(index.getArchives(), (ar1, ar2) -> Integer.compare(ar1.getArchiveId(), ar2.getArchiveId()));
}
public void loadTreeData(Archive archive, File parent, File from) throws IOException
{
//archiveId-fileId-fileName
String[] parts = Files.getNameWithoutExtension(from.getName()).split("-");
assert parts.length == 3;
int archiveId = Integer.parseInt(parts[0]);
int fileId = Integer.parseInt(parts[1]);
int nameHash = (int) Long.parseLong(parts[2], 16);
assert archiveId == archive.getArchiveId();
byte[] data = Files.toByteArray(from);
FSFile file = new FSFile(fileId);
file.setNameHash(nameHash);
file.setContents(data);
archive.addFile(file);
File archiveFile = new File(parent, archive.getArchiveId() + ".rev");
int rev = Integer.parseInt(Files.readFirstLine(archiveFile, Charset.defaultCharset()));
archive.setRevision(rev);
archiveFile = new File(parent, archive.getArchiveId() + ".name");
int name = Integer.parseInt(Files.readFirstLine(archiveFile, Charset.defaultCharset()));
archive.setNameHash(name);
}
public void loadTreeSingleFile(Archive archive, File parent, File from) throws IOException
{
//archiveId-fileId-fileName
String[] parts = Files.getNameWithoutExtension(from.getName()).split("-");
assert parts.length == 3;
int archiveId = Integer.parseInt(parts[0]);
int fileId = Integer.parseInt(parts[1]);
int nameHash = (int) Long.parseLong(parts[2], 16);
assert archiveId == archive.getArchiveId();
FSFile file = new FSFile(fileId);
file.setNameHash(nameHash);
archive.addFile(file);
byte[] contents = Files.toByteArray(from);
file.setContents(contents);
File archiveFile = new File(parent, archive.getArchiveId() + ".rev");
int rev = Integer.parseInt(Files.readFirstLine(archiveFile, Charset.defaultCharset()));
archive.setRevision(rev);
archiveFile = new File(parent, archive.getArchiveId() + ".name");
int name = Integer.parseInt(Files.readFirstLine(archiveFile, Charset.defaultCharset()));
archive.setNameHash(name);
}
public void loadTree(Archive archive, File parent, File from) throws IOException
{
List<FSFile> files = new ArrayList<>();
for (File file : from.listFiles())
{
//fileId-fileName.dat
String[] split = Files.getNameWithoutExtension(file.getName()).split("-");
assert split.length == 2;
int fileId = Integer.parseInt(split[0]);
int fileName = (int) Long.parseLong(split[1], 16);
FSFile f = new FSFile(fileId);
f.setNameHash(fileName);
files.add(f);
byte[] contents = Files.toByteArray(file);
f.setContents(contents);
}
// the filesystem may order these differently (eg, 1, 10, 2)
Collections.sort(files, (f1, f2) -> Integer.compare(f1.getFileId(), f2.getFileId()));
files.forEach(archive::addFile);
File archiveFile = new File(parent, archive.getArchiveId() + ".rev");
int rev = Integer.parseInt(Files.readFirstLine(archiveFile, Charset.defaultCharset()));
archive.setRevision(rev);
archiveFile = new File(parent, archive.getArchiveId() + ".name");
int name = Integer.parseInt(Files.readFirstLine(archiveFile, Charset.defaultCharset()));
archive.setNameHash(name);
}
@Override
public void save(Store store) throws IOException
{
for (Index i : store.getIndexes())
{
saveIndex(i);
}
}
private void saveIndex(Index i) throws IOException
{
File idx = new File(folder, "" + i.getId());
idx.mkdirs();
for (Archive a : i.getArchives())
{
saveArchive(a, idx);
}
File rev = new File(folder, i.getId() + ".rev");
Files.write(Integer.toString(i.getRevision()), rev, Charset.defaultCharset());
}
private void saveArchive(Archive a, File to) throws IOException
{
byte[] data = a.getData();
List<FSFile> files = a.getFiles();
if (data != null)
{
assert files.size() == 1; // this is the maps
FSFile file = files.get(0);
File archiveFile = new File(to, a.getArchiveId() + "-"
+ file.getFileId() + "-" + Integer.toHexString(file.getNameHash()) + ".datc");
Files.write(data, archiveFile);
archiveFile = new File(to, a.getArchiveId() + ".rev");
Files.write(Integer.toString(a.getRevision()), archiveFile, Charset.defaultCharset());
archiveFile = new File(to, a.getArchiveId() + ".name");
Files.write(Integer.toString(a.getNameHash()), archiveFile, Charset.defaultCharset());
return;
}
if (files.size() == 1)
{
FSFile file = files.get(0);
File archiveFile = new File(to, a.getArchiveId() + "-"
+ file.getFileId() + "-" + Integer.toHexString(file.getNameHash()) + ".dat");
byte[] contents = file.getContents();
Files.write(contents, archiveFile);
archiveFile = new File(to, a.getArchiveId() + ".rev");
Files.write(Integer.toString(a.getRevision()), archiveFile, Charset.defaultCharset());
archiveFile = new File(to, a.getArchiveId() + ".name");
Files.write(Integer.toString(a.getNameHash()), archiveFile, Charset.defaultCharset());
return;
}
File archiveFile = new File(to, a.getArchiveId() + ".rev");
Files.write(Integer.toString(a.getRevision()), archiveFile, Charset.defaultCharset());
archiveFile = new File(to, a.getArchiveId() + ".name");
Files.write(Integer.toString(a.getNameHash()), archiveFile, Charset.defaultCharset());
File archiveFolder = new File(to, Integer.toString(a.getArchiveId()));
archiveFolder.mkdirs();
for (FSFile file : files)
{
archiveFile = new File(archiveFolder, file.getFileId() + "-"
+ Integer.toHexString(file.getNameHash()) + ".dat");
byte[] contents = file.getContents();
Files.write(contents, archiveFile);
}
}
}

28
cache/src/test/java/net/runelite/cache/client/CacheClientTest.java vendored
View File

@@ -28,7 +28,6 @@ import java.io.File;
import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletableFuture;
import net.runelite.cache.CacheProperties; import net.runelite.cache.CacheProperties;
import net.runelite.cache.fs.Store; import net.runelite.cache.fs.Store;
import net.runelite.cache.fs.tree.TreeStorage;
import net.runelite.cache.protocol.packets.HandshakeResponseType; import net.runelite.cache.protocol.packets.HandshakeResponseType;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
@@ -72,31 +71,4 @@ public class CacheClientTest
store.save(); store.save();
} }
} }
@Test
@Ignore
public void testTree() throws Exception
{
try (Store store = new Store(new File("C:\\rs\\temp")))
{
TreeStorage storage = new TreeStorage(new File("C:\\rs\\runescape-data\\cache"));
storage.load(store);
CacheClient c = new CacheClient(store, CacheProperties.getRsVersion());
c.connect();
CompletableFuture<HandshakeResponseType> handshake = c.handshake();
HandshakeResponseType result = handshake.get();
logger.info("Handshake result: {}", result);
Assert.assertEquals(HandshakeResponseType.RESPONSE_OK, result);
c.download();
c.close();
storage = new TreeStorage(new File("C:\\rs\\temp\\t"));
storage.save(store);
}
}
} }

47
cache/src/test/java/net/runelite/cache/fs/StoreLoadTest.java vendored
View File

@@ -28,9 +28,7 @@ import com.google.common.io.Files;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import net.runelite.cache.StoreLocation; import net.runelite.cache.StoreLocation;
import net.runelite.cache.fs.tree.TreeStorage;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Rule; import org.junit.Rule;
import org.junit.Test; import org.junit.Test;
import org.junit.rules.TemporaryFolder; import org.junit.rules.TemporaryFolder;
@@ -81,49 +79,4 @@ public class StoreLoadTest
} }
} }
} }
@Test
@Ignore
public void unpackStore() throws IOException
{
File base = StoreLocation.LOCATION;
try (Store store = new Store(base))
{
store.load();
TreeStorage storage = new TreeStorage(folder.newFolder());
storage.save(store);
}
}
@Test
@Ignore
public void loadTree() throws IOException
{
try (Store store = new Store(folder.newFolder()))
{
TreeStorage storage = new TreeStorage(new File("C:\\rs\\temp\\tree"));
storage.load(store);
try (Store store2 = new Store(StoreLocation.LOCATION))
{
store2.load();
Assert.assertEquals(store, store2);
}
}
}
@Test
@Ignore
public void saveTree() throws IOException
{
try (Store store = new Store(new File("d:/rs/07/temp/cache")))
{
store.load();
TreeStorage storage = new TreeStorage(new File("d:/rs/07/temp/tree"));
storage.save(store);
}
}
} }

12
cache/src/test/java/net/runelite/cache/fs/StoreTest.java vendored
View File

@@ -28,7 +28,6 @@ import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.Random; import java.util.Random;
import net.runelite.cache.StoreLocation; import net.runelite.cache.StoreLocation;
import net.runelite.cache.fs.tree.TreeStorage;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Rule; import org.junit.Rule;
import org.junit.Test; import org.junit.Test;
@@ -153,17 +152,6 @@ public class StoreTest
Assert.assertEquals(store, store2); Assert.assertEquals(store, store2);
} }
// Test tree save/load
File tree = folder.newFolder();
Storage treeStorage = new TreeStorage(tree);
treeStorage.save(store);
try (Store store2 = new Store(treeStorage))
{
store2.load();
}
} }
} }
} }