ra/runelite
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

cache: add saveTree/loadTree to save/load cache to/from a normal filesystem

Browse Source
This commit is contained in:
Adam
2017-03-14 14:08:36 -04:00
parent 963d82c903
commit b2327d7e8e
5 changed files with 350 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

166
cache/src/main/java/net/runelite/cache/fs/Archive.java vendored
View File

@@ -22,12 +22,14 @@
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package net.runelite.cache.fs;
import com.google.common.io.Files;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import net.runelite.cache.io.InputStream;
@@ -38,9 +40,9 @@ import org.slf4j.LoggerFactory;
public class Archive
{
private static final Logger logger = LoggerFactory.getLogger(Archive.class);
private Index index; // member of this index
private byte[] data; // raw data from the datafile, compressed/encrypted
private int archiveId;
@@ -51,7 +53,7 @@ public class Archive
private int compression;
private List<File> files = new ArrayList<>();
public Archive(Index index, int id)
{
this.index = index;
@@ -109,18 +111,18 @@ public class Archive
{
this.data = data;
}
public File addFile(int id)
{
File file = new File(this, id);
this.files.add(file);
return file;
}
public void load(InputStream stream, int numberOfFiles, int protocol)
public void loadFiles(InputStream stream, int numberOfFiles, int protocol)
{
int archive = 0;
for (int i = 0; i < numberOfFiles; ++i)
{
int fileId = archive += protocol >= 7 ? stream.readBigSmart() : stream.readUnsignedShort();
@@ -140,7 +142,7 @@ public class Archive
logger.warn("Unable to decrypt archive {}", this);
return;
}
byte[] decompressedData = res.data;
if (this.crc != res.crc)
@@ -164,7 +166,7 @@ public class Archive
loadContents(decompressedData);
this.setData(null); // now that we've loaded it, clean it so it doesn't get written back
}
public void loadContents(byte[] data)
{
logger.trace("Loading contents of archive {} ({} files)", archiveId, files.size());
@@ -217,9 +219,9 @@ public class Archive
for (int id = 0; id < filesCount; ++id)
{
int chunkSize = chunkSizes[id][chunk];
stream.readBytes(fileContents[id], fileOffsets[id], chunkSize);
fileOffsets[id] += chunkSize;
}
}
@@ -238,7 +240,7 @@ public class Archive
logger.trace("Saving contents of archive {}/{} using cached data", index.getId(), archiveId);
return data;
}
OutputStream stream = new OutputStream();
int filesCount = this.getFiles().size();
@@ -276,7 +278,143 @@ public class Archive
return fileData;
}
public void saveTree(java.io.File to) throws IOException
{
if (data != null)
{
assert files.size() == 1; // this is the maps
File file = files.get(0);
java.io.File archiveFile = new java.io.File(to, this.getArchiveId() + "-" + file.getFileId() + "-" + file.getNameHash() + ".datc");
Files.write(data, archiveFile);
archiveFile = new java.io.File(to, this.getArchiveId() + ".rev");
Files.write("" + this.getRevision(), archiveFile, Charset.defaultCharset());
archiveFile = new java.io.File(to, this.getArchiveId() + ".name");
Files.write("" + this.getNameHash(), archiveFile, Charset.defaultCharset());
return;
}
if (files.size() == 1)
{
File file = this.getFiles().get(0);
java.io.File archiveFile = new java.io.File(to, this.getArchiveId() + "-" + file.getFileId() + "-" + file.getNameHash() + ".dat");
byte[] contents = file.getContents();
Files.write(contents, archiveFile);
archiveFile = new java.io.File(to, this.getArchiveId() + ".rev");
Files.write("" + this.getRevision(), archiveFile, Charset.defaultCharset());
archiveFile = new java.io.File(to, this.getArchiveId() + ".name");
Files.write("" + this.getNameHash(), archiveFile, Charset.defaultCharset());
return;
}
java.io.File archiveFile = new java.io.File(to, this.getArchiveId() + ".rev");
Files.write("" + this.getRevision(), archiveFile, Charset.defaultCharset());
archiveFile = new java.io.File(to, this.getArchiveId() + ".name");
Files.write("" + this.getNameHash(), archiveFile, Charset.defaultCharset());
java.io.File archiveFolder = new java.io.File(to, "" + this.getArchiveId());
archiveFolder.mkdirs();
for (File file : files)
{
archiveFile = new java.io.File(archiveFolder, file.getFileId() + "-" + file.getNameHash() + ".dat");
byte[] contents = file.getContents();
Files.write(contents, archiveFile);
}
}
public void loadTreeData(java.io.File parent, java.io.File from) throws IOException
{
//archiveId-fileId-fileName - assumes name isn't negative
String[] parts = Files.getNameWithoutExtension(from.getName()).split("-");
int archiveId = Integer.parseInt(parts[0]);
int fileId = Integer.parseInt(parts[1]);
int nameHash = Integer.parseInt(parts[2]);
assert archiveId == this.getArchiveId();
data = Files.toByteArray(from);
File file = new File(this, fileId);
file.setNameHash(nameHash);
files.add(file);
java.io.File archiveFile = new java.io.File(parent, this.getArchiveId() + ".rev");
int rev = Integer.parseInt(Files.readFirstLine(archiveFile, Charset.defaultCharset()));
this.setRevision(rev);
archiveFile = new java.io.File(parent, this.getArchiveId() + ".name");
int name = Integer.parseInt(Files.readFirstLine(archiveFile, Charset.defaultCharset()));
this.setNameHash(name);
}
public void loadTreeSingleFile(java.io.File parent, java.io.File from) throws IOException
{
//archiveId-fileId-fileName
String[] parts = Files.getNameWithoutExtension(from.getName()).split("-");
int archiveId = Integer.parseInt(parts[0]);
int fileId = Integer.parseInt(parts[1]);
int nameHash = Integer.parseInt(parts[2]);
assert archiveId == this.getArchiveId();
File file = new File(this, fileId);
file.setNameHash(nameHash);
byte[] contents = Files.toByteArray(from);
file.setContents(contents);
files.add(file);
java.io.File archiveFile = new java.io.File(parent, this.getArchiveId() + ".rev");
int rev = Integer.parseInt(Files.readFirstLine(archiveFile, Charset.defaultCharset()));
this.setRevision(rev);
archiveFile = new java.io.File(parent, this.getArchiveId() + ".name");
int name = Integer.parseInt(Files.readFirstLine(archiveFile, Charset.defaultCharset()));
this.setNameHash(name);
}
public void loadTree(java.io.File parent, java.io.File from) throws IOException
{
for (java.io.File file : from.listFiles())
{
//fileId-fileName.dat
String[] split = Files.getNameWithoutExtension(file.getName()).split("-");
int fileId = Integer.parseInt(split[0]);
int fileName = Integer.parseInt(split[1]);
File f = new File(this, fileId);
f.setNameHash(fileName);
byte[] contents = Files.toByteArray(file);
f.setContents(contents);
files.add(f);
}
java.io.File archiveFile = new java.io.File(parent, this.getArchiveId() + ".rev");
int rev = Integer.parseInt(Files.readFirstLine(archiveFile, Charset.defaultCharset()));
this.setRevision(rev);
archiveFile = new java.io.File(parent, this.getArchiveId() + ".name");
int name = Integer.parseInt(Files.readFirstLine(archiveFile, Charset.defaultCharset()));
this.setNameHash(name);
// the filesystem may order these differently (eg, 1, 10, 2)
Collections.sort(files, (f1, f2) -> Integer.compare(f1.getFileId(), f2.getFileId()));
}
public void loadNames(InputStream stream, int numberOfFiles)
{
for (int i = 0; i < numberOfFiles; ++i)

104
cache/src/main/java/net/runelite/cache/fs/Index.java vendored
View File

@@ -22,12 +22,14 @@
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package net.runelite.cache.fs;
import com.google.common.io.Files;
import java.io.Closeable;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import net.runelite.cache.util.Djb2;
@@ -40,11 +42,11 @@ import org.slf4j.LoggerFactory;
public class Index implements Closeable
{
private static final Logger logger = LoggerFactory.getLogger(Index.class);
private final Store store;
private final IndexFile index;
private final int id;
private XteaKeyManager xteaManager;
private int protocol = 7;
@@ -55,14 +57,14 @@ public class Index implements Closeable
private int compression; // compression method of this index's data in 255
private final List<Archive> archives = new ArrayList<>();
public Index(Store store, IndexFile index, int id)
{
this.store = store;
this.index = index;
this.id = id;
}
@Override
public void close() throws IOException
{
@@ -160,19 +162,23 @@ public class Index implements Closeable
{
return archives;
}
public Archive addArchive(int id)
{
Archive archive = new Archive(this, id);
this.archives.add(archive);
return archive;
}
public Archive getArchive(int id)
{
for (Archive a : archives)
{
if (a.getArchiveId() == id)
{
return a;
}
}
return null;
}
@@ -180,25 +186,29 @@ public class Index implements Closeable
{
int hash = Djb2.hash(name);
for (Archive a : archives)
{
if (a.getNameHash() == hash)
{
return a;
}
}
return null;
}
public void load() throws IOException
{
logger.trace("Loading index {}", id);
DataFile dataFile = store.getData();
IndexFile index255 = store.getIndex255();
IndexEntry entry = index255.read(id);
byte[] indexData = dataFile.read(index255.getIndexFileId(), entry.getId(), entry.getSector(), entry.getLength());
DataFileReadResult res = DataFile.decompress(indexData, null);
byte[] data = res.data;
archives.clear();
readIndexData(data);
this.crc = res.crc;
@@ -208,11 +218,11 @@ public class Index implements Closeable
this.loadArchives();
}
public void save() throws IOException
{
saveArchives();
byte[] data = this.writeIndexData();
DataFile dataFile = store.getData();
@@ -226,7 +236,63 @@ public class Index implements Closeable
this.crc = res.crc;
this.whirlpool = res.whirlpool;
}
public void saveTree(java.io.File to) throws IOException
{
java.io.File idx = new java.io.File(to, "" + this.getId());
idx.mkdirs();
for (Archive a : archives)
{
a.saveTree(idx);
}
java.io.File rev = new java.io.File(to, this.getId() + ".rev");
Files.write("" + this.getRevision(), rev, Charset.defaultCharset());
}
public void loadTree(java.io.File parent, java.io.File to) throws IOException
{
for (java.io.File f : to.listFiles())
{
if (f.isDirectory())
{
int id = Integer.parseInt(f.getName());
Archive archive = new Archive(this, id);
archive.loadTree(to, f);
archives.add(archive);
}
else if (f.getName().endsWith(".dat"))
{
// one file. archiveId-fileId-name
String[] parts = Files.getNameWithoutExtension(f.getName()).split("-");
int id = Integer.parseInt(parts[0]);
Archive archive = new Archive(this, id);
archive.loadTreeSingleFile(to, f);
archives.add(archive);
}
else if (f.getName().endsWith(".datc"))
{
// packed data
String[] parts = Files.getNameWithoutExtension(f.getName()).split("-");
int id = Integer.parseInt(parts[0]);
Archive archive = new Archive(this, id);
archive.loadTreeData(to, f);
archives.add(archive);
}
}
String str = Files.readFirstLine(new java.io.File(parent, this.getId() + ".rev"), Charset.defaultCharset());
revision = Integer.parseInt(str);
Collections.sort(archives, (ar1, ar2) -> Integer.compare(ar1.getArchiveId(), ar2.getArchiveId()));
}
public void readIndexData(byte[] data)
{
InputStream stream = new InputStream(data);
@@ -304,7 +370,7 @@ public class Index implements Closeable
archive = 0;
Archive a = this.archives.get(index);
a.load(stream, numberOfFiles[index], protocol);
a.loadFiles(stream, numberOfFiles[index], protocol);
}
if (named)
@@ -317,7 +383,7 @@ public class Index implements Closeable
}
}
}
private void loadArchives() throws IOException
{
// get data from index file
@@ -347,7 +413,7 @@ public class Index implements Closeable
a.decompressAndLoad(null);
}
}
public void saveArchives() throws IOException
{
for (Archive a : archives)
@@ -372,14 +438,14 @@ public class Index implements Closeable
DataFileWriteResult res = data.write(this.id, a.getArchiveId(), compressedData, rev);
this.index.write(new IndexEntry(this.index, a.getArchiveId(), res.sector, res.compressedLength));
logger.trace("Saved archive {}/{} at sector {}, compressed length {}", this.getId(), a.getArchiveId(), res.sector, res.compressedLength);
a.setCrc(res.crc);
a.setWhirlpool(res.whirlpool);
}
}
public byte[] writeIndexData()
{
OutputStream stream = new OutputStream();
@@ -454,7 +520,7 @@ public class Index implements Closeable
for (data = 0; data < this.archives.size(); ++data)
{
Archive a = this.archives.get(data);
int len = a.getFiles().size();
if (protocol >= 7)

67
cache/src/main/java/net/runelite/cache/fs/Store.java vendored
View File

@@ -22,7 +22,6 @@
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package net.runelite.cache.fs;
import java.io.Closeable;
@@ -30,6 +29,7 @@ import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import net.runelite.cache.IndexType;
@@ -43,19 +43,19 @@ public class Store implements Closeable
private static final String MAIN_FILE_CACHE_DAT = "main_file_cache.dat2";
private static final String MAIN_FILE_CACHE_IDX = "main_file_cache.idx";
private final File folder;
private final DataFile data;
private final IndexFile index255;
private final List<Index> indexes = new ArrayList<>();
public Store(File folder) throws IOException
{
this.folder = folder;
data = new DataFile(this, new File(folder, MAIN_FILE_CACHE_DAT));
index255 = new IndexFile(this, 255, new File(folder, MAIN_FILE_CACHE_IDX + "255"));
for (int i = 0; i < index255.getIndexCount(); ++i)
{
this.addIndex(i);
@@ -76,7 +76,9 @@ public class Store implements Closeable
data.close();
index255.close();
for (Index i : indexes)
{
i.close();
}
}
@Override
@@ -105,18 +107,22 @@ public class Store implements Closeable
}
return true;
}
public final Index addIndex(int id) throws FileNotFoundException
{
for (Index i : indexes)
{
if (i.getIndex().getIndexFileId() == id)
{
throw new IllegalArgumentException("index " + id + " already exists");
}
}
IndexFile indexFile = new IndexFile(this, id, new File(folder, MAIN_FILE_CACHE_IDX + id));
Index index = new Index(this, indexFile, id);
this.indexes.add(index);
return index;
}
@@ -125,13 +131,15 @@ public class Store implements Closeable
assert indexes.contains(index);
indexes.remove(index);
}
public void load() throws IOException
{
for (Index i : indexes)
{
i.load();
}
}
public void save() throws IOException
{
logger.debug("Clearing data and indexes in preparation for store save");
@@ -139,10 +147,41 @@ public class Store implements Closeable
data.clear();
for (Index i : indexes)
{
i.clear();
}
for (Index i : indexes)
{
i.save();
}
}
public void saveTree(java.io.File to) throws IOException
{
for (Index i : indexes)
{
i.saveTree(to);
}
}
public void loadTree(java.io.File from) throws IOException
{
for (java.io.File idx : from.listFiles())
{
if (!idx.isDirectory())
{
continue;
}
int id = Integer.parseInt(idx.getName());
IndexFile indexFile = new IndexFile(this, id, new File(folder, MAIN_FILE_CACHE_IDX + id));
Index index = new Index(this, indexFile, id);
index.loadTree(from, idx);
indexes.add(index);
}
Collections.sort(indexes, (idx1, idx2) -> Integer.compare(idx1.getId(), idx2.getId()));
}
public DataFile getData()
@@ -159,7 +198,7 @@ public class Store implements Closeable
{
return indexes;
}
public Index getIndex(IndexType type)
{
return indexes.get(type.getNumber());
@@ -168,8 +207,12 @@ public class Store implements Closeable
public final Index findIndex(int id)
{
for (Index i : indexes)
{
if (i.getId() == id)
{
return i;
}
}
return null;
}
}

43
cache/src/test/java/net/runelite/cache/fs/StoreLoadTest.java vendored
View File

@@ -22,14 +22,13 @@
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package net.runelite.cache.fs;
import com.google.common.io.Files;
import java.io.FileOutputStream;
import java.io.IOException;
import net.runelite.cache.StoreLocation;
import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -55,7 +54,9 @@ public class StoreLoadTest
java.io.File testStoreFile = folder.newFolder();
for (java.io.File f : StoreLocation.LOCATION.listFiles())
{
Files.copy(f, new java.io.File(testStoreFile, f.getName()));
}
Store testStore = new Store(testStoreFile);
testStore.load();
@@ -68,7 +69,8 @@ public class StoreLoadTest
Assert.assertTrue(store.equals(testStore));
}
//@Test
@Test
@Ignore
public void unpackStore() throws IOException
{
java.io.File base = StoreLocation.LOCATION;
@@ -76,29 +78,22 @@ public class StoreLoadTest
{
store.load();
for (Index i : store.getIndexes())
{
java.io.File ifile = new java.io.File(folder.newFolder(), "" + i.getId());
ifile.mkdir();
store.saveTree(folder.newFolder());
}
}
for (Archive a : i.getArchives())
{
java.io.File afile = new java.io.File(ifile, "" + a.getArchiveId());
afile.mkdir();
@Test
@Ignore
public void loadTree() throws IOException
{
Store store = new Store(folder.newFolder());
store.loadTree(new java.io.File("C:\\rs\\temp\\tree"));
for (File f : a.getFiles())
{
java.io.File ffile = new java.io.File(afile, "" + f.getFileId());
try (FileOutputStream fout = new FileOutputStream(ffile))
{
if (f.getContents() != null)
{
fout.write(f.getContents());
}
}
}
}
}
try (Store store2 = new Store(StoreLocation.LOCATION))
{
store2.load();
Assert.assertEquals(store, store2);
}
}
}

68
cache/src/test/java/net/runelite/cache/fs/StoreTest.java vendored
View File

@@ -22,7 +22,6 @@
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package net.runelite.cache.fs;
import java.io.IOException;
@@ -35,9 +34,11 @@ import org.junit.rules.TemporaryFolder;
public class StoreTest
{
private static final int NUMBER_OF_FILES = 1024;
@Rule
public TemporaryFolder folder = StoreLocation.getTemporaryFolder();
@Test
public void testOneFile() throws IOException
{
@@ -50,29 +51,28 @@ public class StoreTest
file.setContents("test".getBytes());
store.save();
try (Store store2 = new Store(folder.getRoot()))
{
store2.load();
Assert.assertEquals(store, store2);
}
}
}
private static final int NUMBER_OF_FILES = 1024;
@Test
public void testManyFiles() throws IOException
{
Random random = new Random(42L);
java.io.File root = folder.newFolder();
try (Store store = new Store(folder.getRoot()))
try (Store store = new Store(root))
{
Index index = store.addIndex(0);
Archive archive = index.addArchive(0);
archive.setNameHash(random.nextInt());
for (int i = 0; i < NUMBER_OF_FILES; ++i)
{
File file = archive.addFile(i);
@@ -81,70 +81,80 @@ public class StoreTest
random.nextBytes(data);
file.setContents(data);
}
store.save();
try (Store store2 = new Store(folder.getRoot()))
try (Store store2 = new Store(root))
{
store2.load();
Assert.assertEquals(store, store2);
}
}
}
@Test
public void testMultipleArchives() throws IOException
{
Random random = new Random(43L);
java.io.File root = folder.newFolder();
try (Store store = new Store(folder.getRoot()))
try (Store store = new Store(root))
{
Index index = store.addIndex(0);
Index index2 = store.addIndex(1);
Archive archive = index.addArchive(0);
archive.setNameHash(random.nextInt());
archive.setNameHash(random.nextInt(Integer.MAX_VALUE));
Archive archive2 = index.addArchive(1);
Archive archive3 = index2.addArchive(0);
for (int i = 0; i < NUMBER_OF_FILES; ++i)
{
File file = archive.addFile(i);
file.setNameHash(random.nextInt());
file.setNameHash(random.nextInt(Integer.MAX_VALUE));
byte[] data = new byte[random.nextInt(1024)];
random.nextBytes(data);
file.setContents(data);
}
for (int i = 0; i < NUMBER_OF_FILES; ++i)
{
File file = archive2.addFile(i);
file.setNameHash(random.nextInt());
file.setNameHash(random.nextInt(Integer.MAX_VALUE));
byte[] data = new byte[random.nextInt(1024)];
random.nextBytes(data);
file.setContents(data);
}
for (int i = 0; i < NUMBER_OF_FILES; ++i)
{
File file = archive3.addFile(i);
file.setNameHash(random.nextInt());
file.setNameHash(random.nextInt(Integer.MAX_VALUE));
byte[] data = new byte[random.nextInt(1024)];
random.nextBytes(data);
file.setContents(data);
}
store.save();
try (Store store2 = new Store(folder.getRoot()))
try (Store store2 = new Store(root))
{
store2.load();
Assert.assertEquals(store, store2);
}
// Test tree save/load
java.io.File tree = folder.newFolder();
store.saveTree(tree);
try (Store store2 = new Store(folder.newFolder()))
{
store2.loadTree(tree);
}
}
}
}