ra/runelite
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

cache: add saveTree/loadTree to save/load cache to/from a normal filesystem

Browse Source
This commit is contained in:
Adam
2017-03-14 14:08:36 -04:00
parent 963d82c903
commit b2327d7e8e
5 changed files with 350 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

142
cache/src/main/java/net/runelite/cache/fs/Archive.java vendored
View File

@@ -22,12 +22,14 @@
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package net.runelite.cache.fs;
import com.google.common.io.Files;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import net.runelite.cache.io.InputStream;
@@ -117,7 +119,7 @@ public class Archive
return file;
}
public void load(InputStream stream, int numberOfFiles, int protocol)
public void loadFiles(InputStream stream, int numberOfFiles, int protocol)
{
int archive = 0;
@@ -277,6 +279,142 @@ public class Archive
return fileData;
}
public void saveTree(java.io.File to) throws IOException
{
if (data != null)
{
assert files.size() == 1; // this is the maps
File file = files.get(0);
java.io.File archiveFile = new java.io.File(to, this.getArchiveId() + "-" + file.getFileId() + "-" + file.getNameHash() + ".datc");
Files.write(data, archiveFile);
archiveFile = new java.io.File(to, this.getArchiveId() + ".rev");
Files.write("" + this.getRevision(), archiveFile, Charset.defaultCharset());
archiveFile = new java.io.File(to, this.getArchiveId() + ".name");
Files.write("" + this.getNameHash(), archiveFile, Charset.defaultCharset());
return;
}
if (files.size() == 1)
{
File file = this.getFiles().get(0);
java.io.File archiveFile = new java.io.File(to, this.getArchiveId() + "-" + file.getFileId() + "-" + file.getNameHash() + ".dat");
byte[] contents = file.getContents();
Files.write(contents, archiveFile);
archiveFile = new java.io.File(to, this.getArchiveId() + ".rev");
Files.write("" + this.getRevision(), archiveFile, Charset.defaultCharset());
archiveFile = new java.io.File(to, this.getArchiveId() + ".name");
Files.write("" + this.getNameHash(), archiveFile, Charset.defaultCharset());
return;
}
java.io.File archiveFile = new java.io.File(to, this.getArchiveId() + ".rev");
Files.write("" + this.getRevision(), archiveFile, Charset.defaultCharset());
archiveFile = new java.io.File(to, this.getArchiveId() + ".name");
Files.write("" + this.getNameHash(), archiveFile, Charset.defaultCharset());
java.io.File archiveFolder = new java.io.File(to, "" + this.getArchiveId());
archiveFolder.mkdirs();
for (File file : files)
{
archiveFile = new java.io.File(archiveFolder, file.getFileId() + "-" + file.getNameHash() + ".dat");
byte[] contents = file.getContents();
Files.write(contents, archiveFile);
}
}
public void loadTreeData(java.io.File parent, java.io.File from) throws IOException
{
//archiveId-fileId-fileName - assumes name isn't negative
String[] parts = Files.getNameWithoutExtension(from.getName()).split("-");
int archiveId = Integer.parseInt(parts[0]);
int fileId = Integer.parseInt(parts[1]);
int nameHash = Integer.parseInt(parts[2]);
assert archiveId == this.getArchiveId();
data = Files.toByteArray(from);
File file = new File(this, fileId);
file.setNameHash(nameHash);
files.add(file);
java.io.File archiveFile = new java.io.File(parent, this.getArchiveId() + ".rev");
int rev = Integer.parseInt(Files.readFirstLine(archiveFile, Charset.defaultCharset()));
this.setRevision(rev);
archiveFile = new java.io.File(parent, this.getArchiveId() + ".name");
int name = Integer.parseInt(Files.readFirstLine(archiveFile, Charset.defaultCharset()));
this.setNameHash(name);
}
public void loadTreeSingleFile(java.io.File parent, java.io.File from) throws IOException
{
//archiveId-fileId-fileName
String[] parts = Files.getNameWithoutExtension(from.getName()).split("-");
int archiveId = Integer.parseInt(parts[0]);
int fileId = Integer.parseInt(parts[1]);
int nameHash = Integer.parseInt(parts[2]);
assert archiveId == this.getArchiveId();
File file = new File(this, fileId);
file.setNameHash(nameHash);
byte[] contents = Files.toByteArray(from);
file.setContents(contents);
files.add(file);
java.io.File archiveFile = new java.io.File(parent, this.getArchiveId() + ".rev");
int rev = Integer.parseInt(Files.readFirstLine(archiveFile, Charset.defaultCharset()));
this.setRevision(rev);
archiveFile = new java.io.File(parent, this.getArchiveId() + ".name");
int name = Integer.parseInt(Files.readFirstLine(archiveFile, Charset.defaultCharset()));
this.setNameHash(name);
}
public void loadTree(java.io.File parent, java.io.File from) throws IOException
{
for (java.io.File file : from.listFiles())
{
//fileId-fileName.dat
String[] split = Files.getNameWithoutExtension(file.getName()).split("-");
int fileId = Integer.parseInt(split[0]);
int fileName = Integer.parseInt(split[1]);
File f = new File(this, fileId);
f.setNameHash(fileName);
byte[] contents = Files.toByteArray(file);
f.setContents(contents);
files.add(f);
}
java.io.File archiveFile = new java.io.File(parent, this.getArchiveId() + ".rev");
int rev = Integer.parseInt(Files.readFirstLine(archiveFile, Charset.defaultCharset()));
this.setRevision(rev);
archiveFile = new java.io.File(parent, this.getArchiveId() + ".name");
int name = Integer.parseInt(Files.readFirstLine(archiveFile, Charset.defaultCharset()));
this.setNameHash(name);
// the filesystem may order these differently (eg, 1, 10, 2)
Collections.sort(files, (f1, f2) -> Integer.compare(f1.getFileId(), f2.getFileId()));
}
public void loadNames(InputStream stream, int numberOfFiles)
{
for (int i = 0; i < numberOfFiles; ++i)

70
cache/src/main/java/net/runelite/cache/fs/Index.java vendored
View File

@@ -22,12 +22,14 @@
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package net.runelite.cache.fs;
import com.google.common.io.Files;
import java.io.Closeable;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import net.runelite.cache.util.Djb2;
@@ -171,8 +173,12 @@ public class Index implements Closeable
public Archive getArchive(int id)
{
for (Archive a : archives)
{
if (a.getArchiveId() == id)
{
return a;
}
}
return null;
}
@@ -180,8 +186,12 @@ public class Index implements Closeable
{
int hash = Djb2.hash(name);
for (Archive a : archives)
{
if (a.getNameHash() == hash)
{
return a;
}
}
return null;
}
@@ -227,6 +237,62 @@ public class Index implements Closeable
this.whirlpool = res.whirlpool;
}
public void saveTree(java.io.File to) throws IOException
{
java.io.File idx = new java.io.File(to, "" + this.getId());
idx.mkdirs();
for (Archive a : archives)
{
a.saveTree(idx);
}
java.io.File rev = new java.io.File(to, this.getId() + ".rev");
Files.write("" + this.getRevision(), rev, Charset.defaultCharset());
}
public void loadTree(java.io.File parent, java.io.File to) throws IOException
{
for (java.io.File f : to.listFiles())
{
if (f.isDirectory())
{
int id = Integer.parseInt(f.getName());
Archive archive = new Archive(this, id);
archive.loadTree(to, f);
archives.add(archive);
}
else if (f.getName().endsWith(".dat"))
{
// one file. archiveId-fileId-name
String[] parts = Files.getNameWithoutExtension(f.getName()).split("-");
int id = Integer.parseInt(parts[0]);
Archive archive = new Archive(this, id);
archive.loadTreeSingleFile(to, f);
archives.add(archive);
}
else if (f.getName().endsWith(".datc"))
{
// packed data
String[] parts = Files.getNameWithoutExtension(f.getName()).split("-");
int id = Integer.parseInt(parts[0]);
Archive archive = new Archive(this, id);
archive.loadTreeData(to, f);
archives.add(archive);
}
}
String str = Files.readFirstLine(new java.io.File(parent, this.getId() + ".rev"), Charset.defaultCharset());
revision = Integer.parseInt(str);
Collections.sort(archives, (ar1, ar2) -> Integer.compare(ar1.getArchiveId(), ar2.getArchiveId()));
}
public void readIndexData(byte[] data)
{
InputStream stream = new InputStream(data);
@@ -304,7 +370,7 @@ public class Index implements Closeable
archive = 0;
Archive a = this.archives.get(index);
a.load(stream, numberOfFiles[index], protocol);
a.loadFiles(stream, numberOfFiles[index], protocol);
}
if (named)

45
cache/src/main/java/net/runelite/cache/fs/Store.java vendored
View File

@@ -22,7 +22,6 @@
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package net.runelite.cache.fs;
import java.io.Closeable;
@@ -30,6 +29,7 @@ import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import net.runelite.cache.IndexType;
@@ -76,7 +76,9 @@ public class Store implements Closeable
data.close();
index255.close();
for (Index i : indexes)
{
i.close();
}
}
@Override
@@ -109,8 +111,12 @@ public class Store implements Closeable
public final Index addIndex(int id) throws FileNotFoundException
{
for (Index i : indexes)
{
if (i.getIndex().getIndexFileId() == id)
{
throw new IllegalArgumentException("index " + id + " already exists");
}
}
IndexFile indexFile = new IndexFile(this, id, new File(folder, MAIN_FILE_CACHE_IDX + id));
Index index = new Index(this, indexFile, id);
@@ -129,7 +135,9 @@ public class Store implements Closeable
public void load() throws IOException
{
for (Index i : indexes)
{
i.load();
}
}
public void save() throws IOException
@@ -139,10 +147,41 @@ public class Store implements Closeable
data.clear();
for (Index i : indexes)
{
i.clear();
}
for (Index i : indexes)
{
i.save();
}
}
public void saveTree(java.io.File to) throws IOException
{
for (Index i : indexes)
{
i.saveTree(to);
}
}
public void loadTree(java.io.File from) throws IOException
{
for (java.io.File idx : from.listFiles())
{
if (!idx.isDirectory())
{
continue;
}
int id = Integer.parseInt(idx.getName());
IndexFile indexFile = new IndexFile(this, id, new File(folder, MAIN_FILE_CACHE_IDX + id));
Index index = new Index(this, indexFile, id);
index.loadTree(from, idx);
indexes.add(index);
}
Collections.sort(indexes, (idx1, idx2) -> Integer.compare(idx1.getId(), idx2.getId()));
}
public DataFile getData()
@@ -168,8 +207,12 @@ public class Store implements Closeable
public final Index findIndex(int id)
{
for (Index i : indexes)
{
if (i.getId() == id)
{
return i;
}
}
return null;
}
}

43
cache/src/test/java/net/runelite/cache/fs/StoreLoadTest.java vendored
View File

@@ -22,14 +22,13 @@
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package net.runelite.cache.fs;
import com.google.common.io.Files;
import java.io.FileOutputStream;
import java.io.IOException;
import net.runelite.cache.StoreLocation;
import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -55,7 +54,9 @@ public class StoreLoadTest
java.io.File testStoreFile = folder.newFolder();
for (java.io.File f : StoreLocation.LOCATION.listFiles())
{
Files.copy(f, new java.io.File(testStoreFile, f.getName()));
}
Store testStore = new Store(testStoreFile);
testStore.load();
@@ -68,7 +69,8 @@ public class StoreLoadTest
Assert.assertTrue(store.equals(testStore));
}
//@Test
@Test
@Ignore
public void unpackStore() throws IOException
{
java.io.File base = StoreLocation.LOCATION;
@@ -76,29 +78,22 @@ public class StoreLoadTest
{
store.load();
for (Index i : store.getIndexes())
{
java.io.File ifile = new java.io.File(folder.newFolder(), "" + i.getId());
ifile.mkdir();
store.saveTree(folder.newFolder());
}
}
for (Archive a : i.getArchives())
{
java.io.File afile = new java.io.File(ifile, "" + a.getArchiveId());
afile.mkdir();
@Test
@Ignore
public void loadTree() throws IOException
{
Store store = new Store(folder.newFolder());
store.loadTree(new java.io.File("C:\\rs\\temp\\tree"));
for (File f : a.getFiles())
{
java.io.File ffile = new java.io.File(afile, "" + f.getFileId());
try (FileOutputStream fout = new FileOutputStream(ffile))
{
if (f.getContents() != null)
{
fout.write(f.getContents());
}
}
}
}
}
try (Store store2 = new Store(StoreLocation.LOCATION))
{
store2.load();
Assert.assertEquals(store, store2);
}
}
}

32
cache/src/test/java/net/runelite/cache/fs/StoreTest.java vendored
View File

@@ -22,7 +22,6 @@
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package net.runelite.cache.fs;
import java.io.IOException;
@@ -35,6 +34,8 @@ import org.junit.rules.TemporaryFolder;
public class StoreTest
{
private static final int NUMBER_OF_FILES = 1024;
@Rule
public TemporaryFolder folder = StoreLocation.getTemporaryFolder();
@@ -60,14 +61,13 @@ public class StoreTest
}
}
private static final int NUMBER_OF_FILES = 1024;
@Test
public void testManyFiles() throws IOException
{
Random random = new Random(42L);
java.io.File root = folder.newFolder();
try (Store store = new Store(folder.getRoot()))
try (Store store = new Store(root))
{
Index index = store.addIndex(0);
Archive archive = index.addArchive(0);
@@ -84,7 +84,7 @@ public class StoreTest
store.save();
try (Store store2 = new Store(folder.getRoot()))
try (Store store2 = new Store(root))
{
store2.load();
@@ -97,14 +97,15 @@ public class StoreTest
public void testMultipleArchives() throws IOException
{
Random random = new Random(43L);
java.io.File root = folder.newFolder();
try (Store store = new Store(folder.getRoot()))
try (Store store = new Store(root))
{
Index index = store.addIndex(0);
Index index2 = store.addIndex(1);
Archive archive = index.addArchive(0);
archive.setNameHash(random.nextInt());
archive.setNameHash(random.nextInt(Integer.MAX_VALUE));
Archive archive2 = index.addArchive(1);
@@ -113,7 +114,7 @@ public class StoreTest
for (int i = 0; i < NUMBER_OF_FILES; ++i)
{
File file = archive.addFile(i);
file.setNameHash(random.nextInt());
file.setNameHash(random.nextInt(Integer.MAX_VALUE));
byte[] data = new byte[random.nextInt(1024)];
random.nextBytes(data);
file.setContents(data);
@@ -122,7 +123,7 @@ public class StoreTest
for (int i = 0; i < NUMBER_OF_FILES; ++i)
{
File file = archive2.addFile(i);
file.setNameHash(random.nextInt());
file.setNameHash(random.nextInt(Integer.MAX_VALUE));
byte[] data = new byte[random.nextInt(1024)];
random.nextBytes(data);
file.setContents(data);
@@ -131,7 +132,7 @@ public class StoreTest
for (int i = 0; i < NUMBER_OF_FILES; ++i)
{
File file = archive3.addFile(i);
file.setNameHash(random.nextInt());
file.setNameHash(random.nextInt(Integer.MAX_VALUE));
byte[] data = new byte[random.nextInt(1024)];
random.nextBytes(data);
file.setContents(data);
@@ -139,12 +140,21 @@ public class StoreTest
store.save();
try (Store store2 = new Store(folder.getRoot()))
try (Store store2 = new Store(root))
{
store2.load();
Assert.assertEquals(store, store2);
}
// Test tree save/load
java.io.File tree = folder.newFolder();
store.saveTree(tree);
try (Store store2 = new Store(folder.newFolder()))
{
store2.loadTree(tree);
}
}
}
}