cache: add saveTree/loadTree to save/load cache to/from a normal filesystem

This commit is contained in:
Adam
2017-03-14 14:08:36 -04:00
parent 963d82c903
commit b2327d7e8e
5 changed files with 350 additions and 98 deletions

View File

@@ -22,12 +22,14 @@
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/ */
package net.runelite.cache.fs; package net.runelite.cache.fs;
import com.google.common.io.Files;
import java.io.IOException; import java.io.IOException;
import java.nio.charset.Charset;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Objects; import java.util.Objects;
import net.runelite.cache.io.InputStream; import net.runelite.cache.io.InputStream;
@@ -38,9 +40,9 @@ import org.slf4j.LoggerFactory;
public class Archive public class Archive
{ {
private static final Logger logger = LoggerFactory.getLogger(Archive.class); private static final Logger logger = LoggerFactory.getLogger(Archive.class);
private Index index; // member of this index private Index index; // member of this index
private byte[] data; // raw data from the datafile, compressed/encrypted private byte[] data; // raw data from the datafile, compressed/encrypted
private int archiveId; private int archiveId;
@@ -51,7 +53,7 @@ public class Archive
private int compression; private int compression;
private List<File> files = new ArrayList<>(); private List<File> files = new ArrayList<>();
public Archive(Index index, int id) public Archive(Index index, int id)
{ {
this.index = index; this.index = index;
@@ -109,18 +111,18 @@ public class Archive
{ {
this.data = data; this.data = data;
} }
public File addFile(int id) public File addFile(int id)
{ {
File file = new File(this, id); File file = new File(this, id);
this.files.add(file); this.files.add(file);
return file; return file;
} }
public void load(InputStream stream, int numberOfFiles, int protocol) public void loadFiles(InputStream stream, int numberOfFiles, int protocol)
{ {
int archive = 0; int archive = 0;
for (int i = 0; i < numberOfFiles; ++i) for (int i = 0; i < numberOfFiles; ++i)
{ {
int fileId = archive += protocol >= 7 ? stream.readBigSmart() : stream.readUnsignedShort(); int fileId = archive += protocol >= 7 ? stream.readBigSmart() : stream.readUnsignedShort();
@@ -140,7 +142,7 @@ public class Archive
logger.warn("Unable to decrypt archive {}", this); logger.warn("Unable to decrypt archive {}", this);
return; return;
} }
byte[] decompressedData = res.data; byte[] decompressedData = res.data;
if (this.crc != res.crc) if (this.crc != res.crc)
@@ -164,7 +166,7 @@ public class Archive
loadContents(decompressedData); loadContents(decompressedData);
this.setData(null); // now that we've loaded it, clean it so it doesn't get written back this.setData(null); // now that we've loaded it, clean it so it doesn't get written back
} }
public void loadContents(byte[] data) public void loadContents(byte[] data)
{ {
logger.trace("Loading contents of archive {} ({} files)", archiveId, files.size()); logger.trace("Loading contents of archive {} ({} files)", archiveId, files.size());
@@ -217,9 +219,9 @@ public class Archive
for (int id = 0; id < filesCount; ++id) for (int id = 0; id < filesCount; ++id)
{ {
int chunkSize = chunkSizes[id][chunk]; int chunkSize = chunkSizes[id][chunk];
stream.readBytes(fileContents[id], fileOffsets[id], chunkSize); stream.readBytes(fileContents[id], fileOffsets[id], chunkSize);
fileOffsets[id] += chunkSize; fileOffsets[id] += chunkSize;
} }
} }
@@ -238,7 +240,7 @@ public class Archive
logger.trace("Saving contents of archive {}/{} using cached data", index.getId(), archiveId); logger.trace("Saving contents of archive {}/{} using cached data", index.getId(), archiveId);
return data; return data;
} }
OutputStream stream = new OutputStream(); OutputStream stream = new OutputStream();
int filesCount = this.getFiles().size(); int filesCount = this.getFiles().size();
@@ -276,7 +278,143 @@ public class Archive
return fileData; return fileData;
} }
public void saveTree(java.io.File to) throws IOException
{
if (data != null)
{
assert files.size() == 1; // this is the maps
File file = files.get(0);
java.io.File archiveFile = new java.io.File(to, this.getArchiveId() + "-" + file.getFileId() + "-" + file.getNameHash() + ".datc");
Files.write(data, archiveFile);
archiveFile = new java.io.File(to, this.getArchiveId() + ".rev");
Files.write("" + this.getRevision(), archiveFile, Charset.defaultCharset());
archiveFile = new java.io.File(to, this.getArchiveId() + ".name");
Files.write("" + this.getNameHash(), archiveFile, Charset.defaultCharset());
return;
}
if (files.size() == 1)
{
File file = this.getFiles().get(0);
java.io.File archiveFile = new java.io.File(to, this.getArchiveId() + "-" + file.getFileId() + "-" + file.getNameHash() + ".dat");
byte[] contents = file.getContents();
Files.write(contents, archiveFile);
archiveFile = new java.io.File(to, this.getArchiveId() + ".rev");
Files.write("" + this.getRevision(), archiveFile, Charset.defaultCharset());
archiveFile = new java.io.File(to, this.getArchiveId() + ".name");
Files.write("" + this.getNameHash(), archiveFile, Charset.defaultCharset());
return;
}
java.io.File archiveFile = new java.io.File(to, this.getArchiveId() + ".rev");
Files.write("" + this.getRevision(), archiveFile, Charset.defaultCharset());
archiveFile = new java.io.File(to, this.getArchiveId() + ".name");
Files.write("" + this.getNameHash(), archiveFile, Charset.defaultCharset());
java.io.File archiveFolder = new java.io.File(to, "" + this.getArchiveId());
archiveFolder.mkdirs();
for (File file : files)
{
archiveFile = new java.io.File(archiveFolder, file.getFileId() + "-" + file.getNameHash() + ".dat");
byte[] contents = file.getContents();
Files.write(contents, archiveFile);
}
}
public void loadTreeData(java.io.File parent, java.io.File from) throws IOException
{
//archiveId-fileId-fileName - assumes name isn't negative
String[] parts = Files.getNameWithoutExtension(from.getName()).split("-");
int archiveId = Integer.parseInt(parts[0]);
int fileId = Integer.parseInt(parts[1]);
int nameHash = Integer.parseInt(parts[2]);
assert archiveId == this.getArchiveId();
data = Files.toByteArray(from);
File file = new File(this, fileId);
file.setNameHash(nameHash);
files.add(file);
java.io.File archiveFile = new java.io.File(parent, this.getArchiveId() + ".rev");
int rev = Integer.parseInt(Files.readFirstLine(archiveFile, Charset.defaultCharset()));
this.setRevision(rev);
archiveFile = new java.io.File(parent, this.getArchiveId() + ".name");
int name = Integer.parseInt(Files.readFirstLine(archiveFile, Charset.defaultCharset()));
this.setNameHash(name);
}
public void loadTreeSingleFile(java.io.File parent, java.io.File from) throws IOException
{
//archiveId-fileId-fileName
String[] parts = Files.getNameWithoutExtension(from.getName()).split("-");
int archiveId = Integer.parseInt(parts[0]);
int fileId = Integer.parseInt(parts[1]);
int nameHash = Integer.parseInt(parts[2]);
assert archiveId == this.getArchiveId();
File file = new File(this, fileId);
file.setNameHash(nameHash);
byte[] contents = Files.toByteArray(from);
file.setContents(contents);
files.add(file);
java.io.File archiveFile = new java.io.File(parent, this.getArchiveId() + ".rev");
int rev = Integer.parseInt(Files.readFirstLine(archiveFile, Charset.defaultCharset()));
this.setRevision(rev);
archiveFile = new java.io.File(parent, this.getArchiveId() + ".name");
int name = Integer.parseInt(Files.readFirstLine(archiveFile, Charset.defaultCharset()));
this.setNameHash(name);
}
public void loadTree(java.io.File parent, java.io.File from) throws IOException
{
for (java.io.File file : from.listFiles())
{
//fileId-fileName.dat
String[] split = Files.getNameWithoutExtension(file.getName()).split("-");
int fileId = Integer.parseInt(split[0]);
int fileName = Integer.parseInt(split[1]);
File f = new File(this, fileId);
f.setNameHash(fileName);
byte[] contents = Files.toByteArray(file);
f.setContents(contents);
files.add(f);
}
java.io.File archiveFile = new java.io.File(parent, this.getArchiveId() + ".rev");
int rev = Integer.parseInt(Files.readFirstLine(archiveFile, Charset.defaultCharset()));
this.setRevision(rev);
archiveFile = new java.io.File(parent, this.getArchiveId() + ".name");
int name = Integer.parseInt(Files.readFirstLine(archiveFile, Charset.defaultCharset()));
this.setNameHash(name);
// the filesystem may order these differently (eg, 1, 10, 2)
Collections.sort(files, (f1, f2) -> Integer.compare(f1.getFileId(), f2.getFileId()));
}
public void loadNames(InputStream stream, int numberOfFiles) public void loadNames(InputStream stream, int numberOfFiles)
{ {
for (int i = 0; i < numberOfFiles; ++i) for (int i = 0; i < numberOfFiles; ++i)

View File

@@ -22,12 +22,14 @@
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/ */
package net.runelite.cache.fs; package net.runelite.cache.fs;
import com.google.common.io.Files;
import java.io.Closeable; import java.io.Closeable;
import java.io.IOException; import java.io.IOException;
import java.nio.charset.Charset;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Objects; import java.util.Objects;
import net.runelite.cache.util.Djb2; import net.runelite.cache.util.Djb2;
@@ -40,11 +42,11 @@ import org.slf4j.LoggerFactory;
public class Index implements Closeable public class Index implements Closeable
{ {
private static final Logger logger = LoggerFactory.getLogger(Index.class); private static final Logger logger = LoggerFactory.getLogger(Index.class);
private final Store store; private final Store store;
private final IndexFile index; private final IndexFile index;
private final int id; private final int id;
private XteaKeyManager xteaManager; private XteaKeyManager xteaManager;
private int protocol = 7; private int protocol = 7;
@@ -55,14 +57,14 @@ public class Index implements Closeable
private int compression; // compression method of this index's data in 255 private int compression; // compression method of this index's data in 255
private final List<Archive> archives = new ArrayList<>(); private final List<Archive> archives = new ArrayList<>();
public Index(Store store, IndexFile index, int id) public Index(Store store, IndexFile index, int id)
{ {
this.store = store; this.store = store;
this.index = index; this.index = index;
this.id = id; this.id = id;
} }
@Override @Override
public void close() throws IOException public void close() throws IOException
{ {
@@ -160,19 +162,23 @@ public class Index implements Closeable
{ {
return archives; return archives;
} }
public Archive addArchive(int id) public Archive addArchive(int id)
{ {
Archive archive = new Archive(this, id); Archive archive = new Archive(this, id);
this.archives.add(archive); this.archives.add(archive);
return archive; return archive;
} }
public Archive getArchive(int id) public Archive getArchive(int id)
{ {
for (Archive a : archives) for (Archive a : archives)
{
if (a.getArchiveId() == id) if (a.getArchiveId() == id)
{
return a; return a;
}
}
return null; return null;
} }
@@ -180,25 +186,29 @@ public class Index implements Closeable
{ {
int hash = Djb2.hash(name); int hash = Djb2.hash(name);
for (Archive a : archives) for (Archive a : archives)
{
if (a.getNameHash() == hash) if (a.getNameHash() == hash)
{
return a; return a;
}
}
return null; return null;
} }
public void load() throws IOException public void load() throws IOException
{ {
logger.trace("Loading index {}", id); logger.trace("Loading index {}", id);
DataFile dataFile = store.getData(); DataFile dataFile = store.getData();
IndexFile index255 = store.getIndex255(); IndexFile index255 = store.getIndex255();
IndexEntry entry = index255.read(id); IndexEntry entry = index255.read(id);
byte[] indexData = dataFile.read(index255.getIndexFileId(), entry.getId(), entry.getSector(), entry.getLength()); byte[] indexData = dataFile.read(index255.getIndexFileId(), entry.getId(), entry.getSector(), entry.getLength());
DataFileReadResult res = DataFile.decompress(indexData, null); DataFileReadResult res = DataFile.decompress(indexData, null);
byte[] data = res.data; byte[] data = res.data;
archives.clear(); archives.clear();
readIndexData(data); readIndexData(data);
this.crc = res.crc; this.crc = res.crc;
@@ -208,11 +218,11 @@ public class Index implements Closeable
this.loadArchives(); this.loadArchives();
} }
public void save() throws IOException public void save() throws IOException
{ {
saveArchives(); saveArchives();
byte[] data = this.writeIndexData(); byte[] data = this.writeIndexData();
DataFile dataFile = store.getData(); DataFile dataFile = store.getData();
@@ -226,7 +236,63 @@ public class Index implements Closeable
this.crc = res.crc; this.crc = res.crc;
this.whirlpool = res.whirlpool; this.whirlpool = res.whirlpool;
} }
public void saveTree(java.io.File to) throws IOException
{
java.io.File idx = new java.io.File(to, "" + this.getId());
idx.mkdirs();
for (Archive a : archives)
{
a.saveTree(idx);
}
java.io.File rev = new java.io.File(to, this.getId() + ".rev");
Files.write("" + this.getRevision(), rev, Charset.defaultCharset());
}
public void loadTree(java.io.File parent, java.io.File to) throws IOException
{
for (java.io.File f : to.listFiles())
{
if (f.isDirectory())
{
int id = Integer.parseInt(f.getName());
Archive archive = new Archive(this, id);
archive.loadTree(to, f);
archives.add(archive);
}
else if (f.getName().endsWith(".dat"))
{
// one file. archiveId-fileId-name
String[] parts = Files.getNameWithoutExtension(f.getName()).split("-");
int id = Integer.parseInt(parts[0]);
Archive archive = new Archive(this, id);
archive.loadTreeSingleFile(to, f);
archives.add(archive);
}
else if (f.getName().endsWith(".datc"))
{
// packed data
String[] parts = Files.getNameWithoutExtension(f.getName()).split("-");
int id = Integer.parseInt(parts[0]);
Archive archive = new Archive(this, id);
archive.loadTreeData(to, f);
archives.add(archive);
}
}
String str = Files.readFirstLine(new java.io.File(parent, this.getId() + ".rev"), Charset.defaultCharset());
revision = Integer.parseInt(str);
Collections.sort(archives, (ar1, ar2) -> Integer.compare(ar1.getArchiveId(), ar2.getArchiveId()));
}
public void readIndexData(byte[] data) public void readIndexData(byte[] data)
{ {
InputStream stream = new InputStream(data); InputStream stream = new InputStream(data);
@@ -304,7 +370,7 @@ public class Index implements Closeable
archive = 0; archive = 0;
Archive a = this.archives.get(index); Archive a = this.archives.get(index);
a.load(stream, numberOfFiles[index], protocol); a.loadFiles(stream, numberOfFiles[index], protocol);
} }
if (named) if (named)
@@ -317,7 +383,7 @@ public class Index implements Closeable
} }
} }
} }
private void loadArchives() throws IOException private void loadArchives() throws IOException
{ {
// get data from index file // get data from index file
@@ -347,7 +413,7 @@ public class Index implements Closeable
a.decompressAndLoad(null); a.decompressAndLoad(null);
} }
} }
public void saveArchives() throws IOException public void saveArchives() throws IOException
{ {
for (Archive a : archives) for (Archive a : archives)
@@ -372,14 +438,14 @@ public class Index implements Closeable
DataFileWriteResult res = data.write(this.id, a.getArchiveId(), compressedData, rev); DataFileWriteResult res = data.write(this.id, a.getArchiveId(), compressedData, rev);
this.index.write(new IndexEntry(this.index, a.getArchiveId(), res.sector, res.compressedLength)); this.index.write(new IndexEntry(this.index, a.getArchiveId(), res.sector, res.compressedLength));
logger.trace("Saved archive {}/{} at sector {}, compressed length {}", this.getId(), a.getArchiveId(), res.sector, res.compressedLength); logger.trace("Saved archive {}/{} at sector {}, compressed length {}", this.getId(), a.getArchiveId(), res.sector, res.compressedLength);
a.setCrc(res.crc); a.setCrc(res.crc);
a.setWhirlpool(res.whirlpool); a.setWhirlpool(res.whirlpool);
} }
} }
public byte[] writeIndexData() public byte[] writeIndexData()
{ {
OutputStream stream = new OutputStream(); OutputStream stream = new OutputStream();
@@ -454,7 +520,7 @@ public class Index implements Closeable
for (data = 0; data < this.archives.size(); ++data) for (data = 0; data < this.archives.size(); ++data)
{ {
Archive a = this.archives.get(data); Archive a = this.archives.get(data);
int len = a.getFiles().size(); int len = a.getFiles().size();
if (protocol >= 7) if (protocol >= 7)

View File

@@ -22,7 +22,6 @@
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/ */
package net.runelite.cache.fs; package net.runelite.cache.fs;
import java.io.Closeable; import java.io.Closeable;
@@ -30,6 +29,7 @@ import java.io.File;
import java.io.FileNotFoundException; import java.io.FileNotFoundException;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Objects; import java.util.Objects;
import net.runelite.cache.IndexType; import net.runelite.cache.IndexType;
@@ -43,19 +43,19 @@ public class Store implements Closeable
private static final String MAIN_FILE_CACHE_DAT = "main_file_cache.dat2"; private static final String MAIN_FILE_CACHE_DAT = "main_file_cache.dat2";
private static final String MAIN_FILE_CACHE_IDX = "main_file_cache.idx"; private static final String MAIN_FILE_CACHE_IDX = "main_file_cache.idx";
private final File folder; private final File folder;
private final DataFile data; private final DataFile data;
private final IndexFile index255; private final IndexFile index255;
private final List<Index> indexes = new ArrayList<>(); private final List<Index> indexes = new ArrayList<>();
public Store(File folder) throws IOException public Store(File folder) throws IOException
{ {
this.folder = folder; this.folder = folder;
data = new DataFile(this, new File(folder, MAIN_FILE_CACHE_DAT)); data = new DataFile(this, new File(folder, MAIN_FILE_CACHE_DAT));
index255 = new IndexFile(this, 255, new File(folder, MAIN_FILE_CACHE_IDX + "255")); index255 = new IndexFile(this, 255, new File(folder, MAIN_FILE_CACHE_IDX + "255"));
for (int i = 0; i < index255.getIndexCount(); ++i) for (int i = 0; i < index255.getIndexCount(); ++i)
{ {
this.addIndex(i); this.addIndex(i);
@@ -76,7 +76,9 @@ public class Store implements Closeable
data.close(); data.close();
index255.close(); index255.close();
for (Index i : indexes) for (Index i : indexes)
{
i.close(); i.close();
}
} }
@Override @Override
@@ -105,18 +107,22 @@ public class Store implements Closeable
} }
return true; return true;
} }
public final Index addIndex(int id) throws FileNotFoundException public final Index addIndex(int id) throws FileNotFoundException
{ {
for (Index i : indexes) for (Index i : indexes)
{
if (i.getIndex().getIndexFileId() == id) if (i.getIndex().getIndexFileId() == id)
{
throw new IllegalArgumentException("index " + id + " already exists"); throw new IllegalArgumentException("index " + id + " already exists");
}
}
IndexFile indexFile = new IndexFile(this, id, new File(folder, MAIN_FILE_CACHE_IDX + id)); IndexFile indexFile = new IndexFile(this, id, new File(folder, MAIN_FILE_CACHE_IDX + id));
Index index = new Index(this, indexFile, id); Index index = new Index(this, indexFile, id);
this.indexes.add(index); this.indexes.add(index);
return index; return index;
} }
@@ -125,13 +131,15 @@ public class Store implements Closeable
assert indexes.contains(index); assert indexes.contains(index);
indexes.remove(index); indexes.remove(index);
} }
public void load() throws IOException public void load() throws IOException
{ {
for (Index i : indexes) for (Index i : indexes)
{
i.load(); i.load();
}
} }
public void save() throws IOException public void save() throws IOException
{ {
logger.debug("Clearing data and indexes in preparation for store save"); logger.debug("Clearing data and indexes in preparation for store save");
@@ -139,10 +147,41 @@ public class Store implements Closeable
data.clear(); data.clear();
for (Index i : indexes) for (Index i : indexes)
{
i.clear(); i.clear();
}
for (Index i : indexes) for (Index i : indexes)
{
i.save(); i.save();
}
}
public void saveTree(java.io.File to) throws IOException
{
for (Index i : indexes)
{
i.saveTree(to);
}
}
public void loadTree(java.io.File from) throws IOException
{
for (java.io.File idx : from.listFiles())
{
if (!idx.isDirectory())
{
continue;
}
int id = Integer.parseInt(idx.getName());
IndexFile indexFile = new IndexFile(this, id, new File(folder, MAIN_FILE_CACHE_IDX + id));
Index index = new Index(this, indexFile, id);
index.loadTree(from, idx);
indexes.add(index);
}
Collections.sort(indexes, (idx1, idx2) -> Integer.compare(idx1.getId(), idx2.getId()));
} }
public DataFile getData() public DataFile getData()
@@ -159,7 +198,7 @@ public class Store implements Closeable
{ {
return indexes; return indexes;
} }
public Index getIndex(IndexType type) public Index getIndex(IndexType type)
{ {
return indexes.get(type.getNumber()); return indexes.get(type.getNumber());
@@ -168,8 +207,12 @@ public class Store implements Closeable
public final Index findIndex(int id) public final Index findIndex(int id)
{ {
for (Index i : indexes) for (Index i : indexes)
{
if (i.getId() == id) if (i.getId() == id)
{
return i; return i;
}
}
return null; return null;
} }
} }

View File

@@ -22,14 +22,13 @@
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/ */
package net.runelite.cache.fs; package net.runelite.cache.fs;
import com.google.common.io.Files; import com.google.common.io.Files;
import java.io.FileOutputStream;
import java.io.IOException; import java.io.IOException;
import net.runelite.cache.StoreLocation; import net.runelite.cache.StoreLocation;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Rule; import org.junit.Rule;
import org.junit.Test; import org.junit.Test;
import org.junit.rules.TemporaryFolder; import org.junit.rules.TemporaryFolder;
@@ -55,7 +54,9 @@ public class StoreLoadTest
java.io.File testStoreFile = folder.newFolder(); java.io.File testStoreFile = folder.newFolder();
for (java.io.File f : StoreLocation.LOCATION.listFiles()) for (java.io.File f : StoreLocation.LOCATION.listFiles())
{
Files.copy(f, new java.io.File(testStoreFile, f.getName())); Files.copy(f, new java.io.File(testStoreFile, f.getName()));
}
Store testStore = new Store(testStoreFile); Store testStore = new Store(testStoreFile);
testStore.load(); testStore.load();
@@ -68,7 +69,8 @@ public class StoreLoadTest
Assert.assertTrue(store.equals(testStore)); Assert.assertTrue(store.equals(testStore));
} }
//@Test @Test
@Ignore
public void unpackStore() throws IOException public void unpackStore() throws IOException
{ {
java.io.File base = StoreLocation.LOCATION; java.io.File base = StoreLocation.LOCATION;
@@ -76,29 +78,22 @@ public class StoreLoadTest
{ {
store.load(); store.load();
for (Index i : store.getIndexes()) store.saveTree(folder.newFolder());
{ }
java.io.File ifile = new java.io.File(folder.newFolder(), "" + i.getId()); }
ifile.mkdir();
for (Archive a : i.getArchives()) @Test
{ @Ignore
java.io.File afile = new java.io.File(ifile, "" + a.getArchiveId()); public void loadTree() throws IOException
afile.mkdir(); {
Store store = new Store(folder.newFolder());
store.loadTree(new java.io.File("C:\\rs\\temp\\tree"));
for (File f : a.getFiles()) try (Store store2 = new Store(StoreLocation.LOCATION))
{ {
java.io.File ffile = new java.io.File(afile, "" + f.getFileId()); store2.load();
try (FileOutputStream fout = new FileOutputStream(ffile))
{ Assert.assertEquals(store, store2);
if (f.getContents() != null)
{
fout.write(f.getContents());
}
}
}
}
}
} }
} }
} }

View File

@@ -22,7 +22,6 @@
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/ */
package net.runelite.cache.fs; package net.runelite.cache.fs;
import java.io.IOException; import java.io.IOException;
@@ -35,9 +34,11 @@ import org.junit.rules.TemporaryFolder;
public class StoreTest public class StoreTest
{ {
private static final int NUMBER_OF_FILES = 1024;
@Rule @Rule
public TemporaryFolder folder = StoreLocation.getTemporaryFolder(); public TemporaryFolder folder = StoreLocation.getTemporaryFolder();
@Test @Test
public void testOneFile() throws IOException public void testOneFile() throws IOException
{ {
@@ -50,29 +51,28 @@ public class StoreTest
file.setContents("test".getBytes()); file.setContents("test".getBytes());
store.save(); store.save();
try (Store store2 = new Store(folder.getRoot())) try (Store store2 = new Store(folder.getRoot()))
{ {
store2.load(); store2.load();
Assert.assertEquals(store, store2); Assert.assertEquals(store, store2);
} }
} }
} }
private static final int NUMBER_OF_FILES = 1024;
@Test @Test
public void testManyFiles() throws IOException public void testManyFiles() throws IOException
{ {
Random random = new Random(42L); Random random = new Random(42L);
java.io.File root = folder.newFolder();
try (Store store = new Store(folder.getRoot())) try (Store store = new Store(root))
{ {
Index index = store.addIndex(0); Index index = store.addIndex(0);
Archive archive = index.addArchive(0); Archive archive = index.addArchive(0);
archive.setNameHash(random.nextInt()); archive.setNameHash(random.nextInt());
for (int i = 0; i < NUMBER_OF_FILES; ++i) for (int i = 0; i < NUMBER_OF_FILES; ++i)
{ {
File file = archive.addFile(i); File file = archive.addFile(i);
@@ -81,70 +81,80 @@ public class StoreTest
random.nextBytes(data); random.nextBytes(data);
file.setContents(data); file.setContents(data);
} }
store.save(); store.save();
try (Store store2 = new Store(folder.getRoot())) try (Store store2 = new Store(root))
{ {
store2.load(); store2.load();
Assert.assertEquals(store, store2); Assert.assertEquals(store, store2);
} }
} }
} }
@Test @Test
public void testMultipleArchives() throws IOException public void testMultipleArchives() throws IOException
{ {
Random random = new Random(43L); Random random = new Random(43L);
java.io.File root = folder.newFolder();
try (Store store = new Store(folder.getRoot())) try (Store store = new Store(root))
{ {
Index index = store.addIndex(0); Index index = store.addIndex(0);
Index index2 = store.addIndex(1); Index index2 = store.addIndex(1);
Archive archive = index.addArchive(0); Archive archive = index.addArchive(0);
archive.setNameHash(random.nextInt()); archive.setNameHash(random.nextInt(Integer.MAX_VALUE));
Archive archive2 = index.addArchive(1); Archive archive2 = index.addArchive(1);
Archive archive3 = index2.addArchive(0); Archive archive3 = index2.addArchive(0);
for (int i = 0; i < NUMBER_OF_FILES; ++i) for (int i = 0; i < NUMBER_OF_FILES; ++i)
{ {
File file = archive.addFile(i); File file = archive.addFile(i);
file.setNameHash(random.nextInt()); file.setNameHash(random.nextInt(Integer.MAX_VALUE));
byte[] data = new byte[random.nextInt(1024)]; byte[] data = new byte[random.nextInt(1024)];
random.nextBytes(data); random.nextBytes(data);
file.setContents(data); file.setContents(data);
} }
for (int i = 0; i < NUMBER_OF_FILES; ++i) for (int i = 0; i < NUMBER_OF_FILES; ++i)
{ {
File file = archive2.addFile(i); File file = archive2.addFile(i);
file.setNameHash(random.nextInt()); file.setNameHash(random.nextInt(Integer.MAX_VALUE));
byte[] data = new byte[random.nextInt(1024)]; byte[] data = new byte[random.nextInt(1024)];
random.nextBytes(data); random.nextBytes(data);
file.setContents(data); file.setContents(data);
} }
for (int i = 0; i < NUMBER_OF_FILES; ++i) for (int i = 0; i < NUMBER_OF_FILES; ++i)
{ {
File file = archive3.addFile(i); File file = archive3.addFile(i);
file.setNameHash(random.nextInt()); file.setNameHash(random.nextInt(Integer.MAX_VALUE));
byte[] data = new byte[random.nextInt(1024)]; byte[] data = new byte[random.nextInt(1024)];
random.nextBytes(data); random.nextBytes(data);
file.setContents(data); file.setContents(data);
} }
store.save(); store.save();
try (Store store2 = new Store(folder.getRoot())) try (Store store2 = new Store(root))
{ {
store2.load(); store2.load();
Assert.assertEquals(store, store2); Assert.assertEquals(store, store2);
} }
// Test tree save/load
java.io.File tree = folder.newFolder();
store.saveTree(tree);
try (Store store2 = new Store(folder.newFolder()))
{
store2.loadTree(tree);
}
} }
} }
} }