cache: add saveTree/loadTree to save/load cache to/from a normal filesystem
This commit is contained in:
@@ -22,12 +22,14 @@
|
|||||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||||
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package net.runelite.cache.fs;
|
package net.runelite.cache.fs;
|
||||||
|
|
||||||
|
import com.google.common.io.Files;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import java.nio.charset.Charset;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
|
import java.util.Collections;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
import net.runelite.cache.io.InputStream;
|
import net.runelite.cache.io.InputStream;
|
||||||
@@ -38,9 +40,9 @@ import org.slf4j.LoggerFactory;
|
|||||||
public class Archive
|
public class Archive
|
||||||
{
|
{
|
||||||
private static final Logger logger = LoggerFactory.getLogger(Archive.class);
|
private static final Logger logger = LoggerFactory.getLogger(Archive.class);
|
||||||
|
|
||||||
private Index index; // member of this index
|
private Index index; // member of this index
|
||||||
|
|
||||||
private byte[] data; // raw data from the datafile, compressed/encrypted
|
private byte[] data; // raw data from the datafile, compressed/encrypted
|
||||||
|
|
||||||
private int archiveId;
|
private int archiveId;
|
||||||
@@ -51,7 +53,7 @@ public class Archive
|
|||||||
private int compression;
|
private int compression;
|
||||||
|
|
||||||
private List<File> files = new ArrayList<>();
|
private List<File> files = new ArrayList<>();
|
||||||
|
|
||||||
public Archive(Index index, int id)
|
public Archive(Index index, int id)
|
||||||
{
|
{
|
||||||
this.index = index;
|
this.index = index;
|
||||||
@@ -109,18 +111,18 @@ public class Archive
|
|||||||
{
|
{
|
||||||
this.data = data;
|
this.data = data;
|
||||||
}
|
}
|
||||||
|
|
||||||
public File addFile(int id)
|
public File addFile(int id)
|
||||||
{
|
{
|
||||||
File file = new File(this, id);
|
File file = new File(this, id);
|
||||||
this.files.add(file);
|
this.files.add(file);
|
||||||
return file;
|
return file;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void load(InputStream stream, int numberOfFiles, int protocol)
|
public void loadFiles(InputStream stream, int numberOfFiles, int protocol)
|
||||||
{
|
{
|
||||||
int archive = 0;
|
int archive = 0;
|
||||||
|
|
||||||
for (int i = 0; i < numberOfFiles; ++i)
|
for (int i = 0; i < numberOfFiles; ++i)
|
||||||
{
|
{
|
||||||
int fileId = archive += protocol >= 7 ? stream.readBigSmart() : stream.readUnsignedShort();
|
int fileId = archive += protocol >= 7 ? stream.readBigSmart() : stream.readUnsignedShort();
|
||||||
@@ -140,7 +142,7 @@ public class Archive
|
|||||||
logger.warn("Unable to decrypt archive {}", this);
|
logger.warn("Unable to decrypt archive {}", this);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
byte[] decompressedData = res.data;
|
byte[] decompressedData = res.data;
|
||||||
|
|
||||||
if (this.crc != res.crc)
|
if (this.crc != res.crc)
|
||||||
@@ -164,7 +166,7 @@ public class Archive
|
|||||||
loadContents(decompressedData);
|
loadContents(decompressedData);
|
||||||
this.setData(null); // now that we've loaded it, clean it so it doesn't get written back
|
this.setData(null); // now that we've loaded it, clean it so it doesn't get written back
|
||||||
}
|
}
|
||||||
|
|
||||||
public void loadContents(byte[] data)
|
public void loadContents(byte[] data)
|
||||||
{
|
{
|
||||||
logger.trace("Loading contents of archive {} ({} files)", archiveId, files.size());
|
logger.trace("Loading contents of archive {} ({} files)", archiveId, files.size());
|
||||||
@@ -217,9 +219,9 @@ public class Archive
|
|||||||
for (int id = 0; id < filesCount; ++id)
|
for (int id = 0; id < filesCount; ++id)
|
||||||
{
|
{
|
||||||
int chunkSize = chunkSizes[id][chunk];
|
int chunkSize = chunkSizes[id][chunk];
|
||||||
|
|
||||||
stream.readBytes(fileContents[id], fileOffsets[id], chunkSize);
|
stream.readBytes(fileContents[id], fileOffsets[id], chunkSize);
|
||||||
|
|
||||||
fileOffsets[id] += chunkSize;
|
fileOffsets[id] += chunkSize;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -238,7 +240,7 @@ public class Archive
|
|||||||
logger.trace("Saving contents of archive {}/{} using cached data", index.getId(), archiveId);
|
logger.trace("Saving contents of archive {}/{} using cached data", index.getId(), archiveId);
|
||||||
return data;
|
return data;
|
||||||
}
|
}
|
||||||
|
|
||||||
OutputStream stream = new OutputStream();
|
OutputStream stream = new OutputStream();
|
||||||
|
|
||||||
int filesCount = this.getFiles().size();
|
int filesCount = this.getFiles().size();
|
||||||
@@ -276,7 +278,143 @@ public class Archive
|
|||||||
|
|
||||||
return fileData;
|
return fileData;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void saveTree(java.io.File to) throws IOException
|
||||||
|
{
|
||||||
|
if (data != null)
|
||||||
|
{
|
||||||
|
assert files.size() == 1; // this is the maps
|
||||||
|
|
||||||
|
File file = files.get(0);
|
||||||
|
|
||||||
|
java.io.File archiveFile = new java.io.File(to, this.getArchiveId() + "-" + file.getFileId() + "-" + file.getNameHash() + ".datc");
|
||||||
|
Files.write(data, archiveFile);
|
||||||
|
|
||||||
|
archiveFile = new java.io.File(to, this.getArchiveId() + ".rev");
|
||||||
|
Files.write("" + this.getRevision(), archiveFile, Charset.defaultCharset());
|
||||||
|
|
||||||
|
archiveFile = new java.io.File(to, this.getArchiveId() + ".name");
|
||||||
|
Files.write("" + this.getNameHash(), archiveFile, Charset.defaultCharset());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (files.size() == 1)
|
||||||
|
{
|
||||||
|
File file = this.getFiles().get(0);
|
||||||
|
|
||||||
|
java.io.File archiveFile = new java.io.File(to, this.getArchiveId() + "-" + file.getFileId() + "-" + file.getNameHash() + ".dat");
|
||||||
|
byte[] contents = file.getContents();
|
||||||
|
|
||||||
|
Files.write(contents, archiveFile);
|
||||||
|
|
||||||
|
archiveFile = new java.io.File(to, this.getArchiveId() + ".rev");
|
||||||
|
Files.write("" + this.getRevision(), archiveFile, Charset.defaultCharset());
|
||||||
|
|
||||||
|
archiveFile = new java.io.File(to, this.getArchiveId() + ".name");
|
||||||
|
Files.write("" + this.getNameHash(), archiveFile, Charset.defaultCharset());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
java.io.File archiveFile = new java.io.File(to, this.getArchiveId() + ".rev");
|
||||||
|
Files.write("" + this.getRevision(), archiveFile, Charset.defaultCharset());
|
||||||
|
|
||||||
|
archiveFile = new java.io.File(to, this.getArchiveId() + ".name");
|
||||||
|
Files.write("" + this.getNameHash(), archiveFile, Charset.defaultCharset());
|
||||||
|
|
||||||
|
java.io.File archiveFolder = new java.io.File(to, "" + this.getArchiveId());
|
||||||
|
archiveFolder.mkdirs();
|
||||||
|
|
||||||
|
for (File file : files)
|
||||||
|
{
|
||||||
|
archiveFile = new java.io.File(archiveFolder, file.getFileId() + "-" + file.getNameHash() + ".dat");
|
||||||
|
byte[] contents = file.getContents();
|
||||||
|
Files.write(contents, archiveFile);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void loadTreeData(java.io.File parent, java.io.File from) throws IOException
|
||||||
|
{
|
||||||
|
//archiveId-fileId-fileName - assumes name isn't negative
|
||||||
|
String[] parts = Files.getNameWithoutExtension(from.getName()).split("-");
|
||||||
|
int archiveId = Integer.parseInt(parts[0]);
|
||||||
|
int fileId = Integer.parseInt(parts[1]);
|
||||||
|
int nameHash = Integer.parseInt(parts[2]);
|
||||||
|
|
||||||
|
assert archiveId == this.getArchiveId();
|
||||||
|
|
||||||
|
data = Files.toByteArray(from);
|
||||||
|
|
||||||
|
File file = new File(this, fileId);
|
||||||
|
file.setNameHash(nameHash);
|
||||||
|
|
||||||
|
files.add(file);
|
||||||
|
|
||||||
|
java.io.File archiveFile = new java.io.File(parent, this.getArchiveId() + ".rev");
|
||||||
|
int rev = Integer.parseInt(Files.readFirstLine(archiveFile, Charset.defaultCharset()));
|
||||||
|
this.setRevision(rev);
|
||||||
|
|
||||||
|
archiveFile = new java.io.File(parent, this.getArchiveId() + ".name");
|
||||||
|
int name = Integer.parseInt(Files.readFirstLine(archiveFile, Charset.defaultCharset()));
|
||||||
|
this.setNameHash(name);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void loadTreeSingleFile(java.io.File parent, java.io.File from) throws IOException
|
||||||
|
{
|
||||||
|
//archiveId-fileId-fileName
|
||||||
|
String[] parts = Files.getNameWithoutExtension(from.getName()).split("-");
|
||||||
|
int archiveId = Integer.parseInt(parts[0]);
|
||||||
|
int fileId = Integer.parseInt(parts[1]);
|
||||||
|
int nameHash = Integer.parseInt(parts[2]);
|
||||||
|
|
||||||
|
assert archiveId == this.getArchiveId();
|
||||||
|
|
||||||
|
File file = new File(this, fileId);
|
||||||
|
file.setNameHash(nameHash);
|
||||||
|
|
||||||
|
byte[] contents = Files.toByteArray(from);
|
||||||
|
file.setContents(contents);
|
||||||
|
|
||||||
|
files.add(file);
|
||||||
|
|
||||||
|
java.io.File archiveFile = new java.io.File(parent, this.getArchiveId() + ".rev");
|
||||||
|
int rev = Integer.parseInt(Files.readFirstLine(archiveFile, Charset.defaultCharset()));
|
||||||
|
this.setRevision(rev);
|
||||||
|
|
||||||
|
archiveFile = new java.io.File(parent, this.getArchiveId() + ".name");
|
||||||
|
int name = Integer.parseInt(Files.readFirstLine(archiveFile, Charset.defaultCharset()));
|
||||||
|
this.setNameHash(name);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void loadTree(java.io.File parent, java.io.File from) throws IOException
|
||||||
|
{
|
||||||
|
for (java.io.File file : from.listFiles())
|
||||||
|
{
|
||||||
|
//fileId-fileName.dat
|
||||||
|
String[] split = Files.getNameWithoutExtension(file.getName()).split("-");
|
||||||
|
int fileId = Integer.parseInt(split[0]);
|
||||||
|
int fileName = Integer.parseInt(split[1]);
|
||||||
|
|
||||||
|
File f = new File(this, fileId);
|
||||||
|
f.setNameHash(fileName);
|
||||||
|
|
||||||
|
byte[] contents = Files.toByteArray(file);
|
||||||
|
f.setContents(contents);
|
||||||
|
|
||||||
|
files.add(f);
|
||||||
|
}
|
||||||
|
|
||||||
|
java.io.File archiveFile = new java.io.File(parent, this.getArchiveId() + ".rev");
|
||||||
|
int rev = Integer.parseInt(Files.readFirstLine(archiveFile, Charset.defaultCharset()));
|
||||||
|
this.setRevision(rev);
|
||||||
|
|
||||||
|
archiveFile = new java.io.File(parent, this.getArchiveId() + ".name");
|
||||||
|
int name = Integer.parseInt(Files.readFirstLine(archiveFile, Charset.defaultCharset()));
|
||||||
|
this.setNameHash(name);
|
||||||
|
|
||||||
|
// the filesystem may order these differently (eg, 1, 10, 2)
|
||||||
|
Collections.sort(files, (f1, f2) -> Integer.compare(f1.getFileId(), f2.getFileId()));
|
||||||
|
}
|
||||||
|
|
||||||
public void loadNames(InputStream stream, int numberOfFiles)
|
public void loadNames(InputStream stream, int numberOfFiles)
|
||||||
{
|
{
|
||||||
for (int i = 0; i < numberOfFiles; ++i)
|
for (int i = 0; i < numberOfFiles; ++i)
|
||||||
|
|||||||
104
cache/src/main/java/net/runelite/cache/fs/Index.java
vendored
104
cache/src/main/java/net/runelite/cache/fs/Index.java
vendored
@@ -22,12 +22,14 @@
|
|||||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||||
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package net.runelite.cache.fs;
|
package net.runelite.cache.fs;
|
||||||
|
|
||||||
|
import com.google.common.io.Files;
|
||||||
import java.io.Closeable;
|
import java.io.Closeable;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import java.nio.charset.Charset;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
|
import java.util.Collections;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
import net.runelite.cache.util.Djb2;
|
import net.runelite.cache.util.Djb2;
|
||||||
@@ -40,11 +42,11 @@ import org.slf4j.LoggerFactory;
|
|||||||
public class Index implements Closeable
|
public class Index implements Closeable
|
||||||
{
|
{
|
||||||
private static final Logger logger = LoggerFactory.getLogger(Index.class);
|
private static final Logger logger = LoggerFactory.getLogger(Index.class);
|
||||||
|
|
||||||
private final Store store;
|
private final Store store;
|
||||||
private final IndexFile index;
|
private final IndexFile index;
|
||||||
private final int id;
|
private final int id;
|
||||||
|
|
||||||
private XteaKeyManager xteaManager;
|
private XteaKeyManager xteaManager;
|
||||||
|
|
||||||
private int protocol = 7;
|
private int protocol = 7;
|
||||||
@@ -55,14 +57,14 @@ public class Index implements Closeable
|
|||||||
private int compression; // compression method of this index's data in 255
|
private int compression; // compression method of this index's data in 255
|
||||||
|
|
||||||
private final List<Archive> archives = new ArrayList<>();
|
private final List<Archive> archives = new ArrayList<>();
|
||||||
|
|
||||||
public Index(Store store, IndexFile index, int id)
|
public Index(Store store, IndexFile index, int id)
|
||||||
{
|
{
|
||||||
this.store = store;
|
this.store = store;
|
||||||
this.index = index;
|
this.index = index;
|
||||||
this.id = id;
|
this.id = id;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void close() throws IOException
|
public void close() throws IOException
|
||||||
{
|
{
|
||||||
@@ -160,19 +162,23 @@ public class Index implements Closeable
|
|||||||
{
|
{
|
||||||
return archives;
|
return archives;
|
||||||
}
|
}
|
||||||
|
|
||||||
public Archive addArchive(int id)
|
public Archive addArchive(int id)
|
||||||
{
|
{
|
||||||
Archive archive = new Archive(this, id);
|
Archive archive = new Archive(this, id);
|
||||||
this.archives.add(archive);
|
this.archives.add(archive);
|
||||||
return archive;
|
return archive;
|
||||||
}
|
}
|
||||||
|
|
||||||
public Archive getArchive(int id)
|
public Archive getArchive(int id)
|
||||||
{
|
{
|
||||||
for (Archive a : archives)
|
for (Archive a : archives)
|
||||||
|
{
|
||||||
if (a.getArchiveId() == id)
|
if (a.getArchiveId() == id)
|
||||||
|
{
|
||||||
return a;
|
return a;
|
||||||
|
}
|
||||||
|
}
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -180,25 +186,29 @@ public class Index implements Closeable
|
|||||||
{
|
{
|
||||||
int hash = Djb2.hash(name);
|
int hash = Djb2.hash(name);
|
||||||
for (Archive a : archives)
|
for (Archive a : archives)
|
||||||
|
{
|
||||||
if (a.getNameHash() == hash)
|
if (a.getNameHash() == hash)
|
||||||
|
{
|
||||||
return a;
|
return a;
|
||||||
|
}
|
||||||
|
}
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void load() throws IOException
|
public void load() throws IOException
|
||||||
{
|
{
|
||||||
logger.trace("Loading index {}", id);
|
logger.trace("Loading index {}", id);
|
||||||
|
|
||||||
DataFile dataFile = store.getData();
|
DataFile dataFile = store.getData();
|
||||||
IndexFile index255 = store.getIndex255();
|
IndexFile index255 = store.getIndex255();
|
||||||
|
|
||||||
IndexEntry entry = index255.read(id);
|
IndexEntry entry = index255.read(id);
|
||||||
byte[] indexData = dataFile.read(index255.getIndexFileId(), entry.getId(), entry.getSector(), entry.getLength());
|
byte[] indexData = dataFile.read(index255.getIndexFileId(), entry.getId(), entry.getSector(), entry.getLength());
|
||||||
DataFileReadResult res = DataFile.decompress(indexData, null);
|
DataFileReadResult res = DataFile.decompress(indexData, null);
|
||||||
byte[] data = res.data;
|
byte[] data = res.data;
|
||||||
|
|
||||||
archives.clear();
|
archives.clear();
|
||||||
|
|
||||||
readIndexData(data);
|
readIndexData(data);
|
||||||
|
|
||||||
this.crc = res.crc;
|
this.crc = res.crc;
|
||||||
@@ -208,11 +218,11 @@ public class Index implements Closeable
|
|||||||
|
|
||||||
this.loadArchives();
|
this.loadArchives();
|
||||||
}
|
}
|
||||||
|
|
||||||
public void save() throws IOException
|
public void save() throws IOException
|
||||||
{
|
{
|
||||||
saveArchives();
|
saveArchives();
|
||||||
|
|
||||||
byte[] data = this.writeIndexData();
|
byte[] data = this.writeIndexData();
|
||||||
|
|
||||||
DataFile dataFile = store.getData();
|
DataFile dataFile = store.getData();
|
||||||
@@ -226,7 +236,63 @@ public class Index implements Closeable
|
|||||||
this.crc = res.crc;
|
this.crc = res.crc;
|
||||||
this.whirlpool = res.whirlpool;
|
this.whirlpool = res.whirlpool;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void saveTree(java.io.File to) throws IOException
|
||||||
|
{
|
||||||
|
java.io.File idx = new java.io.File(to, "" + this.getId());
|
||||||
|
idx.mkdirs();
|
||||||
|
|
||||||
|
for (Archive a : archives)
|
||||||
|
{
|
||||||
|
a.saveTree(idx);
|
||||||
|
}
|
||||||
|
|
||||||
|
java.io.File rev = new java.io.File(to, this.getId() + ".rev");
|
||||||
|
Files.write("" + this.getRevision(), rev, Charset.defaultCharset());
|
||||||
|
}
|
||||||
|
|
||||||
|
public void loadTree(java.io.File parent, java.io.File to) throws IOException
|
||||||
|
{
|
||||||
|
for (java.io.File f : to.listFiles())
|
||||||
|
{
|
||||||
|
if (f.isDirectory())
|
||||||
|
{
|
||||||
|
int id = Integer.parseInt(f.getName());
|
||||||
|
|
||||||
|
Archive archive = new Archive(this, id);
|
||||||
|
archive.loadTree(to, f);
|
||||||
|
archives.add(archive);
|
||||||
|
}
|
||||||
|
else if (f.getName().endsWith(".dat"))
|
||||||
|
{
|
||||||
|
// one file. archiveId-fileId-name
|
||||||
|
String[] parts = Files.getNameWithoutExtension(f.getName()).split("-");
|
||||||
|
|
||||||
|
int id = Integer.parseInt(parts[0]);
|
||||||
|
|
||||||
|
Archive archive = new Archive(this, id);
|
||||||
|
archive.loadTreeSingleFile(to, f);
|
||||||
|
archives.add(archive);
|
||||||
|
}
|
||||||
|
else if (f.getName().endsWith(".datc"))
|
||||||
|
{
|
||||||
|
// packed data
|
||||||
|
String[] parts = Files.getNameWithoutExtension(f.getName()).split("-");
|
||||||
|
|
||||||
|
int id = Integer.parseInt(parts[0]);
|
||||||
|
|
||||||
|
Archive archive = new Archive(this, id);
|
||||||
|
archive.loadTreeData(to, f);
|
||||||
|
archives.add(archive);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
String str = Files.readFirstLine(new java.io.File(parent, this.getId() + ".rev"), Charset.defaultCharset());
|
||||||
|
revision = Integer.parseInt(str);
|
||||||
|
|
||||||
|
Collections.sort(archives, (ar1, ar2) -> Integer.compare(ar1.getArchiveId(), ar2.getArchiveId()));
|
||||||
|
}
|
||||||
|
|
||||||
public void readIndexData(byte[] data)
|
public void readIndexData(byte[] data)
|
||||||
{
|
{
|
||||||
InputStream stream = new InputStream(data);
|
InputStream stream = new InputStream(data);
|
||||||
@@ -304,7 +370,7 @@ public class Index implements Closeable
|
|||||||
archive = 0;
|
archive = 0;
|
||||||
|
|
||||||
Archive a = this.archives.get(index);
|
Archive a = this.archives.get(index);
|
||||||
a.load(stream, numberOfFiles[index], protocol);
|
a.loadFiles(stream, numberOfFiles[index], protocol);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (named)
|
if (named)
|
||||||
@@ -317,7 +383,7 @@ public class Index implements Closeable
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void loadArchives() throws IOException
|
private void loadArchives() throws IOException
|
||||||
{
|
{
|
||||||
// get data from index file
|
// get data from index file
|
||||||
@@ -347,7 +413,7 @@ public class Index implements Closeable
|
|||||||
a.decompressAndLoad(null);
|
a.decompressAndLoad(null);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public void saveArchives() throws IOException
|
public void saveArchives() throws IOException
|
||||||
{
|
{
|
||||||
for (Archive a : archives)
|
for (Archive a : archives)
|
||||||
@@ -372,14 +438,14 @@ public class Index implements Closeable
|
|||||||
|
|
||||||
DataFileWriteResult res = data.write(this.id, a.getArchiveId(), compressedData, rev);
|
DataFileWriteResult res = data.write(this.id, a.getArchiveId(), compressedData, rev);
|
||||||
this.index.write(new IndexEntry(this.index, a.getArchiveId(), res.sector, res.compressedLength));
|
this.index.write(new IndexEntry(this.index, a.getArchiveId(), res.sector, res.compressedLength));
|
||||||
|
|
||||||
logger.trace("Saved archive {}/{} at sector {}, compressed length {}", this.getId(), a.getArchiveId(), res.sector, res.compressedLength);
|
logger.trace("Saved archive {}/{} at sector {}, compressed length {}", this.getId(), a.getArchiveId(), res.sector, res.compressedLength);
|
||||||
|
|
||||||
a.setCrc(res.crc);
|
a.setCrc(res.crc);
|
||||||
a.setWhirlpool(res.whirlpool);
|
a.setWhirlpool(res.whirlpool);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public byte[] writeIndexData()
|
public byte[] writeIndexData()
|
||||||
{
|
{
|
||||||
OutputStream stream = new OutputStream();
|
OutputStream stream = new OutputStream();
|
||||||
@@ -454,7 +520,7 @@ public class Index implements Closeable
|
|||||||
for (data = 0; data < this.archives.size(); ++data)
|
for (data = 0; data < this.archives.size(); ++data)
|
||||||
{
|
{
|
||||||
Archive a = this.archives.get(data);
|
Archive a = this.archives.get(data);
|
||||||
|
|
||||||
int len = a.getFiles().size();
|
int len = a.getFiles().size();
|
||||||
|
|
||||||
if (protocol >= 7)
|
if (protocol >= 7)
|
||||||
|
|||||||
@@ -22,7 +22,6 @@
|
|||||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||||
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package net.runelite.cache.fs;
|
package net.runelite.cache.fs;
|
||||||
|
|
||||||
import java.io.Closeable;
|
import java.io.Closeable;
|
||||||
@@ -30,6 +29,7 @@ import java.io.File;
|
|||||||
import java.io.FileNotFoundException;
|
import java.io.FileNotFoundException;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
|
import java.util.Collections;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
import net.runelite.cache.IndexType;
|
import net.runelite.cache.IndexType;
|
||||||
@@ -43,19 +43,19 @@ public class Store implements Closeable
|
|||||||
|
|
||||||
private static final String MAIN_FILE_CACHE_DAT = "main_file_cache.dat2";
|
private static final String MAIN_FILE_CACHE_DAT = "main_file_cache.dat2";
|
||||||
private static final String MAIN_FILE_CACHE_IDX = "main_file_cache.idx";
|
private static final String MAIN_FILE_CACHE_IDX = "main_file_cache.idx";
|
||||||
|
|
||||||
private final File folder;
|
private final File folder;
|
||||||
private final DataFile data;
|
private final DataFile data;
|
||||||
private final IndexFile index255;
|
private final IndexFile index255;
|
||||||
private final List<Index> indexes = new ArrayList<>();
|
private final List<Index> indexes = new ArrayList<>();
|
||||||
|
|
||||||
public Store(File folder) throws IOException
|
public Store(File folder) throws IOException
|
||||||
{
|
{
|
||||||
this.folder = folder;
|
this.folder = folder;
|
||||||
|
|
||||||
data = new DataFile(this, new File(folder, MAIN_FILE_CACHE_DAT));
|
data = new DataFile(this, new File(folder, MAIN_FILE_CACHE_DAT));
|
||||||
index255 = new IndexFile(this, 255, new File(folder, MAIN_FILE_CACHE_IDX + "255"));
|
index255 = new IndexFile(this, 255, new File(folder, MAIN_FILE_CACHE_IDX + "255"));
|
||||||
|
|
||||||
for (int i = 0; i < index255.getIndexCount(); ++i)
|
for (int i = 0; i < index255.getIndexCount(); ++i)
|
||||||
{
|
{
|
||||||
this.addIndex(i);
|
this.addIndex(i);
|
||||||
@@ -76,7 +76,9 @@ public class Store implements Closeable
|
|||||||
data.close();
|
data.close();
|
||||||
index255.close();
|
index255.close();
|
||||||
for (Index i : indexes)
|
for (Index i : indexes)
|
||||||
|
{
|
||||||
i.close();
|
i.close();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -105,18 +107,22 @@ public class Store implements Closeable
|
|||||||
}
|
}
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
public final Index addIndex(int id) throws FileNotFoundException
|
public final Index addIndex(int id) throws FileNotFoundException
|
||||||
{
|
{
|
||||||
for (Index i : indexes)
|
for (Index i : indexes)
|
||||||
|
{
|
||||||
if (i.getIndex().getIndexFileId() == id)
|
if (i.getIndex().getIndexFileId() == id)
|
||||||
|
{
|
||||||
throw new IllegalArgumentException("index " + id + " already exists");
|
throw new IllegalArgumentException("index " + id + " already exists");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
IndexFile indexFile = new IndexFile(this, id, new File(folder, MAIN_FILE_CACHE_IDX + id));
|
IndexFile indexFile = new IndexFile(this, id, new File(folder, MAIN_FILE_CACHE_IDX + id));
|
||||||
Index index = new Index(this, indexFile, id);
|
Index index = new Index(this, indexFile, id);
|
||||||
|
|
||||||
this.indexes.add(index);
|
this.indexes.add(index);
|
||||||
|
|
||||||
return index;
|
return index;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -125,13 +131,15 @@ public class Store implements Closeable
|
|||||||
assert indexes.contains(index);
|
assert indexes.contains(index);
|
||||||
indexes.remove(index);
|
indexes.remove(index);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void load() throws IOException
|
public void load() throws IOException
|
||||||
{
|
{
|
||||||
for (Index i : indexes)
|
for (Index i : indexes)
|
||||||
|
{
|
||||||
i.load();
|
i.load();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public void save() throws IOException
|
public void save() throws IOException
|
||||||
{
|
{
|
||||||
logger.debug("Clearing data and indexes in preparation for store save");
|
logger.debug("Clearing data and indexes in preparation for store save");
|
||||||
@@ -139,10 +147,41 @@ public class Store implements Closeable
|
|||||||
data.clear();
|
data.clear();
|
||||||
|
|
||||||
for (Index i : indexes)
|
for (Index i : indexes)
|
||||||
|
{
|
||||||
i.clear();
|
i.clear();
|
||||||
|
}
|
||||||
|
|
||||||
for (Index i : indexes)
|
for (Index i : indexes)
|
||||||
|
{
|
||||||
i.save();
|
i.save();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void saveTree(java.io.File to) throws IOException
|
||||||
|
{
|
||||||
|
for (Index i : indexes)
|
||||||
|
{
|
||||||
|
i.saveTree(to);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void loadTree(java.io.File from) throws IOException
|
||||||
|
{
|
||||||
|
for (java.io.File idx : from.listFiles())
|
||||||
|
{
|
||||||
|
if (!idx.isDirectory())
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
int id = Integer.parseInt(idx.getName());
|
||||||
|
IndexFile indexFile = new IndexFile(this, id, new File(folder, MAIN_FILE_CACHE_IDX + id));
|
||||||
|
Index index = new Index(this, indexFile, id);
|
||||||
|
index.loadTree(from, idx);
|
||||||
|
indexes.add(index);
|
||||||
|
}
|
||||||
|
|
||||||
|
Collections.sort(indexes, (idx1, idx2) -> Integer.compare(idx1.getId(), idx2.getId()));
|
||||||
}
|
}
|
||||||
|
|
||||||
public DataFile getData()
|
public DataFile getData()
|
||||||
@@ -159,7 +198,7 @@ public class Store implements Closeable
|
|||||||
{
|
{
|
||||||
return indexes;
|
return indexes;
|
||||||
}
|
}
|
||||||
|
|
||||||
public Index getIndex(IndexType type)
|
public Index getIndex(IndexType type)
|
||||||
{
|
{
|
||||||
return indexes.get(type.getNumber());
|
return indexes.get(type.getNumber());
|
||||||
@@ -168,8 +207,12 @@ public class Store implements Closeable
|
|||||||
public final Index findIndex(int id)
|
public final Index findIndex(int id)
|
||||||
{
|
{
|
||||||
for (Index i : indexes)
|
for (Index i : indexes)
|
||||||
|
{
|
||||||
if (i.getId() == id)
|
if (i.getId() == id)
|
||||||
|
{
|
||||||
return i;
|
return i;
|
||||||
|
}
|
||||||
|
}
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -22,14 +22,13 @@
|
|||||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||||
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package net.runelite.cache.fs;
|
package net.runelite.cache.fs;
|
||||||
|
|
||||||
import com.google.common.io.Files;
|
import com.google.common.io.Files;
|
||||||
import java.io.FileOutputStream;
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import net.runelite.cache.StoreLocation;
|
import net.runelite.cache.StoreLocation;
|
||||||
import org.junit.Assert;
|
import org.junit.Assert;
|
||||||
|
import org.junit.Ignore;
|
||||||
import org.junit.Rule;
|
import org.junit.Rule;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
import org.junit.rules.TemporaryFolder;
|
import org.junit.rules.TemporaryFolder;
|
||||||
@@ -55,7 +54,9 @@ public class StoreLoadTest
|
|||||||
|
|
||||||
java.io.File testStoreFile = folder.newFolder();
|
java.io.File testStoreFile = folder.newFolder();
|
||||||
for (java.io.File f : StoreLocation.LOCATION.listFiles())
|
for (java.io.File f : StoreLocation.LOCATION.listFiles())
|
||||||
|
{
|
||||||
Files.copy(f, new java.io.File(testStoreFile, f.getName()));
|
Files.copy(f, new java.io.File(testStoreFile, f.getName()));
|
||||||
|
}
|
||||||
|
|
||||||
Store testStore = new Store(testStoreFile);
|
Store testStore = new Store(testStoreFile);
|
||||||
testStore.load();
|
testStore.load();
|
||||||
@@ -68,7 +69,8 @@ public class StoreLoadTest
|
|||||||
Assert.assertTrue(store.equals(testStore));
|
Assert.assertTrue(store.equals(testStore));
|
||||||
}
|
}
|
||||||
|
|
||||||
//@Test
|
@Test
|
||||||
|
@Ignore
|
||||||
public void unpackStore() throws IOException
|
public void unpackStore() throws IOException
|
||||||
{
|
{
|
||||||
java.io.File base = StoreLocation.LOCATION;
|
java.io.File base = StoreLocation.LOCATION;
|
||||||
@@ -76,29 +78,22 @@ public class StoreLoadTest
|
|||||||
{
|
{
|
||||||
store.load();
|
store.load();
|
||||||
|
|
||||||
for (Index i : store.getIndexes())
|
store.saveTree(folder.newFolder());
|
||||||
{
|
}
|
||||||
java.io.File ifile = new java.io.File(folder.newFolder(), "" + i.getId());
|
}
|
||||||
ifile.mkdir();
|
|
||||||
|
|
||||||
for (Archive a : i.getArchives())
|
@Test
|
||||||
{
|
@Ignore
|
||||||
java.io.File afile = new java.io.File(ifile, "" + a.getArchiveId());
|
public void loadTree() throws IOException
|
||||||
afile.mkdir();
|
{
|
||||||
|
Store store = new Store(folder.newFolder());
|
||||||
|
store.loadTree(new java.io.File("C:\\rs\\temp\\tree"));
|
||||||
|
|
||||||
for (File f : a.getFiles())
|
try (Store store2 = new Store(StoreLocation.LOCATION))
|
||||||
{
|
{
|
||||||
java.io.File ffile = new java.io.File(afile, "" + f.getFileId());
|
store2.load();
|
||||||
try (FileOutputStream fout = new FileOutputStream(ffile))
|
|
||||||
{
|
Assert.assertEquals(store, store2);
|
||||||
if (f.getContents() != null)
|
|
||||||
{
|
|
||||||
fout.write(f.getContents());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -22,7 +22,6 @@
|
|||||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||||
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package net.runelite.cache.fs;
|
package net.runelite.cache.fs;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
@@ -35,9 +34,11 @@ import org.junit.rules.TemporaryFolder;
|
|||||||
|
|
||||||
public class StoreTest
|
public class StoreTest
|
||||||
{
|
{
|
||||||
|
private static final int NUMBER_OF_FILES = 1024;
|
||||||
|
|
||||||
@Rule
|
@Rule
|
||||||
public TemporaryFolder folder = StoreLocation.getTemporaryFolder();
|
public TemporaryFolder folder = StoreLocation.getTemporaryFolder();
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testOneFile() throws IOException
|
public void testOneFile() throws IOException
|
||||||
{
|
{
|
||||||
@@ -50,29 +51,28 @@ public class StoreTest
|
|||||||
file.setContents("test".getBytes());
|
file.setContents("test".getBytes());
|
||||||
|
|
||||||
store.save();
|
store.save();
|
||||||
|
|
||||||
try (Store store2 = new Store(folder.getRoot()))
|
try (Store store2 = new Store(folder.getRoot()))
|
||||||
{
|
{
|
||||||
store2.load();
|
store2.load();
|
||||||
|
|
||||||
Assert.assertEquals(store, store2);
|
Assert.assertEquals(store, store2);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static final int NUMBER_OF_FILES = 1024;
|
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testManyFiles() throws IOException
|
public void testManyFiles() throws IOException
|
||||||
{
|
{
|
||||||
Random random = new Random(42L);
|
Random random = new Random(42L);
|
||||||
|
java.io.File root = folder.newFolder();
|
||||||
|
|
||||||
try (Store store = new Store(folder.getRoot()))
|
try (Store store = new Store(root))
|
||||||
{
|
{
|
||||||
Index index = store.addIndex(0);
|
Index index = store.addIndex(0);
|
||||||
Archive archive = index.addArchive(0);
|
Archive archive = index.addArchive(0);
|
||||||
archive.setNameHash(random.nextInt());
|
archive.setNameHash(random.nextInt());
|
||||||
|
|
||||||
for (int i = 0; i < NUMBER_OF_FILES; ++i)
|
for (int i = 0; i < NUMBER_OF_FILES; ++i)
|
||||||
{
|
{
|
||||||
File file = archive.addFile(i);
|
File file = archive.addFile(i);
|
||||||
@@ -81,70 +81,80 @@ public class StoreTest
|
|||||||
random.nextBytes(data);
|
random.nextBytes(data);
|
||||||
file.setContents(data);
|
file.setContents(data);
|
||||||
}
|
}
|
||||||
|
|
||||||
store.save();
|
store.save();
|
||||||
|
|
||||||
try (Store store2 = new Store(folder.getRoot()))
|
try (Store store2 = new Store(root))
|
||||||
{
|
{
|
||||||
store2.load();
|
store2.load();
|
||||||
|
|
||||||
Assert.assertEquals(store, store2);
|
Assert.assertEquals(store, store2);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testMultipleArchives() throws IOException
|
public void testMultipleArchives() throws IOException
|
||||||
{
|
{
|
||||||
Random random = new Random(43L);
|
Random random = new Random(43L);
|
||||||
|
java.io.File root = folder.newFolder();
|
||||||
|
|
||||||
try (Store store = new Store(folder.getRoot()))
|
try (Store store = new Store(root))
|
||||||
{
|
{
|
||||||
Index index = store.addIndex(0);
|
Index index = store.addIndex(0);
|
||||||
Index index2 = store.addIndex(1);
|
Index index2 = store.addIndex(1);
|
||||||
|
|
||||||
Archive archive = index.addArchive(0);
|
Archive archive = index.addArchive(0);
|
||||||
archive.setNameHash(random.nextInt());
|
archive.setNameHash(random.nextInt(Integer.MAX_VALUE));
|
||||||
|
|
||||||
Archive archive2 = index.addArchive(1);
|
Archive archive2 = index.addArchive(1);
|
||||||
|
|
||||||
Archive archive3 = index2.addArchive(0);
|
Archive archive3 = index2.addArchive(0);
|
||||||
|
|
||||||
for (int i = 0; i < NUMBER_OF_FILES; ++i)
|
for (int i = 0; i < NUMBER_OF_FILES; ++i)
|
||||||
{
|
{
|
||||||
File file = archive.addFile(i);
|
File file = archive.addFile(i);
|
||||||
file.setNameHash(random.nextInt());
|
file.setNameHash(random.nextInt(Integer.MAX_VALUE));
|
||||||
byte[] data = new byte[random.nextInt(1024)];
|
byte[] data = new byte[random.nextInt(1024)];
|
||||||
random.nextBytes(data);
|
random.nextBytes(data);
|
||||||
file.setContents(data);
|
file.setContents(data);
|
||||||
}
|
}
|
||||||
|
|
||||||
for (int i = 0; i < NUMBER_OF_FILES; ++i)
|
for (int i = 0; i < NUMBER_OF_FILES; ++i)
|
||||||
{
|
{
|
||||||
File file = archive2.addFile(i);
|
File file = archive2.addFile(i);
|
||||||
file.setNameHash(random.nextInt());
|
file.setNameHash(random.nextInt(Integer.MAX_VALUE));
|
||||||
byte[] data = new byte[random.nextInt(1024)];
|
byte[] data = new byte[random.nextInt(1024)];
|
||||||
random.nextBytes(data);
|
random.nextBytes(data);
|
||||||
file.setContents(data);
|
file.setContents(data);
|
||||||
}
|
}
|
||||||
|
|
||||||
for (int i = 0; i < NUMBER_OF_FILES; ++i)
|
for (int i = 0; i < NUMBER_OF_FILES; ++i)
|
||||||
{
|
{
|
||||||
File file = archive3.addFile(i);
|
File file = archive3.addFile(i);
|
||||||
file.setNameHash(random.nextInt());
|
file.setNameHash(random.nextInt(Integer.MAX_VALUE));
|
||||||
byte[] data = new byte[random.nextInt(1024)];
|
byte[] data = new byte[random.nextInt(1024)];
|
||||||
random.nextBytes(data);
|
random.nextBytes(data);
|
||||||
file.setContents(data);
|
file.setContents(data);
|
||||||
}
|
}
|
||||||
|
|
||||||
store.save();
|
store.save();
|
||||||
|
|
||||||
try (Store store2 = new Store(folder.getRoot()))
|
try (Store store2 = new Store(root))
|
||||||
{
|
{
|
||||||
store2.load();
|
store2.load();
|
||||||
|
|
||||||
Assert.assertEquals(store, store2);
|
Assert.assertEquals(store, store2);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Test tree save/load
|
||||||
|
java.io.File tree = folder.newFolder();
|
||||||
|
store.saveTree(tree);
|
||||||
|
|
||||||
|
try (Store store2 = new Store(folder.newFolder()))
|
||||||
|
{
|
||||||
|
store2.loadTree(tree);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user