cache: add saveTree/loadTree to save/load cache to/from a normal filesystem

cache/src/main/java/net/runelite/cache/fs/Archive.java

@@ -22,12 +22,14 @@
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
-
 package net.runelite.cache.fs;
 
+import com.google.common.io.Files;
 import java.io.IOException;
+import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 import net.runelite.cache.io.InputStream;
@@ -117,7 +119,7 @@ public class Archive
 		return file;
 	}
 
-	public void load(InputStream stream, int numberOfFiles, int protocol)
+	public void loadFiles(InputStream stream, int numberOfFiles, int protocol)
 	{
 		int archive = 0;
 
@@ -277,6 +279,142 @@ public class Archive
 		return fileData;
 	}
 
+	public void saveTree(java.io.File to) throws IOException
+	{
+		if (data != null)
+		{
+			assert files.size() == 1; // this is the maps
+
+			File file = files.get(0);
+
+			java.io.File archiveFile = new java.io.File(to, this.getArchiveId() + "-" + file.getFileId() + "-" + file.getNameHash() + ".datc");
+			Files.write(data, archiveFile);
+
+			archiveFile = new java.io.File(to, this.getArchiveId() + ".rev");
+			Files.write("" + this.getRevision(), archiveFile, Charset.defaultCharset());
+
+			archiveFile = new java.io.File(to, this.getArchiveId() + ".name");
+			Files.write("" + this.getNameHash(), archiveFile, Charset.defaultCharset());
+			return;
+		}
+
+		if (files.size() == 1)
+		{
+			File file = this.getFiles().get(0);
+
+			java.io.File archiveFile = new java.io.File(to, this.getArchiveId() + "-" + file.getFileId() + "-" + file.getNameHash() + ".dat");
+			byte[] contents = file.getContents();
+
+			Files.write(contents, archiveFile);
+
+			archiveFile = new java.io.File(to, this.getArchiveId() + ".rev");
+			Files.write("" + this.getRevision(), archiveFile, Charset.defaultCharset());
+
+			archiveFile = new java.io.File(to, this.getArchiveId() + ".name");
+			Files.write("" + this.getNameHash(), archiveFile, Charset.defaultCharset());
+			return;
+		}
+
+		java.io.File archiveFile = new java.io.File(to, this.getArchiveId() + ".rev");
+		Files.write("" + this.getRevision(), archiveFile, Charset.defaultCharset());
+
+		archiveFile = new java.io.File(to, this.getArchiveId() + ".name");
+		Files.write("" + this.getNameHash(), archiveFile, Charset.defaultCharset());
+
+		java.io.File archiveFolder = new java.io.File(to, "" + this.getArchiveId());
+		archiveFolder.mkdirs();
+
+		for (File file : files)
+		{
+			archiveFile = new java.io.File(archiveFolder, file.getFileId() + "-" + file.getNameHash() + ".dat");
+			byte[] contents = file.getContents();
+			Files.write(contents, archiveFile);
+		}
+	}
+
+	public void loadTreeData(java.io.File parent, java.io.File from) throws IOException
+	{
+		//archiveId-fileId-fileName - assumes name isn't negative
+		String[] parts = Files.getNameWithoutExtension(from.getName()).split("-");
+		int archiveId = Integer.parseInt(parts[0]);
+		int fileId = Integer.parseInt(parts[1]);
+		int nameHash = Integer.parseInt(parts[2]);
+
+		assert archiveId == this.getArchiveId();
+
+		data = Files.toByteArray(from);
+
+		File file = new File(this, fileId);
+		file.setNameHash(nameHash);
+
+		files.add(file);
+
+		java.io.File archiveFile = new java.io.File(parent, this.getArchiveId() + ".rev");
+		int rev = Integer.parseInt(Files.readFirstLine(archiveFile, Charset.defaultCharset()));
+		this.setRevision(rev);
+
+		archiveFile = new java.io.File(parent, this.getArchiveId() + ".name");
+		int name = Integer.parseInt(Files.readFirstLine(archiveFile, Charset.defaultCharset()));
+		this.setNameHash(name);
+	}
+
+	public void loadTreeSingleFile(java.io.File parent, java.io.File from) throws IOException
+	{
+		//archiveId-fileId-fileName
+		String[] parts = Files.getNameWithoutExtension(from.getName()).split("-");
+		int archiveId = Integer.parseInt(parts[0]);
+		int fileId = Integer.parseInt(parts[1]);
+		int nameHash = Integer.parseInt(parts[2]);
+
+		assert archiveId == this.getArchiveId();
+
+		File file = new File(this, fileId);
+		file.setNameHash(nameHash);
+
+		byte[] contents = Files.toByteArray(from);
+		file.setContents(contents);
+
+		files.add(file);
+
+		java.io.File archiveFile = new java.io.File(parent, this.getArchiveId() + ".rev");
+		int rev = Integer.parseInt(Files.readFirstLine(archiveFile, Charset.defaultCharset()));
+		this.setRevision(rev);
+
+		archiveFile = new java.io.File(parent, this.getArchiveId() + ".name");
+		int name = Integer.parseInt(Files.readFirstLine(archiveFile, Charset.defaultCharset()));
+		this.setNameHash(name);
+	}
+
+	public void loadTree(java.io.File parent, java.io.File from) throws IOException
+	{
+		for (java.io.File file : from.listFiles())
+		{
+			//fileId-fileName.dat
+			String[] split = Files.getNameWithoutExtension(file.getName()).split("-");
+			int fileId = Integer.parseInt(split[0]);
+			int fileName = Integer.parseInt(split[1]);
+
+			File f = new File(this, fileId);
+			f.setNameHash(fileName);
+
+			byte[] contents = Files.toByteArray(file);
+			f.setContents(contents);
+
+			files.add(f);
+		}
+
+		java.io.File archiveFile = new java.io.File(parent, this.getArchiveId() + ".rev");
+		int rev = Integer.parseInt(Files.readFirstLine(archiveFile, Charset.defaultCharset()));
+		this.setRevision(rev);
+
+		archiveFile = new java.io.File(parent, this.getArchiveId() + ".name");
+		int name = Integer.parseInt(Files.readFirstLine(archiveFile, Charset.defaultCharset()));
+		this.setNameHash(name);
+
+		// the filesystem may order these differently (eg, 1, 10, 2)
+		Collections.sort(files, (f1, f2) -> Integer.compare(f1.getFileId(), f2.getFileId()));
+	}
+
 	public void loadNames(InputStream stream, int numberOfFiles)
 	{
 		for (int i = 0; i < numberOfFiles; ++i)

cache/src/main/java/net/runelite/cache/fs/Index.java

@@ -22,12 +22,14 @@
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
-
 package net.runelite.cache.fs;
 
+import com.google.common.io.Files;
 import java.io.Closeable;
 import java.io.IOException;
+import java.nio.charset.Charset;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 import net.runelite.cache.util.Djb2;
@@ -171,8 +173,12 @@ public class Index implements Closeable
 	public Archive getArchive(int id)
 	{
 		for (Archive a : archives)
+		{
 			if (a.getArchiveId() == id)
+			{
 				return a;
+			}
+		}
 		return null;
 	}
 
@@ -180,8 +186,12 @@ public class Index implements Closeable
 	{
 		int hash = Djb2.hash(name);
 		for (Archive a : archives)
+		{
 			if (a.getNameHash() == hash)
+			{
 				return a;
+			}
+		}
 		return null;
 	}
 
@@ -227,6 +237,62 @@ public class Index implements Closeable
 		this.whirlpool = res.whirlpool;
 	}
 
+	public void saveTree(java.io.File to) throws IOException
+	{
+		java.io.File idx = new java.io.File(to, "" + this.getId());
+		idx.mkdirs();
+
+		for (Archive a : archives)
+		{
+			a.saveTree(idx);
+		}
+
+		java.io.File rev = new java.io.File(to, this.getId() + ".rev");
+		Files.write("" + this.getRevision(), rev, Charset.defaultCharset());
+	}
+
+	public void loadTree(java.io.File parent, java.io.File to) throws IOException
+	{
+		for (java.io.File f : to.listFiles())
+		{
+			if (f.isDirectory())
+			{
+				int id = Integer.parseInt(f.getName());
+
+				Archive archive = new Archive(this, id);
+				archive.loadTree(to, f);
+				archives.add(archive);
+			}
+			else if (f.getName().endsWith(".dat"))
+			{
+				// one file. archiveId-fileId-name
+				String[] parts = Files.getNameWithoutExtension(f.getName()).split("-");
+
+				int id = Integer.parseInt(parts[0]);
+
+				Archive archive = new Archive(this, id);
+				archive.loadTreeSingleFile(to, f);
+				archives.add(archive);
+			}
+			else if (f.getName().endsWith(".datc"))
+			{
+				// packed data
+				String[] parts = Files.getNameWithoutExtension(f.getName()).split("-");
+
+				int id = Integer.parseInt(parts[0]);
+
+				Archive archive = new Archive(this, id);
+				archive.loadTreeData(to, f);
+				archives.add(archive);
+			}
+		}
+
+		String str = Files.readFirstLine(new java.io.File(parent, this.getId() + ".rev"), Charset.defaultCharset());
+		revision = Integer.parseInt(str);
+
+		Collections.sort(archives, (ar1, ar2) -> Integer.compare(ar1.getArchiveId(), ar2.getArchiveId()));
+	}
+
 	public void readIndexData(byte[] data)
 	{
 		InputStream stream = new InputStream(data);
@@ -304,7 +370,7 @@ public class Index implements Closeable
 				archive = 0;
 
 				Archive a = this.archives.get(index);
-				a.load(stream, numberOfFiles[index], protocol);
+				a.loadFiles(stream, numberOfFiles[index], protocol);
 			}
 
 			if (named)

cache/src/main/java/net/runelite/cache/fs/Store.java

@@ -22,7 +22,6 @@
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
-
 package net.runelite.cache.fs;
 
 import java.io.Closeable;
@@ -30,6 +29,7 @@ import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 import net.runelite.cache.IndexType;
@@ -76,8 +76,10 @@ public class Store implements Closeable
 		data.close();
 		index255.close();
 		for (Index i : indexes)
+		{
 			i.close();
 		}
+	}
 
 	@Override
 	public int hashCode()
@@ -109,8 +111,12 @@ public class Store implements Closeable
 	public final Index addIndex(int id) throws FileNotFoundException
 	{
 		for (Index i : indexes)
+		{
 			if (i.getIndex().getIndexFileId() == id)
+			{
 				throw new IllegalArgumentException("index " + id + " already exists");
+			}
+		}
 
 		IndexFile indexFile = new IndexFile(this, id, new File(folder, MAIN_FILE_CACHE_IDX + id));
 		Index index = new Index(this, indexFile, id);
@@ -129,8 +135,10 @@ public class Store implements Closeable
 	public void load() throws IOException
 	{
 		for (Index i : indexes)
+		{
 			i.load();
 		}
+	}
 
 	public void save() throws IOException
 	{
@@ -139,11 +147,42 @@ public class Store implements Closeable
 		data.clear();
 
 		for (Index i : indexes)
+		{
 			i.clear();
+		}
 
 		for (Index i : indexes)
+		{
 			i.save();
 		}
+	}
+
+	public void saveTree(java.io.File to) throws IOException
+	{
+		for (Index i : indexes)
+		{
+			i.saveTree(to);
+		}
+	}
+
+	public void loadTree(java.io.File from) throws IOException
+	{
+		for (java.io.File idx : from.listFiles())
+		{
+			if (!idx.isDirectory())
+			{
+				continue;
+			}
+
+			int id = Integer.parseInt(idx.getName());
+			IndexFile indexFile = new IndexFile(this, id, new File(folder, MAIN_FILE_CACHE_IDX + id));
+			Index index = new Index(this, indexFile, id);
+			index.loadTree(from, idx);
+			indexes.add(index);
+		}
+
+		Collections.sort(indexes, (idx1, idx2) -> Integer.compare(idx1.getId(), idx2.getId()));
+	}
 
 	public DataFile getData()
 	{
@@ -168,8 +207,12 @@ public class Store implements Closeable
 	public final Index findIndex(int id)
 	{
 		for (Index i : indexes)
+		{
 			if (i.getId() == id)
+			{
 				return i;
+			}
+		}
 		return null;
 	}
 }

cache/src/test/java/net/runelite/cache/fs/StoreLoadTest.java

@@ -22,14 +22,13 @@
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
-
 package net.runelite.cache.fs;
 
 import com.google.common.io.Files;
-import java.io.FileOutputStream;
 import java.io.IOException;
 import net.runelite.cache.StoreLocation;
 import org.junit.Assert;
+import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
@@ -55,7 +54,9 @@ public class StoreLoadTest
 
 		java.io.File testStoreFile = folder.newFolder();
 		for (java.io.File f : StoreLocation.LOCATION.listFiles())
+		{
 			Files.copy(f, new java.io.File(testStoreFile, f.getName()));
+		}
 
 		Store testStore = new Store(testStoreFile);
 		testStore.load();
@@ -68,7 +69,8 @@ public class StoreLoadTest
 		Assert.assertTrue(store.equals(testStore));
 	}
 
-	//@Test
+	@Test
+	@Ignore
 	public void unpackStore() throws IOException
 	{
 		java.io.File base = StoreLocation.LOCATION;
@@ -76,29 +78,22 @@ public class StoreLoadTest
 		{
 			store.load();
 
-			for (Index i : store.getIndexes())
+			store.saveTree(folder.newFolder());
-			{
+		}
-				java.io.File ifile = new java.io.File(folder.newFolder(), "" + i.getId());
+	}
-				ifile.mkdir();
 
-				for (Archive a : i.getArchives())
+	@Test
+	@Ignore
+	public void loadTree() throws IOException
 	{
-					java.io.File afile = new java.io.File(ifile, "" + a.getArchiveId());
+		Store store = new Store(folder.newFolder());
-					afile.mkdir();
+		store.loadTree(new java.io.File("C:\\rs\\temp\\tree"));
 
-					for (File f : a.getFiles())
+		try (Store store2 = new Store(StoreLocation.LOCATION))
 		{
-						java.io.File ffile = new java.io.File(afile, "" + f.getFileId());
+			store2.load();
-						try (FileOutputStream fout = new FileOutputStream(ffile))
+
-						{
+			Assert.assertEquals(store, store2);
-							if (f.getContents() != null)
-							{
-								fout.write(f.getContents());
-							}
-						}
-					}
-				}
-			}
 		}
 	}
 }

cache/src/test/java/net/runelite/cache/fs/StoreTest.java

@@ -22,7 +22,6 @@
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
-
 package net.runelite.cache.fs;
 
 import java.io.IOException;
@@ -35,6 +34,8 @@ import org.junit.rules.TemporaryFolder;
 
 public class StoreTest
 {
+	private static final int NUMBER_OF_FILES = 1024;
+
 	@Rule
 	public TemporaryFolder folder = StoreLocation.getTemporaryFolder();
 
@@ -60,14 +61,13 @@ public class StoreTest
 		}
 	}
 
-	private static final int NUMBER_OF_FILES = 1024;
-	
 	@Test
 	public void testManyFiles() throws IOException
 	{
 		Random random = new Random(42L);
+		java.io.File root = folder.newFolder();
 
-		try (Store store = new Store(folder.getRoot()))
+		try (Store store = new Store(root))
 		{
 			Index index = store.addIndex(0);
 			Archive archive = index.addArchive(0);
@@ -84,7 +84,7 @@ public class StoreTest
 
 			store.save();
 
-			try (Store store2 = new Store(folder.getRoot()))
+			try (Store store2 = new Store(root))
 			{
 				store2.load();
 
@@ -97,14 +97,15 @@ public class StoreTest
 	public void testMultipleArchives() throws IOException
 	{
 		Random random = new Random(43L);
+		java.io.File root = folder.newFolder();
 
-		try (Store store = new Store(folder.getRoot()))
+		try (Store store = new Store(root))
 		{
 			Index index = store.addIndex(0);
 			Index index2 = store.addIndex(1);
 
 			Archive archive = index.addArchive(0);
-			archive.setNameHash(random.nextInt());
+			archive.setNameHash(random.nextInt(Integer.MAX_VALUE));
 
 			Archive archive2 = index.addArchive(1);
 
@@ -113,7 +114,7 @@ public class StoreTest
 			for (int i = 0; i < NUMBER_OF_FILES; ++i)
 			{
 				File file = archive.addFile(i);
-				file.setNameHash(random.nextInt());
+				file.setNameHash(random.nextInt(Integer.MAX_VALUE));
 				byte[] data = new byte[random.nextInt(1024)];
 				random.nextBytes(data);
 				file.setContents(data);
@@ -122,7 +123,7 @@ public class StoreTest
 			for (int i = 0; i < NUMBER_OF_FILES; ++i)
 			{
 				File file = archive2.addFile(i);
-				file.setNameHash(random.nextInt());
+				file.setNameHash(random.nextInt(Integer.MAX_VALUE));
 				byte[] data = new byte[random.nextInt(1024)];
 				random.nextBytes(data);
 				file.setContents(data);
@@ -131,7 +132,7 @@ public class StoreTest
 			for (int i = 0; i < NUMBER_OF_FILES; ++i)
 			{
 				File file = archive3.addFile(i);
-				file.setNameHash(random.nextInt());
+				file.setNameHash(random.nextInt(Integer.MAX_VALUE));
 				byte[] data = new byte[random.nextInt(1024)];
 				random.nextBytes(data);
 				file.setContents(data);
@@ -139,12 +140,21 @@ public class StoreTest
 
 			store.save();
 
-			try (Store store2 = new Store(folder.getRoot()))
+			try (Store store2 = new Store(root))
 			{
 				store2.load();
 
 				Assert.assertEquals(store, store2);
 			}
+
+			// Test tree save/load
+			java.io.File tree = folder.newFolder();
+			store.saveTree(tree);
+
+			try (Store store2 = new Store(folder.newFolder()))
+			{
+				store2.loadTree(tree);
+			}
 		}
 	}
 }