Add cache downloader client

cache/src/main/java/net/runelite/cache/downloader/CacheClient.java

@@ -0,0 +1,270 @@
+/*
+ * Copyright (c) 2016, Adam <Adam@sigterm.info>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *    This product includes software developed by Adam <Adam@sigterm.info>
+ * 4. Neither the name of the Adam <Adam@sigterm.info> nor the
+ *    names of its contributors may be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Adam <Adam@sigterm.info> ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Adam <Adam@sigterm.info> BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package net.runelite.cache.downloader;
+
+import io.netty.bootstrap.Bootstrap;
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelFuture;
+import io.netty.channel.ChannelInitializer;
+import io.netty.channel.ChannelOption;
+import io.netty.channel.ChannelPipeline;
+import io.netty.channel.EventLoopGroup;
+import io.netty.channel.nio.NioEventLoopGroup;
+import io.netty.channel.socket.SocketChannel;
+import io.netty.channel.socket.nio.NioSocketChannel;
+import io.netty.handler.proxy.HttpProxyHandler;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.ArrayDeque;
+import java.util.Queue;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import net.runelite.cache.downloader.requests.FileRequest;
+import net.runelite.cache.fs.Archive;
+import net.runelite.cache.fs.DataFileReadResult;
+import net.runelite.cache.fs.Index;
+import net.runelite.cache.fs.Store;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class CacheClient
+{
+	private static final Logger logger = LoggerFactory.getLogger(CacheClient.class);
+
+	private static final String HOST = "oldschool1.runescape.com";
+	private static final int PORT = 43594;
+
+	private static final int CLIENT_REVISION = 115;
+
+	private Store store; // store cache will be written to
+	private int clientRevision;
+
+	private EventLoopGroup group = new NioEventLoopGroup(1);
+	private Channel channel;
+
+	private Queue<PendingFileRequest> requests = new ArrayDeque<>();
+
+	public CacheClient(Store store)
+	{
+		this(store, CLIENT_REVISION);
+	}
+
+	public CacheClient(Store store, int clientRevision)
+	{
+		this.store = store;
+		this.clientRevision = clientRevision;
+	}
+
+	public void connect() throws InterruptedException
+	{
+		Bootstrap b = new Bootstrap();
+		b.group(group)
+			.channel(NioSocketChannel.class)
+			.option(ChannelOption.TCP_NODELAY, true)
+			.handler(new ChannelInitializer<SocketChannel>()
+			{
+				@Override
+				public void initChannel(SocketChannel ch) throws Exception
+				{
+					ChannelPipeline p = ch.pipeline();
+					//p.addFirst(new HttpProxyHandler(new InetSocketAddress("runelite.net", 3128)));
+					p.addLast(new CacheClientHandler(CacheClient.this));
+				}
+			});
+
+		// Start the client.
+		ChannelFuture f = b.connect(HOST, PORT).sync();
+		channel = f.channel();
+	}
+
+	public void stop()
+	{
+		group.shutdownGracefully();
+	}
+
+	public int getClientRevision()
+	{
+		return clientRevision;
+	}
+
+	public void download() throws InterruptedException, ExecutionException, FileNotFoundException
+	{
+		FileResult result = requestFile(255, 255).get();
+		ByteBuf buffer = Unpooled.wrappedBuffer(result.getContents());
+
+		int indexCount = result.getContents().length / 8;
+
+		for (int i = 0; i < indexCount; ++i)
+		{
+			int crc = buffer.readInt();
+			int revision = buffer.readInt();
+
+			if (i == 5)
+				continue; // XXX maps are xtea encrypted
+
+			Index index = store.findIndex(i);
+
+			if (index == null)
+			{
+				logger.info("Index {} does not exist, creating", i);
+			}
+			else if (index.getRevision() != revision)
+			{
+				logger.info("Index {} has the wrong revision (our revision {}, their revision {}", index.getId(), index.getRevision(), revision);
+			}
+			else
+			{
+				logger.info("Index {} is up to date", index.getId());
+				continue;
+			}
+
+			logger.info("Downloading index {}", i);
+
+			FileResult indexFileResult = requestFile(255, i).get();
+
+			logger.info("Downloaded index {}", i);
+
+			if (indexFileResult.getCrc() != crc)
+			{
+				logger.info("Corrupted download for index {}", i);
+				continue;
+			}
+
+			Index oldIndex = null;
+			if (index != null)
+			{
+				store.removeIndex(index);
+
+				oldIndex = index;
+			}
+
+			index = store.addIndex(i);
+
+			index.readIndexData(indexFileResult.getContents());
+			index.setCrc(crc);
+
+			logger.info("Index {} has {} archives", i, index.getArchives().size());
+
+			for (Archive archive : index.getArchives())
+			{
+				Archive oldArchive = oldIndex != null ? oldIndex.getArchive(archive.getArchiveId()) : null;
+
+				if (oldArchive == null || oldArchive.getRevision() != archive.getRevision())
+				{
+					logger.info("Archive {} in index {} is out of date, downloading", archive.getArchiveId(), index.getId());
+
+					FileResult archiveFileResult = requestFile(index.getId(), archive.getArchiveId()).get();
+
+					byte[] contents = archiveFileResult.getContents();
+
+					archive.loadContents(contents);
+					archive.setCompression(archiveFileResult.getCompression());
+				}
+				else
+				{
+					logger.info("Active {} in index {} is up to date", archive.getArchiveId(), index.getId());
+
+					// copy existing contents, this is sort of hackish.
+					byte[] contents = oldArchive.saveContents();
+					archive.loadContents(contents);
+					archive.setCompression(oldArchive.getCompression());
+				}
+			}
+
+			try
+			{
+				store.save(); // save up to this point to disk
+				// XXX if this save takes too long, server closes the connection
+			}
+			catch (IOException ex)
+			{
+				logger.warn("unable to save cache", ex);
+			}
+
+		}
+	}
+
+	public synchronized CompletableFuture<FileResult> requestFile(int index, int fileId)
+	{
+		ByteBuf buf = Unpooled.buffer(4);
+		FileRequest request = new FileRequest(index, fileId);
+		CompletableFuture<FileResult> future = new CompletableFuture<>();
+		PendingFileRequest pf = new PendingFileRequest(request, future);
+
+		buf.writeByte(request.getIndex() == 255 ? 1 : 0);
+		int hash = pf.computeHash();
+		buf.writeByte(hash >> 16);
+		buf.writeByte(hash >> 8);
+		buf.writeByte(hash);
+
+		logger.trace("Sending request for {}/{}", index, fileId);
+
+		channel.writeAndFlush(buf);
+
+		requests.add(pf);
+
+		return future;
+	}
+
+	private PendingFileRequest findRequest(int index, int file)
+	{
+		for (PendingFileRequest pr : requests)
+		{
+			if (pr.getRequest().getIndex() == index && pr.getRequest().getFile() == file)
+			{
+				return pr;
+			}
+		}
+		return null;
+	}
+
+	protected synchronized void onFileFinish(int index, int file, DataFileReadResult dresult)
+	{
+		PendingFileRequest pr = findRequest(index, file);
+
+		if (pr == null)
+		{
+			logger.warn("File download {}/{} with no pending request", index, file);
+			return;
+		}
+
+		requests.remove(pr);
+
+		FileResult result = new FileResult(index, file, dresult.data, dresult.revision, dresult.crc, dresult.whirlpool, dresult.compression);
+
+		logger.debug("File download finished for index {} file {}, length {}", index, file, result.getContents().length);
+
+		pr.getFuture().complete(result);
+	}
+}

cache/src/main/java/net/runelite/cache/downloader/CacheClientHandler.java

@@ -0,0 +1,228 @@
+/*
+ * Copyright (c) 2016, Adam <Adam@sigterm.info>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *    This product includes software developed by Adam <Adam@sigterm.info>
+ * 4. Neither the name of the Adam <Adam@sigterm.info> nor the
+ *    names of its contributors may be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Adam <Adam@sigterm.info> ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Adam <Adam@sigterm.info> BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package net.runelite.cache.downloader;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelInboundHandlerAdapter;
+import io.netty.util.ReferenceCountUtil;
+import net.runelite.cache.downloader.requests.ConnectionInfo;
+import net.runelite.cache.downloader.requests.HelloHandshake;
+import net.runelite.cache.fs.DataFile;
+import net.runelite.cache.fs.DataFileReadResult;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class CacheClientHandler extends ChannelInboundHandlerAdapter
+{
+	private static final Logger logger = LoggerFactory.getLogger(CacheClientHandler.class);
+
+	private CacheClient client;
+	private ClientState state;
+	private ByteBuf buffer = Unpooled.buffer(512);
+
+	public CacheClientHandler(CacheClient client)
+	{
+		this.client = client;
+	}
+
+	@Override
+	public void channelActive(ChannelHandlerContext ctx)
+	{
+		HelloHandshake msg = new HelloHandshake();
+		msg.setRevision(client.getClientRevision());
+
+		ByteBuf message = Unpooled.buffer(5);
+		message.writeByte(msg.getVersion()); // handshake type
+		message.writeInt(msg.getRevision()); // client revision
+
+		ctx.writeAndFlush(message);
+
+		logger.info("Sent handshake with revision {}", msg.getRevision());
+
+		state = ClientState.HANDSHAKING;
+	}
+
+	@Override
+	public void channelRead(ChannelHandlerContext ctx, Object msg)
+	{
+		ByteBuf inbuf = (ByteBuf) msg;
+		buffer.writeBytes(inbuf);
+
+		if (state == ClientState.HANDSHAKING)
+		{
+			int response = buffer.readByte();
+
+			logger.info("Handshake response {}", response);
+
+			if (response != HelloHandshake.RESPONSE_OK)
+			{
+				if (response == HelloHandshake.RESPONSE_OUTDATED)
+					logger.warn("Client version is outdated");
+				else
+					logger.warn("Handshake response error {}", response);
+
+				ctx.close();
+				return;
+			}
+
+			ConnectionInfo cinfo = new ConnectionInfo();
+			ByteBuf outbuf = Unpooled.buffer(4);
+			outbuf.writeByte(cinfo.getVar1());
+			outbuf.writeByte(cinfo.getVar2() >> 16);
+			outbuf.writeByte(cinfo.getVar2() >> 8);
+			outbuf.writeByte(cinfo.getVar2());
+
+			ctx.writeAndFlush(outbuf);
+			state = ClientState.CONNECTED;
+		}
+		else if (state == ClientState.CONNECTED)
+		{
+			ByteBuf copy = buffer.slice();
+
+			int index = copy.readUnsignedByte();
+			int file = copy.readShort();
+			// decompress() starts reading here
+			int compression = copy.readUnsignedByte();
+			int compressedFileSize = copy.readInt();
+
+			int size = compressedFileSize
+				+ 5 // 1 byte compresion type, 4 byte compressed size
+				+ (compression != 0 ? 4 : 0) // compression has leading 4 byte decompressed length
+				;//+ (index != 255 ? 2 : 0); // for the revision
+
+			int breaks = calculateBreaks(size);
+
+			// 3 for index/file
+			if (size + 3 + breaks > buffer.readableBytes())
+			{
+				logger.trace("Index {} archive {}: Not enough data yet {} > {}", index, file, size + 3 + breaks, buffer.readableBytes());
+				return;
+			}
+
+			byte[] compressedData = new byte[size];
+			int compressedDataOffset = 0;
+
+			int totalRead = 3;
+			buffer.skipBytes(3); // skip index/file
+
+			for (int i = 0; i < breaks + 1; ++i)
+			{
+				int bytesInBlock = 512 - (totalRead % 512);
+				int bytesToRead = Math.min(bytesInBlock, size - compressedDataOffset);
+
+				logger.trace("{}/{}: reading block {}/{}, read so far this block: {}, file status: {}/{}",
+					index, file,
+					(totalRead % 512), 512,
+					bytesInBlock,
+					compressedDataOffset, size);
+
+				buffer.getBytes(buffer.readerIndex(), compressedData, compressedDataOffset, bytesToRead);
+				buffer.skipBytes(bytesToRead);
+
+				compressedDataOffset += bytesToRead;
+				totalRead += bytesToRead;
+
+				if (i < breaks)
+				{
+					assert compressedDataOffset < size;
+					int b = buffer.readUnsignedByte();
+					++totalRead;
+					assert b == 0xff;
+				}
+			}
+
+			assert compressedDataOffset == size;
+
+			logger.trace("{}/{}: done downloading file, remaining buffer {}",
+				index, file,
+				buffer.readableBytes());
+			buffer.clear();
+
+			DataFileReadResult result = DataFile.decompress(compressedData);
+
+			client.onFileFinish(index, file, result);
+		}
+
+		buffer.discardReadBytes();
+		ReferenceCountUtil.release(msg);
+	}
+
+	/** Calculate how many breaks there are in the file stream.
+	 * There are calculateBreaks()+1 total chunks in the file stream
+	 *
+	 * File contents are sent in 512 byte chunks, with the first byte of
+	 * each chunk except for the first one being 0xff.
+	 *
+	 * The first chunk has an 8 byte header (index, file, compression,
+	 * compressed size). So, the first chunk can contain 512 - 8 bytes
+	 * of the file, and each chunk after 511 bytes.
+	 *
+	 * The 'size' parameter has the compression type and size included in
+	 * it, since they haven't been read yet by the buffer stream, as
+	 * decompress() reads it, so we use  512 - 3 (because 8-5) = 3
+	 */
+	private int calculateBreaks(int size)
+	{
+		int initialSize = 512 - 3;
+		if (size <= initialSize)
+			return 0; // First in the initial chunk, no breaks
+
+		int left = size - initialSize;
+
+		if (left % 511 == 0)
+		{
+			return (left / 511);
+		}
+		else
+		{
+			// / 511 because 511 bytes of the file per chunk.
+			// + 1 if there is some left over, it still needs its
+			// own chunk
+			return (left / 511) + 1;
+		}
+	}
+
+	@Override
+	public void channelReadComplete(ChannelHandlerContext ctx)
+	{
+		ctx.flush();
+	}
+
+	@Override
+	public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause)
+	{
+		// Close the connection when an exception is raised.
+		cause.printStackTrace();
+		ctx.close();
+	}
+}

cache/src/main/java/net/runelite/cache/downloader/ClientState.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2016, Adam <Adam@sigterm.info>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *    This product includes software developed by Adam <Adam@sigterm.info>
+ * 4. Neither the name of the Adam <Adam@sigterm.info> nor the
+ *    names of its contributors may be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Adam <Adam@sigterm.info> ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Adam <Adam@sigterm.info> BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package net.runelite.cache.downloader;
+
+public enum ClientState
+{
+	HANDSHAKING,
+	CONNECTED
+}

cache/src/main/java/net/runelite/cache/downloader/FileResult.java

@@ -0,0 +1,88 @@
+/*
+ * Copyright (c) 2016, Adam <Adam@sigterm.info>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *    This product includes software developed by Adam <Adam@sigterm.info>
+ * 4. Neither the name of the Adam <Adam@sigterm.info> nor the
+ *    names of its contributors may be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Adam <Adam@sigterm.info> ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Adam <Adam@sigterm.info> BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package net.runelite.cache.downloader;
+
+public class FileResult
+{
+	private final int index;
+	private final int fileId;
+	private final byte[] contents;
+	private final int revision;
+	private final int crc;
+	private final byte[] whirlpool;
+	private final int compression; // compression method used by archive data
+
+	public FileResult(int index, int fileId, byte[] contents, int revision, int crc, byte[] whirlpool, int compression)
+	{
+		this.index = index;
+		this.fileId = fileId;
+		this.contents = contents;
+		this.revision = revision;
+		this.crc = crc;
+		this.whirlpool = whirlpool;
+		this.compression = compression;
+	}
+
+	public int getIndex()
+	{
+		return index;
+	}
+
+	public int getFileId()
+	{
+		return fileId;
+	}
+
+	public byte[] getContents()
+	{
+		return contents;
+	}
+
+	public int getRevision()
+	{
+		return revision;
+	}
+
+	public int getCrc()
+	{
+		return crc;
+	}
+
+	public byte[] getWhirlpool()
+	{
+		return whirlpool;
+	}
+
+	public int getCompression()
+	{
+		return compression;
+	}
+}

cache/src/main/java/net/runelite/cache/downloader/PendingFileRequest.java

@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2016, Adam <Adam@sigterm.info>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *    This product includes software developed by Adam <Adam@sigterm.info>
+ * 4. Neither the name of the Adam <Adam@sigterm.info> nor the
+ *    names of its contributors may be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Adam <Adam@sigterm.info> ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Adam <Adam@sigterm.info> BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package net.runelite.cache.downloader;
+
+import java.util.concurrent.CompletableFuture;
+import net.runelite.cache.downloader.requests.FileRequest;
+
+public class PendingFileRequest
+{
+	private final FileRequest request;
+	private final CompletableFuture<FileResult> future;
+
+	public PendingFileRequest(FileRequest request, CompletableFuture<FileResult> future)
+	{
+		this.request = request;
+		this.future = future;
+	}
+
+	public FileRequest getRequest()
+	{
+		return request;
+	}
+
+	public CompletableFuture<FileResult> getFuture()
+	{
+		return future;
+	}
+
+	public int computeHash()
+	{
+		return (request.getIndex() << 16) | request.getFile();
+	}
+}

cache/src/main/java/net/runelite/cache/downloader/requests/ConnectionInfo.java

@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2016, Adam <Adam@sigterm.info>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *    This product includes software developed by Adam <Adam@sigterm.info>
+ * 4. Neither the name of the Adam <Adam@sigterm.info> nor the
+ *    names of its contributors may be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Adam <Adam@sigterm.info> ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Adam <Adam@sigterm.info> BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package net.runelite.cache.downloader.requests;
+
+public class ConnectionInfo
+{
+	private byte var1 = 3; // I don't know what this is
+	private int var2;
+
+	public byte getVar1()
+	{
+		return var1;
+	}
+
+	public void setVar1(byte var1)
+	{
+		this.var1 = var1;
+	}
+
+	public int getVar2()
+	{
+		return var2;
+	}
+
+	public void setVar2(int var2)
+	{
+		this.var2 = var2;
+	}
+}

cache/src/main/java/net/runelite/cache/downloader/requests/FileRequest.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2016, Adam <Adam@sigterm.info>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *    This product includes software developed by Adam <Adam@sigterm.info>
+ * 4. Neither the name of the Adam <Adam@sigterm.info> nor the
+ *    names of its contributors may be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Adam <Adam@sigterm.info> ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Adam <Adam@sigterm.info> BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package net.runelite.cache.downloader.requests;
+
+public class FileRequest
+{
+	private final int index;
+	private final int file;
+
+	public FileRequest(int index, int file)
+	{
+		this.index = index;
+		this.file = file;
+	}
+
+	public int getIndex()
+	{
+		return index;
+	}
+
+	public int getFile()
+	{
+		return file;
+	}
+}

cache/src/main/java/net/runelite/cache/downloader/requests/HelloHandshake.java

@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2016, Adam <Adam@sigterm.info>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *    This product includes software developed by Adam <Adam@sigterm.info>
+ * 4. Neither the name of the Adam <Adam@sigterm.info> nor the
+ *    names of its contributors may be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Adam <Adam@sigterm.info> ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Adam <Adam@sigterm.info> BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package net.runelite.cache.downloader.requests;
+
+public class HelloHandshake
+{
+	public static final int RESPONSE_OK = 0;
+	public static final int RESPONSE_OUTDATED = 6;
+
+	private byte version = 15; // handshake type
+	private int revision;
+
+	public int getRevision()
+	{
+		return revision;
+	}
+
+	public void setRevision(int revision)
+	{
+		this.revision = revision;
+	}
+	public byte getVersion()
+	{
+		return version;
+	}
+
+	public void setVersion(byte version)
+	{
+		this.version = version;
+	}
+
+}

cache/src/main/java/net/runelite/cache/fs/Index.java

@@ -123,11 +123,21 @@ public class Index implements Closeable
 		return crc;
 	}
 
+	public void setCrc(int crc)
+	{
+		this.crc = crc;
+	}
+
 	public byte[] getWhirlpool()
 	{
 		return whirlpool;
 	}
 
+	public void setWhirlpool(byte[] whirlpool)
+	{
+		this.whirlpool = whirlpool;
+	}
+
 	public IndexFile getIndex()
 	{
 		return index;

cache/src/test/java/net/runelite/cache/downloader/CacheClientTest.java

@@ -0,0 +1,78 @@
+/*
+ * Copyright (c) 2016, Adam <Adam@sigterm.info>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *    This product includes software developed by Adam <Adam@sigterm.info>
+ * 4. Neither the name of the Adam <Adam@sigterm.info> nor the
+ *    names of its contributors may be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Adam <Adam@sigterm.info> ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Adam <Adam@sigterm.info> BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package net.runelite.cache.downloader;
+
+import java.io.File;
+import net.runelite.cache.fs.Store;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.impl.SimpleLogger;
+
+public class CacheClientTest
+{
+	@Before
+	public void before()
+	{
+		 System.setProperty(SimpleLogger.DEFAULT_LOG_LEVEL_KEY, "TRACE");
+	}
+
+	@Test
+	@Ignore
+	public void test() throws Exception
+	{
+		Store store = new Store(new File("d:/temp"));
+		store.load();
+
+		CacheClient c = new CacheClient(store);
+		c.connect();
+
+//		c.requestFile(0, 205).get();
+//		c.requestFile(3, 278).get();
+//		c.requestFile(3, 127).get();
+//		c.requestFile(0, 1210).get();
+
+//		c.requestFile(255, 255).get();
+//		c.requestFile(255, 2).get();
+
+//		c.requestFile(4, 2047);
+
+//		c.requestFile(6, 546);
+
+//		c.requestFile(255, 6).get();
+
+		c.download();
+
+		c.stop();
+
+		store.save();
+	}
+}