From 36acbebd1b598f68139767e80da5653e4c631787 Mon Sep 17 00:00:00 2001 From: Adam Date: Fri, 19 Jan 2018 09:45:36 -0500 Subject: [PATCH] cache service: move database updating to happen during download --- .../runelite/cache/client/CacheClient.java | 5 ++ .../cache/client/DownloadWatcher.java | 4 +- .../http/service/cache/CacheStorage.java | 38 +------------- .../http/service/cache/CacheUpdater.java | 52 ++++++++++++++++--- 4 files changed, 55 insertions(+), 44 deletions(-) diff --git a/cache-client/src/main/java/net/runelite/cache/client/CacheClient.java b/cache-client/src/main/java/net/runelite/cache/client/CacheClient.java index 7e3ff136f6..8fe0f02f59 100644 --- a/cache-client/src/main/java/net/runelite/cache/client/CacheClient.java +++ b/cache-client/src/main/java/net/runelite/cache/client/CacheClient.java @@ -264,6 +264,11 @@ public class CacheClient implements AutoCloseable logger.info("Index {} has {} archives", i, indexData.getArchives().length); + if (watcher != null) + { + watcher.indexComplete(index); + } + for (ArchiveData ad : indexData.getArchives()) { Archive existing = index.getArchive(ad.getId()); diff --git a/cache-client/src/main/java/net/runelite/cache/client/DownloadWatcher.java b/cache-client/src/main/java/net/runelite/cache/client/DownloadWatcher.java index 3924c74541..39812c0796 100644 --- a/cache-client/src/main/java/net/runelite/cache/client/DownloadWatcher.java +++ b/cache-client/src/main/java/net/runelite/cache/client/DownloadWatcher.java @@ -25,9 +25,11 @@ package net.runelite.cache.client; import net.runelite.cache.fs.Archive; +import net.runelite.cache.fs.Index; -@FunctionalInterface public interface DownloadWatcher { + void indexComplete(Index index); + void downloadComplete(Archive archive, byte[] data); } diff --git a/http-service/src/main/java/net/runelite/http/service/cache/CacheStorage.java b/http-service/src/main/java/net/runelite/http/service/cache/CacheStorage.java index 797a4f69c1..9db83a806f 100644 --- a/http-service/src/main/java/net/runelite/http/service/cache/CacheStorage.java +++ b/http-service/src/main/java/net/runelite/http/service/cache/CacheStorage.java @@ -34,14 +34,13 @@ import net.runelite.cache.fs.Store; import net.runelite.cache.index.FileData; import net.runelite.http.service.cache.beans.ArchiveEntry; import net.runelite.http.service.cache.beans.CacheEntry; -import net.runelite.http.service.cache.beans.FileEntry; import net.runelite.http.service.cache.beans.IndexEntry; import org.sql2o.Connection; import org.sql2o.ResultSetIterable; public class CacheStorage implements Storage { - private CacheEntry cacheEntry; + private final CacheEntry cacheEntry; private final CacheDAO cacheDao; private final Connection con; @@ -52,16 +51,6 @@ public class CacheStorage implements Storage this.con = con; } - public CacheEntry getCacheEntry() - { - return cacheEntry; - } - - public void setCacheEntry(CacheEntry cacheEntry) - { - this.cacheEntry = cacheEntry; - } - @Override public void init(Store store) throws IOException { @@ -102,30 +91,7 @@ public class CacheStorage implements Storage @Override public void save(Store store) throws IOException { - for (Index index : store.getIndexes()) - { - IndexEntry entry = cacheDao.findOrCreateIndex(con, cacheEntry, index.getId(), index.getCrc(), index.getRevision()); - // this assumes nothing is associated to the cache yet - cacheDao.associateIndexToCache(con, cacheEntry, entry); - - for (Archive archive : index.getArchives()) - { - ArchiveEntry archiveEntry = cacheDao.findArchive(con, entry, archive.getArchiveId(), - archive.getNameHash(), archive.getCrc(), archive.getRevision()); - if (archiveEntry == null) - { - byte[] hash = archive.getHash(); - archiveEntry = cacheDao.createArchive(con, entry, archive.getArchiveId(), - archive.getNameHash(), archive.getCrc(), archive.getRevision(), hash); - - for (FileData file : archive.getFileData()) - { - cacheDao.associateFileToArchive(con, archiveEntry, file.getId(), file.getNameHash()); - } - } - cacheDao.associateArchiveToIndex(con, archiveEntry, entry); - } - } + throw new UnsupportedOperationException(); } @Override diff --git a/http-service/src/main/java/net/runelite/http/service/cache/CacheUpdater.java b/http-service/src/main/java/net/runelite/http/service/cache/CacheUpdater.java index 8f75f95f64..4d93cbaa5d 100644 --- a/http-service/src/main/java/net/runelite/http/service/cache/CacheUpdater.java +++ b/http-service/src/main/java/net/runelite/http/service/cache/CacheUpdater.java @@ -29,15 +29,21 @@ import io.minio.errors.InvalidEndpointException; import io.minio.errors.InvalidPortException; import java.io.IOException; import java.time.Instant; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import net.runelite.cache.client.CacheClient; +import net.runelite.cache.client.DownloadWatcher; import net.runelite.cache.client.IndexInfo; import net.runelite.cache.fs.Archive; +import net.runelite.cache.fs.Index; import net.runelite.cache.fs.Store; +import net.runelite.cache.index.FileData; import net.runelite.http.api.RuneLiteAPI; +import net.runelite.http.service.cache.beans.ArchiveEntry; import net.runelite.http.service.cache.beans.CacheEntry; import net.runelite.http.service.cache.beans.IndexEntry; import net.runelite.protocol.api.login.HandshakeResponseType; @@ -95,8 +101,45 @@ public class CacheUpdater ExecutorService executor = Executors.newSingleThreadExecutor(); - CacheClient client = new CacheClient(store, rsVersion, - (Archive archive, byte[] data) -> executor.submit(new CacheUploader(minioClient, minioBucket, archive, data))); + CacheEntry newCache = created ? cache : cacheDao.createCache(con, rsVersion, Instant.now()); + + CacheClient client = new CacheClient(store, rsVersion, new DownloadWatcher() + { + private final Map indexEntryMap = new HashMap<>(); + + @Override + public void indexComplete(Index index) + { + IndexEntry entry = cacheDao.findOrCreateIndex(con, newCache, index.getId(), index.getCrc(), index.getRevision()); + // this assumes nothing is associated to the cache yet + cacheDao.associateIndexToCache(con, newCache, entry); + indexEntryMap.put(index, entry); + } + + @Override + public void downloadComplete(Archive archive, byte[] data) + { + executor.submit(new CacheUploader(minioClient, minioBucket, archive, data)); + + IndexEntry entry = indexEntryMap.get(archive.getIndex()); + ArchiveEntry archiveEntry = cacheDao.findArchive(con, entry, archive.getArchiveId(), + archive.getNameHash(), archive.getCrc(), archive.getRevision()); + if (archiveEntry == null) + { + byte[] hash = archive.getHash(); + archiveEntry = cacheDao.createArchive(con, entry, archive.getArchiveId(), + archive.getNameHash(), archive.getCrc(), archive.getRevision(), hash); + + for (FileData file : archive.getFileData()) + { + cacheDao.associateFileToArchive(con, archiveEntry, file.getId(), file.getNameHash()); + } + } + cacheDao.associateArchiveToIndex(con, archiveEntry, entry); + + archive.setFileData(null); // don't need this anymore + } + }); client.connect(); HandshakeResponseType result = client.handshake().join(); @@ -117,11 +160,6 @@ public class CacheUpdater client.download(); - CacheEntry newCache = created ? cache : cacheDao.createCache(con, rsVersion, Instant.now()); - - storage.setCacheEntry(newCache); - store.save(); - // ensure objects are added to the store before they become // visible in the database executor.shutdown();