mobeigi
/
filebot
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Rewrite caching

This commit is contained in:
Reinhard Pointner 2016-03-06 18:11:30 +00:00
parent 139e3eef6c
commit 500a4972e1
15 changed files with 257 additions and 172 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.classpath
View File

@ -8,7 +8,7 @@
<classpathentry kind="lib" path="lib/jars/jacksum.jar"/>
<classpathentry kind="lib" path="lib/jars/simmetrics.jar"/>
<classpathentry kind="lib" path="lib/jars/xmlrpc.jar"/>
<classpathentry kind="lib" path="lib/ivy/jar/ehcache.jar"/>
<classpathentry kind="lib" path="lib/ivy/jar/ehcache.jar" sourcepath="lib/ivy/source/ehcache.jar"/>
<classpathentry kind="lib" path="lib/ivy/jar/glazedlists_java15.jar"/>
<classpathentry kind="lib" path="lib/ivy/jar/icu4j.jar"/>
<classpathentry kind="lib" path="lib/ivy/jar/jna.jar"/>

106
source/net/filebot/Cache.java
View File

@ -1,75 +1,91 @@
package net.filebot;
import java.io.Serializable;
import java.util.Arrays;
import java.util.logging.Level;
import java.util.logging.Logger;
import static net.filebot.Logging.*;
import java.io.Serializable;
import java.time.Duration;
import java.util.Arrays;
import java.util.concurrent.Callable;
import java.util.function.Predicate;
import net.sf.ehcache.CacheManager;
import net.sf.ehcache.Element;
public class Cache {
public static final String EPHEMERAL = "ephemeral-memory";
public static final String PERSISTENT = "persistent-memory";
public static Cache getCache(String name) {
return new Cache(CacheManager.getInstance().getCache(name));
public static Cache getCache(String name, CacheType type) {
return CacheManager.getInstance().getCache(name.toLowerCase(), type);
}
private final net.sf.ehcache.Cache cache;
protected Cache(net.sf.ehcache.Cache cache) {
public Cache(net.sf.ehcache.Cache cache) {
this.cache = cache;
}
public Object get(Object key) {
try {
return cache.get(key).getObjectValue();
} catch (Exception e) {
debug.warning(format("Bad cache state: %s => %s", key, e));
}
return null;
}
public Object computeIf(Object key, Predicate<Element> condition, Callable<?> callable) throws Exception {
// get if present
try {
Element element = cache.get(key);
if (element != null && condition.test(element)) {
return element.getObjectValue();
}
} catch (Exception e) {
debug.warning(format("Bad cache state: %s => %s", key, e));
}
// compute if absent
Object value = callable.call();
try {
cache.put(new Element(key, value));
} catch (Exception e) {
debug.warning(format("Bad cache state: %s => %s", key, e));
}
return value;
}
public Object computeIfAbsent(Object key, Callable<?> callable) throws Exception {
return computeIf(key, Element::isExpired, callable);
}
public Object computeIfStale(Object key, Duration expirationTime, Callable<?> callable) throws Exception {
return computeIf(key, isStale(expirationTime), callable);
}
private Predicate<Element> isStale(Duration expirationTime) {
return (element) -> element.isExpired() || System.currentTimeMillis() - element.getLatestOfCreationAndUpdateTime() < expirationTime.toMillis();
}
public void put(Object key, Object value) {
try {
cache.put(new Element(key, value));
} catch (Throwable e) {
Logger.getLogger(Cache.class.getName()).log(Level.WARNING, e.getMessage());
remove(key); // fail-safe
}
}
public Object get(Object key) {
return get(key, Object.class);
}
public <T> T get(Object key, Class<T> type) {
try {
Element element = cache.get(key);
if (element != null && key.equals(element.getKey())) {
return type.cast(element.getValue());
}
} catch (Exception e) {
Logger.getLogger(Cache.class.getName()).log(Level.WARNING, e.getMessage());
remove(key); // fail-safe
debug.warning(format("Bad cache state: %s => %s", key, e));
}
return null;
}
public void remove(Object key) {
try {
cache.remove(key);
} catch (Exception e1) {
Logger.getLogger(Cache.class.getName()).log(Level.SEVERE, e1.getMessage());
try {
Logger.getLogger(Cache.class.getName()).log(Level.INFO, "Cached data has become invalid: Clearing cache now");
cache.removeAll();
} catch (Exception e2) {
Logger.getLogger(Cache.class.getName()).log(Level.SEVERE, e2.getMessage());
try {
Logger.getLogger(Cache.class.getName()).log(Level.INFO, "Cache has become invalid: Reset all caches");
cache.getCacheManager().clearAll();
} catch (Exception e3) {
Logger.getLogger(Cache.class.getName()).log(Level.SEVERE, e3.getMessage());
}
}
} catch (Exception e) {
debug.warning(format("Bad cache state: %s => %s", key, e));
}
}
@Deprecated
public <T> T get(Object key, Class<T> type) {
return type.cast(get(key));
}
@Deprecated
public static class Key implements Serializable {
protected Object[] fields;

136
source/net/filebot/CacheManager.java Normal file
View File

@ -0,0 +1,136 @@
package net.filebot;
import static net.filebot.Logging.*;
import static net.filebot.Settings.*;
import static net.filebot.util.FileUtilities.*;
import java.io.File;
import java.io.IOException;
import java.nio.channels.FileChannel;
import java.nio.channels.FileLock;
import java.nio.charset.Charset;
import java.nio.file.StandardOpenOption;
import java.util.Scanner;
import net.sf.ehcache.CacheException;
import net.sf.ehcache.config.Configuration;
import net.sf.ehcache.config.DiskStoreConfiguration;
public class CacheManager {
private static final CacheManager instance = new CacheManager();
public static CacheManager getInstance() {
return instance;
}
private final net.sf.ehcache.CacheManager manager;
public CacheManager() {
try {
this.manager = net.sf.ehcache.CacheManager.create(getConfiguration());
} catch (IOException e) {
throw new CacheException(e);
}
}
public Cache getCache(String name, CacheType type) {
String cacheName = name + "_" + type.ordinal();
if (!manager.cacheExists(cacheName)) {
debug.config("Create cache: " + cacheName);
manager.addCache(new net.sf.ehcache.Cache(type.getConfiguration(cacheName)));
}
return new Cache(manager.getCache(cacheName));
}
public void clearAll() {
manager.clearAll();
}
private Configuration getConfiguration() throws IOException {
Configuration config = new Configuration();
config.addDiskStore(getDiskStoreConfiguration());
return config;
}
private DiskStoreConfiguration getDiskStoreConfiguration() throws IOException {
// prepare cache folder for this application instance
File cacheRoot = getApplicationCache().getCanonicalFile();
for (int i = 0; true; i++) {
File cache = new File(cacheRoot, Integer.toString(i));
// make sure cache is readable and writable
createFolders(cache);
final File lockFile = new File(cache, ".lock");
boolean isNewCache = !lockFile.exists();
final FileChannel channel = FileChannel.open(lockFile.toPath(), StandardOpenOption.CREATE, StandardOpenOption.READ, StandardOpenOption.WRITE);
final FileLock lock = channel.tryLock();
if (lock != null) {
debug.config(format("Using persistent disk cache %s", cache));
int applicationRevision = getApplicationRevisionNumber();
int cacheRevision = 0;
try {
cacheRevision = new Scanner(channel, "UTF-8").nextInt();
} catch (Exception e) {
// ignore
}
if (cacheRevision != applicationRevision && applicationRevision > 0 && !isNewCache) {
debug.config(format("App version (r%d) does not match cache version (r%d): reset cache", applicationRevision, cacheRevision));
// tag cache with new revision number
isNewCache = true;
// delete all files related to previous cache instances
for (File it : getChildren(cache)) {
if (!it.equals(lockFile)) {
delete(cache);
}
}
}
if (isNewCache) {
// set new cache revision
channel.position(0);
channel.write(Charset.forName("UTF-8").encode(String.valueOf(applicationRevision)));
channel.truncate(channel.position());
}
// make sure to orderly shutdown cache
Runtime.getRuntime().addShutdownHook(new Thread() {
@Override
public void run() {
try {
manager.shutdown();
} catch (Exception e) {
// ignore, shutting down anyway
}
try {
lock.release();
} catch (Exception e) {
// ignore, shutting down anyway
}
try {
channel.close();
} catch (Exception e) {
// ignore, shutting down anyway
}
}
});
// cache for this application instance is successfully set up and locked
return new DiskStoreConfiguration().path(cache.getPath());
}
// try next lock file
channel.close();
}
}
}

32
source/net/filebot/CacheType.java Normal file
View File

@ -0,0 +1,32 @@
package net.filebot;
import java.time.Duration;
import net.sf.ehcache.config.CacheConfiguration;
public enum CacheType {
Persistent(Duration.ofDays(180), true),
Monthly(Duration.ofDays(60), true),
Weekly(Duration.ofDays(12), true),
Daily(Duration.ofDays(1), true),
Ephemeral(Duration.ofHours(2), false);
final long timeToLiveSeconds;
final boolean diskPersistent;
CacheType(Duration timeToLive, boolean diskPersistent) {
this.timeToLiveSeconds = timeToLive.getSeconds();
this.diskPersistent = diskPersistent;
}
CacheConfiguration getConfiguration(String name) {
// Strategy.LOCALTEMPSWAP is not restartable so we can't but use the deprecated disk persistent code (see http://stackoverflow.com/a/24623527/1514467)
return new CacheConfiguration().name(name).maxEntriesLocalHeap(diskPersistent ? 200 : 0).maxEntriesLocalDisk(0).eternal(false).timeToLiveSeconds(timeToLiveSeconds).timeToIdleSeconds(timeToLiveSeconds).overflowToDisk(diskPersistent).diskPersistent(diskPersistent);
}
}

96
source/net/filebot/Main.java
View File

@ -19,8 +19,6 @@ import java.net.URI;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.channels.FileLock;
import java.nio.charset.Charset;
import java.nio.file.StandardOpenOption;
import java.security.CodeSource;
import java.security.Permission;
@ -30,7 +28,6 @@ import java.security.Policy;
import java.security.ProtectionDomain;
import java.util.Locale;
import java.util.Properties;
import java.util.Scanner;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Stream;
@ -62,7 +59,6 @@ import net.filebot.util.PreferencesMap.PreferencesEntry;
import net.filebot.util.TeePrintStream;
import net.filebot.web.CachedResource;
import net.miginfocom.swing.MigLayout;
import net.sf.ehcache.CacheManager;
import org.kohsuke.args4j.CmdLineException;
import org.w3c.dom.NodeList;
@ -100,7 +96,6 @@ public class Main {
}
}
initializeCache();
CacheManager.getInstance().clearAll();
}
@ -136,7 +131,7 @@ public class Main {
createFolders(getApplicationTempFolder());
// initialize this stuff before anything else
initializeCache();
CacheManager.getInstance();
initializeSecurityManager();
// update system properties
@ -443,95 +438,6 @@ public class Main {
window.setBounds(x, y, width, height);
}
/**
* Shutdown ehcache properly, so that disk-persistent stores can actually be saved to disk
*/
private static void initializeCache() throws Exception {
// prepare cache folder for this application instance
File cacheRoot = getApplicationCache();
createFolders(cacheRoot);
try {
for (int i = 0; true; i++) {
File cache = new File(cacheRoot, String.format("%d", i));
// make sure cache is accessible
createFolders(cache);
final File lockFile = new File(cache, ".lock");
boolean isNewCache = !lockFile.exists();
final FileChannel channel = FileChannel.open(lockFile.toPath(), StandardOpenOption.CREATE, StandardOpenOption.READ, StandardOpenOption.WRITE);
final FileLock lock = channel.tryLock();
if (lock != null) {
// setup cache dir for ehcache
System.setProperty("ehcache.disk.store.dir", cache.getAbsolutePath());
int applicationRevision = getApplicationRevisionNumber();
int cacheRevision = 0;
try {
cacheRevision = new Scanner(channel, "UTF-8").nextInt();
} catch (Exception e) {
// ignore
}
if (cacheRevision != applicationRevision && applicationRevision > 0 && !isNewCache) {
Logger.getLogger(Main.class.getName()).log(Level.WARNING, format("App version (r%d) does not match cache version (r%d): reset cache", applicationRevision, cacheRevision));
// tag cache with new revision number
isNewCache = true;
// delete all files related to previous cache instances
for (File it : getChildren(cache)) {
if (!it.equals(lockFile)) {
delete(cache);
}
}
}
if (isNewCache) {
// set new cache revision
channel.position(0);
channel.write(Charset.forName("UTF-8").encode(String.valueOf(applicationRevision)));
channel.truncate(channel.position());
}
// make sure to orderly shutdown cache
Runtime.getRuntime().addShutdownHook(new Thread() {
@Override
public void run() {
try {
CacheManager.getInstance().shutdown();
} catch (Exception e) {
// ignore, shutting down anyway
}
try {
lock.release();
} catch (Exception e) {
// ignore, shutting down anyway
}
try {
channel.close();
} catch (Exception e) {
// ignore, shutting down anyway
}
}
});
// cache for this application instance is successfully set up and locked
return;
}
// try next lock file
channel.close();
}
} catch (Exception e) {
throw new Exception("Failed to initialize cache: " + e.toString(), e);
}
}
/**
* Initialize default SecurityManager and grant all permissions via security policy. Initialization is required in order to run {@link ExpressionFormat} in a secure sandbox.
*/

7
source/net/filebot/format/MediaBindingBean.java
View File

@ -34,6 +34,8 @@ import java.util.TreeSet;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;
import net.filebot.Cache;
import net.filebot.CacheType;
import net.filebot.Language;
import net.filebot.MediaTypes;
import net.filebot.MetaAttributeView;
@ -412,7 +414,7 @@ public class MediaBindingBean {
}
@Define("crc32")
public String getCRC32() throws IOException, InterruptedException {
public String getCRC32() throws Exception {
// use inferred media file
File inferredMediaFile = getInferredMediaFile();
@ -444,7 +446,8 @@ public class MediaBindingBean {
}
// calculate checksum from file
return crc32(inferredMediaFile);
Cache cache = Cache.getCache("crc32", CacheType.Ephemeral);
return (String) cache.computeIfAbsent(inferredMediaFile, () -> crc32(inferredMediaFile));
}
@Define("fn")

18
source/net/filebot/hash/VerificationUtilities.java
View File

@ -10,8 +10,6 @@ import java.util.Map.Entry;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import net.filebot.Cache;
public final class VerificationUtilities {
/**
@ -106,23 +104,9 @@ public final class VerificationUtilities {
}
public static String crc32(File file) throws IOException, InterruptedException {
// try to get checksum from cache
Cache cache = Cache.getCache(Cache.EPHEMERAL);
String hash = cache.get(file, String.class);
if (hash != null) {
return hash;
}
// compute and cache checksum
hash = computeHash(file, HashType.SFV);
cache.put(file, hash);
return hash;
return computeHash(file, HashType.SFV);
}
/**
* Dummy constructor to prevent instantiation.
*/
private VerificationUtilities() {
throw new UnsupportedOperationException();
}

2
source/net/filebot/ui/MainFrame.java
View File

@ -38,6 +38,7 @@ import javax.swing.border.LineBorder;
import javax.swing.event.ListSelectionEvent;
import javax.swing.event.ListSelectionListener;
import net.filebot.CacheManager;
import net.filebot.Settings;
import net.filebot.cli.GroovyPad;
import net.filebot.mac.MacAppUtilities;
@ -52,7 +53,6 @@ import net.filebot.util.ui.DefaultFancyListCellRenderer;
import net.filebot.util.ui.ShadowBorder;
import net.filebot.util.ui.SwingUI;
import net.miginfocom.swing.MigLayout;
import net.sf.ehcache.CacheManager;
public class MainFrame extends JFrame {

8
source/net/filebot/ui/rename/EpisodeListMatcher.java
View File

@ -39,6 +39,7 @@ import javax.swing.Action;
import javax.swing.SwingUtilities;
import net.filebot.Cache;
import net.filebot.CacheType;
import net.filebot.Settings;
import net.filebot.similarity.CommonSequenceMatcher;
import net.filebot.similarity.EpisodeMatcher;
@ -56,16 +57,17 @@ class EpisodeListMatcher implements AutoCompleteMatcher {
private boolean useAnimeIndex;
private boolean useSeriesIndex;
// remember user selections
private Cache persistentSelectionMemory;
// only allow one fetch session at a time so later requests can make use of cached results
private final Object providerLock = new Object();
// remember user selections
private final Cache persistentSelectionMemory = Cache.getCache(Cache.PERSISTENT);
public EpisodeListMatcher(EpisodeListProvider provider, boolean useSeriesIndex, boolean useAnimeIndex) {
this.provider = provider;
this.useSeriesIndex = useSeriesIndex;
this.useAnimeIndex = useAnimeIndex;
this.persistentSelectionMemory = Cache.getCache("selection_" + provider.getName(), CacheType.Persistent);
}
protected SearchResult selectSearchResult(final String query, final List<SearchResult> searchResults, Map<String, SearchResult> selectionMemory, boolean autodetection, final Component parent) throws Exception {

3
source/net/filebot/web/AcoustIDClient.java
View File

@ -28,6 +28,7 @@ import java.util.logging.Logger;
import javax.swing.Icon;
import net.filebot.Cache;
import net.filebot.CacheType;
import net.filebot.ResourceManager;
public class AcoustIDClient implements MusicIdentificationService {
@ -51,7 +52,7 @@ public class AcoustIDClient implements MusicIdentificationService {
}
public Cache getCache() {
return Cache.getCache("web-datasource-lv3");
return Cache.getCache(getName(), CacheType.Monthly);
}
@Override

5
source/net/filebot/web/AnidbClient.java
View File

@ -30,7 +30,8 @@ import java.util.zip.GZIPInputStream;
import javax.swing.Icon;
import net.filebot.Cache;
import net.filebot.CacheManager;
import net.filebot.CacheType;
import net.filebot.ResourceManager;
import org.jsoup.Jsoup;
@ -78,7 +79,7 @@ public class AnidbClient extends AbstractEpisodeListProvider {
@Override
public ResultCache getCache() {
return new ResultCache(getName(), Cache.getCache("web-datasource-lv2"));
return new ResultCache(getName(), CacheManager.getInstance().getCache(getName(), CacheType.Weekly));
}
@Override

5
source/net/filebot/web/OpenSubtitlesClient.java
View File

@ -30,6 +30,7 @@ import javax.swing.Icon;
import net.filebot.Cache;
import net.filebot.Cache.Key;
import net.filebot.CacheType;
import net.filebot.ResourceManager;
import net.filebot.media.MediaDetection;
import net.filebot.mediainfo.MediaInfo;
@ -85,7 +86,7 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS
}
public ResultCache getCache() {
return new ResultCache(getName(), Cache.getCache("web-datasource"));
return new ResultCache(getName(), Cache.getCache(getName(), CacheType.Daily));
}
@Override
@ -603,7 +604,7 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS
*/
@SuppressWarnings("unchecked")
protected synchronized Map<String, String> getSubLanguageMap() throws IOException {
Cache cache = Cache.getCache("web-datasource-lv2");
Cache cache = Cache.getCache(getName(), CacheType.Persistent);
String cacheKey = getClass().getName() + ".subLanguageMap";
// try to get language map from cache

3
source/net/filebot/web/ShooterSubtitles.java
View File

@ -27,6 +27,7 @@ import java.util.Map;
import javax.swing.Icon;
import net.filebot.Cache;
import net.filebot.CacheType;
import net.filebot.ResourceManager;
import org.json.simple.JSONArray;
@ -88,7 +89,7 @@ public class ShooterSubtitles implements VideoHashSubtitleService {
param.put("format", "json");
param.put("lang", LANGUAGE_CHINESE.equals(languageName) ? "Chn" : "Eng");
Cache cache = Cache.getCache("web-datasource");
Cache cache = Cache.getCache(getName(), CacheType.Daily);
String key = endpoint.toString() + param.toString();
SubtitleDescriptor[] value = cache.get(key, SubtitleDescriptor[].class);
if (value != null) {

3
source/net/filebot/web/TVMazeClient.java
View File

@ -16,6 +16,7 @@ import java.util.Objects;
import javax.swing.Icon;
import net.filebot.Cache;
import net.filebot.CacheType;
import net.filebot.ResourceManager;
public class TVMazeClient extends AbstractEpisodeListProvider {
@ -52,7 +53,7 @@ public class TVMazeClient extends AbstractEpisodeListProvider {
@Override
public ResultCache getCache() {
return new ResultCache(getName(), Cache.getCache("web-datasource"));
return new ResultCache(getName(), Cache.getCache(getName(), CacheType.Daily));
}
@Override

3
source/net/filebot/web/TheTVDBClient.java
View File

@ -28,6 +28,7 @@ import java.util.logging.Logger;
import javax.swing.Icon;
import net.filebot.Cache;
import net.filebot.CacheType;
import net.filebot.ResourceManager;
import net.filebot.util.FileUtilities;
import net.filebot.web.TheTVDBClient.BannerDescriptor.BannerProperty;
@ -77,7 +78,7 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
@Override
public ResultCache getCache() {
return new ResultCache(getName(), Cache.getCache("web-datasource"));
return new ResultCache(getName(), Cache.getCache(getName(), CacheType.Daily));
}
public String getLanguageCode(Locale locale) {