Experiment with new CachedResource framework
This commit is contained in:
parent
500a4972e1
commit
bbed902c63
|
@ -1,28 +1,9 @@
|
|||
<ehcache xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="ehcache.xsd" updateCheck="false">
|
||||
<!--
|
||||
Persistent disk store location
|
||||
-->
|
||||
<diskStore path="ehcache.disk.store.dir" />
|
||||
|
||||
<!--
|
||||
Mandatory Default Cache configuration. These settings will be applied to caches
|
||||
created pragmatically using CacheManager.add(String cacheName)
|
||||
-->
|
||||
<diskStore path="java.io.tmpdir" />
|
||||
|
||||
<defaultCache
|
||||
maxElementsInMemory="100"
|
||||
eternal="false"
|
||||
timeToIdleSeconds="120"
|
||||
timeToLiveSeconds="120"
|
||||
overflowToDisk="false"
|
||||
diskPersistent="false"
|
||||
memoryStoreEvictionPolicy="LRU"
|
||||
/>
|
||||
|
||||
<!--
|
||||
Short-lived (24 hours) persistent disk cache for web responses
|
||||
-->
|
||||
<cache name="web-datasource"
|
||||
maxElementsInMemory="200"
|
||||
maxElementsInMemory="400"
|
||||
maxElementsOnDisk="80000"
|
||||
eternal="false"
|
||||
timeToIdleSeconds="86400"
|
||||
|
@ -32,73 +13,4 @@
|
|||
memoryStoreEvictionPolicy="LRU"
|
||||
/>
|
||||
|
||||
<!--
|
||||
Long-lived (2 weeks) persistent disk cache for web responses
|
||||
-->
|
||||
<cache name="web-datasource-lv2"
|
||||
maxElementsInMemory="200"
|
||||
maxElementsOnDisk="80000"
|
||||
eternal="false"
|
||||
timeToIdleSeconds="1209600"
|
||||
timeToLiveSeconds="1209600"
|
||||
overflowToDisk="true"
|
||||
diskPersistent="true"
|
||||
memoryStoreEvictionPolicy="LRU"
|
||||
/>
|
||||
|
||||
<!--
|
||||
Long-lived (2 months) persistent disk cache for web responses (that can be updated via If-Modified or If-None-Match)
|
||||
-->
|
||||
<cache name="web-datasource-lv3"
|
||||
maxElementsInMemory="100"
|
||||
maxElementsOnDisk="200000"
|
||||
eternal="false"
|
||||
timeToIdleSeconds="5256000"
|
||||
timeToLiveSeconds="5256000"
|
||||
overflowToDisk="true"
|
||||
diskPersistent="true"
|
||||
memoryStoreEvictionPolicy="LRU"
|
||||
/>
|
||||
|
||||
<!--
|
||||
Very long-lived cache (4 months) anime/series lists, movie index, etc
|
||||
-->
|
||||
<cache name="web-persistent-datasource"
|
||||
maxElementsInMemory="100"
|
||||
maxElementsOnDisk="80000"
|
||||
eternal="false"
|
||||
timeToIdleSeconds="10512000"
|
||||
timeToLiveSeconds="10512000"
|
||||
overflowToDisk="true"
|
||||
diskPersistent="true"
|
||||
memoryStoreEvictionPolicy="LRU"
|
||||
/>
|
||||
|
||||
<!--
|
||||
Simple memory cache. Time to live is 4 months.
|
||||
-->
|
||||
<cache name="persistent-memory"
|
||||
maxElementsInMemory="100"
|
||||
maxElementsOnDisk="50000"
|
||||
eternal="false"
|
||||
timeToIdleSeconds="10512000"
|
||||
timeToLiveSeconds="10512000"
|
||||
overflowToDisk="true"
|
||||
diskPersistent="true"
|
||||
memoryStoreEvictionPolicy="LRU"
|
||||
/>
|
||||
|
||||
<!--
|
||||
Simple memory cache. Time to live is 2 hours.
|
||||
-->
|
||||
<cache name="ephemeral-memory"
|
||||
maxElementsInMemory="50000"
|
||||
eternal="false"
|
||||
timeToIdleSeconds="7200"
|
||||
timeToLiveSeconds="7200"
|
||||
overflowToDisk="false"
|
||||
diskPersistent="false"
|
||||
memoryStoreEvictionPolicy="LRU"
|
||||
/>
|
||||
|
||||
</ehcache>
|
||||
|
|
|
@ -1,13 +1,16 @@
|
|||
package net.filebot;
|
||||
|
||||
import static java.nio.charset.StandardCharsets.*;
|
||||
import static net.filebot.Logging.*;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.net.URL;
|
||||
import java.time.Duration;
|
||||
import java.util.Arrays;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.function.Predicate;
|
||||
|
||||
import net.filebot.web.CachedResource2;
|
||||
import net.filebot.web.FloodLimit;
|
||||
import net.sf.ehcache.Element;
|
||||
|
||||
public class Cache {
|
||||
|
@ -24,51 +27,60 @@ public class Cache {
|
|||
|
||||
public Object get(Object key) {
|
||||
try {
|
||||
return cache.get(key).getObjectValue();
|
||||
Element element = cache.get(key);
|
||||
if (element != null) {
|
||||
return element.getObjectValue();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
debug.warning(format("Bad cache state: %s => %s", key, e));
|
||||
e.printStackTrace();
|
||||
debug.warning(format("Cache get: %s => %s", key, e));
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public Object computeIf(Object key, Predicate<Element> condition, Callable<?> callable) throws Exception {
|
||||
public Object computeIf(Object key, Predicate<Element> condition, Compute<?> compute) throws Exception {
|
||||
// get if present
|
||||
Element element = null;
|
||||
try {
|
||||
Element element = cache.get(key);
|
||||
element = cache.get(key);
|
||||
if (element != null && condition.test(element)) {
|
||||
return element.getObjectValue();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
debug.warning(format("Bad cache state: %s => %s", key, e));
|
||||
debug.warning(format("Cache get: %s => %s", key, e));
|
||||
}
|
||||
|
||||
// compute if absent
|
||||
Object value = callable.call();
|
||||
Object value = compute.apply(element);
|
||||
try {
|
||||
cache.put(new Element(key, value));
|
||||
} catch (Exception e) {
|
||||
debug.warning(format("Bad cache state: %s => %s", key, e));
|
||||
debug.warning(format("Cache put: %s => %s", key, e));
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
public Object computeIfAbsent(Object key, Callable<?> callable) throws Exception {
|
||||
return computeIf(key, Element::isExpired, callable);
|
||||
public Object computeIfAbsent(Object key, Compute<?> compute) throws Exception {
|
||||
return computeIf(key, isAbsent(), compute);
|
||||
}
|
||||
|
||||
public Object computeIfStale(Object key, Duration expirationTime, Callable<?> callable) throws Exception {
|
||||
return computeIf(key, isStale(expirationTime), callable);
|
||||
public Object computeIfStale(Object key, Duration expirationTime, Compute<?> compute) throws Exception {
|
||||
return computeIf(key, isStale(expirationTime), compute);
|
||||
}
|
||||
|
||||
private Predicate<Element> isStale(Duration expirationTime) {
|
||||
return (element) -> element.isExpired() || System.currentTimeMillis() - element.getLatestOfCreationAndUpdateTime() < expirationTime.toMillis();
|
||||
public Predicate<Element> isAbsent() {
|
||||
return (element) -> element.getObjectValue() == null;
|
||||
}
|
||||
|
||||
public Predicate<Element> isStale(Duration expirationTime) {
|
||||
return (element) -> System.currentTimeMillis() - element.getLatestOfCreationAndUpdateTime() < expirationTime.toMillis();
|
||||
}
|
||||
|
||||
public void put(Object key, Object value) {
|
||||
try {
|
||||
cache.put(new Element(key, value));
|
||||
} catch (Exception e) {
|
||||
debug.warning(format("Bad cache state: %s => %s", key, e));
|
||||
debug.warning(format("Cache put: %s => %s", key, e));
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -76,10 +88,15 @@ public class Cache {
|
|||
try {
|
||||
cache.remove(key);
|
||||
} catch (Exception e) {
|
||||
debug.warning(format("Bad cache state: %s => %s", key, e));
|
||||
debug.warning(format("Cache remove: %s => %s", key, e));
|
||||
}
|
||||
}
|
||||
|
||||
@FunctionalInterface
|
||||
public interface Compute<R> {
|
||||
R apply(Element element) throws Exception;
|
||||
}
|
||||
|
||||
@Deprecated
|
||||
public <T> T get(Object key, Class<T> type) {
|
||||
return type.cast(get(key));
|
||||
|
@ -114,4 +131,8 @@ public class Cache {
|
|||
}
|
||||
}
|
||||
|
||||
public CachedResource2<String, String> resource(String url, Duration expirationTime, FloodLimit limit) {
|
||||
return new CachedResource2<String, String>(url, URL::new, CachedResource2.fetchIfModified(limit), CachedResource2.decode(UTF_8), expirationTime, this);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -447,7 +447,7 @@ public class MediaBindingBean {
|
|||
|
||||
// calculate checksum from file
|
||||
Cache cache = Cache.getCache("crc32", CacheType.Ephemeral);
|
||||
return (String) cache.computeIfAbsent(inferredMediaFile, () -> crc32(inferredMediaFile));
|
||||
return (String) cache.computeIfAbsent(inferredMediaFile, element -> crc32(inferredMediaFile));
|
||||
}
|
||||
|
||||
@Define("fn")
|
||||
|
|
|
@ -0,0 +1,123 @@
|
|||
package net.filebot.web;
|
||||
|
||||
import static net.filebot.Logging.*;
|
||||
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.net.URL;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.charset.Charset;
|
||||
import java.time.Duration;
|
||||
import java.util.concurrent.Callable;
|
||||
|
||||
import net.filebot.Cache;
|
||||
|
||||
public class CachedResource2<K, R> {
|
||||
|
||||
public static final int DEFAULT_RETRY_LIMIT = 2;
|
||||
public static final Duration DEFAULT_RETRY_DELAY = Duration.ofSeconds(2);
|
||||
|
||||
protected final K key;
|
||||
|
||||
protected final Source<K> source;
|
||||
protected final Fetch fetch;
|
||||
protected final Parse<R> parse;
|
||||
|
||||
protected final Duration expirationTime;
|
||||
|
||||
protected final int retryCountLimit;
|
||||
protected final long retryWaitTime;
|
||||
|
||||
protected final Cache cache;
|
||||
|
||||
public CachedResource2(K key, Source<K> source, Fetch fetch, Parse<R> parse, Duration expirationTime, Cache cache) {
|
||||
this(key, source, fetch, parse, DEFAULT_RETRY_LIMIT, DEFAULT_RETRY_DELAY, expirationTime, cache);
|
||||
}
|
||||
|
||||
public CachedResource2(K key, Source<K> source, Fetch fetch, Parse<R> parse, int retryCountLimit, Duration retryWaitTime, Duration expirationTime, Cache cache) {
|
||||
this.key = key;
|
||||
this.source = source;
|
||||
this.fetch = fetch;
|
||||
this.parse = parse;
|
||||
this.expirationTime = expirationTime;
|
||||
this.retryCountLimit = retryCountLimit;
|
||||
this.retryWaitTime = retryWaitTime.toMillis();
|
||||
this.cache = cache;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
public synchronized R get() throws Exception {
|
||||
return (R) cache.computeIfStale(key, expirationTime, element -> {
|
||||
URL resource = source.source(key);
|
||||
long lastModified = element == null ? 0 : element.getLatestOfCreationAndUpdateTime();
|
||||
|
||||
debug.fine(format("Fetch %s (If-Modified-Since: %tc)", resource, lastModified));
|
||||
|
||||
try {
|
||||
ByteBuffer data = retry(() -> fetch.fetch(resource, lastModified), retryCountLimit, lastModified);
|
||||
|
||||
// 304 Not Modified
|
||||
if (data == null && element != null && element.getObjectValue() != null) {
|
||||
return element.getObjectValue();
|
||||
}
|
||||
|
||||
return parse.parse(data);
|
||||
} catch (IOException e) {
|
||||
debug.fine(format("Fetch failed => %s", e));
|
||||
|
||||
// use previously cached data if possible
|
||||
if (element == null || element.getObjectValue() == null) {
|
||||
throw e;
|
||||
}
|
||||
return element.getObjectKey();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
protected <T> T retry(Callable<T> callable, int retryCount, long retryWaitTime) throws Exception {
|
||||
try {
|
||||
return callable.call();
|
||||
} catch (FileNotFoundException e) {
|
||||
// resource does not exist, do not retry
|
||||
throw e;
|
||||
} catch (IOException e) {
|
||||
// retry or rethrow exception
|
||||
if (retryCount > 0) {
|
||||
throw e;
|
||||
}
|
||||
Thread.sleep(retryWaitTime);
|
||||
return retry(callable, retryCount - 1, retryWaitTime * 2);
|
||||
}
|
||||
}
|
||||
|
||||
@FunctionalInterface
|
||||
public interface Source<K> {
|
||||
URL source(K key) throws Exception;
|
||||
}
|
||||
|
||||
@FunctionalInterface
|
||||
public interface Fetch {
|
||||
ByteBuffer fetch(URL url, long lastModified) throws Exception;
|
||||
}
|
||||
|
||||
@FunctionalInterface
|
||||
public interface Parse<R> {
|
||||
R parse(ByteBuffer bytes) throws Exception;
|
||||
}
|
||||
|
||||
public static Parse<String> decode(Charset charset) {
|
||||
return (bb) -> charset.decode(bb).toString();
|
||||
}
|
||||
|
||||
public static Fetch fetchIfModified(FloodLimit limit) {
|
||||
return (url, lastModified) -> {
|
||||
try {
|
||||
limit.acquirePermit();
|
||||
return WebRequest.fetchIfModified(url, lastModified);
|
||||
} catch (FileNotFoundException e) {
|
||||
return ByteBuffer.allocate(0);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
}
|
|
@ -2,16 +2,13 @@ package net.filebot.web;
|
|||
|
||||
import static java.util.Collections.*;
|
||||
import static java.util.stream.Collectors.*;
|
||||
import static net.filebot.Logging.*;
|
||||
import static net.filebot.util.JsonUtilities.*;
|
||||
import static net.filebot.util.StringUtilities.*;
|
||||
import static net.filebot.web.WebRequest.*;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.net.URL;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.charset.Charset;
|
||||
import java.time.Duration;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.time.format.DateTimeParseException;
|
||||
import java.time.temporal.ChronoField;
|
||||
|
@ -27,27 +24,22 @@ import java.util.Map.Entry;
|
|||
import java.util.TreeMap;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.function.Function;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import javax.swing.Icon;
|
||||
|
||||
import net.filebot.Cache;
|
||||
import net.filebot.CacheType;
|
||||
import net.filebot.ResourceManager;
|
||||
import net.filebot.web.TMDbClient.MovieInfo;
|
||||
import net.filebot.web.TMDbClient.MovieInfo.MovieProperty;
|
||||
import net.filebot.web.TMDbClient.Person;
|
||||
import net.sf.ehcache.Cache;
|
||||
import net.sf.ehcache.CacheManager;
|
||||
|
||||
public class OMDbClient implements MovieIdentificationService {
|
||||
|
||||
private static final FloodLimit REQUEST_LIMIT = new FloodLimit(20, 10, TimeUnit.SECONDS);
|
||||
|
||||
private final String protocol = "http";
|
||||
private final String host = "www.omdbapi.com";
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return "OMDb";
|
||||
|
@ -70,7 +62,7 @@ public class OMDbClient implements MovieIdentificationService {
|
|||
}
|
||||
|
||||
@Override
|
||||
public List<Movie> searchMovie(String query, Locale locale) throws IOException {
|
||||
public List<Movie> searchMovie(String query, Locale locale) throws Exception {
|
||||
// query by name with year filter if possible
|
||||
Matcher nameYear = Pattern.compile("(.+)\\b(19\\d{2}|20\\d{2})$").matcher(query);
|
||||
if (nameYear.matches()) {
|
||||
|
@ -80,14 +72,14 @@ public class OMDbClient implements MovieIdentificationService {
|
|||
}
|
||||
}
|
||||
|
||||
public List<Movie> searchMovie(String movieName, int movieYear) throws IOException {
|
||||
public List<Movie> searchMovie(String movieName, int movieYear) throws Exception {
|
||||
Map<String, Object> param = new LinkedHashMap<String, Object>(2);
|
||||
param.put("s", movieName);
|
||||
if (movieYear > 0) {
|
||||
param.put("y", movieYear);
|
||||
}
|
||||
|
||||
Map<?, ?> response = request(param, REQUEST_LIMIT);
|
||||
Map<?, ?> response = request(param);
|
||||
|
||||
List<Movie> result = new ArrayList<Movie>();
|
||||
for (Object it : getArray(response, "Search")) {
|
||||
|
@ -141,40 +133,16 @@ public class OMDbClient implements MovieIdentificationService {
|
|||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
public Map<?, ?> request(Map<String, Object> parameters, final FloodLimit limit) throws IOException {
|
||||
URL url = new URL(protocol, host, "/?" + encodeParameters(parameters, true));
|
||||
public Map<?, ?> request(Map<String, Object> parameters) throws Exception {
|
||||
String url = "http://www.omdbapi.com/?" + encodeParameters(parameters, true);
|
||||
|
||||
CachedResource<String> json = new CachedResource<String>(url.toString(), String.class, CachedResource.ONE_WEEK) {
|
||||
Cache cache = Cache.getCache(getName(), CacheType.Weekly);
|
||||
String json = cache.resource(url, Duration.ofDays(7), REQUEST_LIMIT).get();
|
||||
|
||||
@Override
|
||||
public String process(ByteBuffer data) throws Exception {
|
||||
return Charset.forName("UTF-8").decode(data).toString();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected ByteBuffer fetchData(URL url, long lastModified) throws IOException {
|
||||
try {
|
||||
if (limit != null) {
|
||||
limit.acquirePermit();
|
||||
}
|
||||
return super.fetchData(url, lastModified);
|
||||
} catch (FileNotFoundException e) {
|
||||
return ByteBuffer.allocate(0);
|
||||
} catch (InterruptedException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Cache getCache() {
|
||||
return CacheManager.getInstance().getCache("web-datasource-lv2");
|
||||
}
|
||||
};
|
||||
|
||||
return asMap(readJson(json.get()));
|
||||
return asMap(readJson(json));
|
||||
}
|
||||
|
||||
public Map<String, String> getMovieInfo(Integer i, String t, String y, boolean tomatoes) throws IOException {
|
||||
public Map<String, String> getMovieInfo(Integer i, String t, String y, boolean tomatoes) throws Exception {
|
||||
// e.g. http://www.imdbapi.com/?i=tt0379786&r=xml&tomatoes=true
|
||||
Map<String, Object> param = new LinkedHashMap<String, Object>(2);
|
||||
if (i != null) {
|
||||
|
@ -188,10 +156,10 @@ public class OMDbClient implements MovieIdentificationService {
|
|||
}
|
||||
param.put("tomatoes", String.valueOf(tomatoes));
|
||||
|
||||
return getInfoMap(request(param, REQUEST_LIMIT));
|
||||
return getInfoMap(request(param));
|
||||
}
|
||||
|
||||
public MovieInfo getMovieInfo(Movie movie) throws IOException {
|
||||
public MovieInfo getMovieInfo(Movie movie) throws Exception {
|
||||
Map<String, String> data = movie.getImdbId() > 0 ? getMovieInfo(movie.getImdbId(), null, null, false) : getMovieInfo(null, movie.getName(), String.valueOf(movie.getYear()), false);
|
||||
|
||||
// sanity check
|
||||
|
@ -243,7 +211,7 @@ public class OMDbClient implements MovieIdentificationService {
|
|||
}
|
||||
}
|
||||
} catch (DateTimeParseException e) {
|
||||
Logger.getLogger(OMDbClient.class.getName()).log(Level.WARNING, String.format("Bad date: %s: %s", value, e.getMessage()));
|
||||
debug.warning(format("Bad date: %s =~ %s => %s", value, format, e));
|
||||
}
|
||||
}
|
||||
return null;
|
||||
|
|
|
@ -4,10 +4,6 @@ import static org.junit.Assert.*;
|
|||
|
||||
import java.io.IOException;
|
||||
|
||||
import net.sf.ehcache.CacheManager;
|
||||
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
|
||||
public class AcoustIDClientTest {
|
||||
|
@ -38,10 +34,4 @@ public class AcoustIDClientTest {
|
|||
assertEquals("聽媽媽的話", info.getTitle());
|
||||
}
|
||||
|
||||
@BeforeClass
|
||||
@AfterClass
|
||||
public static void clearCache() {
|
||||
CacheManager.getInstance().clearAll();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -24,12 +24,12 @@ public class OMDbClientTest {
|
|||
|
||||
@Test
|
||||
public void searchMovie2() throws Exception {
|
||||
List<Movie> results = client.searchMovie("The Illusionist", null);
|
||||
List<Movie> results = client.searchMovie("The Terminator", null);
|
||||
Movie movie = results.get(0);
|
||||
|
||||
assertEquals("The Illusionist", movie.getName());
|
||||
assertEquals(2006, movie.getYear());
|
||||
assertEquals(443543, movie.getImdbId(), 0);
|
||||
assertEquals("The Terminator", movie.getName());
|
||||
assertEquals(1984, movie.getYear());
|
||||
assertEquals(88247, movie.getImdbId());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
Loading…
Reference in New Issue