Experiment with new CachedResource framework

source/ehcache.xml

@@ -1,28 +1,9 @@
 <ehcache xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="ehcache.xsd" updateCheck="false">
-	<!--
-	Persistent disk store location
-	-->
-	<diskStore path="ehcache.disk.store.dir" />
 
-	<!--
+	<diskStore path="java.io.tmpdir" />
-	Mandatory Default Cache configuration. These settings will be applied to caches
+
-	created pragmatically using CacheManager.add(String cacheName)
-	-->
 	<defaultCache
-		maxElementsInMemory="100"
+		maxElementsInMemory="400"
-		eternal="false"
-		timeToIdleSeconds="120"
-		timeToLiveSeconds="120"
-		overflowToDisk="false"
-		diskPersistent="false"
-		memoryStoreEvictionPolicy="LRU"
-	/>
-
-	<!--
-	Short-lived (24 hours) persistent disk cache for web responses
-	-->
-	<cache name="web-datasource"
-		maxElementsInMemory="200"
 		maxElementsOnDisk="80000"
 		eternal="false"
 		timeToIdleSeconds="86400"
@@ -32,73 +13,4 @@
 		memoryStoreEvictionPolicy="LRU"
 	/>
 
-	<!--
-	Long-lived (2 weeks) persistent disk cache for web responses
-	-->
-	<cache name="web-datasource-lv2"
-		maxElementsInMemory="200"
-		maxElementsOnDisk="80000"
-		eternal="false"
-		timeToIdleSeconds="1209600"
-		timeToLiveSeconds="1209600"
-		overflowToDisk="true"
-		diskPersistent="true"
-		memoryStoreEvictionPolicy="LRU"
-	/>
-
-	<!--
-	Long-lived (2 months) persistent disk cache for web responses (that can be updated via If-Modified or If-None-Match)
-	-->
-	<cache name="web-datasource-lv3"
-		maxElementsInMemory="100"
-		maxElementsOnDisk="200000"
-		eternal="false"
-		timeToIdleSeconds="5256000"
-		timeToLiveSeconds="5256000"
-		overflowToDisk="true"
-		diskPersistent="true"
-		memoryStoreEvictionPolicy="LRU"
-	/>
-
-	<!--
-	Very long-lived cache (4 months) anime/series lists, movie index, etc
-	-->
-	<cache name="web-persistent-datasource"
-		maxElementsInMemory="100"
-		maxElementsOnDisk="80000"
-		eternal="false"
-		timeToIdleSeconds="10512000"
-		timeToLiveSeconds="10512000"
-		overflowToDisk="true"
-		diskPersistent="true"
-		memoryStoreEvictionPolicy="LRU"
-	/>
-
-	<!--
-	Simple memory cache. Time to live is 4 months.
-	-->
-	<cache name="persistent-memory"
-		maxElementsInMemory="100"
-		maxElementsOnDisk="50000"
-		eternal="false"
-		timeToIdleSeconds="10512000"
-		timeToLiveSeconds="10512000"
-		overflowToDisk="true"
-		diskPersistent="true"
-		memoryStoreEvictionPolicy="LRU"
-	/>
-
-	<!--
-	Simple memory cache. Time to live is 2 hours.
-	-->
-	<cache name="ephemeral-memory"
-		maxElementsInMemory="50000"
-		eternal="false"
-		timeToIdleSeconds="7200"
-		timeToLiveSeconds="7200"
-		overflowToDisk="false"
-		diskPersistent="false"
-		memoryStoreEvictionPolicy="LRU"
-	/>
-
 </ehcache>

source/net/filebot/Cache.java

@@ -1,13 +1,16 @@
 package net.filebot;
 
+import static java.nio.charset.StandardCharsets.*;
 import static net.filebot.Logging.*;
 
 import java.io.Serializable;
+import java.net.URL;
 import java.time.Duration;
 import java.util.Arrays;
-import java.util.concurrent.Callable;
 import java.util.function.Predicate;
 
+import net.filebot.web.CachedResource2;
+import net.filebot.web.FloodLimit;
 import net.sf.ehcache.Element;
 
 public class Cache {
@@ -24,51 +27,60 @@ public class Cache {
 
 	public Object get(Object key) {
 		try {
-			return cache.get(key).getObjectValue();
+			Element element = cache.get(key);
+			if (element != null) {
+				return element.getObjectValue();
+			}
 		} catch (Exception e) {
-			debug.warning(format("Bad cache state: %s => %s", key, e));
+			e.printStackTrace();
+			debug.warning(format("Cache get: %s => %s", key, e));
 		}
 		return null;
 	}
 
-	public Object computeIf(Object key, Predicate<Element> condition, Callable<?> callable) throws Exception {
+	public Object computeIf(Object key, Predicate<Element> condition, Compute<?> compute) throws Exception {
 		// get if present
+		Element element = null;
 		try {
-			Element element = cache.get(key);
+			element = cache.get(key);
 			if (element != null && condition.test(element)) {
 				return element.getObjectValue();
 			}
 		} catch (Exception e) {
-			debug.warning(format("Bad cache state: %s => %s", key, e));
+			debug.warning(format("Cache get: %s => %s", key, e));
 		}
 
 		// compute if absent
-		Object value = callable.call();
+		Object value = compute.apply(element);
 		try {
 			cache.put(new Element(key, value));
 		} catch (Exception e) {
-			debug.warning(format("Bad cache state: %s => %s", key, e));
+			debug.warning(format("Cache put: %s => %s", key, e));
 		}
 		return value;
 	}
 
-	public Object computeIfAbsent(Object key, Callable<?> callable) throws Exception {
+	public Object computeIfAbsent(Object key, Compute<?> compute) throws Exception {
-		return computeIf(key, Element::isExpired, callable);
+		return computeIf(key, isAbsent(), compute);
 	}
 
-	public Object computeIfStale(Object key, Duration expirationTime, Callable<?> callable) throws Exception {
+	public Object computeIfStale(Object key, Duration expirationTime, Compute<?> compute) throws Exception {
-		return computeIf(key, isStale(expirationTime), callable);
+		return computeIf(key, isStale(expirationTime), compute);
 	}
 
-	private Predicate<Element> isStale(Duration expirationTime) {
+	public Predicate<Element> isAbsent() {
-		return (element) -> element.isExpired() || System.currentTimeMillis() - element.getLatestOfCreationAndUpdateTime() < expirationTime.toMillis();
+		return (element) -> element.getObjectValue() == null;
+	}
+
+	public Predicate<Element> isStale(Duration expirationTime) {
+		return (element) -> System.currentTimeMillis() - element.getLatestOfCreationAndUpdateTime() < expirationTime.toMillis();
 	}
 
 	public void put(Object key, Object value) {
 		try {
 			cache.put(new Element(key, value));
 		} catch (Exception e) {
-			debug.warning(format("Bad cache state: %s => %s", key, e));
+			debug.warning(format("Cache put: %s => %s", key, e));
 		}
 	}
 
@@ -76,10 +88,15 @@ public class Cache {
 		try {
 			cache.remove(key);
 		} catch (Exception e) {
-			debug.warning(format("Bad cache state: %s => %s", key, e));
+			debug.warning(format("Cache remove: %s => %s", key, e));
 		}
 	}
 
+	@FunctionalInterface
+	public interface Compute<R> {
+		R apply(Element element) throws Exception;
+	}
+
 	@Deprecated
 	public <T> T get(Object key, Class<T> type) {
 		return type.cast(get(key));
@@ -114,4 +131,8 @@ public class Cache {
 		}
 	}
 
+	public CachedResource2<String, String> resource(String url, Duration expirationTime, FloodLimit limit) {
+		return new CachedResource2<String, String>(url, URL::new, CachedResource2.fetchIfModified(limit), CachedResource2.decode(UTF_8), expirationTime, this);
+	}
+
 }

source/net/filebot/format/MediaBindingBean.java

@@ -447,7 +447,7 @@ public class MediaBindingBean {
 
 		// calculate checksum from file
 		Cache cache = Cache.getCache("crc32", CacheType.Ephemeral);
-		return (String) cache.computeIfAbsent(inferredMediaFile, () -> crc32(inferredMediaFile));
+		return (String) cache.computeIfAbsent(inferredMediaFile, element -> crc32(inferredMediaFile));
 	}
 
 	@Define("fn")

source/net/filebot/web/CachedResource2.java

@@ -0,0 +1,123 @@
+package net.filebot.web;
+
+import static net.filebot.Logging.*;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URL;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.time.Duration;
+import java.util.concurrent.Callable;
+
+import net.filebot.Cache;
+
+public class CachedResource2<K, R> {
+
+	public static final int DEFAULT_RETRY_LIMIT = 2;
+	public static final Duration DEFAULT_RETRY_DELAY = Duration.ofSeconds(2);
+
+	protected final K key;
+
+	protected final Source<K> source;
+	protected final Fetch fetch;
+	protected final Parse<R> parse;
+
+	protected final Duration expirationTime;
+
+	protected final int retryCountLimit;
+	protected final long retryWaitTime;
+
+	protected final Cache cache;
+
+	public CachedResource2(K key, Source<K> source, Fetch fetch, Parse<R> parse, Duration expirationTime, Cache cache) {
+		this(key, source, fetch, parse, DEFAULT_RETRY_LIMIT, DEFAULT_RETRY_DELAY, expirationTime, cache);
+	}
+
+	public CachedResource2(K key, Source<K> source, Fetch fetch, Parse<R> parse, int retryCountLimit, Duration retryWaitTime, Duration expirationTime, Cache cache) {
+		this.key = key;
+		this.source = source;
+		this.fetch = fetch;
+		this.parse = parse;
+		this.expirationTime = expirationTime;
+		this.retryCountLimit = retryCountLimit;
+		this.retryWaitTime = retryWaitTime.toMillis();
+		this.cache = cache;
+	}
+
+	@SuppressWarnings("unchecked")
+	public synchronized R get() throws Exception {
+		return (R) cache.computeIfStale(key, expirationTime, element -> {
+			URL resource = source.source(key);
+			long lastModified = element == null ? 0 : element.getLatestOfCreationAndUpdateTime();
+
+			debug.fine(format("Fetch %s (If-Modified-Since: %tc)", resource, lastModified));
+
+			try {
+				ByteBuffer data = retry(() -> fetch.fetch(resource, lastModified), retryCountLimit, lastModified);
+
+				// 304 Not Modified
+				if (data == null && element != null && element.getObjectValue() != null) {
+					return element.getObjectValue();
+				}
+
+				return parse.parse(data);
+			} catch (IOException e) {
+				debug.fine(format("Fetch failed => %s", e));
+
+				// use previously cached data if possible
+				if (element == null || element.getObjectValue() == null) {
+					throw e;
+				}
+				return element.getObjectKey();
+			}
+		});
+	}
+
+	protected <T> T retry(Callable<T> callable, int retryCount, long retryWaitTime) throws Exception {
+		try {
+			return callable.call();
+		} catch (FileNotFoundException e) {
+			// resource does not exist, do not retry
+			throw e;
+		} catch (IOException e) {
+			// retry or rethrow exception
+			if (retryCount > 0) {
+				throw e;
+			}
+			Thread.sleep(retryWaitTime);
+			return retry(callable, retryCount - 1, retryWaitTime * 2);
+		}
+	}
+
+	@FunctionalInterface
+	public interface Source<K> {
+		URL source(K key) throws Exception;
+	}
+
+	@FunctionalInterface
+	public interface Fetch {
+		ByteBuffer fetch(URL url, long lastModified) throws Exception;
+	}
+
+	@FunctionalInterface
+	public interface Parse<R> {
+		R parse(ByteBuffer bytes) throws Exception;
+	}
+
+	public static Parse<String> decode(Charset charset) {
+		return (bb) -> charset.decode(bb).toString();
+	}
+
+	public static Fetch fetchIfModified(FloodLimit limit) {
+		return (url, lastModified) -> {
+			try {
+				limit.acquirePermit();
+				return WebRequest.fetchIfModified(url, lastModified);
+			} catch (FileNotFoundException e) {
+				return ByteBuffer.allocate(0);
+			}
+		};
+	}
+
+}

source/net/filebot/web/OMDbClient.java

@@ -2,16 +2,13 @@ package net.filebot.web;
 
 import static java.util.Collections.*;
 import static java.util.stream.Collectors.*;
+import static net.filebot.Logging.*;
 import static net.filebot.util.JsonUtilities.*;
 import static net.filebot.util.StringUtilities.*;
 import static net.filebot.web.WebRequest.*;
 
 import java.io.File;
-import java.io.FileNotFoundException;
+import java.time.Duration;
-import java.io.IOException;
-import java.net.URL;
-import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
 import java.time.format.DateTimeFormatter;
 import java.time.format.DateTimeParseException;
 import java.time.temporal.ChronoField;
@@ -27,27 +24,22 @@ import java.util.Map.Entry;
 import java.util.TreeMap;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
-import java.util.logging.Level;
-import java.util.logging.Logger;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import javax.swing.Icon;
 
+import net.filebot.Cache;
+import net.filebot.CacheType;
 import net.filebot.ResourceManager;
 import net.filebot.web.TMDbClient.MovieInfo;
 import net.filebot.web.TMDbClient.MovieInfo.MovieProperty;
 import net.filebot.web.TMDbClient.Person;
-import net.sf.ehcache.Cache;
-import net.sf.ehcache.CacheManager;
 
 public class OMDbClient implements MovieIdentificationService {
 
 	private static final FloodLimit REQUEST_LIMIT = new FloodLimit(20, 10, TimeUnit.SECONDS);
 
-	private final String protocol = "http";
-	private final String host = "www.omdbapi.com";
-
 	@Override
 	public String getName() {
 		return "OMDb";
@@ -70,7 +62,7 @@ public class OMDbClient implements MovieIdentificationService {
 	}
 
 	@Override
-	public List<Movie> searchMovie(String query, Locale locale) throws IOException {
+	public List<Movie> searchMovie(String query, Locale locale) throws Exception {
 		// query by name with year filter if possible
 		Matcher nameYear = Pattern.compile("(.+)\\b(19\\d{2}|20\\d{2})$").matcher(query);
 		if (nameYear.matches()) {
@@ -80,14 +72,14 @@ public class OMDbClient implements MovieIdentificationService {
 		}
 	}
 
-	public List<Movie> searchMovie(String movieName, int movieYear) throws IOException {
+	public List<Movie> searchMovie(String movieName, int movieYear) throws Exception {
 		Map<String, Object> param = new LinkedHashMap<String, Object>(2);
 		param.put("s", movieName);
 		if (movieYear > 0) {
 			param.put("y", movieYear);
 		}
 
-		Map<?, ?> response = request(param, REQUEST_LIMIT);
+		Map<?, ?> response = request(param);
 
 		List<Movie> result = new ArrayList<Movie>();
 		for (Object it : getArray(response, "Search")) {
@@ -141,40 +133,16 @@ public class OMDbClient implements MovieIdentificationService {
 		throw new UnsupportedOperationException();
 	}
 
-	public Map<?, ?> request(Map<String, Object> parameters, final FloodLimit limit) throws IOException {
+	public Map<?, ?> request(Map<String, Object> parameters) throws Exception {
-		URL url = new URL(protocol, host, "/?" + encodeParameters(parameters, true));
+		String url = "http://www.omdbapi.com/?" + encodeParameters(parameters, true);
 
-		CachedResource<String> json = new CachedResource<String>(url.toString(), String.class, CachedResource.ONE_WEEK) {
+		Cache cache = Cache.getCache(getName(), CacheType.Weekly);
+		String json = cache.resource(url, Duration.ofDays(7), REQUEST_LIMIT).get();
 
-			@Override
+		return asMap(readJson(json));
-			public String process(ByteBuffer data) throws Exception {
-				return Charset.forName("UTF-8").decode(data).toString();
 	}
 
-			@Override
+	public Map<String, String> getMovieInfo(Integer i, String t, String y, boolean tomatoes) throws Exception {
-			protected ByteBuffer fetchData(URL url, long lastModified) throws IOException {
-				try {
-					if (limit != null) {
-						limit.acquirePermit();
-					}
-					return super.fetchData(url, lastModified);
-				} catch (FileNotFoundException e) {
-					return ByteBuffer.allocate(0);
-				} catch (InterruptedException e) {
-					throw new RuntimeException(e);
-				}
-			}
-
-			@Override
-			protected Cache getCache() {
-				return CacheManager.getInstance().getCache("web-datasource-lv2");
-			}
-		};
-
-		return asMap(readJson(json.get()));
-	}
-
-	public Map<String, String> getMovieInfo(Integer i, String t, String y, boolean tomatoes) throws IOException {
 		// e.g. http://www.imdbapi.com/?i=tt0379786&r=xml&tomatoes=true
 		Map<String, Object> param = new LinkedHashMap<String, Object>(2);
 		if (i != null) {
@@ -188,10 +156,10 @@ public class OMDbClient implements MovieIdentificationService {
 		}
 		param.put("tomatoes", String.valueOf(tomatoes));
 
-		return getInfoMap(request(param, REQUEST_LIMIT));
+		return getInfoMap(request(param));
 	}
 
-	public MovieInfo getMovieInfo(Movie movie) throws IOException {
+	public MovieInfo getMovieInfo(Movie movie) throws Exception {
 		Map<String, String> data = movie.getImdbId() > 0 ? getMovieInfo(movie.getImdbId(), null, null, false) : getMovieInfo(null, movie.getName(), String.valueOf(movie.getYear()), false);
 
 		// sanity check
@@ -243,7 +211,7 @@ public class OMDbClient implements MovieIdentificationService {
 					}
 				}
 			} catch (DateTimeParseException e) {
-				Logger.getLogger(OMDbClient.class.getName()).log(Level.WARNING, String.format("Bad date: %s: %s", value, e.getMessage()));
+				debug.warning(format("Bad date: %s =~ %s => %s", value, format, e));
 			}
 		}
 		return null;

test/net/filebot/web/AcoustIDClientTest.java

@@ -4,10 +4,6 @@ import static org.junit.Assert.*;
 
 import java.io.IOException;
 
-import net.sf.ehcache.CacheManager;
-
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
 import org.junit.Test;
 
 public class AcoustIDClientTest {
@@ -38,10 +34,4 @@ public class AcoustIDClientTest {
 		assertEquals("聽媽媽的話", info.getTitle());
 	}
 
-	@BeforeClass
-	@AfterClass
-	public static void clearCache() {
-		CacheManager.getInstance().clearAll();
-	}
-
 }

test/net/filebot/web/OMDbClientTest.java

@@ -24,12 +24,12 @@ public class OMDbClientTest {
 
 	@Test
 	public void searchMovie2() throws Exception {
-		List<Movie> results = client.searchMovie("The Illusionist", null);
+		List<Movie> results = client.searchMovie("The Terminator", null);
 		Movie movie = results.get(0);
 
-		assertEquals("The Illusionist", movie.getName());
+		assertEquals("The Terminator", movie.getName());
-		assertEquals(2006, movie.getYear());
+		assertEquals(1984, movie.getYear());
-		assertEquals(443543, movie.getImdbId(), 0);
+		assertEquals(88247, movie.getImdbId());
 	}
 
 	@Test