mobeigi
/
filebot
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Experiment with new CachedResource framework

This commit is contained in:
Reinhard Pointner 2016-03-07 15:36:13 +00:00
parent bc2b96d09b
commit 7d3b099c07
8 changed files with 88 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
source/net/filebot/Cache.java
View File

@ -6,15 +6,12 @@ import static net.filebot.Logging.*;
import java.io.Serializable;
import java.net.URL;
import java.nio.ByteBuffer;
import java.time.Duration;
import java.util.Arrays;
import java.util.function.Predicate;
import net.filebot.CachedResource2.Fetch;
import net.filebot.CachedResource2.Resource;
import net.filebot.CachedResource2.Transform;
import net.filebot.web.FloodLimit;
import net.sf.ehcache.Element;
import org.w3c.dom.Document;
@ -28,20 +25,20 @@ public class Cache {
return CacheManager.getInstance().getCache(name.toLowerCase(), type);
}
public <R> CachedResource2<String, R> resource(String key, Resource<String> source, Fetch fetch, Transform<ByteBuffer, ? extends Object> parse, Transform<? super Object, R> cast, Duration expirationTime) {
return new CachedResource2<String, R>(key, source, fetch, parse, cast, expirationTime, this);
public CachedResource2<String, String> text(String key, Transform<String, URL> resource, Duration expirationTime, Fetch fetch) {
return new CachedResource2<String, String>(key, resource, fetch, getText(UTF_8), String.class::cast, expirationTime, this);
}
public CachedResource2<String, String> text(String url, Duration expirationTime, FloodLimit limit) {
return new CachedResource2<String, String>(url, URL::new, withPermit(fetchIfModified(), r -> limit.acquirePermit() != null), getText(UTF_8), String.class::cast, expirationTime, this);
public CachedResource2<String, Document> xml(String key, Transform<String, URL> resource, Duration expirationTime) {
return new CachedResource2<String, Document>(key, resource, fetchIfModified(), validateXml(getText(UTF_8)), getXml(String.class::cast), expirationTime, this);
}
public CachedResource2<String, Document> xml(String key, Resource<String> source, Duration expirationTime) {
return new CachedResource2<String, Document>(key, source, fetchIfModified(), validateXml(getText(UTF_8)), getXml(String.class::cast), expirationTime, this);
public CachedResource2<String, Object> json(String key, Transform<String, URL> resource, Duration expirationTime) {
return json(key, resource, expirationTime, fetchIfModified());
}
public CachedResource2<String, Object> json(String key, Resource<String> source, Duration expirationTime) {
return new CachedResource2<String, Object>(key, source, fetchIfModified(), validateJson(getText(UTF_8)), getJson(String.class::cast), expirationTime, this);
public CachedResource2<String, Object> json(String key, Transform<String, URL> resource, Duration expirationTime, Fetch fetch) {
return new CachedResource2<String, Object>(key, resource, fetch, validateJson(getText(UTF_8)), getJson(String.class::cast), expirationTime, this);
}
private final net.sf.ehcache.Cache cache;

48
source/net/filebot/CachedResource2.java
View File

@ -8,6 +8,9 @@ import java.net.URL;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import java.time.Duration;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
import net.filebot.util.JsonUtilities;
@ -22,7 +25,7 @@ public class CachedResource2<K, R> {
private K key;
private Resource<K> source;
private Transform<K, URL> resource;
private Fetch fetch;
private Transform<ByteBuffer, ? extends Object> parse;
private Transform<? super Object, R> cast;
@ -34,13 +37,13 @@ public class CachedResource2<K, R> {
private final Cache cache;
public CachedResource2(K key, Resource<K> source, Fetch fetch, Transform<ByteBuffer, ? extends Object> parse, Transform<? super Object, R> cast, Duration expirationTime, Cache cache) {
this(key, source, fetch, parse, cast, DEFAULT_RETRY_LIMIT, DEFAULT_RETRY_DELAY, expirationTime, cache);
public CachedResource2(K key, Transform<K, URL> resource, Fetch fetch, Transform<ByteBuffer, ? extends Object> parse, Transform<? super Object, R> cast, Duration expirationTime, Cache cache) {
this(key, resource, fetch, parse, cast, DEFAULT_RETRY_LIMIT, DEFAULT_RETRY_DELAY, expirationTime, cache);
}
public CachedResource2(K key, Resource<K> source, Fetch fetch, Transform<ByteBuffer, ? extends Object> parse, Transform<? super Object, R> cast, int retryCountLimit, Duration retryWaitTime, Duration expirationTime, Cache cache) {
public CachedResource2(K key, Transform<K, URL> resource, Fetch fetch, Transform<ByteBuffer, ? extends Object> parse, Transform<? super Object, R> cast, int retryCountLimit, Duration retryWaitTime, Duration expirationTime, Cache cache) {
this.key = key;
this.source = source;
this.resource = resource;
this.fetch = fetch;
this.parse = parse;
this.cast = cast;
@ -52,13 +55,11 @@ public class CachedResource2<K, R> {
public synchronized R get() throws Exception {
Object value = cache.computeIfStale(key, expirationTime, element -> {
URL resource = source.source(key);
URL url = resource.transform(key);
long lastModified = element == null ? 0 : element.getLatestOfCreationAndUpdateTime();
debug.fine(format("Fetch %s (If-Modified-Since: %tc)", resource, lastModified));
try {
ByteBuffer data = retry(() -> fetch.fetch(resource, lastModified), retryCountLimit, retryWaitTime);
ByteBuffer data = retry(() -> fetch.fetch(url, lastModified), retryCountLimit, retryWaitTime);
// 304 Not Modified
if (data == null && element != null && element.getObjectValue() != null) {
@ -96,11 +97,6 @@ public class CachedResource2<K, R> {
}
}
@FunctionalInterface
public interface Resource<K> {
URL source(K key) throws Exception;
}
@FunctionalInterface
public interface Fetch {
ByteBuffer fetch(URL url, long lastModified) throws Exception;
@ -151,6 +147,7 @@ public class CachedResource2<K, R> {
public static Fetch fetchIfModified() {
return (url, lastModified) -> {
try {
debug.fine(format("Fetch %s (If-Modified-Since: %tc)", url, lastModified));
return WebRequest.fetchIfModified(url, lastModified);
} catch (FileNotFoundException e) {
debug.warning(format("Resource not found: %s => %s", url, e));
@ -159,6 +156,29 @@ public class CachedResource2<K, R> {
};
}
public static Fetch fetchIfNoneMatch(Cache etagStorage) {
return (url, lastModified) -> {
Map<String, List<String>> responseHeaders = new HashMap<String, List<String>>();
String etagKey = url.toString();
Object etagValue = etagStorage.get(etagKey);
try {
debug.fine(format("Fetch %s (If-None-Match: %s, If-Modified-Since: %tc)", url, etagValue, lastModified));
return WebRequest.fetch(url, lastModified, etagValue, null, responseHeaders);
} catch (FileNotFoundException e) {
debug.warning(format("Resource not found: %s => %s", url, e));
return ByteBuffer.allocate(0);
} finally {
List<String> value = responseHeaders.get("ETag");
if (value != null && value.size() > 0 && !value.contains(etagValue)) {
debug.finest(format("ETag %s", value));
etagStorage.put(etagKey, value.get(0));
}
}
};
}
public static Fetch withPermit(Fetch fetch, Permit<?> permit) {
return (url, lastModified) -> {
if (permit.acquirePermit(url)) {

4
source/net/filebot/cli/ScriptShellMethods.java
View File

@ -276,11 +276,11 @@ public class ScriptShellMethods {
}
public static ByteBuffer get(URL self) throws IOException {
return WebRequest.fetch(self, 0, null, null);
return WebRequest.fetch(self, 0, null, null, null);
}
public static ByteBuffer get(URL self, Map<String, String> requestParameters) throws IOException {
return WebRequest.fetch(self, 0, requestParameters, null);
return WebRequest.fetch(self, 0, null, requestParameters, null);
}
public static ByteBuffer post(URL self, Map<String, ?> parameters, Map<String, String> requestParameters) throws IOException {

51
source/net/filebot/web/ETagCachedResource.java
View File

@ -1,51 +0,0 @@
package net.filebot.web;
import java.io.IOException;
import java.io.Serializable;
import java.net.URL;
import java.nio.ByteBuffer;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import net.sf.ehcache.Cache;
import net.sf.ehcache.CacheManager;
import net.sf.ehcache.Element;
public abstract class ETagCachedResource<T extends Serializable> extends CachedResource<T> {
public ETagCachedResource(String resource, Class<T> type) {
super(resource, type, ONE_WEEK, 2, 1000);
}
public ETagCachedResource(String resource, Class<T> type, long expirationTime, int retryCountLimit, long retryWaitTime) {
super(resource, type, expirationTime, retryCountLimit, retryWaitTime);
}
@Override
protected ByteBuffer fetchData(URL url, long lastModified) throws IOException {
String etagKey = "ETag" + ":" + url.toString();
Element etag = getCache().get(etagKey);
Map<String, String> requestParameters = new HashMap<String, String>();
if (etag != null && etag.getObjectValue() != null) {
requestParameters.put("If-None-Match", etag.getObjectValue().toString());
}
// If-Modified-Since must not be set if If-None-Match is set
Map<String, List<String>> responseHeaders = new HashMap<String, List<String>>();
ByteBuffer data = WebRequest.fetch(url, requestParameters.size() > 0 ? -1 : lastModified, requestParameters, responseHeaders);
if (responseHeaders.containsKey("ETag")) {
getCache().put(new Element(etagKey, responseHeaders.get("ETag").get(0)));
}
return data;
}
@Override
protected Cache getCache() {
return CacheManager.getInstance().getCache("web-datasource-lv3");
}
}

13
source/net/filebot/web/OMDbClient.java
View File

@ -2,13 +2,14 @@ package net.filebot.web;
import static java.util.Collections.*;
import static java.util.stream.Collectors.*;
import static net.filebot.CachedResource2.*;
import static net.filebot.Logging.*;
import static net.filebot.util.JsonUtilities.*;
import static net.filebot.util.StringUtilities.*;
import static net.filebot.web.WebRequest.*;
import java.io.File;
import java.time.Duration;
import java.net.URL;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.time.temporal.ChronoField;
@ -134,12 +135,16 @@ public class OMDbClient implements MovieIdentificationService {
}
public Map<?, ?> request(Map<String, Object> parameters) throws Exception {
String url = "http://www.omdbapi.com/?" + encodeParameters(parameters, true);
String key = '?' + encodeParameters(parameters, true);
Cache cache = Cache.getCache(getName(), CacheType.Weekly);
String json = cache.text(url, Duration.ofDays(7), REQUEST_LIMIT).get();
Object json = cache.json(key, s -> getResource(s), Cache.ONE_WEEK, withPermit(fetchIfModified(), r -> REQUEST_LIMIT.acquirePermit() != null)).get();
return asMap(readJson(json));
return asMap(json);
}
public URL getResource(String file) throws Exception {
return new URL("http://www.omdbapi.com/" + file);
}
public Map<String, String> getMovieInfo(Integer i, String t, String y, boolean tomatoes) throws Exception {

54
source/net/filebot/web/TMDbClient.java
View File

@ -2,17 +2,15 @@ package net.filebot.web;
import static java.util.Arrays.*;
import static java.util.Collections.*;
import static net.filebot.CachedResource2.*;
import static net.filebot.util.StringUtilities.*;
import static net.filebot.web.WebRequest.*;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.Serializable;
import java.net.URI;
import java.net.URL;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import java.util.AbstractList;
import java.util.ArrayList;
import java.util.Collection;
@ -33,6 +31,8 @@ import java.util.regex.Pattern;
import javax.swing.Icon;
import net.filebot.Cache;
import net.filebot.CacheType;
import net.filebot.Language;
import net.filebot.ResourceManager;
import net.filebot.web.TMDbClient.MovieInfo.MovieProperty;
@ -67,7 +67,7 @@ public class TMDbClient implements MovieIdentificationService {
}
@Override
public List<Movie> searchMovie(String query, Locale locale) throws IOException {
public List<Movie> searchMovie(String query, Locale locale) throws Exception {
// query by name with year filter if possible
Matcher nameYear = Pattern.compile("(.+)\\b\\(?(19\\d{2}|20\\d{2})\\)?$").matcher(query.trim());
if (nameYear.matches()) {
@ -77,7 +77,7 @@ public class TMDbClient implements MovieIdentificationService {
}
}
public List<Movie> searchMovie(String movieName, int movieYear, Locale locale, boolean extendedInfo) throws IOException {
public List<Movie> searchMovie(String movieName, int movieYear, Locale locale, boolean extendedInfo) throws Exception {
// ignore queries that are too short to yield good results
if (movieName.length() < 3 && !(movieName.length() >= 1 && movieYear > 0)) {
return emptyList();
@ -155,7 +155,7 @@ public class TMDbClient implements MovieIdentificationService {
}
@Override
public Movie getMovieDescriptor(Movie id, Locale locale) throws IOException {
public Movie getMovieDescriptor(Movie id, Locale locale) throws Exception {
if (id.getTmdbId() > 0 || id.getImdbId() > 0) {
MovieInfo info = getMovieInfo(id, locale, false);
if (info != null) {
@ -175,7 +175,7 @@ public class TMDbClient implements MovieIdentificationService {
throw new UnsupportedOperationException();
}
public MovieInfo getMovieInfo(Movie movie, Locale locale, boolean extendedInfo) throws IOException {
public MovieInfo getMovieInfo(Movie movie, Locale locale, boolean extendedInfo) throws Exception {
try {
if (movie.getTmdbId() > 0) {
return getMovieInfo(String.valueOf(movie.getTmdbId()), locale, extendedInfo);
@ -188,7 +188,7 @@ public class TMDbClient implements MovieIdentificationService {
return null;
}
public MovieInfo getMovieInfo(String id, Locale locale, boolean extendedInfo) throws IOException {
public MovieInfo getMovieInfo(String id, Locale locale, boolean extendedInfo) throws Exception {
JSONObject response = request("movie/" + id, extendedInfo ? singletonMap("append_to_response", "alternative_titles,releases,casts,trailers") : null, locale, REQUEST_LIMIT);
Map<MovieProperty, String> fields = new EnumMap<MovieProperty, String>(MovieProperty.class);
@ -325,7 +325,7 @@ public class TMDbClient implements MovieIdentificationService {
return new MovieInfo(fields, alternativeTitles, genres, certifications, spokenLanguages, productionCountries, productionCompanies, cast, trailers);
}
public List<Artwork> getArtwork(String id) throws IOException {
public List<Artwork> getArtwork(String id) throws Exception {
// http://api.themoviedb.org/3/movie/11/images
JSONObject config = request("configuration", null, null, REQUEST_LIMIT);
String baseUrl = (String) ((JSONObject) config.get("images")).get("base_url");
@ -350,7 +350,7 @@ public class TMDbClient implements MovieIdentificationService {
return artwork;
}
public JSONObject request(String resource, Map<String, Object> parameters, Locale locale, final FloodLimit limit) throws IOException {
public JSONObject request(String resource, Map<String, Object> parameters, Locale locale, final FloodLimit limit) throws Exception {
// default parameters
LinkedHashMap<String, Object> data = new LinkedHashMap<String, Object>();
if (parameters != null) {
@ -371,37 +371,23 @@ public class TMDbClient implements MovieIdentificationService {
}
data.put("api_key", apikey);
URL url = new URL("http", host, "/" + version + "/" + resource + "?" + encodeParameters(data, true));
String key = resource + '?' + encodeParameters(data, true);
CachedResource<String> json = new ETagCachedResource<String>(url.toString(), String.class) {
Cache etagStorage = Cache.getCache("etag", CacheType.Monthly);
Cache cache = Cache.getCache(getName(), CacheType.Monthly);
String json = cache.text(key, s -> getResource(s), Cache.ONE_WEEK, withPermit(fetchIfNoneMatch(etagStorage), r -> REQUEST_LIMIT.acquirePermit() != null)).get();
@Override
public String process(ByteBuffer data) throws Exception {
return Charset.forName("UTF-8").decode(data).toString();
}
@Override
protected ByteBuffer fetchData(URL url, long lastModified) throws IOException {
try {
if (limit != null) {
limit.acquirePermit();
}
return super.fetchData(url, lastModified);
} catch (FileNotFoundException e) {
return ByteBuffer.allocate(0);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
}
};
JSONObject object = (JSONObject) JSONValue.parse(json.get());
JSONObject object = (JSONObject) JSONValue.parse(json);
if (object == null || object.isEmpty()) {
throw new FileNotFoundException("Resource not found: " + url);
throw new FileNotFoundException("Resource not found: " + getResource(key));
}
return object;
}
public URL getResource(String file) throws Exception {
return new URL("http", host, "/" + version + "/" + file);
}
protected List<JSONObject> jsonList(final Object array) {
return new AbstractList<JSONObject>() {

14
source/net/filebot/web/WebRequest.java
View File

@ -97,17 +97,25 @@ public final class WebRequest {
}
public static ByteBuffer fetch(URL resource) throws IOException {
return fetch(resource, 0, null, null);
return fetch(resource, 0, null, null, null);
}
public static ByteBuffer fetchIfModified(URL resource, long ifModifiedSince) throws IOException {
return fetch(resource, ifModifiedSince, null, null);
return fetch(resource, ifModifiedSince, null, null, null);
}
public static ByteBuffer fetch(URL url, long ifModifiedSince, Map<String, String> requestParameters, Map<String, List<String>> responseParameters) throws IOException {
public static ByteBuffer fetchIfNoneMatch(URL resource, Object etag) throws IOException {
return fetch(resource, 0, etag, null, null);
}
public static ByteBuffer fetch(URL url, long ifModifiedSince, Object etag, Map<String, String> requestParameters, Map<String, List<String>> responseParameters) throws IOException {
URLConnection connection = url.openConnection();
if (ifModifiedSince > 0) {
connection.setIfModifiedSince(ifModifiedSince);
} else if (etag != null) {
// If-Modified-Since must not be set if If-None-Match is set
connection.addRequestProperty("If-None-Match", etag.toString());
}
try {

8
test/net/filebot/web/TMDbClientTest.java
View File

@ -14,7 +14,7 @@ import org.junit.Test;
public class TMDbClientTest {
private final TMDbClient tmdb = new TMDbClient("66308fb6e3fd850dde4c7d21df2e8306");
TMDbClient tmdb = new TMDbClient("66308fb6e3fd850dde4c7d21df2e8306");
@Test
public void searchByName() throws Exception {
@ -74,9 +74,9 @@ public class TMDbClientTest {
MovieInfo movie = tmdb.getMovieInfo(new Movie(null, 0, 418279, -1), Locale.ENGLISH, true);
assertEquals("Transformers", movie.getName());
assertEquals("2007-07-02", movie.getReleased().toString());
assertEquals("2007-06-27", movie.getReleased().toString());
assertEquals("PG-13", movie.getCertification());
assertEquals("[en, es]", movie.getSpokenLanguages().toString());
assertEquals("[es, en]", movie.getSpokenLanguages().toString());
assertEquals("Shia LaBeouf", movie.getActors().get(0));
assertEquals("Michael Bay", movie.getDirector());
}
@ -85,7 +85,7 @@ public class TMDbClientTest {
public void getArtwork() throws Exception {
List<Artwork> artwork = tmdb.getArtwork("tt0418279");
assertEquals("backdrops", artwork.get(0).getCategory());
assertEquals("http://image.tmdb.org/t/p/original/dXTeZELpoVMDOTTLnNoCpsCngwW.jpg", artwork.get(0).getUrl().toString());
assertEquals("http://image.tmdb.org/t/p/original/ac0HwGJIU3GxjjGujlIjLJmAGPR.jpg", artwork.get(0).getUrl().toString());
}
@Ignore