From a46a3e48a87203c12f0675d239f4d228a2e50479 Mon Sep 17 00:00:00 2001 From: Reinhard Pointner Date: Mon, 7 Mar 2016 11:32:00 +0000 Subject: [PATCH] Experiment with new CachedResource framework --- source/net/filebot/Cache.java | 22 +++--- source/net/filebot/web/CachedResource2.java | 19 ++++++ source/net/filebot/web/CachedXmlResource.java | 67 ------------------- source/net/filebot/web/OMDbClient.java | 2 +- source/net/filebot/web/TVMazeClient.java | 18 +++-- source/net/filebot/web/TheTVDBClient.java | 9 +-- test/net/filebot/web/TheTVDBClientTest.java | 2 +- 7 files changed, 50 insertions(+), 89 deletions(-) delete mode 100644 source/net/filebot/web/CachedXmlResource.java diff --git a/source/net/filebot/Cache.java b/source/net/filebot/Cache.java index 9e2c9965..d230a7dc 100644 --- a/source/net/filebot/Cache.java +++ b/source/net/filebot/Cache.java @@ -20,10 +20,24 @@ import org.w3c.dom.Document; public class Cache { + public static final Duration ONE_DAY = Duration.ofDays(1); + public static Cache getCache(String name, CacheType type) { return CacheManager.getInstance().getCache(name.toLowerCase(), type); } + public Resource text(String url, Duration expirationTime, FloodLimit limit) { + return new CachedResource2(url, URL::new, withPermit(fetchIfModified(), r -> limit.acquirePermit() != null), getText(UTF_8), String.class::cast, expirationTime, this); + } + + public Resource xml(String key, Source source, Duration expirationTime) { + return new CachedResource2(key, source, fetchIfModified(), validateXml(getText(UTF_8)), getXml(String.class::cast), expirationTime, this); + } + + public Resource json(String key, Source source, Duration expirationTime) { + return new CachedResource2(key, source, fetchIfModified(), validateJson(getText(UTF_8)), getJson(String.class::cast), expirationTime, this); + } + private final net.sf.ehcache.Cache cache; public Cache(net.sf.ehcache.Cache cache) { @@ -136,12 +150,4 @@ public class Cache { } } - public Resource xml(String key, Source source, Duration expirationTime) { - return new CachedResource2(key, source, fetchIfModified(), validateXml(getText(UTF_8)), getXml(String.class::cast), DEFAULT_RETRY_LIMIT, DEFAULT_RETRY_DELAY, expirationTime, this); - } - - public Resource resource(String url, Duration expirationTime, FloodLimit limit) { - return new CachedResource2(url, URL::new, withPermit(fetchIfModified(), r -> limit.acquirePermit() != null), getText(UTF_8), String.class::cast, DEFAULT_RETRY_LIMIT, DEFAULT_RETRY_DELAY, expirationTime, this); - } - } diff --git a/source/net/filebot/web/CachedResource2.java b/source/net/filebot/web/CachedResource2.java index 7a8780a7..f6076e6f 100644 --- a/source/net/filebot/web/CachedResource2.java +++ b/source/net/filebot/web/CachedResource2.java @@ -11,6 +11,7 @@ import java.time.Duration; import java.util.concurrent.Callable; import net.filebot.Cache; +import net.filebot.util.JsonUtilities; import org.w3c.dom.Document; @@ -33,6 +34,10 @@ public class CachedResource2 implements Resource { protected final Cache cache; + public CachedResource2(K key, Source source, Fetch fetch, Transform parse, Transform cast, Duration expirationTime, Cache cache) { + this(key, source, fetch, parse, cast, DEFAULT_RETRY_LIMIT, DEFAULT_RETRY_DELAY, expirationTime, cache); + } + public CachedResource2(K key, Source source, Fetch fetch, Transform parse, Transform cast, int retryCountLimit, Duration retryWaitTime, Duration expirationTime, Cache cache) { this.key = key; this.source = source; @@ -124,12 +129,26 @@ public class CachedResource2 implements Resource { }; } + public static Transform validateJson(Transform parse) { + return (object) -> { + String json = parse.transform(object); + JsonUtilities.readJson(json); + return json; + }; + } + public static Transform getXml(Transform parse) { return (object) -> { return WebRequest.getDocument(parse.transform(object)); }; } + public static Transform getJson(Transform parse) { + return (object) -> { + return JsonUtilities.readJson(parse.transform(object)); + }; + } + public static Fetch fetchIfModified() { return (url, lastModified) -> { try { diff --git a/source/net/filebot/web/CachedXmlResource.java b/source/net/filebot/web/CachedXmlResource.java deleted file mode 100644 index f7f4c0a9..00000000 --- a/source/net/filebot/web/CachedXmlResource.java +++ /dev/null @@ -1,67 +0,0 @@ -package net.filebot.web; - -import java.io.IOException; -import java.io.StringReader; -import java.net.URL; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; - -import javax.xml.parsers.SAXParserFactory; - -import net.sf.ehcache.Cache; -import net.sf.ehcache.CacheManager; - -import org.w3c.dom.Document; -import org.xml.sax.InputSource; -import org.xml.sax.SAXException; -import org.xml.sax.XMLReader; -import org.xml.sax.helpers.DefaultHandler; - -public class CachedXmlResource extends AbstractCachedResource { - - public CachedXmlResource(String resource) { - super(resource, String.class, ONE_DAY, 2, 1000); - } - - @Override - protected Cache getCache() { - return CacheManager.getInstance().getCache("web-datasource-lv3"); - } - - public Document getDocument() throws IOException { - try { - return WebRequest.getDocument(get()); - } catch (Exception e) { - throw new IOException(String.format("Error while loading XML resource: %s (%s)", getResourceLocation(resource), e.getMessage())); - } - } - - @Override - public String process(String data) throws Exception { - // make sure xml data is valid and well-formed before caching it - SAXParserFactory sax = SAXParserFactory.newInstance(); - sax.setValidating(false); - sax.setNamespaceAware(false); - - XMLReader reader = sax.newSAXParser().getXMLReader(); - reader.setErrorHandler(new DefaultHandler()); // unwind on error - try { - reader.parse(new InputSource(new StringReader(data))); - } catch (SAXException e) { - throw new IOException(String.format("Malformed XML: %s (%s)", getResourceLocation(resource), e.getMessage())); - } - - return data; - } - - @Override - protected String fetchData(URL url, long lastModified) throws IOException { - ByteBuffer data = WebRequest.fetchIfModified(url, lastModified); - - if (data == null) - return null; // not modified - - return StandardCharsets.UTF_8.decode(data).toString(); - } - -} diff --git a/source/net/filebot/web/OMDbClient.java b/source/net/filebot/web/OMDbClient.java index ced8575d..87b7a3a6 100644 --- a/source/net/filebot/web/OMDbClient.java +++ b/source/net/filebot/web/OMDbClient.java @@ -137,7 +137,7 @@ public class OMDbClient implements MovieIdentificationService { String url = "http://www.omdbapi.com/?" + encodeParameters(parameters, true); Cache cache = Cache.getCache(getName(), CacheType.Weekly); - String json = cache.resource(url, Duration.ofDays(7), REQUEST_LIMIT).get(); + String json = cache.text(url, Duration.ofDays(7), REQUEST_LIMIT).get(); return asMap(readJson(json)); } diff --git a/source/net/filebot/web/TVMazeClient.java b/source/net/filebot/web/TVMazeClient.java index b90a1724..1c9392af 100644 --- a/source/net/filebot/web/TVMazeClient.java +++ b/source/net/filebot/web/TVMazeClient.java @@ -5,8 +5,8 @@ import static java.util.stream.Collectors.*; import static net.filebot.util.JsonUtilities.*; import static net.filebot.web.WebRequest.*; -import java.io.IOException; import java.net.URI; +import java.net.URL; import java.util.List; import java.util.Locale; import java.util.Objects; @@ -55,11 +55,11 @@ public class TVMazeClient extends AbstractEpisodeListProvider { } @Override - public List fetchSearchResult(String query, Locale locale) throws IOException { + public List fetchSearchResult(String query, Locale locale) throws Exception { // e.g. http://api.tvmaze.com/search/shows?q=girls Object response = request("search/shows?q=" + encode(query, true)); - // TODO: FUTURE WORK: consider adding TVmaze aka titles for each result, e.g. http://api.tvmaze.com/shows/1/akas + // FUTURE WORK: consider adding TVmaze aka titles for each result, e.g. http://api.tvmaze.com/shows/1/akas return streamJsonObjects(response).map(it -> { Object show = it.get("show"); Integer id = getInteger(show, "id"); @@ -69,7 +69,7 @@ public class TVMazeClient extends AbstractEpisodeListProvider { }).collect(toList()); } - protected SeriesInfo fetchSeriesInfo(TVMazeSearchResult show, SortOrder sortOrder, Locale locale) throws IOException { + protected SeriesInfo fetchSeriesInfo(TVMazeSearchResult show, SortOrder sortOrder, Locale locale) throws Exception { // e.g. http://api.tvmaze.com/shows/1 Object response = request("shows/" + show.getId()); @@ -110,8 +110,14 @@ public class TVMazeClient extends AbstractEpisodeListProvider { return new SeriesData(seriesInfo, episodes); } - public Object request(String resource) throws IOException { - return new CachedJsonResource("http://api.tvmaze.com/" + resource).getJsonObject(); + protected Object request(String resource) throws Exception { + Cache cache = Cache.getCache(getName(), CacheType.Monthly); + Resource json = cache.json(resource, s -> getResource(resource), Cache.ONE_DAY); + return json.get(); + } + + protected URL getResource(String resource) throws Exception { + return new URL("http://api.tvmaze.com/" + resource); } @Override diff --git a/source/net/filebot/web/TheTVDBClient.java b/source/net/filebot/web/TheTVDBClient.java index a028dbb4..a3c1dd5e 100644 --- a/source/net/filebot/web/TheTVDBClient.java +++ b/source/net/filebot/web/TheTVDBClient.java @@ -38,8 +38,6 @@ import org.w3c.dom.Node; public class TheTVDBClient extends AbstractEpisodeListProvider { - private final String host = "www.thetvdb.com"; - private final Map mirrors = MirrorType.newMap(); private final String apikey; @@ -301,10 +299,9 @@ public class TheTVDBClient extends AbstractEpisodeListProvider { } } - protected Document getXmlResource(MirrorType mirror, String path) throws Exception { + protected Document getXmlResource(MirrorType mirror, String resource) throws Exception { Cache cache = Cache.getCache(getName(), CacheType.Monthly); - Duration expirationTime = Duration.ofDays(1); - Resource xml = cache.xml(path, s -> getResource(mirror, s), expirationTime); + Resource xml = cache.xml(resource, s -> getResource(mirror, s), Cache.ONE_DAY); return xml.get(); } @@ -362,7 +359,7 @@ public class TheTVDBClient extends AbstractEpisodeListProvider { @Override public URI getEpisodeListLink(SearchResult searchResult) { - return URI.create("http://" + host + "/?tab=seasonall&id=" + ((TheTVDBSearchResult) searchResult).getSeriesId()); + return URI.create("http://www.thetvdb.com/?tab=seasonall&id=" + ((TheTVDBSearchResult) searchResult).getSeriesId()); } /** diff --git a/test/net/filebot/web/TheTVDBClientTest.java b/test/net/filebot/web/TheTVDBClientTest.java index 13d99709..3bdc9f7a 100644 --- a/test/net/filebot/web/TheTVDBClientTest.java +++ b/test/net/filebot/web/TheTVDBClientTest.java @@ -15,7 +15,7 @@ import org.junit.Test; public class TheTVDBClientTest { - private TheTVDBClient thetvdb = new TheTVDBClient("BA864DEE427E384A"); + TheTVDBClient thetvdb = new TheTVDBClient("BA864DEE427E384A"); @Test public void search() throws Exception {