From 28df8ff69a9d53ee13cb3a2920f448c2327e2539 Mon Sep 17 00:00:00 2001 From: Reinhard Pointner Date: Wed, 20 Nov 2013 02:53:36 +0000 Subject: [PATCH] * fully support ETag caching mechanism in TheMovieDB client --- source/ehcache.xml | 2 +- .../filebot/web/CachedXmlResource.java | 4 ++ .../filebot/web/ETagCachedResource.java | 51 +++++++++++++++++++ .../sourceforge/filebot/web/FloodLimit.java | 29 ++++++----- .../sourceforge/filebot/web/TMDbClient.java | 20 ++++---- .../sourceforge/filebot/web/WebRequest.java | 12 +++-- 6 files changed, 90 insertions(+), 28 deletions(-) create mode 100644 source/net/sourceforge/filebot/web/ETagCachedResource.java diff --git a/source/ehcache.xml b/source/ehcache.xml index 05e4e2aa..8100f96d 100644 --- a/source/ehcache.xml +++ b/source/ehcache.xml @@ -50,7 +50,7 @@ Very long-lived cache (4 months) anime/series lists, movie index, etc --> { super(resource, String.class, ONE_WEEK, 2, 1000); } + public CachedXmlResource(String resource, long expirationTime, int retryCountLimit, long retryWaitTime) { + super(resource, String.class, expirationTime, retryCountLimit, retryWaitTime); + } + @Override protected Cache getCache() { return CacheManager.getInstance().getCache("web-persistent-datasource"); diff --git a/source/net/sourceforge/filebot/web/ETagCachedResource.java b/source/net/sourceforge/filebot/web/ETagCachedResource.java new file mode 100644 index 00000000..98daaf26 --- /dev/null +++ b/source/net/sourceforge/filebot/web/ETagCachedResource.java @@ -0,0 +1,51 @@ +package net.sourceforge.filebot.web; + +import java.io.IOException; +import java.io.Serializable; +import java.net.URL; +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import net.sf.ehcache.Cache; +import net.sf.ehcache.CacheManager; +import net.sf.ehcache.Element; + +public abstract class ETagCachedResource extends CachedResource { + + public ETagCachedResource(String resource, Class type) { + super(resource, type, ONE_WEEK, 2, 1000); + } + + public ETagCachedResource(String resource, Class type, long expirationTime, int retryCountLimit, long retryWaitTime) { + super(resource, type, expirationTime, retryCountLimit, retryWaitTime); + } + + @Override + protected ByteBuffer fetchData(URL url, long lastModified) throws IOException { + String etagKey = "ETag" + ":" + url.toString(); + Element etag = getCache().get(etagKey); + + Map requestParameters = new HashMap(); + if (etag != null && etag.getObjectValue() != null) { + requestParameters.put("If-None-Match", etag.getObjectValue().toString()); + } + + // If-Modified-Since must not be set if If-None-Match is set + Map> responseHeaders = new HashMap>(); + ByteBuffer data = WebRequest.fetch(url, requestParameters.size() > 0 ? -1 : lastModified, requestParameters, responseHeaders); + + if (data != null && responseHeaders.containsKey("ETag")) { + getCache().put(new Element(etagKey, responseHeaders.get("ETag").get(0))); + } + + return data; + } + + @Override + protected Cache getCache() { + return CacheManager.getInstance().getCache("web-persistent-datasource"); + } + +} diff --git a/source/net/sourceforge/filebot/web/FloodLimit.java b/source/net/sourceforge/filebot/web/FloodLimit.java index 86af6fc5..0beab143 100644 --- a/source/net/sourceforge/filebot/web/FloodLimit.java +++ b/source/net/sourceforge/filebot/web/FloodLimit.java @@ -1,36 +1,37 @@ - package net.sourceforge.filebot.web; - +import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; - public class FloodLimit { - + private final Semaphore permits; private final ScheduledThreadPoolExecutor timer = new ScheduledThreadPoolExecutor(1); - + private final long releaseDelay; private final TimeUnit timeUnit; - - + public FloodLimit(int permitLimit, long releaseDelay, TimeUnit timeUnit) { this.permits = new Semaphore(permitLimit, true); this.releaseDelay = releaseDelay; this.timeUnit = timeUnit; } - - - public void acquirePermit() throws InterruptedException { + + public ScheduledFuture acquirePermit() throws InterruptedException { permits.acquire(); - timer.schedule(new ReleasePermit(), releaseDelay, timeUnit); + return timer.schedule(new ReleasePermit(), releaseDelay, timeUnit); } - - + + public void releaseNow(ScheduledFuture future) { + if (future.cancel(false)) { + permits.release(); + } + } + private class ReleasePermit implements Runnable { - + @Override public void run() { permits.release(); diff --git a/source/net/sourceforge/filebot/web/TMDbClient.java b/source/net/sourceforge/filebot/web/TMDbClient.java index 16ea1bfc..f5dd1b76 100644 --- a/source/net/sourceforge/filebot/web/TMDbClient.java +++ b/source/net/sourceforge/filebot/web/TMDbClient.java @@ -21,14 +21,13 @@ import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Scanner; +import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; import javax.swing.Icon; -import net.sf.ehcache.Cache; -import net.sf.ehcache.CacheManager; import net.sourceforge.filebot.ResourceManager; import net.sourceforge.filebot.web.TMDbClient.MovieInfo.MovieProperty; import net.sourceforge.filebot.web.TMDbClient.Person.PersonProperty; @@ -250,7 +249,7 @@ public class TMDbClient implements MovieIdentificationService { URL url = new URL("http", host, "/" + version + "/" + resource + "?" + encodeParameters(data, true)); - CachedResource json = new CachedResource(url.toString(), String.class) { + CachedResource json = new ETagCachedResource(url.toString(), String.class) { @Override public String process(ByteBuffer data) throws Exception { @@ -260,21 +259,22 @@ public class TMDbClient implements MovieIdentificationService { @Override protected ByteBuffer fetchData(URL url, long lastModified) throws IOException { try { + ScheduledFuture permit = null; if (limit != null) { - limit.acquirePermit(); + permit = limit.acquirePermit(); } - return super.fetchData(url, lastModified); + + ByteBuffer data = super.fetchData(url, lastModified); + if (data == null && limit != null && permit != null) { + limit.releaseNow(permit); + } + return data; } catch (FileNotFoundException e) { return ByteBuffer.allocate(0); } catch (InterruptedException e) { throw new RuntimeException(e); } } - - @Override - protected Cache getCache() { - return CacheManager.getInstance().getCache("web-datasource"); - } }; JSONObject object = (JSONObject) JSONValue.parse(json.get()); diff --git a/source/net/sourceforge/filebot/web/WebRequest.java b/source/net/sourceforge/filebot/web/WebRequest.java index d324ba61..a1dc934a 100644 --- a/source/net/sourceforge/filebot/web/WebRequest.java +++ b/source/net/sourceforge/filebot/web/WebRequest.java @@ -16,6 +16,7 @@ import java.nio.charset.Charset; import java.security.GeneralSecurityException; import java.security.SecureRandom; import java.security.cert.X509Certificate; +import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.logging.Level; @@ -110,14 +111,14 @@ public final class WebRequest { } public static ByteBuffer fetch(URL resource) throws IOException { - return fetch(resource, 0, null); + return fetch(resource, 0, null, null); } public static ByteBuffer fetchIfModified(URL resource, long ifModifiedSince) throws IOException { - return fetch(resource, ifModifiedSince, null); + return fetch(resource, ifModifiedSince, null, null); } - public static ByteBuffer fetch(URL url, long ifModifiedSince, Map requestParameters) throws IOException { + public static ByteBuffer fetch(URL url, long ifModifiedSince, Map requestParameters, Map> responseParameters) throws IOException { URLConnection connection = url.openConnection(); if (ifModifiedSince > 0) { connection.setIfModifiedSince(ifModifiedSince); @@ -146,6 +147,11 @@ public final class WebRequest { in = new InflaterInputStream(in, new Inflater(true)); } + // store response headers + if (responseParameters != null) { + responseParameters.putAll(connection.getHeaderFields()); + } + ByteBufferOutputStream buffer = new ByteBufferOutputStream(contentLength >= 0 ? contentLength : 4 * 1024); try { // read all