mobeigi
/
filebot
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

* fully support ETag caching mechanism in TheMovieDB client

This commit is contained in:
Reinhard Pointner 2013-11-20 02:53:36 +00:00
parent f85d706dce
commit 28df8ff69a
6 changed files with 90 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
source/ehcache.xml
View File

@ -50,7 +50,7 @@
Very long-lived cache (4 months) anime/series lists, movie index, etc
-->
<cache name="web-persistent-datasource"
maxElementsInMemory="50"
maxElementsInMemory="200"
maxElementsOnDisk="50000"
eternal="false"
timeToIdleSeconds="10512000"

4
source/net/sourceforge/filebot/web/CachedXmlResource.java
View File

@ -17,6 +17,10 @@ public class CachedXmlResource extends AbstractCachedResource<String, String> {
super(resource, String.class, ONE_WEEK, 2, 1000);
}
public CachedXmlResource(String resource, long expirationTime, int retryCountLimit, long retryWaitTime) {
super(resource, String.class, expirationTime, retryCountLimit, retryWaitTime);
}
@Override
protected Cache getCache() {
return CacheManager.getInstance().getCache("web-persistent-datasource");

51
source/net/sourceforge/filebot/web/ETagCachedResource.java Normal file
View File

@ -0,0 +1,51 @@
package net.sourceforge.filebot.web;
import java.io.IOException;
import java.io.Serializable;
import java.net.URL;
import java.nio.ByteBuffer;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import net.sf.ehcache.Cache;
import net.sf.ehcache.CacheManager;
import net.sf.ehcache.Element;
public abstract class ETagCachedResource<T extends Serializable> extends CachedResource<T> {
public ETagCachedResource(String resource, Class<T> type) {
super(resource, type, ONE_WEEK, 2, 1000);
}
public ETagCachedResource(String resource, Class<T> type, long expirationTime, int retryCountLimit, long retryWaitTime) {
super(resource, type, expirationTime, retryCountLimit, retryWaitTime);
}
@Override
protected ByteBuffer fetchData(URL url, long lastModified) throws IOException {
String etagKey = "ETag" + ":" + url.toString();
Element etag = getCache().get(etagKey);
Map<String, String> requestParameters = new HashMap<String, String>();
if (etag != null && etag.getObjectValue() != null) {
requestParameters.put("If-None-Match", etag.getObjectValue().toString());
}
// If-Modified-Since must not be set if If-None-Match is set
Map<String, List<String>> responseHeaders = new HashMap<String, List<String>>();
ByteBuffer data = WebRequest.fetch(url, requestParameters.size() > 0 ? -1 : lastModified, requestParameters, responseHeaders);
if (data != null && responseHeaders.containsKey("ETag")) {
getCache().put(new Element(etagKey, responseHeaders.get("ETag").get(0)));
}
return data;
}
@Override
protected Cache getCache() {
return CacheManager.getInstance().getCache("web-persistent-datasource");
}
}

29
source/net/sourceforge/filebot/web/FloodLimit.java
View File

@ -1,36 +1,37 @@
package net.sourceforge.filebot.web;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
public class FloodLimit {
private final Semaphore permits;
private final ScheduledThreadPoolExecutor timer = new ScheduledThreadPoolExecutor(1);
private final long releaseDelay;
private final TimeUnit timeUnit;
public FloodLimit(int permitLimit, long releaseDelay, TimeUnit timeUnit) {
this.permits = new Semaphore(permitLimit, true);
this.releaseDelay = releaseDelay;
this.timeUnit = timeUnit;
}
public void acquirePermit() throws InterruptedException {
public ScheduledFuture<?> acquirePermit() throws InterruptedException {
permits.acquire();
timer.schedule(new ReleasePermit(), releaseDelay, timeUnit);
return timer.schedule(new ReleasePermit(), releaseDelay, timeUnit);
}
public void releaseNow(ScheduledFuture<?> future) {
if (future.cancel(false)) {
permits.release();
}
}
private class ReleasePermit implements Runnable {
@Override
public void run() {
permits.release();

20
source/net/sourceforge/filebot/web/TMDbClient.java
View File

@ -21,14 +21,13 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Scanner;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.swing.Icon;
import net.sf.ehcache.Cache;
import net.sf.ehcache.CacheManager;
import net.sourceforge.filebot.ResourceManager;
import net.sourceforge.filebot.web.TMDbClient.MovieInfo.MovieProperty;
import net.sourceforge.filebot.web.TMDbClient.Person.PersonProperty;
@ -250,7 +249,7 @@ public class TMDbClient implements MovieIdentificationService {
URL url = new URL("http", host, "/" + version + "/" + resource + "?" + encodeParameters(data, true));
CachedResource<String> json = new CachedResource<String>(url.toString(), String.class) {
CachedResource<String> json = new ETagCachedResource<String>(url.toString(), String.class) {
@Override
public String process(ByteBuffer data) throws Exception {
@ -260,21 +259,22 @@ public class TMDbClient implements MovieIdentificationService {
@Override
protected ByteBuffer fetchData(URL url, long lastModified) throws IOException {
try {
ScheduledFuture<?> permit = null;
if (limit != null) {
limit.acquirePermit();
permit = limit.acquirePermit();
}
return super.fetchData(url, lastModified);
ByteBuffer data = super.fetchData(url, lastModified);
if (data == null && limit != null && permit != null) {
limit.releaseNow(permit);
}
return data;
} catch (FileNotFoundException e) {
return ByteBuffer.allocate(0);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
}
@Override
protected Cache getCache() {
return CacheManager.getInstance().getCache("web-datasource");
}
};
JSONObject object = (JSONObject) JSONValue.parse(json.get());

12
source/net/sourceforge/filebot/web/WebRequest.java
View File

@ -16,6 +16,7 @@ import java.nio.charset.Charset;
import java.security.GeneralSecurityException;
import java.security.SecureRandom;
import java.security.cert.X509Certificate;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.logging.Level;
@ -110,14 +111,14 @@ public final class WebRequest {
}
public static ByteBuffer fetch(URL resource) throws IOException {
return fetch(resource, 0, null);
return fetch(resource, 0, null, null);
}
public static ByteBuffer fetchIfModified(URL resource, long ifModifiedSince) throws IOException {
return fetch(resource, ifModifiedSince, null);
return fetch(resource, ifModifiedSince, null, null);
}
public static ByteBuffer fetch(URL url, long ifModifiedSince, Map<String, String> requestParameters) throws IOException {
public static ByteBuffer fetch(URL url, long ifModifiedSince, Map<String, String> requestParameters, Map<String, List<String>> responseParameters) throws IOException {
URLConnection connection = url.openConnection();
if (ifModifiedSince > 0) {
connection.setIfModifiedSince(ifModifiedSince);
@ -146,6 +147,11 @@ public final class WebRequest {
in = new InflaterInputStream(in, new Inflater(true));
}
// store response headers
if (responseParameters != null) {
responseParameters.putAll(connection.getHeaderFields());
}
ByteBufferOutputStream buffer = new ByteBufferOutputStream(contentLength >= 0 ? contentLength : 4 * 1024);
try {
// read all