* added caching for TMDB and IMDB
This commit is contained in:
parent
43ee82a9fa
commit
f9c823581c
|
@ -0,0 +1,47 @@
|
|||
|
||||
package net.sourceforge.filebot.web;
|
||||
|
||||
|
||||
import static net.sourceforge.filebot.web.WebRequest.*;
|
||||
import static net.sourceforge.tuned.FileUtilities.*;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.Reader;
|
||||
import java.net.URL;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.charset.Charset;
|
||||
|
||||
import net.sf.ehcache.Cache;
|
||||
import net.sf.ehcache.CacheManager;
|
||||
|
||||
|
||||
public class CachedPage extends CachedResource<String> {
|
||||
|
||||
public CachedPage(URL url) {
|
||||
super(url.toString(), String.class, 2 * 24 * 60 * 60 * 1000); // 48h update interval
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
protected Cache getCache() {
|
||||
return CacheManager.getInstance().getCache("web-datasource");
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public String process(ByteBuffer data) throws Exception {
|
||||
return Charset.forName("UTF-16BE").decode(data).toString();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
protected ByteBuffer fetchData(URL url, long lastModified) throws IOException {
|
||||
return Charset.forName("UTF-16BE").encode(readAll(openConnection(url)));
|
||||
}
|
||||
|
||||
|
||||
protected Reader openConnection(URL url) throws IOException {
|
||||
return getReader(url.openConnection());
|
||||
}
|
||||
|
||||
}
|
|
@ -18,20 +18,28 @@ import net.sf.ehcache.Element;
|
|||
|
||||
public abstract class CachedResource<T extends Serializable> {
|
||||
|
||||
private Cache cache;
|
||||
private String resource;
|
||||
private Class<T> type;
|
||||
private long expirationTime;
|
||||
|
||||
|
||||
public CachedResource(String resource, Class<T> type, long expirationTime) {
|
||||
this.cache = CacheManager.getInstance().getCache("web-persistent-datasource");
|
||||
this.resource = resource;
|
||||
this.type = type;
|
||||
this.expirationTime = expirationTime;
|
||||
}
|
||||
|
||||
|
||||
protected Cache getCache() {
|
||||
return CacheManager.getInstance().getCache("web-persistent-datasource");
|
||||
}
|
||||
|
||||
|
||||
protected ByteBuffer fetchData(URL url, long lastModified) throws IOException {
|
||||
return fetchIfModified(url, lastModified);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Convert resource data into usable data
|
||||
*/
|
||||
|
@ -44,7 +52,7 @@ public abstract class CachedResource<T extends Serializable> {
|
|||
long lastUpdateTime = 0;
|
||||
|
||||
try {
|
||||
element = cache.get(cacheKey);
|
||||
element = getCache().get(cacheKey);
|
||||
if (element != null) {
|
||||
lastUpdateTime = element.getLatestOfCreationAndUpdateTime();
|
||||
}
|
||||
|
@ -58,7 +66,7 @@ public abstract class CachedResource<T extends Serializable> {
|
|||
}
|
||||
|
||||
// fetch and process resource
|
||||
ByteBuffer data = fetchIfModified(new URL(resource), element != null ? lastUpdateTime : 0);
|
||||
ByteBuffer data = fetchData(new URL(resource), element != null ? lastUpdateTime : 0);
|
||||
|
||||
if (data != null) {
|
||||
try {
|
||||
|
@ -69,7 +77,7 @@ public abstract class CachedResource<T extends Serializable> {
|
|||
}
|
||||
|
||||
try {
|
||||
cache.put(element);
|
||||
getCache().put(element);
|
||||
} catch (Exception e) {
|
||||
Logger.getLogger(getClass().getName()).log(Level.WARNING, e.getMessage());
|
||||
}
|
||||
|
@ -77,4 +85,5 @@ public abstract class CachedResource<T extends Serializable> {
|
|||
// update cached data and last-updated time
|
||||
return type.cast(element.getValue());
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -7,6 +7,7 @@ import static net.sourceforge.tuned.XPathUtilities.*;
|
|||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.Reader;
|
||||
import java.net.URL;
|
||||
import java.net.URLConnection;
|
||||
import java.util.ArrayList;
|
||||
|
@ -111,12 +112,20 @@ public class IMDbClient implements MovieIdentificationService {
|
|||
|
||||
|
||||
protected Document parsePage(URL url) throws IOException, SAXException {
|
||||
CachedPage page = new CachedPage(url) {
|
||||
|
||||
@Override
|
||||
protected Reader openConnection(URL url) throws IOException {
|
||||
URLConnection connection = url.openConnection();
|
||||
|
||||
// IMDb refuses default user agent (Java/1.6.0_12)
|
||||
connection.addRequestProperty("User-Agent", "Mozilla");
|
||||
|
||||
return getHtmlDocument(connection);
|
||||
return getReader(connection);
|
||||
}
|
||||
};
|
||||
|
||||
return getHtmlDocument(page.get());
|
||||
}
|
||||
|
||||
|
||||
|
|
|
@ -5,7 +5,6 @@ package net.sourceforge.filebot.web;
|
|||
import static java.util.Arrays.*;
|
||||
import static java.util.Collections.*;
|
||||
import static net.sourceforge.filebot.web.WebRequest.*;
|
||||
import static net.sourceforge.tuned.FileUtilities.*;
|
||||
import static net.sourceforge.tuned.XPathUtilities.*;
|
||||
|
||||
import java.io.File;
|
||||
|
@ -29,9 +28,6 @@ import org.w3c.dom.Document;
|
|||
import org.w3c.dom.Node;
|
||||
import org.xml.sax.SAXException;
|
||||
|
||||
import net.sf.ehcache.Cache;
|
||||
import net.sf.ehcache.CacheManager;
|
||||
import net.sf.ehcache.Element;
|
||||
import net.sourceforge.filebot.ResourceManager;
|
||||
import net.sourceforge.filebot.web.TMDbClient.Artwork.ArtworkProperty;
|
||||
import net.sourceforge.filebot.web.TMDbClient.MovieInfo.MovieProperty;
|
||||
|
@ -132,19 +128,7 @@ public class TMDbClient implements MovieIdentificationService {
|
|||
|
||||
|
||||
protected Document fetchResource(String method, String parameter, Locale locale) throws IOException, SAXException {
|
||||
URL url = getResourceLocation(method, parameter, locale);
|
||||
|
||||
Cache cache = CacheManager.getInstance().getCache("web-persistent-datasource");
|
||||
Element element = cache.get(url.toString());
|
||||
if (element != null) {
|
||||
return WebRequest.getDocument((String) element.getValue());
|
||||
}
|
||||
|
||||
String xml = readAll(WebRequest.getReader(url.openConnection()));
|
||||
Document dom = getDocument(xml);
|
||||
|
||||
cache.put(new Element(url.toString(), xml));
|
||||
return dom;
|
||||
return getDocument(new CachedPage(getResourceLocation(method, parameter, locale)).get());
|
||||
}
|
||||
|
||||
|
||||
|
|
|
@ -87,6 +87,11 @@ public final class WebRequest {
|
|||
}
|
||||
|
||||
|
||||
public static Document getHtmlDocument(String html) throws SAXException, IOException {
|
||||
return getHtmlDocument(new StringReader(html));
|
||||
}
|
||||
|
||||
|
||||
public static Document getDocument(URL url) throws IOException, SAXException {
|
||||
return getDocument(url.openConnection());
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue