* added caching for TMDB and IMDB

source/net/sourceforge/filebot/web/CachedPage.java

@@ -0,0 +1,47 @@
+
+package net.sourceforge.filebot.web;
+
+
+import static net.sourceforge.filebot.web.WebRequest.*;
+import static net.sourceforge.tuned.FileUtilities.*;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.net.URL;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+
+import net.sf.ehcache.Cache;
+import net.sf.ehcache.CacheManager;
+
+
+public class CachedPage extends CachedResource<String> {
+	
+	public CachedPage(URL url) {
+		super(url.toString(), String.class, 2 * 24 * 60 * 60 * 1000); // 48h update interval
+	}
+	
+	
+	@Override
+	protected Cache getCache() {
+		return CacheManager.getInstance().getCache("web-datasource");
+	}
+	
+	
+	@Override
+	public String process(ByteBuffer data) throws Exception {
+		return Charset.forName("UTF-16BE").decode(data).toString();
+	}
+	
+	
+	@Override
+	protected ByteBuffer fetchData(URL url, long lastModified) throws IOException {
+		return Charset.forName("UTF-16BE").encode(readAll(openConnection(url)));
+	}
+	
+	
+	protected Reader openConnection(URL url) throws IOException {
+		return getReader(url.openConnection());
+	}
+	
+}

source/net/sourceforge/filebot/web/CachedResource.java

@@ -18,20 +18,28 @@ import net.sf.ehcache.Element;
 
 public abstract class CachedResource<T extends Serializable> {
 	
-	private Cache cache;
 	private String resource;
 	private Class<T> type;
 	private long expirationTime;
 	
 	
 	public CachedResource(String resource, Class<T> type, long expirationTime) {
-		this.cache = CacheManager.getInstance().getCache("web-persistent-datasource");
 		this.resource = resource;
 		this.type = type;
 		this.expirationTime = expirationTime;
 	}
 	
 	
+	protected Cache getCache() {
+		return CacheManager.getInstance().getCache("web-persistent-datasource");
+	}
+	
+	
+	protected ByteBuffer fetchData(URL url, long lastModified) throws IOException {
+		return fetchIfModified(url, lastModified);
+	}
+	
+	
 	/**
 	 * Convert resource data into usable data
 	 */
@@ -44,7 +52,7 @@ public abstract class CachedResource<T extends Serializable> {
 		long lastUpdateTime = 0;
 		
 		try {
-			element = cache.get(cacheKey);
+			element = getCache().get(cacheKey);
 			if (element != null) {
 				lastUpdateTime = element.getLatestOfCreationAndUpdateTime();
 			}
@@ -58,7 +66,7 @@ public abstract class CachedResource<T extends Serializable> {
 		}
 		
 		// fetch and process resource
-		ByteBuffer data = fetchIfModified(new URL(resource), element != null ? lastUpdateTime : 0);
+		ByteBuffer data = fetchData(new URL(resource), element != null ? lastUpdateTime : 0);
 		
 		if (data != null) {
 			try {
@@ -69,7 +77,7 @@ public abstract class CachedResource<T extends Serializable> {
 		}
 		
 		try {
-			cache.put(element);
+			getCache().put(element);
 		} catch (Exception e) {
 			Logger.getLogger(getClass().getName()).log(Level.WARNING, e.getMessage());
 		}
@@ -77,4 +85,5 @@ public abstract class CachedResource<T extends Serializable> {
 		// update cached data and last-updated time
 		return type.cast(element.getValue());
 	}
+	
 }

source/net/sourceforge/filebot/web/IMDbClient.java

@@ -7,6 +7,7 @@ import static net.sourceforge.tuned.XPathUtilities.*;
 
 import java.io.File;
 import java.io.IOException;
+import java.io.Reader;
 import java.net.URL;
 import java.net.URLConnection;
 import java.util.ArrayList;
@@ -111,12 +112,20 @@ public class IMDbClient implements MovieIdentificationService {
 	
 	
 	protected Document parsePage(URL url) throws IOException, SAXException {
+		CachedPage page = new CachedPage(url) {
+			
+			@Override
+			protected Reader openConnection(URL url) throws IOException {
 				URLConnection connection = url.openConnection();
 				
 				// IMDb refuses default user agent (Java/1.6.0_12)
 				connection.addRequestProperty("User-Agent", "Mozilla");
 				
-		return getHtmlDocument(connection);
+				return getReader(connection);
+			}
+		};
+		
+		return getHtmlDocument(page.get());
 	}
 	
 	

source/net/sourceforge/filebot/web/TMDbClient.java

@@ -5,7 +5,6 @@ package net.sourceforge.filebot.web;
 import static java.util.Arrays.*;
 import static java.util.Collections.*;
 import static net.sourceforge.filebot.web.WebRequest.*;
-import static net.sourceforge.tuned.FileUtilities.*;
 import static net.sourceforge.tuned.XPathUtilities.*;
 
 import java.io.File;
@@ -29,9 +28,6 @@ import org.w3c.dom.Document;
 import org.w3c.dom.Node;
 import org.xml.sax.SAXException;
 
-import net.sf.ehcache.Cache;
-import net.sf.ehcache.CacheManager;
-import net.sf.ehcache.Element;
 import net.sourceforge.filebot.ResourceManager;
 import net.sourceforge.filebot.web.TMDbClient.Artwork.ArtworkProperty;
 import net.sourceforge.filebot.web.TMDbClient.MovieInfo.MovieProperty;
@@ -132,19 +128,7 @@ public class TMDbClient implements MovieIdentificationService {
 	
 	
 	protected Document fetchResource(String method, String parameter, Locale locale) throws IOException, SAXException {
-		URL url = getResourceLocation(method, parameter, locale);
+		return getDocument(new CachedPage(getResourceLocation(method, parameter, locale)).get());
-		
-		Cache cache = CacheManager.getInstance().getCache("web-persistent-datasource");
-		Element element = cache.get(url.toString());
-		if (element != null) {
-			return WebRequest.getDocument((String) element.getValue());
-		}
-		
-		String xml = readAll(WebRequest.getReader(url.openConnection()));
-		Document dom = getDocument(xml);
-		
-		cache.put(new Element(url.toString(), xml));
-		return dom;
 	}
 	
 	

source/net/sourceforge/filebot/web/WebRequest.java

@@ -87,6 +87,11 @@ public final class WebRequest {
 	}
 	
 	
+	public static Document getHtmlDocument(String html) throws SAXException, IOException {
+		return getHtmlDocument(new StringReader(html));
+	}
+	
+	
 	public static Document getDocument(URL url) throws IOException, SAXException {
 		return getDocument(url.openConnection());
 	}