* use extensive caching for all TheTVDB data and request resources only if modified

source/ehcache.xml

@@ -51,7 +51,7 @@
 	-->
 	<cache name="web-persistent-datasource"
 		maxElementsInMemory="50"
-		maxElementsOnDisk="5000"
+		maxElementsOnDisk="50000"
 		eternal="false"
 		timeToIdleSeconds="5259000"
 		timeToLiveSeconds="5259000"

source/net/sourceforge/filebot/web/AbstractCachedResource.java

@@ -0,0 +1,135 @@
+package net.sourceforge.filebot.web;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URL;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import net.sf.ehcache.Cache;
+import net.sf.ehcache.Element;
+
+public abstract class AbstractCachedResource<R, T extends Serializable> {
+
+	private String resource;
+	private Class<T> type;
+	private long expirationTime;
+
+	private int retryCountLimit;
+	private long retryWaitTime;
+
+	public AbstractCachedResource(String resource, Class<T> type, long expirationTime, int retryCountLimit, long retryWaitTime) {
+		this.resource = resource;
+		this.type = type;
+		this.expirationTime = expirationTime;
+		this.retryCountLimit = retryCountLimit;
+		this.retryWaitTime = retryWaitTime;
+	}
+
+	/**
+	 * Convert resource data into usable data
+	 */
+	protected abstract R fetchData(URL url, long lastModified) throws IOException;
+
+	protected abstract T process(R data) throws Exception;
+
+	protected abstract Cache getCache();
+
+	public synchronized T get() throws IOException {
+		String cacheKey = type.getName() + ":" + resource.toString();
+		Element element = null;
+		long lastUpdateTime = 0;
+
+		try {
+			element = getCache().get(cacheKey);
+
+			// sanity check ehcache diskcache problems
+			if (element != null && !cacheKey.equals(element.getKey().toString())) {
+				element = null;
+			}
+
+			if (element != null) {
+				lastUpdateTime = element.getLatestOfCreationAndUpdateTime();
+			}
+
+			// fetch from cache
+			if (element != null && System.currentTimeMillis() - lastUpdateTime < expirationTime) {
+				return type.cast(element.getValue());
+			}
+		} catch (Exception e) {
+			Logger.getLogger(getClass().getName()).log(Level.FINEST, e.getMessage());
+		}
+
+		// fetch and process resource
+		R data = null;
+		T product = null;
+		IOException networkException = null;
+
+		try {
+			long lastModified = element != null ? lastUpdateTime : 0;
+			URL url = getResourceLocation(resource);
+			data = fetch(url, lastModified, element != null ? 0 : retryCountLimit);
+		} catch (IOException e) {
+			networkException = e;
+		} catch (InterruptedException e) {
+			throw new RuntimeException(e);
+		}
+
+		if (data != null) {
+			try {
+				product = process(data);
+				element = new Element(cacheKey, product);
+			} catch (Exception e) {
+				throw new IOException(e);
+			}
+		} else {
+			try {
+				if (element != null) {
+					product = type.cast(element.getValue());
+				}
+			} catch (Exception e) {
+				Logger.getLogger(getClass().getName()).log(Level.FINEST, e.getMessage());
+			}
+		}
+
+		try {
+			if (element != null) {
+				getCache().put(element);
+			}
+		} catch (Exception e) {
+			Logger.getLogger(getClass().getName()).log(Level.FINEST, e.getMessage());
+		}
+
+		// throw network error only if we can't use previously cached data
+		if (networkException != null) {
+			if (product == null) {
+				throw networkException;
+			}
+
+			// just log error and continue with cached data
+			Logger.getLogger(getClass().getName()).log(Level.WARNING, networkException.toString());
+		}
+
+		return product;
+	}
+
+	protected URL getResourceLocation(String resource) throws IOException {
+		return new URL(resource);
+	}
+
+	protected R fetch(URL url, long lastModified, int retries) throws IOException, InterruptedException {
+		for (int i = 0; retries < 0 || i <= retries; i++) {
+			try {
+				if (i > 0) {
+					Thread.sleep(retryWaitTime);
+				}
+				return fetchData(url, lastModified);
+			} catch (IOException e) {
+				if (i >= 0 && i >= retries) {
+					throw e;
+				}
+			}
+		}
+		return null; // can't happen
+	}
+}

source/net/sourceforge/filebot/web/CachedPage.java

@@ -1,45 +1,36 @@
-
 package net.sourceforge.filebot.web;
 
-
 import static net.sourceforge.filebot.web.WebRequest.*;
 import static net.sourceforge.tuned.FileUtilities.*;
 
 import java.io.IOException;
 import java.io.Reader;
 import java.net.URL;
-import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
 
 import net.sf.ehcache.Cache;
 import net.sf.ehcache.CacheManager;
 
-
+public class CachedPage extends AbstractCachedResource<String, String> {
-public class CachedPage extends CachedResource<String> {
 
 	public CachedPage(URL url) {
-		super(url.toString(), String.class, 2 * 24 * 60 * 60 * 1000); // 48h update interval
+		super(url.toString(), String.class, 24 * 60 * 60 * 1000, 0, 0); // 24h update interval
 	}
 
-	
 	@Override
 	protected Cache getCache() {
 		return CacheManager.getInstance().getCache("web-datasource");
 	}
 
-	
 	@Override
-	public String process(ByteBuffer data) throws Exception {
+	public String process(String data) throws Exception {
-		return Charset.forName("UTF-16BE").decode(data).toString();
+		return data;
 	}
 
-	
 	@Override
-	protected ByteBuffer fetchData(URL url, long lastModified) throws IOException {
+	protected String fetchData(URL url, long lastModified) throws IOException {
-		return Charset.forName("UTF-16BE").encode(readAll(openConnection(url)));
+		return readAll(openConnection(url));
 	}
 
-	
 	protected Reader openConnection(URL url) throws IOException {
 		return getReader(url.openConnection());
 	}

source/net/sourceforge/filebot/web/CachedResource.java

@@ -1,158 +1,35 @@
-
 package net.sourceforge.filebot.web;
 
-
-import static net.sourceforge.filebot.web.WebRequest.*;
-
 import java.io.IOException;
 import java.io.Serializable;
 import java.net.URL;
 import java.nio.ByteBuffer;
-import java.util.logging.Level;
-import java.util.logging.Logger;
 
 import net.sf.ehcache.Cache;
 import net.sf.ehcache.CacheManager;
-import net.sf.ehcache.Element;
-
-
-public abstract class CachedResource<T extends Serializable> {
-	
-	private String resource;
-	private Class<T> type;
-	private long expirationTime;
-	
-	private int retryCountLimit;
-	private long retryWaitTime;
 
+public abstract class CachedResource<T extends Serializable> extends AbstractCachedResource<ByteBuffer, T> {
 
 	public CachedResource(String resource, Class<T> type) {
 		this(resource, type, Long.MAX_VALUE);
 	}
 
-	
 	public CachedResource(String resource, Class<T> type, long expirationTime) {
-		this(resource, type, expirationTime, 2, 1000);
+		this(resource, type, expirationTime, 2, 1000); // 3 retries in 1s intervals by default
 	}
 
-	
 	public CachedResource(String resource, Class<T> type, long expirationTime, int retryCountLimit, long retryWaitTime) {
-		this.resource = resource;
+		super(resource, type, expirationTime, retryCountLimit, retryWaitTime);
-		this.type = type;
-		this.expirationTime = expirationTime;
-		this.retryCountLimit = retryCountLimit;
-		this.retryWaitTime = retryWaitTime;
 	}
 
-	
+	@Override
 	protected Cache getCache() {
 		return CacheManager.getInstance().getCache("web-persistent-datasource");
 	}
 
-	
+	@Override
 	protected ByteBuffer fetchData(URL url, long lastModified) throws IOException {
-		return fetchIfModified(url, lastModified);
+		return WebRequest.fetchIfModified(url, lastModified);
 	}
 
-	
-	/**
-	 * Convert resource data into usable data
-	 */
-	public abstract T process(ByteBuffer data) throws Exception;
-	
-	
-	public synchronized T get() throws IOException {
-		String cacheKey = type.getName() + ":" + resource.toString();
-		Element element = null;
-		long lastUpdateTime = 0;
-		
-		try {
-			element = getCache().get(cacheKey);
-			
-			// sanity check ehcache diskcache problems
-			if (element != null && !cacheKey.equals(element.getKey().toString())) {
-				element = null;
-			}
-			
-			if (element != null) {
-				lastUpdateTime = element.getLatestOfCreationAndUpdateTime();
-			}
-			
-			// fetch from cache
-			if (element != null && System.currentTimeMillis() - lastUpdateTime < expirationTime) {
-				return type.cast(element.getValue());
-			}
-		} catch (Exception e) {
-			Logger.getLogger(getClass().getName()).log(Level.FINEST, e.getMessage());
-		}
-		
-		// fetch and process resource
-		ByteBuffer data = null;
-		T product = null;
-		IOException networkException = null;
-		
-		try {
-			long lastModified = element != null ? lastUpdateTime : 0;
-			URL url = new URL(resource);
-			data = fetch(url, lastModified, element != null ? 0 : retryCountLimit);
-		} catch (IOException e) {
-			networkException = e;
-		} catch (InterruptedException e) {
-			throw new RuntimeException(e);
-		}
-		
-		if (data != null) {
-			try {
-				product = process(data);
-				element = new Element(cacheKey, product);
-			} catch (Exception e) {
-				throw new IOException(e);
-			}
-		} else {
-			try {
-				if (element != null) {
-					product = type.cast(element.getValue());
-				}
-			} catch (Exception e) {
-				Logger.getLogger(getClass().getName()).log(Level.FINEST, e.getMessage());
-			}
-		}
-		
-		try {
-			if (element != null) {
-				getCache().put(element);
-			}
-		} catch (Exception e) {
-			Logger.getLogger(getClass().getName()).log(Level.FINEST, e.getMessage());
-		}
-		
-		// throw network error only if we can't use previously cached data
-		if (networkException != null) {
-			if (product == null) {
-				throw networkException;
-			}
-			
-			// just log error and continue with cached data
-			Logger.getLogger(getClass().getName()).log(Level.WARNING, networkException.toString());
-		}
-		
-		return product;
-	}
-	
-	
-	protected ByteBuffer fetch(URL url, long lastModified, int retries) throws IOException, InterruptedException {
-		for (int i = 0; retries < 0 || i <= retries; i++) {
-			try {
-				if (i > 0) {
-					Thread.sleep(retryWaitTime);
-				}
-				return fetchData(url, lastModified);
-			} catch (IOException e) {
-				if (i >= 0 && i >= retries) {
-					throw e;
-				}
-			}
-		}
-		return null; // can't happen
-	}
 }

source/net/sourceforge/filebot/web/CachedXmlResource.java

@@ -0,0 +1,48 @@
+package net.sourceforge.filebot.web;
+
+import java.io.IOException;
+import java.net.URL;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+
+import net.sf.ehcache.Cache;
+import net.sf.ehcache.CacheManager;
+
+import org.w3c.dom.Document;
+import org.xml.sax.SAXException;
+
+public class CachedXmlResource extends AbstractCachedResource<String, String> {
+
+	public CachedXmlResource(String resource) {
+		super(resource, String.class, 24 * 60 * 60 * 1000, 2, 1000);
+	}
+
+	@Override
+	protected Cache getCache() {
+		return CacheManager.getInstance().getCache("web-persistent-datasource");
+	}
+
+	public Document getDocument() throws IOException {
+		try {
+			return WebRequest.getDocument(get());
+		} catch (SAXException e) {
+			throw new RuntimeException(e);
+		}
+	}
+
+	@Override
+	public String process(String data) throws Exception {
+		return data;
+	}
+
+	@Override
+	protected String fetchData(URL url, long lastModified) throws IOException {
+		ByteBuffer data = WebRequest.fetchIfModified(url, lastModified);
+
+		if (data == null)
+			return null; // not modified
+
+		return Charset.forName("UTF-8").decode(data).toString();
+	}
+
+}

source/net/sourceforge/filebot/web/TheTVDBClient.java

@@ -3,13 +3,17 @@ package net.sourceforge.filebot.web;
 import static java.util.Arrays.*;
 import static net.sourceforge.filebot.web.EpisodeUtilities.*;
 import static net.sourceforge.filebot.web.WebRequest.*;
+import static net.sourceforge.tuned.FileUtilities.*;
 import static net.sourceforge.tuned.XPathUtilities.*;
 
 import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStreamReader;
 import java.io.Serializable;
 import java.net.MalformedURLException;
 import java.net.URI;
 import java.net.URL;
+import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.EnumMap;
 import java.util.EnumSet;
@@ -25,12 +29,12 @@ import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
 
 import javax.swing.Icon;
-import javax.xml.parsers.DocumentBuilderFactory;
 
 import net.sourceforge.filebot.Cache;
 import net.sourceforge.filebot.ResourceManager;
 import net.sourceforge.filebot.web.TheTVDBClient.BannerDescriptor.BannerProperty;
 import net.sourceforge.filebot.web.TheTVDBClient.SeriesInfo.SeriesProperty;
+import net.sourceforge.tuned.ByteBufferInputStream;
 import net.sourceforge.tuned.FileUtilities;
 
 import org.w3c.dom.Document;
@@ -95,8 +99,7 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
 	@Override
 	public List<SearchResult> fetchSearchResult(String query, Locale locale) throws Exception {
 		// perform online search
-		URL url = getResource(MirrorType.SEARCH, "/api/GetSeries.php?seriesname=" + encode(query, true) + "&language=" + getLanguageCode(locale));
+		Document dom = getXmlResource(MirrorType.SEARCH, "/api/GetSeries.php?seriesname=" + encode(query, true) + "&language=" + getLanguageCode(locale));
-		Document dom = getDocument(url);
 
 		List<Node> nodes = selectNodes("Data/Series", dom);
 		Map<Integer, TheTVDBSearchResult> resultSet = new LinkedHashMap<Integer, TheTVDBSearchResult>();
@@ -182,31 +185,52 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
 		return episodes;
 	}
 
-	public Document getSeriesRecord(TheTVDBSearchResult searchResult, String languageCode) throws Exception {
+	public Document getSeriesRecord(final TheTVDBSearchResult searchResult, final String languageCode) throws Exception {
-		URL seriesRecord = getResource(MirrorType.ZIP, "/api/" + apikey + "/series/" + searchResult.getSeriesId() + "/all/" + languageCode + ".zip");
+		final String path = "/api/" + apikey + "/series/" + searchResult.getSeriesId() + "/all/" + languageCode + ".zip";
+		final MirrorType mirror = MirrorType.ZIP;
 
-		try {
+		CachedXmlResource record = new CachedXmlResource(path) {
-
+			@Override
-			ZipInputStream zipInputStream = new ZipInputStream(seriesRecord.openStream());
+			protected URL getResourceLocation(String resource) throws IOException {
-			ZipEntry zipEntry;
+				return getResourceURL(mirror, path);
-
-			try {
-				String seriesRecordName = languageCode + ".xml";
-
-				while ((zipEntry = zipInputStream.getNextEntry()) != null) {
-					if (seriesRecordName.equals(zipEntry.getName())) {
-						return DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(zipInputStream);
-					}
-				}
-
-				// zip file must contain the series record
-				throw new FileNotFoundException(String.format("Archive must contain %s: %s", seriesRecordName, seriesRecord));
-			} finally {
-				zipInputStream.close();
 			}
-		} catch (FileNotFoundException e) {
+
-			throw new FileNotFoundException(String.format("Series record not found: %s [%s]: %s", searchResult.getName(), languageCode, seriesRecord));
+			@Override
-		}
+			protected String fetchData(URL url, long lastModified) throws IOException {
+				try {
+					ByteBuffer data = WebRequest.fetchIfModified(url, lastModified);
+					if (data == null)
+						return null; // not modified
+
+					ZipInputStream zipInputStream = new ZipInputStream(new ByteBufferInputStream(data));
+					ZipEntry zipEntry;
+
+					try {
+						String seriesRecordName = languageCode + ".xml";
+
+						while ((zipEntry = zipInputStream.getNextEntry()) != null) {
+							if (seriesRecordName.equals(zipEntry.getName())) {
+								return readAll(new InputStreamReader(zipInputStream, "UTF-8"));
+							}
+						}
+
+						// zip file must contain the series record
+						throw new FileNotFoundException(String.format("Archive must contain %s: %s", seriesRecordName, getResourceURL(mirror, path)));
+					} finally {
+						zipInputStream.close();
+					}
+				} catch (FileNotFoundException e) {
+					throw new FileNotFoundException(String.format("Series record not found: %s [%s]: %s", searchResult.getName(), languageCode, getResourceURL(mirror, path)));
+				}
+			}
+
+			@Override
+			public String process(String data) throws Exception {
+				return data;
+			}
+		};
+
+		return record.getDocument();
 	}
 
 	public TheTVDBSearchResult lookupByID(int id, Locale locale) throws Exception {
@@ -216,10 +240,8 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
 		}
 
 		try {
-			URL baseRecordLocation = getResource(MirrorType.XML, "/api/" + apikey + "/series/" + id + "/all/" + getLanguageCode(locale) + ".xml");
+			Document dom = getXmlResource(MirrorType.XML, "/api/" + apikey + "/series/" + id + "/all/" + getLanguageCode(locale) + ".xml");
-			Document baseRecord = getDocument(baseRecordLocation);
+			String name = selectString("//SeriesName", dom);
-
-			String name = selectString("//SeriesName", baseRecord);
 
 			TheTVDBSearchResult series = new TheTVDBSearchResult(name, id);
 			getCache().putData("lookupByID", id, locale, series);
@@ -237,8 +259,7 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
 			return cachedItem;
 		}
 
-		URL query = getResource(null, "/api/GetSeriesByRemoteID.php?imdbid=" + imdbid + "&language=" + getLanguageCode(locale));
+		Document dom = getXmlResource(null, "/api/GetSeriesByRemoteID.php?imdbid=" + imdbid + "&language=" + getLanguageCode(locale));
-		Document dom = getDocument(query);
 
 		String id = selectString("//seriesid", dom);
 		String name = selectString("//SeriesName", dom);
@@ -256,7 +277,7 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
 		return URI.create("http://" + host + "/?tab=seasonall&id=" + ((TheTVDBSearchResult) searchResult).getSeriesId());
 	}
 
-	protected String getMirror(MirrorType mirrorType) throws Exception {
+	protected String getMirror(MirrorType mirrorType) throws IOException {
 		synchronized (mirrors) {
 			if (mirrors.isEmpty()) {
 				// try cache first
@@ -272,7 +293,7 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
 				}
 
 				// initialize mirrors
-				Document dom = getDocument(getResource(null, "/api/" + apikey + "/mirrors.xml"));
+				Document dom = getXmlResource(null, "/api/" + apikey + "/mirrors.xml");
 
 				// all mirrors by type
 				Map<MirrorType, List<String>> mirrorListMap = new EnumMap<MirrorType, List<String>>(MirrorType.class);
@@ -312,7 +333,19 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
 		}
 	}
 
-	protected URL getResource(MirrorType mirrorType, String path) throws Exception {
+	protected Document getXmlResource(final MirrorType mirrorType, final String path) throws IOException {
+		CachedXmlResource resource = new CachedXmlResource(path) {
+
+			protected URL getResourceLocation(String path) throws IOException {
+				return getResourceURL(mirrorType, path);
+			};
+		};
+
+		// fetch data or retrieve from cache
+		return resource.getDocument();
+	}
+
+	protected URL getResourceURL(MirrorType mirrorType, String path) throws IOException {
 		if (mirrorType != null) {
 			// use mirror
 			String mirror = getMirror(mirrorType);
@@ -373,13 +406,13 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
 			return cachedItem;
 		}
 
-		Document dom = getDocument(getResource(MirrorType.XML, "/api/" + apikey + "/series/" + searchResult.seriesId + "/" + getLanguageCode(locale) + ".xml"));
+		Document dom = getXmlResource(MirrorType.XML, "/api/" + apikey + "/series/" + searchResult.seriesId + "/" + getLanguageCode(locale) + ".xml");
 
 		Node node = selectNode("//Series", dom);
 		Map<SeriesProperty, String> fields = new EnumMap<SeriesProperty, String>(SeriesProperty.class);
 
 		// remember banner mirror
-		fields.put(SeriesProperty.BannerMirror, getResource(MirrorType.BANNER, "/banners/").toString());
+		fields.put(SeriesProperty.BannerMirror, getResourceURL(MirrorType.BANNER, "/banners/").toString());
 
 		// copy values from xml
 		for (SeriesProperty key : SeriesProperty.values()) {
@@ -603,7 +636,7 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
 			return asList(cachedList);
 		}
 
-		Document dom = getDocument(getResource(MirrorType.XML, "/api/" + apikey + "/series/" + series.seriesId + "/banners.xml"));
+		Document dom = getXmlResource(MirrorType.XML, "/api/" + apikey + "/series/" + series.seriesId + "/banners.xml");
 
 		List<Node> nodes = selectNodes("//Banner", dom);
 		List<BannerDescriptor> banners = new ArrayList<BannerDescriptor>();
@@ -613,7 +646,7 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
 				Map<BannerProperty, String> item = new EnumMap<BannerProperty, String>(BannerProperty.class);
 
 				// insert banner mirror
-				item.put(BannerProperty.BannerMirror, getResource(MirrorType.BANNER, "/banners/").toString());
+				item.put(BannerProperty.BannerMirror, getResourceURL(MirrorType.BANNER, "/banners/").toString());
 
 				// copy values from xml
 				for (BannerProperty key : BannerProperty.values()) {