From 1fea44ad9e8340e8833bc909b9fbf07c8e9da461 Mon Sep 17 00:00:00 2001 From: Reinhard Pointner Date: Tue, 15 Oct 2013 07:12:42 +0000 Subject: [PATCH] * use extensive caching for all TheTVDB data and request resources only if modified --- source/ehcache.xml | 2 +- .../filebot/web/AbstractCachedResource.java | 135 ++++++++++++++++ .../sourceforge/filebot/web/CachedPage.java | 31 ++-- .../filebot/web/CachedResource.java | 145 ++---------------- .../filebot/web/CachedXmlResource.java | 48 ++++++ .../filebot/web/TheTVDBClient.java | 111 +++++++++----- 6 files changed, 278 insertions(+), 194 deletions(-) create mode 100644 source/net/sourceforge/filebot/web/AbstractCachedResource.java create mode 100644 source/net/sourceforge/filebot/web/CachedXmlResource.java diff --git a/source/ehcache.xml b/source/ehcache.xml index 5ebea897..692b3677 100644 --- a/source/ehcache.xml +++ b/source/ehcache.xml @@ -51,7 +51,7 @@ --> { + + private String resource; + private Class type; + private long expirationTime; + + private int retryCountLimit; + private long retryWaitTime; + + public AbstractCachedResource(String resource, Class type, long expirationTime, int retryCountLimit, long retryWaitTime) { + this.resource = resource; + this.type = type; + this.expirationTime = expirationTime; + this.retryCountLimit = retryCountLimit; + this.retryWaitTime = retryWaitTime; + } + + /** + * Convert resource data into usable data + */ + protected abstract R fetchData(URL url, long lastModified) throws IOException; + + protected abstract T process(R data) throws Exception; + + protected abstract Cache getCache(); + + public synchronized T get() throws IOException { + String cacheKey = type.getName() + ":" + resource.toString(); + Element element = null; + long lastUpdateTime = 0; + + try { + element = getCache().get(cacheKey); + + // sanity check ehcache diskcache problems + if (element != null && !cacheKey.equals(element.getKey().toString())) { + element = null; + } + + if (element != null) { + lastUpdateTime = element.getLatestOfCreationAndUpdateTime(); + } + + // fetch from cache + if (element != null && System.currentTimeMillis() - lastUpdateTime < expirationTime) { + return type.cast(element.getValue()); + } + } catch (Exception e) { + Logger.getLogger(getClass().getName()).log(Level.FINEST, e.getMessage()); + } + + // fetch and process resource + R data = null; + T product = null; + IOException networkException = null; + + try { + long lastModified = element != null ? lastUpdateTime : 0; + URL url = getResourceLocation(resource); + data = fetch(url, lastModified, element != null ? 0 : retryCountLimit); + } catch (IOException e) { + networkException = e; + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + + if (data != null) { + try { + product = process(data); + element = new Element(cacheKey, product); + } catch (Exception e) { + throw new IOException(e); + } + } else { + try { + if (element != null) { + product = type.cast(element.getValue()); + } + } catch (Exception e) { + Logger.getLogger(getClass().getName()).log(Level.FINEST, e.getMessage()); + } + } + + try { + if (element != null) { + getCache().put(element); + } + } catch (Exception e) { + Logger.getLogger(getClass().getName()).log(Level.FINEST, e.getMessage()); + } + + // throw network error only if we can't use previously cached data + if (networkException != null) { + if (product == null) { + throw networkException; + } + + // just log error and continue with cached data + Logger.getLogger(getClass().getName()).log(Level.WARNING, networkException.toString()); + } + + return product; + } + + protected URL getResourceLocation(String resource) throws IOException { + return new URL(resource); + } + + protected R fetch(URL url, long lastModified, int retries) throws IOException, InterruptedException { + for (int i = 0; retries < 0 || i <= retries; i++) { + try { + if (i > 0) { + Thread.sleep(retryWaitTime); + } + return fetchData(url, lastModified); + } catch (IOException e) { + if (i >= 0 && i >= retries) { + throw e; + } + } + } + return null; // can't happen + } +} diff --git a/source/net/sourceforge/filebot/web/CachedPage.java b/source/net/sourceforge/filebot/web/CachedPage.java index 4ae9ef17..f8a5ef4d 100644 --- a/source/net/sourceforge/filebot/web/CachedPage.java +++ b/source/net/sourceforge/filebot/web/CachedPage.java @@ -1,47 +1,38 @@ - package net.sourceforge.filebot.web; - import static net.sourceforge.filebot.web.WebRequest.*; import static net.sourceforge.tuned.FileUtilities.*; import java.io.IOException; import java.io.Reader; import java.net.URL; -import java.nio.ByteBuffer; -import java.nio.charset.Charset; import net.sf.ehcache.Cache; import net.sf.ehcache.CacheManager; +public class CachedPage extends AbstractCachedResource { -public class CachedPage extends CachedResource { - public CachedPage(URL url) { - super(url.toString(), String.class, 2 * 24 * 60 * 60 * 1000); // 48h update interval + super(url.toString(), String.class, 24 * 60 * 60 * 1000, 0, 0); // 24h update interval } - - + @Override protected Cache getCache() { return CacheManager.getInstance().getCache("web-datasource"); } - - + @Override - public String process(ByteBuffer data) throws Exception { - return Charset.forName("UTF-16BE").decode(data).toString(); + public String process(String data) throws Exception { + return data; } - - + @Override - protected ByteBuffer fetchData(URL url, long lastModified) throws IOException { - return Charset.forName("UTF-16BE").encode(readAll(openConnection(url))); + protected String fetchData(URL url, long lastModified) throws IOException { + return readAll(openConnection(url)); } - - + protected Reader openConnection(URL url) throws IOException { return getReader(url.openConnection()); } - + } diff --git a/source/net/sourceforge/filebot/web/CachedResource.java b/source/net/sourceforge/filebot/web/CachedResource.java index ea919131..f710fbe0 100644 --- a/source/net/sourceforge/filebot/web/CachedResource.java +++ b/source/net/sourceforge/filebot/web/CachedResource.java @@ -1,158 +1,35 @@ - package net.sourceforge.filebot.web; - -import static net.sourceforge.filebot.web.WebRequest.*; - import java.io.IOException; import java.io.Serializable; import java.net.URL; import java.nio.ByteBuffer; -import java.util.logging.Level; -import java.util.logging.Logger; import net.sf.ehcache.Cache; import net.sf.ehcache.CacheManager; -import net.sf.ehcache.Element; +public abstract class CachedResource extends AbstractCachedResource { -public abstract class CachedResource { - - private String resource; - private Class type; - private long expirationTime; - - private int retryCountLimit; - private long retryWaitTime; - - public CachedResource(String resource, Class type) { this(resource, type, Long.MAX_VALUE); } - - + public CachedResource(String resource, Class type, long expirationTime) { - this(resource, type, expirationTime, 2, 1000); + this(resource, type, expirationTime, 2, 1000); // 3 retries in 1s intervals by default } - - + public CachedResource(String resource, Class type, long expirationTime, int retryCountLimit, long retryWaitTime) { - this.resource = resource; - this.type = type; - this.expirationTime = expirationTime; - this.retryCountLimit = retryCountLimit; - this.retryWaitTime = retryWaitTime; + super(resource, type, expirationTime, retryCountLimit, retryWaitTime); } - - + + @Override protected Cache getCache() { return CacheManager.getInstance().getCache("web-persistent-datasource"); } - - + + @Override protected ByteBuffer fetchData(URL url, long lastModified) throws IOException { - return fetchIfModified(url, lastModified); - } - - - /** - * Convert resource data into usable data - */ - public abstract T process(ByteBuffer data) throws Exception; - - - public synchronized T get() throws IOException { - String cacheKey = type.getName() + ":" + resource.toString(); - Element element = null; - long lastUpdateTime = 0; - - try { - element = getCache().get(cacheKey); - - // sanity check ehcache diskcache problems - if (element != null && !cacheKey.equals(element.getKey().toString())) { - element = null; - } - - if (element != null) { - lastUpdateTime = element.getLatestOfCreationAndUpdateTime(); - } - - // fetch from cache - if (element != null && System.currentTimeMillis() - lastUpdateTime < expirationTime) { - return type.cast(element.getValue()); - } - } catch (Exception e) { - Logger.getLogger(getClass().getName()).log(Level.FINEST, e.getMessage()); - } - - // fetch and process resource - ByteBuffer data = null; - T product = null; - IOException networkException = null; - - try { - long lastModified = element != null ? lastUpdateTime : 0; - URL url = new URL(resource); - data = fetch(url, lastModified, element != null ? 0 : retryCountLimit); - } catch (IOException e) { - networkException = e; - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - - if (data != null) { - try { - product = process(data); - element = new Element(cacheKey, product); - } catch (Exception e) { - throw new IOException(e); - } - } else { - try { - if (element != null) { - product = type.cast(element.getValue()); - } - } catch (Exception e) { - Logger.getLogger(getClass().getName()).log(Level.FINEST, e.getMessage()); - } - } - - try { - if (element != null) { - getCache().put(element); - } - } catch (Exception e) { - Logger.getLogger(getClass().getName()).log(Level.FINEST, e.getMessage()); - } - - // throw network error only if we can't use previously cached data - if (networkException != null) { - if (product == null) { - throw networkException; - } - - // just log error and continue with cached data - Logger.getLogger(getClass().getName()).log(Level.WARNING, networkException.toString()); - } - - return product; - } - - - protected ByteBuffer fetch(URL url, long lastModified, int retries) throws IOException, InterruptedException { - for (int i = 0; retries < 0 || i <= retries; i++) { - try { - if (i > 0) { - Thread.sleep(retryWaitTime); - } - return fetchData(url, lastModified); - } catch (IOException e) { - if (i >= 0 && i >= retries) { - throw e; - } - } - } - return null; // can't happen + return WebRequest.fetchIfModified(url, lastModified); } + } diff --git a/source/net/sourceforge/filebot/web/CachedXmlResource.java b/source/net/sourceforge/filebot/web/CachedXmlResource.java new file mode 100644 index 00000000..4684e486 --- /dev/null +++ b/source/net/sourceforge/filebot/web/CachedXmlResource.java @@ -0,0 +1,48 @@ +package net.sourceforge.filebot.web; + +import java.io.IOException; +import java.net.URL; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; + +import net.sf.ehcache.Cache; +import net.sf.ehcache.CacheManager; + +import org.w3c.dom.Document; +import org.xml.sax.SAXException; + +public class CachedXmlResource extends AbstractCachedResource { + + public CachedXmlResource(String resource) { + super(resource, String.class, 24 * 60 * 60 * 1000, 2, 1000); + } + + @Override + protected Cache getCache() { + return CacheManager.getInstance().getCache("web-persistent-datasource"); + } + + public Document getDocument() throws IOException { + try { + return WebRequest.getDocument(get()); + } catch (SAXException e) { + throw new RuntimeException(e); + } + } + + @Override + public String process(String data) throws Exception { + return data; + } + + @Override + protected String fetchData(URL url, long lastModified) throws IOException { + ByteBuffer data = WebRequest.fetchIfModified(url, lastModified); + + if (data == null) + return null; // not modified + + return Charset.forName("UTF-8").decode(data).toString(); + } + +} diff --git a/source/net/sourceforge/filebot/web/TheTVDBClient.java b/source/net/sourceforge/filebot/web/TheTVDBClient.java index 75f6dda5..3adac03c 100644 --- a/source/net/sourceforge/filebot/web/TheTVDBClient.java +++ b/source/net/sourceforge/filebot/web/TheTVDBClient.java @@ -3,13 +3,17 @@ package net.sourceforge.filebot.web; import static java.util.Arrays.*; import static net.sourceforge.filebot.web.EpisodeUtilities.*; import static net.sourceforge.filebot.web.WebRequest.*; +import static net.sourceforge.tuned.FileUtilities.*; import static net.sourceforge.tuned.XPathUtilities.*; import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStreamReader; import java.io.Serializable; import java.net.MalformedURLException; import java.net.URI; import java.net.URL; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.EnumMap; import java.util.EnumSet; @@ -25,12 +29,12 @@ import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; import javax.swing.Icon; -import javax.xml.parsers.DocumentBuilderFactory; import net.sourceforge.filebot.Cache; import net.sourceforge.filebot.ResourceManager; import net.sourceforge.filebot.web.TheTVDBClient.BannerDescriptor.BannerProperty; import net.sourceforge.filebot.web.TheTVDBClient.SeriesInfo.SeriesProperty; +import net.sourceforge.tuned.ByteBufferInputStream; import net.sourceforge.tuned.FileUtilities; import org.w3c.dom.Document; @@ -95,8 +99,7 @@ public class TheTVDBClient extends AbstractEpisodeListProvider { @Override public List fetchSearchResult(String query, Locale locale) throws Exception { // perform online search - URL url = getResource(MirrorType.SEARCH, "/api/GetSeries.php?seriesname=" + encode(query, true) + "&language=" + getLanguageCode(locale)); - Document dom = getDocument(url); + Document dom = getXmlResource(MirrorType.SEARCH, "/api/GetSeries.php?seriesname=" + encode(query, true) + "&language=" + getLanguageCode(locale)); List nodes = selectNodes("Data/Series", dom); Map resultSet = new LinkedHashMap(); @@ -182,31 +185,52 @@ public class TheTVDBClient extends AbstractEpisodeListProvider { return episodes; } - public Document getSeriesRecord(TheTVDBSearchResult searchResult, String languageCode) throws Exception { - URL seriesRecord = getResource(MirrorType.ZIP, "/api/" + apikey + "/series/" + searchResult.getSeriesId() + "/all/" + languageCode + ".zip"); + public Document getSeriesRecord(final TheTVDBSearchResult searchResult, final String languageCode) throws Exception { + final String path = "/api/" + apikey + "/series/" + searchResult.getSeriesId() + "/all/" + languageCode + ".zip"; + final MirrorType mirror = MirrorType.ZIP; - try { - - ZipInputStream zipInputStream = new ZipInputStream(seriesRecord.openStream()); - ZipEntry zipEntry; - - try { - String seriesRecordName = languageCode + ".xml"; - - while ((zipEntry = zipInputStream.getNextEntry()) != null) { - if (seriesRecordName.equals(zipEntry.getName())) { - return DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(zipInputStream); - } - } - - // zip file must contain the series record - throw new FileNotFoundException(String.format("Archive must contain %s: %s", seriesRecordName, seriesRecord)); - } finally { - zipInputStream.close(); + CachedXmlResource record = new CachedXmlResource(path) { + @Override + protected URL getResourceLocation(String resource) throws IOException { + return getResourceURL(mirror, path); } - } catch (FileNotFoundException e) { - throw new FileNotFoundException(String.format("Series record not found: %s [%s]: %s", searchResult.getName(), languageCode, seriesRecord)); - } + + @Override + protected String fetchData(URL url, long lastModified) throws IOException { + try { + ByteBuffer data = WebRequest.fetchIfModified(url, lastModified); + if (data == null) + return null; // not modified + + ZipInputStream zipInputStream = new ZipInputStream(new ByteBufferInputStream(data)); + ZipEntry zipEntry; + + try { + String seriesRecordName = languageCode + ".xml"; + + while ((zipEntry = zipInputStream.getNextEntry()) != null) { + if (seriesRecordName.equals(zipEntry.getName())) { + return readAll(new InputStreamReader(zipInputStream, "UTF-8")); + } + } + + // zip file must contain the series record + throw new FileNotFoundException(String.format("Archive must contain %s: %s", seriesRecordName, getResourceURL(mirror, path))); + } finally { + zipInputStream.close(); + } + } catch (FileNotFoundException e) { + throw new FileNotFoundException(String.format("Series record not found: %s [%s]: %s", searchResult.getName(), languageCode, getResourceURL(mirror, path))); + } + } + + @Override + public String process(String data) throws Exception { + return data; + } + }; + + return record.getDocument(); } public TheTVDBSearchResult lookupByID(int id, Locale locale) throws Exception { @@ -216,10 +240,8 @@ public class TheTVDBClient extends AbstractEpisodeListProvider { } try { - URL baseRecordLocation = getResource(MirrorType.XML, "/api/" + apikey + "/series/" + id + "/all/" + getLanguageCode(locale) + ".xml"); - Document baseRecord = getDocument(baseRecordLocation); - - String name = selectString("//SeriesName", baseRecord); + Document dom = getXmlResource(MirrorType.XML, "/api/" + apikey + "/series/" + id + "/all/" + getLanguageCode(locale) + ".xml"); + String name = selectString("//SeriesName", dom); TheTVDBSearchResult series = new TheTVDBSearchResult(name, id); getCache().putData("lookupByID", id, locale, series); @@ -237,8 +259,7 @@ public class TheTVDBClient extends AbstractEpisodeListProvider { return cachedItem; } - URL query = getResource(null, "/api/GetSeriesByRemoteID.php?imdbid=" + imdbid + "&language=" + getLanguageCode(locale)); - Document dom = getDocument(query); + Document dom = getXmlResource(null, "/api/GetSeriesByRemoteID.php?imdbid=" + imdbid + "&language=" + getLanguageCode(locale)); String id = selectString("//seriesid", dom); String name = selectString("//SeriesName", dom); @@ -256,7 +277,7 @@ public class TheTVDBClient extends AbstractEpisodeListProvider { return URI.create("http://" + host + "/?tab=seasonall&id=" + ((TheTVDBSearchResult) searchResult).getSeriesId()); } - protected String getMirror(MirrorType mirrorType) throws Exception { + protected String getMirror(MirrorType mirrorType) throws IOException { synchronized (mirrors) { if (mirrors.isEmpty()) { // try cache first @@ -272,7 +293,7 @@ public class TheTVDBClient extends AbstractEpisodeListProvider { } // initialize mirrors - Document dom = getDocument(getResource(null, "/api/" + apikey + "/mirrors.xml")); + Document dom = getXmlResource(null, "/api/" + apikey + "/mirrors.xml"); // all mirrors by type Map> mirrorListMap = new EnumMap>(MirrorType.class); @@ -312,7 +333,19 @@ public class TheTVDBClient extends AbstractEpisodeListProvider { } } - protected URL getResource(MirrorType mirrorType, String path) throws Exception { + protected Document getXmlResource(final MirrorType mirrorType, final String path) throws IOException { + CachedXmlResource resource = new CachedXmlResource(path) { + + protected URL getResourceLocation(String path) throws IOException { + return getResourceURL(mirrorType, path); + }; + }; + + // fetch data or retrieve from cache + return resource.getDocument(); + } + + protected URL getResourceURL(MirrorType mirrorType, String path) throws IOException { if (mirrorType != null) { // use mirror String mirror = getMirror(mirrorType); @@ -373,13 +406,13 @@ public class TheTVDBClient extends AbstractEpisodeListProvider { return cachedItem; } - Document dom = getDocument(getResource(MirrorType.XML, "/api/" + apikey + "/series/" + searchResult.seriesId + "/" + getLanguageCode(locale) + ".xml")); + Document dom = getXmlResource(MirrorType.XML, "/api/" + apikey + "/series/" + searchResult.seriesId + "/" + getLanguageCode(locale) + ".xml"); Node node = selectNode("//Series", dom); Map fields = new EnumMap(SeriesProperty.class); // remember banner mirror - fields.put(SeriesProperty.BannerMirror, getResource(MirrorType.BANNER, "/banners/").toString()); + fields.put(SeriesProperty.BannerMirror, getResourceURL(MirrorType.BANNER, "/banners/").toString()); // copy values from xml for (SeriesProperty key : SeriesProperty.values()) { @@ -603,7 +636,7 @@ public class TheTVDBClient extends AbstractEpisodeListProvider { return asList(cachedList); } - Document dom = getDocument(getResource(MirrorType.XML, "/api/" + apikey + "/series/" + series.seriesId + "/banners.xml")); + Document dom = getXmlResource(MirrorType.XML, "/api/" + apikey + "/series/" + series.seriesId + "/banners.xml"); List nodes = selectNodes("//Banner", dom); List banners = new ArrayList(); @@ -613,7 +646,7 @@ public class TheTVDBClient extends AbstractEpisodeListProvider { Map item = new EnumMap(BannerProperty.class); // insert banner mirror - item.put(BannerProperty.BannerMirror, getResource(MirrorType.BANNER, "/banners/").toString()); + item.put(BannerProperty.BannerMirror, getResourceURL(MirrorType.BANNER, "/banners/").toString()); // copy values from xml for (BannerProperty key : BannerProperty.values()) {