* use extensive caching for all TheTVDB data and request resources only if modified

This commit is contained in:
Reinhard Pointner 2013-10-15 07:12:42 +00:00
parent a130725d74
commit 1fea44ad9e
6 changed files with 278 additions and 194 deletions

View File

@ -51,7 +51,7 @@
-->
<cache name="web-persistent-datasource"
maxElementsInMemory="50"
maxElementsOnDisk="5000"
maxElementsOnDisk="50000"
eternal="false"
timeToIdleSeconds="5259000"
timeToLiveSeconds="5259000"

View File

@ -0,0 +1,135 @@
package net.sourceforge.filebot.web;
import java.io.IOException;
import java.io.Serializable;
import java.net.URL;
import java.util.logging.Level;
import java.util.logging.Logger;
import net.sf.ehcache.Cache;
import net.sf.ehcache.Element;
public abstract class AbstractCachedResource<R, T extends Serializable> {
private String resource;
private Class<T> type;
private long expirationTime;
private int retryCountLimit;
private long retryWaitTime;
public AbstractCachedResource(String resource, Class<T> type, long expirationTime, int retryCountLimit, long retryWaitTime) {
this.resource = resource;
this.type = type;
this.expirationTime = expirationTime;
this.retryCountLimit = retryCountLimit;
this.retryWaitTime = retryWaitTime;
}
/**
* Convert resource data into usable data
*/
protected abstract R fetchData(URL url, long lastModified) throws IOException;
protected abstract T process(R data) throws Exception;
protected abstract Cache getCache();
public synchronized T get() throws IOException {
String cacheKey = type.getName() + ":" + resource.toString();
Element element = null;
long lastUpdateTime = 0;
try {
element = getCache().get(cacheKey);
// sanity check ehcache diskcache problems
if (element != null && !cacheKey.equals(element.getKey().toString())) {
element = null;
}
if (element != null) {
lastUpdateTime = element.getLatestOfCreationAndUpdateTime();
}
// fetch from cache
if (element != null && System.currentTimeMillis() - lastUpdateTime < expirationTime) {
return type.cast(element.getValue());
}
} catch (Exception e) {
Logger.getLogger(getClass().getName()).log(Level.FINEST, e.getMessage());
}
// fetch and process resource
R data = null;
T product = null;
IOException networkException = null;
try {
long lastModified = element != null ? lastUpdateTime : 0;
URL url = getResourceLocation(resource);
data = fetch(url, lastModified, element != null ? 0 : retryCountLimit);
} catch (IOException e) {
networkException = e;
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
if (data != null) {
try {
product = process(data);
element = new Element(cacheKey, product);
} catch (Exception e) {
throw new IOException(e);
}
} else {
try {
if (element != null) {
product = type.cast(element.getValue());
}
} catch (Exception e) {
Logger.getLogger(getClass().getName()).log(Level.FINEST, e.getMessage());
}
}
try {
if (element != null) {
getCache().put(element);
}
} catch (Exception e) {
Logger.getLogger(getClass().getName()).log(Level.FINEST, e.getMessage());
}
// throw network error only if we can't use previously cached data
if (networkException != null) {
if (product == null) {
throw networkException;
}
// just log error and continue with cached data
Logger.getLogger(getClass().getName()).log(Level.WARNING, networkException.toString());
}
return product;
}
protected URL getResourceLocation(String resource) throws IOException {
return new URL(resource);
}
protected R fetch(URL url, long lastModified, int retries) throws IOException, InterruptedException {
for (int i = 0; retries < 0 || i <= retries; i++) {
try {
if (i > 0) {
Thread.sleep(retryWaitTime);
}
return fetchData(url, lastModified);
} catch (IOException e) {
if (i >= 0 && i >= retries) {
throw e;
}
}
}
return null; // can't happen
}
}

View File

@ -1,47 +1,38 @@
package net.sourceforge.filebot.web;
import static net.sourceforge.filebot.web.WebRequest.*;
import static net.sourceforge.tuned.FileUtilities.*;
import java.io.IOException;
import java.io.Reader;
import java.net.URL;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import net.sf.ehcache.Cache;
import net.sf.ehcache.CacheManager;
public class CachedPage extends AbstractCachedResource<String, String> {
public class CachedPage extends CachedResource<String> {
public CachedPage(URL url) {
super(url.toString(), String.class, 2 * 24 * 60 * 60 * 1000); // 48h update interval
super(url.toString(), String.class, 24 * 60 * 60 * 1000, 0, 0); // 24h update interval
}
@Override
protected Cache getCache() {
return CacheManager.getInstance().getCache("web-datasource");
}
@Override
public String process(ByteBuffer data) throws Exception {
return Charset.forName("UTF-16BE").decode(data).toString();
public String process(String data) throws Exception {
return data;
}
@Override
protected ByteBuffer fetchData(URL url, long lastModified) throws IOException {
return Charset.forName("UTF-16BE").encode(readAll(openConnection(url)));
protected String fetchData(URL url, long lastModified) throws IOException {
return readAll(openConnection(url));
}
protected Reader openConnection(URL url) throws IOException {
return getReader(url.openConnection());
}
}

View File

@ -1,158 +1,35 @@
package net.sourceforge.filebot.web;
import static net.sourceforge.filebot.web.WebRequest.*;
import java.io.IOException;
import java.io.Serializable;
import java.net.URL;
import java.nio.ByteBuffer;
import java.util.logging.Level;
import java.util.logging.Logger;
import net.sf.ehcache.Cache;
import net.sf.ehcache.CacheManager;
import net.sf.ehcache.Element;
public abstract class CachedResource<T extends Serializable> extends AbstractCachedResource<ByteBuffer, T> {
public abstract class CachedResource<T extends Serializable> {
private String resource;
private Class<T> type;
private long expirationTime;
private int retryCountLimit;
private long retryWaitTime;
public CachedResource(String resource, Class<T> type) {
this(resource, type, Long.MAX_VALUE);
}
public CachedResource(String resource, Class<T> type, long expirationTime) {
this(resource, type, expirationTime, 2, 1000);
this(resource, type, expirationTime, 2, 1000); // 3 retries in 1s intervals by default
}
public CachedResource(String resource, Class<T> type, long expirationTime, int retryCountLimit, long retryWaitTime) {
this.resource = resource;
this.type = type;
this.expirationTime = expirationTime;
this.retryCountLimit = retryCountLimit;
this.retryWaitTime = retryWaitTime;
super(resource, type, expirationTime, retryCountLimit, retryWaitTime);
}
@Override
protected Cache getCache() {
return CacheManager.getInstance().getCache("web-persistent-datasource");
}
@Override
protected ByteBuffer fetchData(URL url, long lastModified) throws IOException {
return fetchIfModified(url, lastModified);
}
/**
* Convert resource data into usable data
*/
public abstract T process(ByteBuffer data) throws Exception;
public synchronized T get() throws IOException {
String cacheKey = type.getName() + ":" + resource.toString();
Element element = null;
long lastUpdateTime = 0;
try {
element = getCache().get(cacheKey);
// sanity check ehcache diskcache problems
if (element != null && !cacheKey.equals(element.getKey().toString())) {
element = null;
}
if (element != null) {
lastUpdateTime = element.getLatestOfCreationAndUpdateTime();
}
// fetch from cache
if (element != null && System.currentTimeMillis() - lastUpdateTime < expirationTime) {
return type.cast(element.getValue());
}
} catch (Exception e) {
Logger.getLogger(getClass().getName()).log(Level.FINEST, e.getMessage());
}
// fetch and process resource
ByteBuffer data = null;
T product = null;
IOException networkException = null;
try {
long lastModified = element != null ? lastUpdateTime : 0;
URL url = new URL(resource);
data = fetch(url, lastModified, element != null ? 0 : retryCountLimit);
} catch (IOException e) {
networkException = e;
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
if (data != null) {
try {
product = process(data);
element = new Element(cacheKey, product);
} catch (Exception e) {
throw new IOException(e);
}
} else {
try {
if (element != null) {
product = type.cast(element.getValue());
}
} catch (Exception e) {
Logger.getLogger(getClass().getName()).log(Level.FINEST, e.getMessage());
}
}
try {
if (element != null) {
getCache().put(element);
}
} catch (Exception e) {
Logger.getLogger(getClass().getName()).log(Level.FINEST, e.getMessage());
}
// throw network error only if we can't use previously cached data
if (networkException != null) {
if (product == null) {
throw networkException;
}
// just log error and continue with cached data
Logger.getLogger(getClass().getName()).log(Level.WARNING, networkException.toString());
}
return product;
}
protected ByteBuffer fetch(URL url, long lastModified, int retries) throws IOException, InterruptedException {
for (int i = 0; retries < 0 || i <= retries; i++) {
try {
if (i > 0) {
Thread.sleep(retryWaitTime);
}
return fetchData(url, lastModified);
} catch (IOException e) {
if (i >= 0 && i >= retries) {
throw e;
}
}
}
return null; // can't happen
return WebRequest.fetchIfModified(url, lastModified);
}
}

View File

@ -0,0 +1,48 @@
package net.sourceforge.filebot.web;
import java.io.IOException;
import java.net.URL;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import net.sf.ehcache.Cache;
import net.sf.ehcache.CacheManager;
import org.w3c.dom.Document;
import org.xml.sax.SAXException;
public class CachedXmlResource extends AbstractCachedResource<String, String> {
public CachedXmlResource(String resource) {
super(resource, String.class, 24 * 60 * 60 * 1000, 2, 1000);
}
@Override
protected Cache getCache() {
return CacheManager.getInstance().getCache("web-persistent-datasource");
}
public Document getDocument() throws IOException {
try {
return WebRequest.getDocument(get());
} catch (SAXException e) {
throw new RuntimeException(e);
}
}
@Override
public String process(String data) throws Exception {
return data;
}
@Override
protected String fetchData(URL url, long lastModified) throws IOException {
ByteBuffer data = WebRequest.fetchIfModified(url, lastModified);
if (data == null)
return null; // not modified
return Charset.forName("UTF-8").decode(data).toString();
}
}

View File

@ -3,13 +3,17 @@ package net.sourceforge.filebot.web;
import static java.util.Arrays.*;
import static net.sourceforge.filebot.web.EpisodeUtilities.*;
import static net.sourceforge.filebot.web.WebRequest.*;
import static net.sourceforge.tuned.FileUtilities.*;
import static net.sourceforge.tuned.XPathUtilities.*;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Serializable;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.EnumMap;
import java.util.EnumSet;
@ -25,12 +29,12 @@ import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import javax.swing.Icon;
import javax.xml.parsers.DocumentBuilderFactory;
import net.sourceforge.filebot.Cache;
import net.sourceforge.filebot.ResourceManager;
import net.sourceforge.filebot.web.TheTVDBClient.BannerDescriptor.BannerProperty;
import net.sourceforge.filebot.web.TheTVDBClient.SeriesInfo.SeriesProperty;
import net.sourceforge.tuned.ByteBufferInputStream;
import net.sourceforge.tuned.FileUtilities;
import org.w3c.dom.Document;
@ -95,8 +99,7 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
@Override
public List<SearchResult> fetchSearchResult(String query, Locale locale) throws Exception {
// perform online search
URL url = getResource(MirrorType.SEARCH, "/api/GetSeries.php?seriesname=" + encode(query, true) + "&language=" + getLanguageCode(locale));
Document dom = getDocument(url);
Document dom = getXmlResource(MirrorType.SEARCH, "/api/GetSeries.php?seriesname=" + encode(query, true) + "&language=" + getLanguageCode(locale));
List<Node> nodes = selectNodes("Data/Series", dom);
Map<Integer, TheTVDBSearchResult> resultSet = new LinkedHashMap<Integer, TheTVDBSearchResult>();
@ -182,31 +185,52 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
return episodes;
}
public Document getSeriesRecord(TheTVDBSearchResult searchResult, String languageCode) throws Exception {
URL seriesRecord = getResource(MirrorType.ZIP, "/api/" + apikey + "/series/" + searchResult.getSeriesId() + "/all/" + languageCode + ".zip");
public Document getSeriesRecord(final TheTVDBSearchResult searchResult, final String languageCode) throws Exception {
final String path = "/api/" + apikey + "/series/" + searchResult.getSeriesId() + "/all/" + languageCode + ".zip";
final MirrorType mirror = MirrorType.ZIP;
try {
ZipInputStream zipInputStream = new ZipInputStream(seriesRecord.openStream());
ZipEntry zipEntry;
try {
String seriesRecordName = languageCode + ".xml";
while ((zipEntry = zipInputStream.getNextEntry()) != null) {
if (seriesRecordName.equals(zipEntry.getName())) {
return DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(zipInputStream);
}
}
// zip file must contain the series record
throw new FileNotFoundException(String.format("Archive must contain %s: %s", seriesRecordName, seriesRecord));
} finally {
zipInputStream.close();
CachedXmlResource record = new CachedXmlResource(path) {
@Override
protected URL getResourceLocation(String resource) throws IOException {
return getResourceURL(mirror, path);
}
} catch (FileNotFoundException e) {
throw new FileNotFoundException(String.format("Series record not found: %s [%s]: %s", searchResult.getName(), languageCode, seriesRecord));
}
@Override
protected String fetchData(URL url, long lastModified) throws IOException {
try {
ByteBuffer data = WebRequest.fetchIfModified(url, lastModified);
if (data == null)
return null; // not modified
ZipInputStream zipInputStream = new ZipInputStream(new ByteBufferInputStream(data));
ZipEntry zipEntry;
try {
String seriesRecordName = languageCode + ".xml";
while ((zipEntry = zipInputStream.getNextEntry()) != null) {
if (seriesRecordName.equals(zipEntry.getName())) {
return readAll(new InputStreamReader(zipInputStream, "UTF-8"));
}
}
// zip file must contain the series record
throw new FileNotFoundException(String.format("Archive must contain %s: %s", seriesRecordName, getResourceURL(mirror, path)));
} finally {
zipInputStream.close();
}
} catch (FileNotFoundException e) {
throw new FileNotFoundException(String.format("Series record not found: %s [%s]: %s", searchResult.getName(), languageCode, getResourceURL(mirror, path)));
}
}
@Override
public String process(String data) throws Exception {
return data;
}
};
return record.getDocument();
}
public TheTVDBSearchResult lookupByID(int id, Locale locale) throws Exception {
@ -216,10 +240,8 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
}
try {
URL baseRecordLocation = getResource(MirrorType.XML, "/api/" + apikey + "/series/" + id + "/all/" + getLanguageCode(locale) + ".xml");
Document baseRecord = getDocument(baseRecordLocation);
String name = selectString("//SeriesName", baseRecord);
Document dom = getXmlResource(MirrorType.XML, "/api/" + apikey + "/series/" + id + "/all/" + getLanguageCode(locale) + ".xml");
String name = selectString("//SeriesName", dom);
TheTVDBSearchResult series = new TheTVDBSearchResult(name, id);
getCache().putData("lookupByID", id, locale, series);
@ -237,8 +259,7 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
return cachedItem;
}
URL query = getResource(null, "/api/GetSeriesByRemoteID.php?imdbid=" + imdbid + "&language=" + getLanguageCode(locale));
Document dom = getDocument(query);
Document dom = getXmlResource(null, "/api/GetSeriesByRemoteID.php?imdbid=" + imdbid + "&language=" + getLanguageCode(locale));
String id = selectString("//seriesid", dom);
String name = selectString("//SeriesName", dom);
@ -256,7 +277,7 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
return URI.create("http://" + host + "/?tab=seasonall&id=" + ((TheTVDBSearchResult) searchResult).getSeriesId());
}
protected String getMirror(MirrorType mirrorType) throws Exception {
protected String getMirror(MirrorType mirrorType) throws IOException {
synchronized (mirrors) {
if (mirrors.isEmpty()) {
// try cache first
@ -272,7 +293,7 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
}
// initialize mirrors
Document dom = getDocument(getResource(null, "/api/" + apikey + "/mirrors.xml"));
Document dom = getXmlResource(null, "/api/" + apikey + "/mirrors.xml");
// all mirrors by type
Map<MirrorType, List<String>> mirrorListMap = new EnumMap<MirrorType, List<String>>(MirrorType.class);
@ -312,7 +333,19 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
}
}
protected URL getResource(MirrorType mirrorType, String path) throws Exception {
protected Document getXmlResource(final MirrorType mirrorType, final String path) throws IOException {
CachedXmlResource resource = new CachedXmlResource(path) {
protected URL getResourceLocation(String path) throws IOException {
return getResourceURL(mirrorType, path);
};
};
// fetch data or retrieve from cache
return resource.getDocument();
}
protected URL getResourceURL(MirrorType mirrorType, String path) throws IOException {
if (mirrorType != null) {
// use mirror
String mirror = getMirror(mirrorType);
@ -373,13 +406,13 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
return cachedItem;
}
Document dom = getDocument(getResource(MirrorType.XML, "/api/" + apikey + "/series/" + searchResult.seriesId + "/" + getLanguageCode(locale) + ".xml"));
Document dom = getXmlResource(MirrorType.XML, "/api/" + apikey + "/series/" + searchResult.seriesId + "/" + getLanguageCode(locale) + ".xml");
Node node = selectNode("//Series", dom);
Map<SeriesProperty, String> fields = new EnumMap<SeriesProperty, String>(SeriesProperty.class);
// remember banner mirror
fields.put(SeriesProperty.BannerMirror, getResource(MirrorType.BANNER, "/banners/").toString());
fields.put(SeriesProperty.BannerMirror, getResourceURL(MirrorType.BANNER, "/banners/").toString());
// copy values from xml
for (SeriesProperty key : SeriesProperty.values()) {
@ -603,7 +636,7 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
return asList(cachedList);
}
Document dom = getDocument(getResource(MirrorType.XML, "/api/" + apikey + "/series/" + series.seriesId + "/banners.xml"));
Document dom = getXmlResource(MirrorType.XML, "/api/" + apikey + "/series/" + series.seriesId + "/banners.xml");
List<Node> nodes = selectNodes("//Banner", dom);
List<BannerDescriptor> banners = new ArrayList<BannerDescriptor>();
@ -613,7 +646,7 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
Map<BannerProperty, String> item = new EnumMap<BannerProperty, String>(BannerProperty.class);
// insert banner mirror
item.put(BannerProperty.BannerMirror, getResource(MirrorType.BANNER, "/banners/").toString());
item.put(BannerProperty.BannerMirror, getResourceURL(MirrorType.BANNER, "/banners/").toString());
// copy values from xml
for (BannerProperty key : BannerProperty.values()) {