* unified caching for all providers

* added caching of search results
This commit is contained in:
Reinhard Pointner 2011-11-26 09:44:49 +00:00
parent 73cf4a9b2f
commit 2bf426dedd
8 changed files with 254 additions and 87 deletions

View File

@ -4,8 +4,8 @@ package net.sourceforge.filebot.web;
import static net.sourceforge.filebot.web.EpisodeUtilities.*;
import java.io.Serializable;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Locale;
import java.util.logging.Level;
@ -29,23 +29,54 @@ public abstract class AbstractEpisodeListProvider implements EpisodeListProvider
}
protected abstract List<SearchResult> fetchSearchResult(String query, Locale locale) throws Exception;
protected abstract List<Episode> fetchEpisodeList(SearchResult searchResult, Locale locale) throws Exception;
public List<SearchResult> search(String query) throws Exception {
return search(query, getDefaultLocale());
}
public List<SearchResult> search(String query, Locale locale) throws Exception {
ResultCache cache = getCache();
List<SearchResult> results = (cache != null) ? cache.getSearchResult(query, locale) : null;
if (results != null) {
return results;
}
// perform actual search
results = fetchSearchResult(query, locale);
// cache results and return
return (cache != null) ? cache.putSearchResult(query, locale, results) : results;
}
public List<Episode> getEpisodeList(SearchResult searchResult) throws Exception {
return getEpisodeList(searchResult, getDefaultLocale());
}
public List<Episode> getEpisodeList(SearchResult searchResult, int season) throws Exception {
return getEpisodeList(searchResult, season, getDefaultLocale());
public List<Episode> getEpisodeList(SearchResult searchResult, Locale locale) throws Exception {
ResultCache cache = getCache();
List<Episode> episodes = (cache != null) ? cache.getEpisodeList(searchResult, locale) : null;
if (episodes != null) {
return episodes;
}
// perform actual search
episodes = fetchEpisodeList(searchResult, locale);
// cache results and return
return (cache != null) ? cache.putEpisodeList(searchResult, locale, episodes) : episodes;
}
public Locale getDefaultLocale() {
return Locale.ENGLISH;
public List<Episode> getEpisodeList(SearchResult searchResult, int season) throws Exception {
return getEpisodeList(searchResult, season, getDefaultLocale());
}
@ -62,6 +93,16 @@ public abstract class AbstractEpisodeListProvider implements EpisodeListProvider
}
public Locale getDefaultLocale() {
return Locale.ENGLISH;
}
public ResultCache getCache() {
return null;
}
protected static class ResultCache {
private final String id;
@ -74,18 +115,25 @@ public abstract class AbstractEpisodeListProvider implements EpisodeListProvider
}
public void putSearchResult(String key, Collection<? extends SearchResult> value) {
try {
cache.put(new Element(key(id, "SearchResult", key), value.toArray(new SearchResult[0])));
} catch (Exception e) {
Logger.getLogger(AbstractEpisodeListProvider.class.getName()).log(Level.WARNING, e.getMessage(), e);
}
protected String normalize(String query) {
return query == null ? null : query.trim().toLowerCase();
}
public List<SearchResult> getSearchResult(String key) {
public <T extends SearchResult> List<T> putSearchResult(String query, Locale locale, List<T> value) {
try {
Element element = cache.get(key(id, "SearchResult", key));
cache.put(new Element(new Key(id, normalize(query), locale), value.toArray(new SearchResult[0])));
} catch (Exception e) {
Logger.getLogger(AbstractEpisodeListProvider.class.getName()).log(Level.WARNING, e.getMessage(), e);
}
return value;
}
public List<SearchResult> getSearchResult(String query, Locale locale) {
try {
Element element = cache.get(new Key(id, normalize(query), locale));
if (element != null) {
return Arrays.asList(((SearchResult[]) element.getValue()));
}
@ -97,18 +145,20 @@ public abstract class AbstractEpisodeListProvider implements EpisodeListProvider
}
public void putEpisodeList(int key, Locale language, List<Episode> episodes) {
public List<Episode> putEpisodeList(SearchResult key, Locale locale, List<Episode> episodes) {
try {
cache.put(new Element(key(id, "EpisodeList", key, language.getLanguage()), episodes.toArray(new Episode[0])));
cache.put(new Element(new Key(id, key, locale), episodes.toArray(new Episode[0])));
} catch (Exception e) {
Logger.getLogger(AbstractEpisodeListProvider.class.getName()).log(Level.WARNING, e.getMessage(), e);
}
return episodes;
}
public List<Episode> getEpisodeList(int key, Locale language) {
public List<Episode> getEpisodeList(SearchResult key, Locale locale) {
try {
Element element = cache.get(key(id, "EpisodeList", key, language.getLanguage()));
Element element = cache.get(new Key(id, key, locale));
if (element != null) {
return Arrays.asList((Episode[]) element.getValue());
}
@ -120,8 +170,30 @@ public abstract class AbstractEpisodeListProvider implements EpisodeListProvider
}
private String key(Object... key) {
return Arrays.toString(key);
private static class Key implements Serializable {
protected Object[] fields;
public Key(Object... fields) {
this.fields = fields;
}
@Override
public int hashCode() {
return Arrays.hashCode(fields);
}
@Override
public boolean equals(Object other) {
if (other instanceof Key) {
return Arrays.equals(this.fields, ((Key) other).fields);
}
return false;
}
}
}

View File

@ -33,7 +33,6 @@ import net.sourceforge.filebot.ResourceManager;
public class AnidbClient extends AbstractEpisodeListProvider {
private final String host = "anidb.net";
private final ResultCache cache = new ResultCache(host, CacheManager.getInstance().getCache("web-persistent-datasource"));
private final String client;
private final int clientver;
@ -69,8 +68,21 @@ public class AnidbClient extends AbstractEpisodeListProvider {
}
@Override
public ResultCache getCache() {
return new ResultCache(host, CacheManager.getInstance().getCache("web-persistent-datasource"));
}
@Override
public List<SearchResult> search(String query, final Locale locale) throws Exception {
// bypass automatic caching since search is based on locally cached data anyway
return fetchSearchResult(query, locale);
}
@Override
public List<SearchResult> fetchSearchResult(String query, final Locale locale) throws Exception {
LocalSearch<AnidbSearchResult> index = new LocalSearch<AnidbSearchResult>(getAnimeTitles()) {
@Override
@ -84,17 +96,12 @@ public class AnidbClient extends AbstractEpisodeListProvider {
@Override
public List<Episode> getEpisodeList(SearchResult searchResult, Locale language) throws Exception {
public List<Episode> fetchEpisodeList(SearchResult searchResult, Locale language) throws Exception {
AnidbSearchResult anime = (AnidbSearchResult) searchResult;
// e.g. http://api.anidb.net:9001/httpapi?request=anime&client=filebot&clientver=1&protover=1&aid=4521
URL url = new URL("http", "api." + host, 9001, "/httpapi?request=anime&client=" + client + "&clientver=" + clientver + "&protover=1&aid=" + anime.getAnimeId());
// try cache first
List<Episode> episodes = cache.getEpisodeList(anime.getAnimeId(), language);
if (episodes != null)
return episodes;
// get anime page as xml
Document dom = getDocument(url);
@ -105,7 +112,7 @@ public class AnidbClient extends AbstractEpisodeListProvider {
animeTitle = selectString("//titles/title[@type='main']", dom);
}
episodes = new ArrayList<Episode>(25);
List<Episode> episodes = new ArrayList<Episode>(25);
for (Node node : selectNodes("//episode", dom)) {
Integer number = getIntegerContent("epno", node);
@ -127,10 +134,7 @@ public class AnidbClient extends AbstractEpisodeListProvider {
sortEpisodes(episodes);
// sanity check
if (episodes.size() > 0) {
// populate cache
cache.putEpisodeList(anime.getAnimeId(), language, episodes);
} else {
if (episodes.isEmpty()) {
// anime page xml doesn't work sometimes
throw new RuntimeException(String.format("Failed to parse episode data from xml: %s (%d)", anime, anime.getAnimeId()));
}
@ -163,13 +167,15 @@ public class AnidbClient extends AbstractEpisodeListProvider {
}
@SuppressWarnings("unchecked")
protected List<AnidbSearchResult> getAnimeTitles() throws Exception {
URL url = new URL("http", host, "/api/animetitles.dat.gz");
ResultCache cache = getCache();
@SuppressWarnings("unchecked")
List<AnidbSearchResult> anime = (List) cache.getSearchResult(null);
if (anime != null)
List<AnidbSearchResult> anime = (List) cache.getSearchResult(null, Locale.ROOT);
if (anime != null) {
return anime;
}
// <aid>|<type>|<language>|<title>
// type: 1=primary title (one per anime), 2=synonyms (multiple per anime), 3=shorttitles (multiple per anime), 4=official title (one per language)
@ -216,9 +222,7 @@ public class AnidbClient extends AbstractEpisodeListProvider {
}
// populate cache
cache.putSearchResult(null, anime);
return anime;
return cache.putSearchResult(null, Locale.ROOT, anime);
}
@ -260,6 +264,23 @@ public class AnidbClient extends AbstractEpisodeListProvider {
public String getOfficialTitle(String key) {
return officialTitle != null ? officialTitle.get(key) : null;
}
@Override
public int hashCode() {
return aid;
}
@Override
public boolean equals(Object object) {
if (object instanceof AnidbSearchResult) {
AnidbSearchResult other = (AnidbSearchResult) object;
return this.aid == other.aid;
}
return false;
}
}
}

View File

@ -10,7 +10,12 @@ import java.util.Arrays;
public class HyperLink extends SearchResult {
private final URL url;
protected URL url;
protected HyperLink() {
// used by serializer
}
public HyperLink(String name, URL url) {

View File

@ -30,7 +30,6 @@ import net.sourceforge.filebot.ResourceManager;
public class IMDbClient extends AbstractEpisodeListProvider {
private final String host = "www.imdb.com";
private final ResultCache cache = new ResultCache(host, CacheManager.getInstance().getCache("web-datasource"));
@Override
@ -46,7 +45,13 @@ public class IMDbClient extends AbstractEpisodeListProvider {
@Override
public List<SearchResult> search(String query, Locale locale) throws IOException, SAXException {
public ResultCache getCache() {
return new ResultCache(host, CacheManager.getInstance().getCache("web-datasource"));
}
@Override
public List<SearchResult> fetchSearchResult(String query, Locale locale) throws IOException, SAXException {
URL searchUrl = new URL("http", host, "/find?s=tt&q=" + encode(query));
Document dom = getHtmlDocument(openConnection(searchUrl));
@ -80,19 +85,15 @@ public class IMDbClient extends AbstractEpisodeListProvider {
@Override
public List<Episode> getEpisodeList(SearchResult searchResult, Locale locale) throws IOException, SAXException {
public List<Episode> fetchEpisodeList(SearchResult searchResult, Locale locale) throws IOException, SAXException {
Movie movie = (Movie) searchResult;
List<Episode> episodes = cache.getEpisodeList(movie.getImdbId(), Locale.ROOT);
if (episodes != null)
return episodes;
Document dom = getHtmlDocument(openConnection(getEpisodeListLink(searchResult).toURL()));
String seriesName = normalizeName(selectString("//H1/A", dom));
Date year = new Date(movie.getYear(), 0, 0);
List<Node> nodes = selectNodes("//TABLE//H3/A[preceding-sibling::text()]", dom);
episodes = new ArrayList<Episode>(nodes.size());
List<Episode> episodes = new ArrayList<Episode>(nodes.size());
for (Node node : nodes) {
String title = getTextContent(node);
@ -107,7 +108,6 @@ public class IMDbClient extends AbstractEpisodeListProvider {
episodes.add(new Episode(seriesName, year, season, episode, title, null, null, airdate));
}
cache.putEpisodeList(movie.getImdbId(), Locale.ROOT, episodes);
return episodes;
}

View File

@ -7,8 +7,13 @@ import java.util.Arrays;
public class Movie extends SearchResult {
protected final int year;
protected final int imdbId;
protected int year;
protected int imdbId;
protected Movie() {
// used by serializer
}
public Movie(String name, int year, int imdbId) {

View File

@ -28,7 +28,6 @@ import net.sourceforge.filebot.ResourceManager;
public class SerienjunkiesClient extends AbstractEpisodeListProvider {
private final String host = "api.serienjunkies.de";
private final ResultCache cache = new ResultCache(host, CacheManager.getInstance().getCache("web-datasource"));
private final String apikey;
@ -57,7 +56,20 @@ public class SerienjunkiesClient extends AbstractEpisodeListProvider {
@Override
public List<SearchResult> search(String query, Locale locale) throws Exception {
public ResultCache getCache() {
return new ResultCache(host, CacheManager.getInstance().getCache("web-datasource"));
}
@Override
public List<SearchResult> search(String query, final Locale locale) throws Exception {
// bypass automatic caching since search is based on locally cached data anyway
return fetchSearchResult(query, locale);
}
@Override
public List<SearchResult> fetchSearchResult(String query, Locale locale) throws Exception {
LocalSearch<SerienjunkiesSearchResult> index = new LocalSearch<SerienjunkiesSearchResult>(getSeriesTitles()) {
@Override
@ -71,10 +83,13 @@ public class SerienjunkiesClient extends AbstractEpisodeListProvider {
protected List<SerienjunkiesSearchResult> getSeriesTitles() throws IOException {
ResultCache cache = getCache();
@SuppressWarnings("unchecked")
List<SerienjunkiesSearchResult> seriesList = (List) cache.getSearchResult(null);
if (seriesList != null)
List<SerienjunkiesSearchResult> seriesList = (List) cache.getSearchResult(null, Locale.ROOT);
if (seriesList != null) {
return seriesList;
}
// fetch series data
seriesList = new ArrayList<SerienjunkiesSearchResult>();
@ -95,23 +110,16 @@ public class SerienjunkiesClient extends AbstractEpisodeListProvider {
}
// populate cache
cache.putSearchResult(null, seriesList);
return seriesList;
return cache.putSearchResult(null, Locale.ROOT, seriesList);
}
@Override
public List<Episode> getEpisodeList(SearchResult searchResult, Locale locale) throws IOException {
public List<Episode> fetchEpisodeList(SearchResult searchResult, Locale locale) throws IOException {
SerienjunkiesSearchResult series = (SerienjunkiesSearchResult) searchResult;
// try cache first
List<Episode> episodes = cache.getEpisodeList(series.getSeriesId(), Locale.GERMAN);
if (episodes != null)
return episodes;
// fetch episode data
episodes = new ArrayList<Episode>(25);
List<Episode> episodes = new ArrayList<Episode>(25);
String seriesName = locale.equals(Locale.GERMAN) && series.getGermanTitle() != null ? series.getGermanTitle() : series.getMainTitle();
JSONObject data = (JSONObject) request("/allepisodes.php?d=" + apikey + "&q=" + series.getSeriesId());
@ -128,9 +136,6 @@ public class SerienjunkiesClient extends AbstractEpisodeListProvider {
episodes.add(new Episode(seriesName, series.getStartDate(), season, episode, title, i + 1, null, airdate));
}
// populate cache
cache.putEpisodeList(series.getSeriesId(), Locale.GERMAN, episodes);
// make sure episodes are in ordered correctly
sortEpisodes(episodes);
@ -224,6 +229,23 @@ public class SerienjunkiesClient extends AbstractEpisodeListProvider {
public Date getStartDate() {
return startDate;
}
@Override
public int hashCode() {
return sid;
}
@Override
public boolean equals(Object object) {
if (object instanceof SerienjunkiesSearchResult) {
SerienjunkiesSearchResult other = (SerienjunkiesSearchResult) object;
return this.sid == other.sid;
}
return false;
}
}
}

View File

@ -26,7 +26,6 @@ import net.sourceforge.filebot.ResourceManager;
public class TVRageClient extends AbstractEpisodeListProvider {
private final String host = "services.tvrage.com";
private final ResultCache cache = new ResultCache(host, CacheManager.getInstance().getCache("web-datasource"));
@Override
@ -42,7 +41,13 @@ public class TVRageClient extends AbstractEpisodeListProvider {
@Override
public List<SearchResult> search(String query, Locale locale) throws IOException, SAXException {
public ResultCache getCache() {
return new ResultCache(host, CacheManager.getInstance().getCache("web-datasource"));
}
@Override
public List<SearchResult> fetchSearchResult(String query, Locale locale) throws IOException, SAXException {
URL searchUrl = new URL("http", host, "/feeds/full_search.php?show=" + encode(query));
Document dom = getDocument(searchUrl);
@ -62,20 +67,16 @@ public class TVRageClient extends AbstractEpisodeListProvider {
@Override
public List<Episode> getEpisodeList(SearchResult searchResult, Locale locale) throws IOException, SAXException {
public List<Episode> fetchEpisodeList(SearchResult searchResult, Locale locale) throws IOException, SAXException {
TVRageSearchResult series = (TVRageSearchResult) searchResult;
List<Episode> episodes = cache.getEpisodeList(series.getSeriesId(), Locale.ENGLISH);
if (episodes != null)
return episodes;
URL episodeListUrl = new URL("http", host, "/feeds/full_show_info.php?sid=" + series.getSeriesId());
Document dom = getDocument(episodeListUrl);
String seriesName = selectString("Show/name", dom);
Date seriesStartDate = Date.parse(selectString("Show/started", dom), "MMM/dd/yyyy");
episodes = new ArrayList<Episode>(25);
List<Episode> episodes = new ArrayList<Episode>(25);
List<Episode> specials = new ArrayList<Episode>(5);
// episodes and specials
@ -101,7 +102,6 @@ public class TVRageClient extends AbstractEpisodeListProvider {
// add specials at the end
episodes.addAll(specials);
cache.putEpisodeList(series.getSeriesId(), Locale.ENGLISH, episodes);
return episodes;
}
@ -127,8 +127,13 @@ public class TVRageClient extends AbstractEpisodeListProvider {
public static class TVRageSearchResult extends SearchResult {
private final int showId;
private final String link;
protected int showId;
protected String link;
protected TVRageSearchResult() {
// used by serializer
}
public TVRageSearchResult(String name, int showId, String link) {
@ -147,6 +152,22 @@ public class TVRageClient extends AbstractEpisodeListProvider {
return link;
}
@Override
public int hashCode() {
return showId;
}
@Override
public boolean equals(Object object) {
if (object instanceof TVRageSearchResult) {
TVRageSearchResult other = (TVRageSearchResult) object;
return this.showId == other.showId;
}
return false;
}
}
}

View File

@ -37,7 +37,6 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
private final String host = "www.thetvdb.com";
private final Map<MirrorType, String> mirrors = new EnumMap<MirrorType, String>(MirrorType.class);
private final ResultCache cache = new ResultCache(host, CacheManager.getInstance().getCache("web-datasource"));
private final String apikey;
@ -75,7 +74,13 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
@Override
public List<SearchResult> search(String query, Locale language) throws Exception {
public ResultCache getCache() {
return new ResultCache(host, CacheManager.getInstance().getCache("web-datasource"));
}
@Override
public List<SearchResult> fetchSearchResult(String query, Locale language) throws Exception {
// perform online search
URL url = getResource(null, "/api/GetSeries.php?seriesname=" + encode(query) + "&language=" + language.getLanguage());
Document dom = getDocument(url);
@ -97,12 +102,8 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
@Override
public List<Episode> getEpisodeList(SearchResult searchResult, Locale language) throws Exception {
public List<Episode> fetchEpisodeList(SearchResult searchResult, Locale language) throws Exception {
TheTVDBSearchResult series = (TheTVDBSearchResult) searchResult;
List<Episode> episodes = cache.getEpisodeList(series.getSeriesId(), language);
if (episodes != null)
return episodes;
Document seriesRecord = getSeriesRecord(series, language);
@ -112,7 +113,7 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
List<Node> nodes = selectNodes("Data/Episode", seriesRecord);
episodes = new ArrayList<Episode>(nodes.size());
List<Episode> episodes = new ArrayList<Episode>(nodes.size());
List<Episode> specials = new ArrayList<Episode>(5);
for (Node node : nodes) {
@ -156,7 +157,6 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
// add specials at the end
episodes.addAll(specials);
cache.putEpisodeList(series.getSeriesId(), language, episodes);
return episodes;
}
@ -266,7 +266,12 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
public static class TheTVDBSearchResult extends SearchResult {
private final int seriesId;
protected int seriesId;
protected TheTVDBSearchResult() {
// used by serializer
}
public TheTVDBSearchResult(String seriesName, int seriesId) {
@ -279,6 +284,22 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
return seriesId;
}
@Override
public int hashCode() {
return seriesId;
}
@Override
public boolean equals(Object object) {
if (object instanceof TheTVDBSearchResult) {
TheTVDBSearchResult other = (TheTVDBSearchResult) object;
return this.seriesId == other.seriesId;
}
return false;
}
}