From 2bf426dedd3eb33d4ec80ab8ccf695f944b152d7 Mon Sep 17 00:00:00 2001 From: Reinhard Pointner Date: Sat, 26 Nov 2011 09:44:49 +0000 Subject: [PATCH] * unified caching for all providers * added caching of search results --- .../web/AbstractEpisodeListProvider.java | 110 +++++++++++++++--- .../sourceforge/filebot/web/AnidbClient.java | 57 ++++++--- .../sourceforge/filebot/web/HyperLink.java | 7 +- .../sourceforge/filebot/web/IMDbClient.java | 18 +-- source/net/sourceforge/filebot/web/Movie.java | 9 +- .../filebot/web/SerienjunkiesClient.java | 56 ++++++--- .../sourceforge/filebot/web/TVRageClient.java | 43 +++++-- .../filebot/web/TheTVDBClient.java | 41 +++++-- 8 files changed, 254 insertions(+), 87 deletions(-) diff --git a/source/net/sourceforge/filebot/web/AbstractEpisodeListProvider.java b/source/net/sourceforge/filebot/web/AbstractEpisodeListProvider.java index 25a0b020..6060bc2b 100644 --- a/source/net/sourceforge/filebot/web/AbstractEpisodeListProvider.java +++ b/source/net/sourceforge/filebot/web/AbstractEpisodeListProvider.java @@ -4,8 +4,8 @@ package net.sourceforge.filebot.web; import static net.sourceforge.filebot.web.EpisodeUtilities.*; +import java.io.Serializable; import java.util.Arrays; -import java.util.Collection; import java.util.List; import java.util.Locale; import java.util.logging.Level; @@ -29,23 +29,54 @@ public abstract class AbstractEpisodeListProvider implements EpisodeListProvider } + protected abstract List fetchSearchResult(String query, Locale locale) throws Exception; + + + protected abstract List fetchEpisodeList(SearchResult searchResult, Locale locale) throws Exception; + + public List search(String query) throws Exception { return search(query, getDefaultLocale()); } + public List search(String query, Locale locale) throws Exception { + ResultCache cache = getCache(); + List results = (cache != null) ? cache.getSearchResult(query, locale) : null; + if (results != null) { + return results; + } + + // perform actual search + results = fetchSearchResult(query, locale); + + // cache results and return + return (cache != null) ? cache.putSearchResult(query, locale, results) : results; + } + + public List getEpisodeList(SearchResult searchResult) throws Exception { return getEpisodeList(searchResult, getDefaultLocale()); } - public List getEpisodeList(SearchResult searchResult, int season) throws Exception { - return getEpisodeList(searchResult, season, getDefaultLocale()); + public List getEpisodeList(SearchResult searchResult, Locale locale) throws Exception { + ResultCache cache = getCache(); + List episodes = (cache != null) ? cache.getEpisodeList(searchResult, locale) : null; + if (episodes != null) { + return episodes; + } + + // perform actual search + episodes = fetchEpisodeList(searchResult, locale); + + // cache results and return + return (cache != null) ? cache.putEpisodeList(searchResult, locale, episodes) : episodes; } - public Locale getDefaultLocale() { - return Locale.ENGLISH; + public List getEpisodeList(SearchResult searchResult, int season) throws Exception { + return getEpisodeList(searchResult, season, getDefaultLocale()); } @@ -62,6 +93,16 @@ public abstract class AbstractEpisodeListProvider implements EpisodeListProvider } + public Locale getDefaultLocale() { + return Locale.ENGLISH; + } + + + public ResultCache getCache() { + return null; + } + + protected static class ResultCache { private final String id; @@ -74,18 +115,25 @@ public abstract class AbstractEpisodeListProvider implements EpisodeListProvider } - public void putSearchResult(String key, Collection value) { - try { - cache.put(new Element(key(id, "SearchResult", key), value.toArray(new SearchResult[0]))); - } catch (Exception e) { - Logger.getLogger(AbstractEpisodeListProvider.class.getName()).log(Level.WARNING, e.getMessage(), e); - } + protected String normalize(String query) { + return query == null ? null : query.trim().toLowerCase(); } - public List getSearchResult(String key) { + public List putSearchResult(String query, Locale locale, List value) { try { - Element element = cache.get(key(id, "SearchResult", key)); + cache.put(new Element(new Key(id, normalize(query), locale), value.toArray(new SearchResult[0]))); + } catch (Exception e) { + Logger.getLogger(AbstractEpisodeListProvider.class.getName()).log(Level.WARNING, e.getMessage(), e); + } + + return value; + } + + + public List getSearchResult(String query, Locale locale) { + try { + Element element = cache.get(new Key(id, normalize(query), locale)); if (element != null) { return Arrays.asList(((SearchResult[]) element.getValue())); } @@ -97,18 +145,20 @@ public abstract class AbstractEpisodeListProvider implements EpisodeListProvider } - public void putEpisodeList(int key, Locale language, List episodes) { + public List putEpisodeList(SearchResult key, Locale locale, List episodes) { try { - cache.put(new Element(key(id, "EpisodeList", key, language.getLanguage()), episodes.toArray(new Episode[0]))); + cache.put(new Element(new Key(id, key, locale), episodes.toArray(new Episode[0]))); } catch (Exception e) { Logger.getLogger(AbstractEpisodeListProvider.class.getName()).log(Level.WARNING, e.getMessage(), e); } + + return episodes; } - public List getEpisodeList(int key, Locale language) { + public List getEpisodeList(SearchResult key, Locale locale) { try { - Element element = cache.get(key(id, "EpisodeList", key, language.getLanguage())); + Element element = cache.get(new Key(id, key, locale)); if (element != null) { return Arrays.asList((Episode[]) element.getValue()); } @@ -120,8 +170,30 @@ public abstract class AbstractEpisodeListProvider implements EpisodeListProvider } - private String key(Object... key) { - return Arrays.toString(key); + private static class Key implements Serializable { + + protected Object[] fields; + + + public Key(Object... fields) { + this.fields = fields; + } + + + @Override + public int hashCode() { + return Arrays.hashCode(fields); + } + + + @Override + public boolean equals(Object other) { + if (other instanceof Key) { + return Arrays.equals(this.fields, ((Key) other).fields); + } + + return false; + } } } diff --git a/source/net/sourceforge/filebot/web/AnidbClient.java b/source/net/sourceforge/filebot/web/AnidbClient.java index 7c76351e..49cf07ff 100644 --- a/source/net/sourceforge/filebot/web/AnidbClient.java +++ b/source/net/sourceforge/filebot/web/AnidbClient.java @@ -33,7 +33,6 @@ import net.sourceforge.filebot.ResourceManager; public class AnidbClient extends AbstractEpisodeListProvider { private final String host = "anidb.net"; - private final ResultCache cache = new ResultCache(host, CacheManager.getInstance().getCache("web-persistent-datasource")); private final String client; private final int clientver; @@ -69,8 +68,21 @@ public class AnidbClient extends AbstractEpisodeListProvider { } + @Override + public ResultCache getCache() { + return new ResultCache(host, CacheManager.getInstance().getCache("web-persistent-datasource")); + } + + @Override public List search(String query, final Locale locale) throws Exception { + // bypass automatic caching since search is based on locally cached data anyway + return fetchSearchResult(query, locale); + } + + + @Override + public List fetchSearchResult(String query, final Locale locale) throws Exception { LocalSearch index = new LocalSearch(getAnimeTitles()) { @Override @@ -84,17 +96,12 @@ public class AnidbClient extends AbstractEpisodeListProvider { @Override - public List getEpisodeList(SearchResult searchResult, Locale language) throws Exception { + public List fetchEpisodeList(SearchResult searchResult, Locale language) throws Exception { AnidbSearchResult anime = (AnidbSearchResult) searchResult; // e.g. http://api.anidb.net:9001/httpapi?request=anime&client=filebot&clientver=1&protover=1&aid=4521 URL url = new URL("http", "api." + host, 9001, "/httpapi?request=anime&client=" + client + "&clientver=" + clientver + "&protover=1&aid=" + anime.getAnimeId()); - // try cache first - List episodes = cache.getEpisodeList(anime.getAnimeId(), language); - if (episodes != null) - return episodes; - // get anime page as xml Document dom = getDocument(url); @@ -105,7 +112,7 @@ public class AnidbClient extends AbstractEpisodeListProvider { animeTitle = selectString("//titles/title[@type='main']", dom); } - episodes = new ArrayList(25); + List episodes = new ArrayList(25); for (Node node : selectNodes("//episode", dom)) { Integer number = getIntegerContent("epno", node); @@ -127,10 +134,7 @@ public class AnidbClient extends AbstractEpisodeListProvider { sortEpisodes(episodes); // sanity check - if (episodes.size() > 0) { - // populate cache - cache.putEpisodeList(anime.getAnimeId(), language, episodes); - } else { + if (episodes.isEmpty()) { // anime page xml doesn't work sometimes throw new RuntimeException(String.format("Failed to parse episode data from xml: %s (%d)", anime, anime.getAnimeId())); } @@ -163,13 +167,15 @@ public class AnidbClient extends AbstractEpisodeListProvider { } + @SuppressWarnings("unchecked") protected List getAnimeTitles() throws Exception { URL url = new URL("http", host, "/api/animetitles.dat.gz"); + ResultCache cache = getCache(); - @SuppressWarnings("unchecked") - List anime = (List) cache.getSearchResult(null); - if (anime != null) + List anime = (List) cache.getSearchResult(null, Locale.ROOT); + if (anime != null) { return anime; + } // ||| // type: 1=primary title (one per anime), 2=synonyms (multiple per anime), 3=shorttitles (multiple per anime), 4=official title (one per language) @@ -216,9 +222,7 @@ public class AnidbClient extends AbstractEpisodeListProvider { } // populate cache - cache.putSearchResult(null, anime); - - return anime; + return cache.putSearchResult(null, Locale.ROOT, anime); } @@ -260,6 +264,23 @@ public class AnidbClient extends AbstractEpisodeListProvider { public String getOfficialTitle(String key) { return officialTitle != null ? officialTitle.get(key) : null; } + + + @Override + public int hashCode() { + return aid; + } + + + @Override + public boolean equals(Object object) { + if (object instanceof AnidbSearchResult) { + AnidbSearchResult other = (AnidbSearchResult) object; + return this.aid == other.aid; + } + + return false; + } } } diff --git a/source/net/sourceforge/filebot/web/HyperLink.java b/source/net/sourceforge/filebot/web/HyperLink.java index ac69b146..b3dbc519 100644 --- a/source/net/sourceforge/filebot/web/HyperLink.java +++ b/source/net/sourceforge/filebot/web/HyperLink.java @@ -10,7 +10,12 @@ import java.util.Arrays; public class HyperLink extends SearchResult { - private final URL url; + protected URL url; + + + protected HyperLink() { + // used by serializer + } public HyperLink(String name, URL url) { diff --git a/source/net/sourceforge/filebot/web/IMDbClient.java b/source/net/sourceforge/filebot/web/IMDbClient.java index b8451999..27bfe606 100644 --- a/source/net/sourceforge/filebot/web/IMDbClient.java +++ b/source/net/sourceforge/filebot/web/IMDbClient.java @@ -30,7 +30,6 @@ import net.sourceforge.filebot.ResourceManager; public class IMDbClient extends AbstractEpisodeListProvider { private final String host = "www.imdb.com"; - private final ResultCache cache = new ResultCache(host, CacheManager.getInstance().getCache("web-datasource")); @Override @@ -46,7 +45,13 @@ public class IMDbClient extends AbstractEpisodeListProvider { @Override - public List<SearchResult> search(String query, Locale locale) throws IOException, SAXException { + public ResultCache getCache() { + return new ResultCache(host, CacheManager.getInstance().getCache("web-datasource")); + } + + + @Override + public List<SearchResult> fetchSearchResult(String query, Locale locale) throws IOException, SAXException { URL searchUrl = new URL("http", host, "/find?s=tt&q=" + encode(query)); Document dom = getHtmlDocument(openConnection(searchUrl)); @@ -80,19 +85,15 @@ public class IMDbClient extends AbstractEpisodeListProvider { @Override - public List<Episode> getEpisodeList(SearchResult searchResult, Locale locale) throws IOException, SAXException { + public List<Episode> fetchEpisodeList(SearchResult searchResult, Locale locale) throws IOException, SAXException { Movie movie = (Movie) searchResult; - List<Episode> episodes = cache.getEpisodeList(movie.getImdbId(), Locale.ROOT); - if (episodes != null) - return episodes; - Document dom = getHtmlDocument(openConnection(getEpisodeListLink(searchResult).toURL())); String seriesName = normalizeName(selectString("//H1/A", dom)); Date year = new Date(movie.getYear(), 0, 0); List<Node> nodes = selectNodes("//TABLE//H3/A[preceding-sibling::text()]", dom); - episodes = new ArrayList<Episode>(nodes.size()); + List<Episode> episodes = new ArrayList<Episode>(nodes.size()); for (Node node : nodes) { String title = getTextContent(node); @@ -107,7 +108,6 @@ public class IMDbClient extends AbstractEpisodeListProvider { episodes.add(new Episode(seriesName, year, season, episode, title, null, null, airdate)); } - cache.putEpisodeList(movie.getImdbId(), Locale.ROOT, episodes); return episodes; } diff --git a/source/net/sourceforge/filebot/web/Movie.java b/source/net/sourceforge/filebot/web/Movie.java index 5a8866a7..fd726334 100644 --- a/source/net/sourceforge/filebot/web/Movie.java +++ b/source/net/sourceforge/filebot/web/Movie.java @@ -7,8 +7,13 @@ import java.util.Arrays; public class Movie extends SearchResult { - protected final int year; - protected final int imdbId; + protected int year; + protected int imdbId; + + + protected Movie() { + // used by serializer + } public Movie(String name, int year, int imdbId) { diff --git a/source/net/sourceforge/filebot/web/SerienjunkiesClient.java b/source/net/sourceforge/filebot/web/SerienjunkiesClient.java index 129387e6..d7f2b4a1 100644 --- a/source/net/sourceforge/filebot/web/SerienjunkiesClient.java +++ b/source/net/sourceforge/filebot/web/SerienjunkiesClient.java @@ -28,7 +28,6 @@ import net.sourceforge.filebot.ResourceManager; public class SerienjunkiesClient extends AbstractEpisodeListProvider { private final String host = "api.serienjunkies.de"; - private final ResultCache cache = new ResultCache(host, CacheManager.getInstance().getCache("web-datasource")); private final String apikey; @@ -57,7 +56,20 @@ public class SerienjunkiesClient extends AbstractEpisodeListProvider { @Override - public List<SearchResult> search(String query, Locale locale) throws Exception { + public ResultCache getCache() { + return new ResultCache(host, CacheManager.getInstance().getCache("web-datasource")); + } + + + @Override + public List<SearchResult> search(String query, final Locale locale) throws Exception { + // bypass automatic caching since search is based on locally cached data anyway + return fetchSearchResult(query, locale); + } + + + @Override + public List<SearchResult> fetchSearchResult(String query, Locale locale) throws Exception { LocalSearch<SerienjunkiesSearchResult> index = new LocalSearch<SerienjunkiesSearchResult>(getSeriesTitles()) { @Override @@ -71,10 +83,13 @@ public class SerienjunkiesClient extends AbstractEpisodeListProvider { protected List<SerienjunkiesSearchResult> getSeriesTitles() throws IOException { + ResultCache cache = getCache(); + @SuppressWarnings("unchecked") - List<SerienjunkiesSearchResult> seriesList = (List) cache.getSearchResult(null); - if (seriesList != null) + List<SerienjunkiesSearchResult> seriesList = (List) cache.getSearchResult(null, Locale.ROOT); + if (seriesList != null) { return seriesList; + } // fetch series data seriesList = new ArrayList<SerienjunkiesSearchResult>(); @@ -95,23 +110,16 @@ public class SerienjunkiesClient extends AbstractEpisodeListProvider { } // populate cache - cache.putSearchResult(null, seriesList); - - return seriesList; + return cache.putSearchResult(null, Locale.ROOT, seriesList); } @Override - public List<Episode> getEpisodeList(SearchResult searchResult, Locale locale) throws IOException { + public List<Episode> fetchEpisodeList(SearchResult searchResult, Locale locale) throws IOException { SerienjunkiesSearchResult series = (SerienjunkiesSearchResult) searchResult; - // try cache first - List<Episode> episodes = cache.getEpisodeList(series.getSeriesId(), Locale.GERMAN); - if (episodes != null) - return episodes; - // fetch episode data - episodes = new ArrayList<Episode>(25); + List<Episode> episodes = new ArrayList<Episode>(25); String seriesName = locale.equals(Locale.GERMAN) && series.getGermanTitle() != null ? series.getGermanTitle() : series.getMainTitle(); JSONObject data = (JSONObject) request("/allepisodes.php?d=" + apikey + "&q=" + series.getSeriesId()); @@ -128,9 +136,6 @@ public class SerienjunkiesClient extends AbstractEpisodeListProvider { episodes.add(new Episode(seriesName, series.getStartDate(), season, episode, title, i + 1, null, airdate)); } - // populate cache - cache.putEpisodeList(series.getSeriesId(), Locale.GERMAN, episodes); - // make sure episodes are in ordered correctly sortEpisodes(episodes); @@ -224,6 +229,23 @@ public class SerienjunkiesClient extends AbstractEpisodeListProvider { public Date getStartDate() { return startDate; } + + + @Override + public int hashCode() { + return sid; + } + + + @Override + public boolean equals(Object object) { + if (object instanceof SerienjunkiesSearchResult) { + SerienjunkiesSearchResult other = (SerienjunkiesSearchResult) object; + return this.sid == other.sid; + } + + return false; + } } } diff --git a/source/net/sourceforge/filebot/web/TVRageClient.java b/source/net/sourceforge/filebot/web/TVRageClient.java index 59fa3636..1b0a7ad1 100644 --- a/source/net/sourceforge/filebot/web/TVRageClient.java +++ b/source/net/sourceforge/filebot/web/TVRageClient.java @@ -26,7 +26,6 @@ import net.sourceforge.filebot.ResourceManager; public class TVRageClient extends AbstractEpisodeListProvider { private final String host = "services.tvrage.com"; - private final ResultCache cache = new ResultCache(host, CacheManager.getInstance().getCache("web-datasource")); @Override @@ -42,7 +41,13 @@ public class TVRageClient extends AbstractEpisodeListProvider { @Override - public List<SearchResult> search(String query, Locale locale) throws IOException, SAXException { + public ResultCache getCache() { + return new ResultCache(host, CacheManager.getInstance().getCache("web-datasource")); + } + + + @Override + public List<SearchResult> fetchSearchResult(String query, Locale locale) throws IOException, SAXException { URL searchUrl = new URL("http", host, "/feeds/full_search.php?show=" + encode(query)); Document dom = getDocument(searchUrl); @@ -62,20 +67,16 @@ public class TVRageClient extends AbstractEpisodeListProvider { @Override - public List<Episode> getEpisodeList(SearchResult searchResult, Locale locale) throws IOException, SAXException { + public List<Episode> fetchEpisodeList(SearchResult searchResult, Locale locale) throws IOException, SAXException { TVRageSearchResult series = (TVRageSearchResult) searchResult; - List<Episode> episodes = cache.getEpisodeList(series.getSeriesId(), Locale.ENGLISH); - if (episodes != null) - return episodes; - URL episodeListUrl = new URL("http", host, "/feeds/full_show_info.php?sid=" + series.getSeriesId()); Document dom = getDocument(episodeListUrl); String seriesName = selectString("Show/name", dom); Date seriesStartDate = Date.parse(selectString("Show/started", dom), "MMM/dd/yyyy"); - episodes = new ArrayList<Episode>(25); + List<Episode> episodes = new ArrayList<Episode>(25); List<Episode> specials = new ArrayList<Episode>(5); // episodes and specials @@ -101,7 +102,6 @@ public class TVRageClient extends AbstractEpisodeListProvider { // add specials at the end episodes.addAll(specials); - cache.putEpisodeList(series.getSeriesId(), Locale.ENGLISH, episodes); return episodes; } @@ -127,8 +127,13 @@ public class TVRageClient extends AbstractEpisodeListProvider { public static class TVRageSearchResult extends SearchResult { - private final int showId; - private final String link; + protected int showId; + protected String link; + + + protected TVRageSearchResult() { + // used by serializer + } public TVRageSearchResult(String name, int showId, String link) { @@ -147,6 +152,22 @@ public class TVRageClient extends AbstractEpisodeListProvider { return link; } + + @Override + public int hashCode() { + return showId; + } + + + @Override + public boolean equals(Object object) { + if (object instanceof TVRageSearchResult) { + TVRageSearchResult other = (TVRageSearchResult) object; + return this.showId == other.showId; + } + + return false; + } } } diff --git a/source/net/sourceforge/filebot/web/TheTVDBClient.java b/source/net/sourceforge/filebot/web/TheTVDBClient.java index bab435b8..7efc2ebf 100644 --- a/source/net/sourceforge/filebot/web/TheTVDBClient.java +++ b/source/net/sourceforge/filebot/web/TheTVDBClient.java @@ -37,7 +37,6 @@ public class TheTVDBClient extends AbstractEpisodeListProvider { private final String host = "www.thetvdb.com"; private final Map<MirrorType, String> mirrors = new EnumMap<MirrorType, String>(MirrorType.class); - private final ResultCache cache = new ResultCache(host, CacheManager.getInstance().getCache("web-datasource")); private final String apikey; @@ -75,7 +74,13 @@ public class TheTVDBClient extends AbstractEpisodeListProvider { @Override - public List<SearchResult> search(String query, Locale language) throws Exception { + public ResultCache getCache() { + return new ResultCache(host, CacheManager.getInstance().getCache("web-datasource")); + } + + + @Override + public List<SearchResult> fetchSearchResult(String query, Locale language) throws Exception { // perform online search URL url = getResource(null, "/api/GetSeries.php?seriesname=" + encode(query) + "&language=" + language.getLanguage()); Document dom = getDocument(url); @@ -97,12 +102,8 @@ public class TheTVDBClient extends AbstractEpisodeListProvider { @Override - public List<Episode> getEpisodeList(SearchResult searchResult, Locale language) throws Exception { + public List<Episode> fetchEpisodeList(SearchResult searchResult, Locale language) throws Exception { TheTVDBSearchResult series = (TheTVDBSearchResult) searchResult; - List<Episode> episodes = cache.getEpisodeList(series.getSeriesId(), language); - - if (episodes != null) - return episodes; Document seriesRecord = getSeriesRecord(series, language); @@ -112,7 +113,7 @@ public class TheTVDBClient extends AbstractEpisodeListProvider { List<Node> nodes = selectNodes("Data/Episode", seriesRecord); - episodes = new ArrayList<Episode>(nodes.size()); + List<Episode> episodes = new ArrayList<Episode>(nodes.size()); List<Episode> specials = new ArrayList<Episode>(5); for (Node node : nodes) { @@ -156,7 +157,6 @@ public class TheTVDBClient extends AbstractEpisodeListProvider { // add specials at the end episodes.addAll(specials); - cache.putEpisodeList(series.getSeriesId(), language, episodes); return episodes; } @@ -266,7 +266,12 @@ public class TheTVDBClient extends AbstractEpisodeListProvider { public static class TheTVDBSearchResult extends SearchResult { - private final int seriesId; + protected int seriesId; + + + protected TheTVDBSearchResult() { + // used by serializer + } public TheTVDBSearchResult(String seriesName, int seriesId) { @@ -279,6 +284,22 @@ public class TheTVDBClient extends AbstractEpisodeListProvider { return seriesId; } + + @Override + public int hashCode() { + return seriesId; + } + + + @Override + public boolean equals(Object object) { + if (object instanceof TheTVDBSearchResult) { + TheTVDBSearchResult other = (TheTVDBSearchResult) object; + return this.seriesId == other.seriesId; + } + + return false; + } }