From 7cc54fc59ed94160d7df11a8fcd37a9241c1fe4d Mon Sep 17 00:00:00 2001 From: Reinhard Pointner Date: Wed, 25 Jul 2012 04:38:22 +0000 Subject: [PATCH] * added lots of caching to OpenSubtitlesClient (not much tested yet because the xml-rpc ws is broken again...) --- source/net/sourceforge/filebot/Cache.java | 5 + .../filebot/media/MediaDetection.java | 2 +- .../filebot/web/OpenSubtitlesClient.java | 201 +++++++++++++++--- .../web/OpenSubtitlesSubtitleDescriptor.java | 4 +- .../filebot/web/OpenSubtitlesXmlRpc.java | 3 +- 5 files changed, 184 insertions(+), 31 deletions(-) diff --git a/source/net/sourceforge/filebot/Cache.java b/source/net/sourceforge/filebot/Cache.java index 67744e49..290184cb 100644 --- a/source/net/sourceforge/filebot/Cache.java +++ b/source/net/sourceforge/filebot/Cache.java @@ -35,6 +35,11 @@ public class Cache { } + public Object get(Object key) { + return get(key, Object.class); + } + + public T get(Object key, Class type) { try { Element element = cache.get(key); diff --git a/source/net/sourceforge/filebot/media/MediaDetection.java b/source/net/sourceforge/filebot/media/MediaDetection.java index ff63af55..225c557e 100644 --- a/source/net/sourceforge/filebot/media/MediaDetection.java +++ b/source/net/sourceforge/filebot/media/MediaDetection.java @@ -311,7 +311,7 @@ public class MediaDetection { } } } catch (Exception e) { - Logger.getLogger(MediaDetection.class.getName()).log(Level.WARNING, e.getMessage()); + Logger.getLogger(MediaDetection.class.getName()).log(Level.WARNING, hashLookupService.getName() + ": " + e.getMessage()); } } diff --git a/source/net/sourceforge/filebot/web/OpenSubtitlesClient.java b/source/net/sourceforge/filebot/web/OpenSubtitlesClient.java index c9f2a19c..e00e8354 100644 --- a/source/net/sourceforge/filebot/web/OpenSubtitlesClient.java +++ b/source/net/sourceforge/filebot/web/OpenSubtitlesClient.java @@ -13,6 +13,7 @@ import java.net.URI; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.LinkedList; @@ -26,14 +27,12 @@ import java.util.logging.Logger; import javax.swing.Icon; -import redstone.xmlrpc.XmlRpcException; - -import net.sf.ehcache.Cache; -import net.sf.ehcache.CacheManager; -import net.sf.ehcache.Element; +import net.sourceforge.filebot.Cache; +import net.sourceforge.filebot.Cache.Key; import net.sourceforge.filebot.ResourceManager; import net.sourceforge.filebot.web.OpenSubtitlesXmlRpc.Query; import net.sourceforge.tuned.Timer; +import redstone.xmlrpc.XmlRpcException; /** @@ -76,24 +75,42 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS } + public ResultCache getCache() { + return new ResultCache("opensubtitles.org", Cache.getCache("web-datasource")); + } + + @Override public List search(String query) throws Exception { + List result = getCache().getSearchResult("search", query, null); + if (result != null) { + return result; + } + // require login login(); try { // search for movies / series List resultSet = xmlrpc.searchMoviesOnIMDB(query); - return asList(resultSet.toArray(new SearchResult[0])); + result = asList(resultSet.toArray(new SearchResult[0])); } catch (ClassCastException e) { // unexpected xmlrpc responses (e.g. error messages instead of results) will trigger this throw new XmlRpcException("Illegal XMLRPC response on searchMoviesOnIMDB"); } + + getCache().putSearchResult("search", query, null, result); + return result; } @Override public List getSubtitleList(SearchResult searchResult, String languageName) throws Exception { + List subtitles = getCache().getSubtitleDescriptorList(searchResult, languageName); + if (subtitles != null) { + return subtitles; + } + // singleton array with or empty array int imdbid = ((Movie) searchResult).getImdbId(); String[] languageFilter = languageName != null ? new String[] { getSubLanguageID(languageName) } : new String[0]; @@ -102,18 +119,20 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS login(); // get subtitle list - SubtitleDescriptor[] subtitles = xmlrpc.searchSubtitles(imdbid, languageFilter).toArray(new SubtitleDescriptor[0]); + subtitles = asList(xmlrpc.searchSubtitles(imdbid, languageFilter).toArray(new SubtitleDescriptor[0])); - return asList(subtitles); + getCache().putSubtitleDescriptorList(searchResult, languageName, subtitles); + return subtitles; } + @Override public Map> getSubtitleList(File[] files, String languageName) throws Exception { // singleton array with or empty array String[] languageFilter = languageName != null ? new String[] { getSubLanguageID(languageName) } : new String[0]; // remember hash for each file - Map hashMap = new HashMap(files.length); + Map hashMap = new HashMap(files.length); Map> resultMap = new HashMap>(files.length); // create hash query for each file @@ -123,12 +142,22 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS // add query if (file.length() > HASH_CHUNK_SIZE) { String movieHash = computeHash(file); - queryList.add(Query.forHash(movieHash, file.length(), languageFilter)); - hashMap.put(movieHash, file); + Query query = Query.forHash(movieHash, file.length(), languageFilter); + + // check hash + List cachedResults = getCache().getSubtitleDescriptorList(query, languageName); + if (cachedResults == null) { + queryList.add(query); + hashMap.put(query, file); + } else { + resultMap.put(file, cachedResults); + } } // prepare result map - resultMap.put(file, new LinkedList()); + if (resultMap.get(file) == null) { + resultMap.put(file, new LinkedList()); + } } if (queryList.size() > 0) { @@ -143,11 +172,15 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS // submit query and map results to given files for (OpenSubtitlesSubtitleDescriptor subtitle : xmlrpc.searchSubtitles(batch)) { // get file for hash - File file = hashMap.get(subtitle.getMovieHash()); + File file = hashMap.get(Query.forHash(subtitle.getMovieHash(), subtitle.getMovieByteSize(), languageFilter)); // add subtitle resultMap.get(file).add(subtitle); } + + for (Query query : batch) { + getCache().putSubtitleDescriptorList(query, languageName, resultMap.get(hashMap.get(query))); + } } } @@ -176,21 +209,38 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS } + @SuppressWarnings({ "unchecked", "rawtypes" }) @Override public List searchMovie(String query, Locale locale) throws Exception { + List result = getCache().getSearchResult("searchMovie", query, locale); + if (result != null) { + return (List) result; + } + // require login login(); - return xmlrpc.searchMoviesOnIMDB(query); + List movies = xmlrpc.searchMoviesOnIMDB(query); + + getCache().putSearchResult("searchMovie", query, locale, movies); + return movies; } @Override public Movie getMovieDescriptor(int imdbid, Locale locale) throws Exception { + Movie result = getCache().getData("getMovieDescriptor", imdbid, locale, Movie.class); + if (result != null) { + return result; + } + // require login login(); - return xmlrpc.getIMDBMovieDetails(imdbid); + Movie movie = xmlrpc.getIMDBMovieDetails(imdbid); + + getCache().putData("getMovieDescriptor", imdbid, locale, movie); + return movie; } @@ -209,7 +259,15 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS for (File file : movieFiles) { if (file.length() > HASH_CHUNK_SIZE) { - hashMap.put(computeHash(file), file); // map file by hash + String hash = computeHash(file); + + Movie entry = getCache().getData("getMovieDescriptor", hash, locale, Movie.class); + if (entry == null) { + hashMap.put(hash, file); // map file by hash + } else { + result.put(file, entry); + } + } } @@ -223,8 +281,9 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS for (int bn = 0; bn < ceil((float) hashes.size() / batchSize); bn++) { List batch = hashes.subList(bn * batchSize, min((bn * batchSize) + batchSize, hashes.size())); - for (Entry entry : xmlrpc.checkMovieHash(batch).entrySet()) { - result.put(hashMap.get(entry.getKey()), entry.getValue()); + for (Entry it : xmlrpc.checkMovieHash(batch).entrySet()) { + result.put(hashMap.get(it.getKey()), it.getValue()); + getCache().putData("getMovieDescriptor", it.getKey(), locale, it.getValue()); } } } @@ -283,7 +342,6 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS logoutTimer.cancel(); } - protected final Timer logoutTimer = new Timer() { @Override @@ -298,13 +356,12 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS */ @SuppressWarnings("unchecked") protected synchronized Map getSubLanguageMap() throws Exception { - Cache cache = CacheManager.getInstance().getCache("web-persistent-datasource"); + Cache cache = Cache.getCache("web-persistent-datasource"); String cacheKey = getClass().getName() + ".subLanguageMap"; - Element element = cache.get(cacheKey); - Map subLanguageMap; + Map subLanguageMap = cache.get(cacheKey, Map.class); - if (element == null) { + if (subLanguageMap == null) { subLanguageMap = new HashMap(); // fetch language data @@ -317,10 +374,7 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS subLanguageMap.put("brazilian", "pob"); // cache data - cache.put(new Element(cacheKey, subLanguageMap)); - } else { - // use cached entry - subLanguageMap = (Map) element.getValue(); + cache.put(cacheKey, subLanguageMap); } return subLanguageMap; @@ -348,4 +402,97 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS return null; } + + protected static class ResultCache { + + private final String id; + private final Cache cache; + + + public ResultCache(String id, Cache cache) { + this.id = id; + this.cache = cache; + } + + + protected String normalize(String query) { + return query == null ? null : query.trim().toLowerCase(); + } + + + public List putSearchResult(String method, String query, Locale locale, List value) { + try { + cache.put(new Key(id, normalize(query)), value.toArray(new SearchResult[0])); + } catch (Exception e) { + Logger.getLogger(OpenSubtitlesClient.class.getName()).log(Level.WARNING, e.getMessage()); + } + + return value; + } + + + @SuppressWarnings("unchecked") + public List getSearchResult(String method, String query, Locale locale) { + try { + SearchResult[] array = cache.get(new Key(id, normalize(query)), SearchResult[].class); + if (array != null) { + return Arrays.asList(array); + } + } catch (Exception e) { + Logger.getLogger(OpenSubtitlesClient.class.getName()).log(Level.WARNING, e.getMessage(), e); + } + + return null; + } + + + public List putSubtitleDescriptorList(Object key, String locale, List subtitles) { + try { + cache.put(new Key(id, key, locale), subtitles.toArray(new SubtitleDescriptor[0])); + } catch (Exception e) { + Logger.getLogger(OpenSubtitlesClient.class.getName()).log(Level.WARNING, e.getMessage()); + } + + return subtitles; + } + + + public List getSubtitleDescriptorList(Object key, String locale) { + try { + SubtitleDescriptor[] descriptors = cache.get(new Key(id, key, locale), SubtitleDescriptor[].class); + if (descriptors != null) { + return Arrays.asList(descriptors); + } + } catch (Exception e) { + Logger.getLogger(OpenSubtitlesClient.class.getName()).log(Level.WARNING, e.getMessage(), e); + } + + return null; + } + + + public void putData(Object category, Object key, Locale locale, Object object) { + try { + cache.put(new Key(id, category, locale, key), object); + } catch (Exception e) { + Logger.getLogger(OpenSubtitlesClient.class.getName()).log(Level.WARNING, e.getMessage()); + } + } + + + public T getData(Object category, Object key, Locale locale, Class type) { + try { + T value = cache.get(new Key(id, category, locale, key), type); + if (value != null) { + return value; + } + } catch (Exception e) { + Logger.getLogger(OpenSubtitlesClient.class.getName()).log(Level.WARNING, e.getMessage(), e); + } + + return null; + } + + } + } diff --git a/source/net/sourceforge/filebot/web/OpenSubtitlesSubtitleDescriptor.java b/source/net/sourceforge/filebot/web/OpenSubtitlesSubtitleDescriptor.java index 87a932ba..447d776b 100644 --- a/source/net/sourceforge/filebot/web/OpenSubtitlesSubtitleDescriptor.java +++ b/source/net/sourceforge/filebot/web/OpenSubtitlesSubtitleDescriptor.java @@ -4,6 +4,7 @@ package net.sourceforge.filebot.web; import java.io.IOException; import java.io.InputStream; +import java.io.Serializable; import java.net.URL; import java.nio.ByteBuffer; import java.util.EnumMap; @@ -21,7 +22,7 @@ import net.sourceforge.tuned.FileUtilities; * * @see OpenSubtitlesXmlRpc */ -public class OpenSubtitlesSubtitleDescriptor implements SubtitleDescriptor { +public class OpenSubtitlesSubtitleDescriptor implements SubtitleDescriptor, Serializable { public static enum Property { IDSubtitle, @@ -75,7 +76,6 @@ public class OpenSubtitlesSubtitleDescriptor implements SubtitleDescriptor { } } - private final Map properties; diff --git a/source/net/sourceforge/filebot/web/OpenSubtitlesXmlRpc.java b/source/net/sourceforge/filebot/web/OpenSubtitlesXmlRpc.java index 4baeadcf..088a8250 100644 --- a/source/net/sourceforge/filebot/web/OpenSubtitlesXmlRpc.java +++ b/source/net/sourceforge/filebot/web/OpenSubtitlesXmlRpc.java @@ -7,6 +7,7 @@ import static net.sourceforge.tuned.StringUtilities.*; import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.Serializable; import java.net.MalformedURLException; import java.net.URI; import java.net.URL; @@ -358,7 +359,7 @@ public class OpenSubtitlesXmlRpc { } - public static final class Query extends HashMap { + public static final class Query extends HashMap implements Serializable { private Query(String imdbid, String... sublanguageids) { put("imdbid", imdbid);