Experiment with new CachedResource framework

This commit is contained in:
Reinhard Pointner 2016-03-09 19:26:03 +00:00
parent 96b653da0a
commit 4390752fc0
8 changed files with 226 additions and 515 deletions

View File

@ -6,10 +6,8 @@ import static java.util.stream.Collectors.*;
import static net.filebot.CachedResource.*; import static net.filebot.CachedResource.*;
import static net.filebot.Logging.*; import static net.filebot.Logging.*;
import java.io.Serializable;
import java.net.URL; import java.net.URL;
import java.time.Duration; import java.time.Duration;
import java.util.Arrays;
import java.util.List; import java.util.List;
import java.util.function.Function; import java.util.function.Function;
import java.util.function.Predicate; import java.util.function.Predicate;
@ -168,38 +166,4 @@ public class Cache {
} }
} }
@Deprecated
public <T> T get(Object key, Class<T> type) {
return type.cast(get(key));
}
@Deprecated
public static class Key implements Serializable {
protected Object[] fields;
public Key(Object... fields) {
this.fields = fields;
}
@Override
public int hashCode() {
return Arrays.hashCode(fields);
}
@Override
public boolean equals(Object other) {
if (other instanceof Key) {
return Arrays.equals(this.fields, ((Key) other).fields);
}
return false;
}
@Override
public String toString() {
return Arrays.toString(fields);
}
}
} }

View File

@ -39,6 +39,7 @@ import javax.swing.Action;
import javax.swing.SwingUtilities; import javax.swing.SwingUtilities;
import net.filebot.Cache; import net.filebot.Cache;
import net.filebot.Cache.TypedCache;
import net.filebot.CacheType; import net.filebot.CacheType;
import net.filebot.Settings; import net.filebot.Settings;
import net.filebot.similarity.CommonSequenceMatcher; import net.filebot.similarity.CommonSequenceMatcher;
@ -58,7 +59,7 @@ class EpisodeListMatcher implements AutoCompleteMatcher {
private boolean useSeriesIndex; private boolean useSeriesIndex;
// remember user selections // remember user selections
private Cache persistentSelectionMemory; private TypedCache<SearchResult> persistentSelectionMemory;
// only allow one fetch session at a time so later requests can make use of cached results // only allow one fetch session at a time so later requests can make use of cached results
private final Object providerLock = new Object(); private final Object providerLock = new Object();
@ -67,7 +68,7 @@ class EpisodeListMatcher implements AutoCompleteMatcher {
this.provider = provider; this.provider = provider;
this.useSeriesIndex = useSeriesIndex; this.useSeriesIndex = useSeriesIndex;
this.useAnimeIndex = useAnimeIndex; this.useAnimeIndex = useAnimeIndex;
this.persistentSelectionMemory = Cache.getCache("selection_" + provider.getName(), CacheType.Persistent); this.persistentSelectionMemory = Cache.getCache("selection_" + provider.getName(), CacheType.Persistent).cast(SearchResult.class);
} }
protected SearchResult selectSearchResult(final String query, final List<SearchResult> searchResults, Map<String, SearchResult> selectionMemory, boolean autodetection, final Component parent) throws Exception { protected SearchResult selectSearchResult(final String query, final List<SearchResult> searchResults, Map<String, SearchResult> selectionMemory, boolean autodetection, final Component parent) throws Exception {
@ -132,7 +133,7 @@ class EpisodeListMatcher implements AutoCompleteMatcher {
// check persistent memory // check persistent memory
if (autodetection) { if (autodetection) {
SearchResult persistentSelection = persistentSelectionMemory.get(query, SearchResult.class); SearchResult persistentSelection = persistentSelectionMemory.get(query);
if (persistentSelection != null) { if (persistentSelection != null) {
return persistentSelection; return persistentSelection;
} }

View File

@ -76,34 +76,24 @@ public class AcoustIDClient implements MusicIdentificationService {
return results; return results;
} }
public String lookup(int duration, String fingerprint) throws IOException, InterruptedException { public String lookup(int duration, String fingerprint) throws Exception {
Map<String, String> postParam = new LinkedHashMap<String, String>(); Map<String, String> postParam = new LinkedHashMap<String, String>();
postParam.put("duration", String.valueOf(duration)); postParam.put("duration", String.valueOf(duration));
postParam.put("fingerprint", fingerprint); postParam.put("fingerprint", fingerprint);
String cacheKey = postParam.toString(); // e.g.
Cache cache = getCache(); // http://api.acoustid.org/v2/lookup?client=8XaBELgH&meta=recordings+releasegroups+compress&duration=641&fingerprint=AQABz0qUkZK4oOfhL-CPc4e5C_wW2H2QH9uDL4cvoT8UNQ-eHtsE8cceeFJx-LiiHT-aPzhxoc-Opj_eI5d2hOFyMJRzfDk-QSsu7fBxqZDMHcfxPfDIoPWxv9C1o3yg44d_3Df2GJaUQeeR-cb2HfaPNsdxHj2PJnpwPMN3aPcEMzd-_MeB_Ej4D_CLP8ghHjkJv_jh_UDuQ8xnILwunPg6hF2R8HgzvLhxHVYP_ziJX0eKPnIE1UePMByDJyg7wz_6yELsB8n4oDmDa0Gv40hf6D3CE3_wH6HFaxCPUD9-hNeF5MfWEP3SCGym4-SxnXiGs0mRjEXD6fgl4LmKWrSChzzC33ge9PB3otyJMk-IVC6R8MTNwD9qKQ_CC8kPv4THzEGZS8GPI3x0iGVUxC1hRSizC5VzoamYDi-uR7iKPhGSI82PkiWeB_eHijvsaIWfBCWH5AjjCfVxZ1TQ3CvCTclGnEMfHbnZFA8pjD6KXwd__Cn-Y8e_I9cq6CR-4S9KLXqQcsxxoWh3eMxiHI6TIzyPv0M43YHz4yte-Cv-4D16Hv9F9C9SPUdyGtZRHV-OHEeeGD--BKcjVLOK_NCDXMfx44dzHEiOZ0Z44Rf6DH5R3uiPj4d_PKolJNyRJzyu4_CTD2WOvzjKH9GPb4cUP1Av9EuQd8fGCFee4JlRHi18xQh96NLxkCgfWFKOH6WGeoe4I3za4c5hTscTPEZTES1x8kE-9MQPjT8a8gh5fPgQZtqCFj9MDvp6fDx6NCd07bjx7MLR9AhtnFnQ70GjOcV0opmm4zpY3SOa7HiwdTtyHa6NC4e-HN-OfC5-OP_gLe2QDxfUCz_0w9l65HiPAz9-IaGOUA7-4MZ5CWFOlIfe4yUa6AiZGxf6w0fFxsjTOdC6Itbh4mGD63iPH9-RFy909XAMj7mC5_BvlDyO6kGTZKJxHUd4NDwuZUffw_5RMsde5CWkJAgXnDReNEaP6DTOQ65yaD88HoeX8fge-DSeHo9Qa8cTHc80I-_RoHxx_UHeBxrJw62Q34Kd7MEfpCcu6BLeB1ePw6OO4sOF_sHhmB504WWDZiEu8sKPpkcfCT9xfej0o0lr4T5yNJeOvjmu40w-TDmqHXmYgfFhFy_M7tD1o0cO_B2ms2j-ACEEQgQgAIwzTgAGmBIKIImNQAABwgQATAlhDGCCEIGIIM4BaBgwQBogEBIOESEIA8ARI5xAhxEFmAGAMCKAURKQQpQzRAAkCCBQEAKkQYIYIQQxCixCDADCABMAE0gpJIgyxhEDiCKCCIGAEIgJIQByAhFgGACCACMRQEyBAoxQiHiCBCFOECQFAIgAABR2QAgFjCDMA0AUMIoAIMChQghChASGEGeYEAIAIhgBSErnJPPEGWYAMgw05AhiiGHiBBBGGSCQcQgwRYJwhDDhgCSCSSEIQYwILoyAjAIigBFEUQK8gAYAQ5BCAAjkjCCAEEMZAUQAZQCjCCkpCgFMCCiIcVIAZZgilAQAiSHQECOcQAQIc4QClAHAjDDGkAGAMUoBgyhihgEChFCAAWEIEYwIJYwViAAlHCBIGEIEAEIQAoBwwgwiEBAEEEOoEwBY4wRwxAhBgAcKAESIQAwwIowRFhoBhAE
String response = cache.get(cacheKey, String.class);
if (response != null) {
return response;
}
// respect rate limit return (String) getCache().computeIf(postParam.toString(), Cache.isAbsent(), it -> {
REQUEST_LIMIT.acquirePermit(); REQUEST_LIMIT.acquirePermit();
// http://api.acoustid.org/v2/lookup?client=8XaBELgH&meta=recordings+releasegroups+compress&duration=641&fingerprint=AQABz0qUkZK4oOfhL-CPc4e5C_wW2H2QH9uDL4cvoT8UNQ-eHtsE8cceeFJx-LiiHT-aPzhxoc-Opj_eI5d2hOFyMJRzfDk-QSsu7fBxqZDMHcfxPfDIoPWxv9C1o3yg44d_3Df2GJaUQeeR-cb2HfaPNsdxHj2PJnpwPMN3aPcEMzd-_MeB_Ej4D_CLP8ghHjkJv_jh_UDuQ8xnILwunPg6hF2R8HgzvLhxHVYP_ziJX0eKPnIE1UePMByDJyg7wz_6yELsB8n4oDmDa0Gv40hf6D3CE3_wH6HFaxCPUD9-hNeF5MfWEP3SCGym4-SxnXiGs0mRjEXD6fgl4LmKWrSChzzC33ge9PB3otyJMk-IVC6R8MTNwD9qKQ_CC8kPv4THzEGZS8GPI3x0iGVUxC1hRSizC5VzoamYDi-uR7iKPhGSI82PkiWeB_eHijvsaIWfBCWH5AjjCfVxZ1TQ3CvCTclGnEMfHbnZFA8pjD6KXwd__Cn-Y8e_I9cq6CR-4S9KLXqQcsxxoWh3eMxiHI6TIzyPv0M43YHz4yte-Cv-4D16Hv9F9C9SPUdyGtZRHV-OHEeeGD--BKcjVLOK_NCDXMfx44dzHEiOZ0Z44Rf6DH5R3uiPj4d_PKolJNyRJzyu4_CTD2WOvzjKH9GPb4cUP1Av9EuQd8fGCFee4JlRHi18xQh96NLxkCgfWFKOH6WGeoe4I3za4c5hTscTPEZTES1x8kE-9MQPjT8a8gh5fPgQZtqCFj9MDvp6fDx6NCd07bjx7MLR9AhtnFnQ70GjOcV0opmm4zpY3SOa7HiwdTtyHa6NC4e-HN-OfC5-OP_gLe2QDxfUCz_0w9l65HiPAz9-IaGOUA7-4MZ5CWFOlIfe4yUa6AiZGxf6w0fFxsjTOdC6Itbh4mGD63iPH9-RFy909XAMj7mC5_BvlDyO6kGTZKJxHUd4NDwuZUffw_5RMsde5CWkJAgXnDReNEaP6DTOQ65yaD88HoeX8fge-DSeHo9Qa8cTHc80I-_RoHxx_UHeBxrJw62Q34Kd7MEfpCcu6BLeB1ePw6OO4sOF_sHhmB504WWDZiEu8sKPpkcfCT9xfej0o0lr4T5yNJeOvjmu40w-TDmqHXmYgfFhFy_M7tD1o0cO_B2ms2j-ACEEQgQgAIwzTgAGmBIKIImNQAABwgQATAlhDGCCEIGIIM4BaBgwQBogEBIOESEIA8ARI5xAhxEFmAGAMCKAURKQQpQzRAAkCCBQEAKkQYIYIQQxCixCDADCABMAE0gpJIgyxhEDiCKCCIGAEIgJIQByAhFgGACCACMRQEyBAoxQiHiCBCFOECQFAIgAABR2QAgFjCDMA0AUMIoAIMChQghChASGEGeYEAIAIhgBSErnJPPEGWYAMgw05AhiiGHiBBBGGSCQcQgwRYJwhDDhgCSCSSEIQYwILoyAjAIigBFEUQK8gAYAQ5BCAAjkjCCAEEMZAUQAZQCjCCkpCgFMCCiIcVIAZZgilAQAiSHQECOcQAQIc4QClAHAjDDGkAGAMUoBgyhihgEChFCAAWEIEYwIJYwViAAlHCBIGEIEAEIQAoBwwgwiEBAEEEOoEwBY4wRwxAhBgAcKAESIQAwwIowRFhoBhAE
URL url = new URL("http://api.acoustid.org/v2/lookup?client=" + apikey + "&meta=recordings+releases+releasegroups+tracks+compress"); URL url = new URL("http://api.acoustid.org/v2/lookup?client=" + apikey + "&meta=recordings+releases+releasegroups+tracks+compress");
// enable compression for request and response
Map<String, String> requestParam = new HashMap<String, String>(); Map<String, String> requestParam = new HashMap<String, String>();
requestParam.put("Content-Encoding", "gzip"); requestParam.put("Content-Encoding", "gzip");
requestParam.put("Accept-Encoding", "gzip"); requestParam.put("Accept-Encoding", "gzip");
// submit return UTF_8.decode(post(url, postParam, requestParam)).toString();
response = UTF_8.decode(post(url, postParam, requestParam)).toString(); });
cache.put(cacheKey, response);
return response;
} }
public AudioTrack parseResult(String json, final int targetDuration) throws IOException { public AudioTrack parseResult(String json, final int targetDuration) throws IOException {

View File

@ -1,35 +1,32 @@
package net.filebot.web; package net.filebot.web;
import static java.lang.Math.*;
import static java.util.Arrays.*; import static java.util.Arrays.*;
import static java.util.Collections.*; import static java.util.Collections.*;
import static java.util.stream.Collectors.*; import static java.util.stream.Collectors.*;
import static net.filebot.Logging.*;
import static net.filebot.util.FileUtilities.*; import static net.filebot.util.FileUtilities.*;
import static net.filebot.web.OpenSubtitlesHasher.*; import static net.filebot.web.OpenSubtitlesHasher.*;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.net.URI; import java.net.URI;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection; import java.util.Collection;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.Locale; import java.util.Locale;
import java.util.Map; import java.util.Map;
import java.util.Map.Entry; import java.util.Map.Entry;
import java.util.Set; import java.util.Set;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import java.util.logging.Level; import java.util.logging.Level;
import java.util.logging.Logger; import java.util.logging.Logger;
import javax.swing.Icon; import javax.swing.Icon;
import net.filebot.Cache; import net.filebot.Cache;
import net.filebot.Cache.Key; import net.filebot.Cache.TypedCache;
import net.filebot.CacheType; import net.filebot.CacheType;
import net.filebot.ResourceManager; import net.filebot.ResourceManager;
import net.filebot.media.MediaDetection; import net.filebot.media.MediaDetection;
@ -58,6 +55,21 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS
this.xmlrpc = new OpenSubtitlesXmlRpcWithRetryAndFloodLimit(String.format("%s v%s", name, version), 2, 3000); this.xmlrpc = new OpenSubtitlesXmlRpcWithRetryAndFloodLimit(String.format("%s v%s", name, version), 2, 3000);
} }
@Override
public String getName() {
return "OpenSubtitles";
}
@Override
public Icon getIcon() {
return ResourceManager.getIcon("search.opensubtitles");
}
@Override
public URI getLink() {
return URI.create("http://www.opensubtitles.org");
}
public synchronized void setUser(String username, String password_md5) { public synchronized void setUser(String username, String password_md5) {
// cancel previous session // cancel previous session
this.logout(); this.logout();
@ -71,51 +83,46 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS
} }
@Override @Override
public String getName() { public List<SubtitleSearchResult> search(String query) throws Exception {
return "OpenSubtitles"; throw new UnsupportedOperationException("XMLRPC::SearchMoviesOnIMDB has been banned due to abuse");
} }
@Override @Override
public URI getLink() { public List<Movie> searchMovie(String query, Locale locale) throws Exception {
return URI.create("http://www.opensubtitles.org"); throw new UnsupportedOperationException("XMLRPC::SearchMoviesOnIMDB has been banned due to abuse");
}
@Override
public Icon getIcon() {
return ResourceManager.getIcon("search.opensubtitles");
}
public ResultCache getCache() {
return new ResultCache(getName(), Cache.getCache(getName(), CacheType.Daily));
}
@Override
public synchronized List<SubtitleSearchResult> search(String query) throws Exception {
throw new UnsupportedOperationException(); // XMLRPC::SearchMoviesOnIMDB is not allowed due to abuse
} }
@Override @Override
public synchronized List<SubtitleSearchResult> guess(String tag) throws Exception { public synchronized List<SubtitleSearchResult> guess(String tag) throws Exception {
List<SubtitleSearchResult> subtitles = getCache().getSearchResult("guess", tag);
if (subtitles != null) {
return subtitles;
}
// require login // require login
return getSearchCache("tag").computeIf(tag, Cache.isAbsent(), it -> {
login(); login();
return xmlrpc.guessMovie(singleton(tag)).getOrDefault(tag, emptyList());
subtitles = xmlrpc.guessMovie(singleton(tag)).getOrDefault(tag, emptyList()); });
getCache().putSearchResult("guess", tag, subtitles);
return subtitles;
} }
public synchronized List<SubtitleDescriptor> getSubtitleList(SubtitleSearchResult searchResult, String languageName) throws Exception { public synchronized List<SubtitleSearchResult> searchIMDB(String query) throws Exception {
// require login
return getSearchCache("query").computeIf(query, Cache.isAbsent(), it -> {
login();
return xmlrpc.searchMoviesOnIMDB(query);
});
}
public synchronized List<SubtitleDescriptor> getSubtitleList(Query query) throws Exception {
// require login
return getSubtitlesCache().computeIf(query, Cache.isAbsent(), it -> {
login();
return xmlrpc.searchSubtitles(singleton(query));
});
}
public List<SubtitleDescriptor> getSubtitleList(SubtitleSearchResult searchResult, String languageName) throws Exception {
return getSubtitleList(searchResult, -1, -1, languageName); return getSubtitleList(searchResult, -1, -1, languageName);
} }
@Override @Override
public synchronized List<SubtitleDescriptor> getSubtitleList(SubtitleSearchResult searchResult, int[][] episodeFilter, String languageName) throws Exception { public List<SubtitleDescriptor> getSubtitleList(SubtitleSearchResult searchResult, int[][] episodeFilter, String languageName) throws Exception {
// no filter // no filter
if (episodeFilter == null || episodeFilter.length == 0) { if (episodeFilter == null || episodeFilter.length == 0) {
return getSubtitleList(searchResult, -1, -1, languageName); return getSubtitleList(searchResult, -1, -1, languageName);
@ -156,41 +163,31 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS
} }
public synchronized List<SubtitleDescriptor> getSubtitleList(SubtitleSearchResult searchResult, int season, int episode, String languageName) throws Exception { public synchronized List<SubtitleDescriptor> getSubtitleList(SubtitleSearchResult searchResult, int season, int episode, String languageName) throws Exception {
int imdbid = ((Movie) searchResult).getImdbId(); Query query = Query.forImdbId(searchResult.getImdbId(), season, episode, getLanguageFilter(languageName));
String[] languageFilter = languageName != null ? new String[] { getSubLanguageID(languageName, true) } : new String[0];
Query query = Query.forImdbId(imdbid, season, episode, languageFilter);
List<SubtitleDescriptor> subtitles = getCache().getSubtitleDescriptorList(query);
if (subtitles != null) {
return subtitles;
}
// require login // require login
return getSubtitlesCache().computeIf(query, Cache.isAbsent(), it -> {
login(); login();
return xmlrpc.searchSubtitles(singleton(query));
// get subtitle list });
subtitles = asList(xmlrpc.searchSubtitles(imdbid, season, episode, languageFilter).toArray(new SubtitleDescriptor[0]));
getCache().putSubtitleDescriptorList(query, subtitles);
return subtitles;
} }
@Override @Override
public Map<File, List<SubtitleDescriptor>> getSubtitleList(File[] files, String languageName) throws Exception { public Map<File, List<SubtitleDescriptor>> getSubtitleList(File[] files, String languageName) throws Exception {
Map<File, List<SubtitleDescriptor>> results = new HashMap<File, List<SubtitleDescriptor>>(files.length); Map<File, List<SubtitleDescriptor>> results = new HashMap<File, List<SubtitleDescriptor>>(files.length);
Set<File> remainingFiles = new LinkedHashSet<File>(asList(files)); Set<File> remainingFiles = new HashSet<File>(asList(files));
// lookup subtitles by hash // lookup subtitles by hash
if (remainingFiles.size() > 0) { if (remainingFiles.size() > 0) {
results.putAll(getSubtitleListByHash(remainingFiles.toArray(new File[0]), languageName)); results.putAll(getSubtitleListByHash(remainingFiles.toArray(new File[0]), languageName));
} }
for (Entry<File, List<SubtitleDescriptor>> it : results.entrySet()) { // remove files for which subtitles have already been found
if (it.getValue().size() > 0) { results.forEach((k, v) -> {
remainingFiles.remove(it.getKey()); if (v.size() > 0) {
} remainingFiles.remove(k);
} }
});
// lookup subtitles by tag // lookup subtitles by tag
if (remainingFiles.size() > 0) { if (remainingFiles.size() > 0) {
@ -200,149 +197,51 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS
return results; return results;
} }
// max numbers of queries to submit in a single XML-RPC request, but currently only batchSize == 1 is supported protected Map<File, List<SubtitleDescriptor>> getSubtitleList(File[] files, Function<File, Query> queryMapper) throws Exception {
private final int batchSize = 1; Map<File, List<SubtitleDescriptor>> results = new HashMap<File, List<SubtitleDescriptor>>(files.length);
public synchronized Map<File, List<SubtitleDescriptor>> getSubtitleListByHash(File[] files, String languageName) throws Exception { // dispatch query for all hashes
// singleton array with or empty array for (File f : files) {
String[] languageFilter = languageName != null ? new String[] { getSubLanguageID(languageName, true) } : new String[0]; Query query = queryMapper.apply(f);
// remember hash for each file
Map<Query, File> hashMap = new HashMap<Query, File>(files.length);
Map<File, List<SubtitleDescriptor>> resultMap = new HashMap<File, List<SubtitleDescriptor>>(files.length);
// create hash query for each file
List<Query> hashQueryList = new ArrayList<Query>(files.length);
for (File file : files) {
Query query = file.length() > HASH_CHUNK_SIZE ? Query.forHash(computeHash(file), file.length(), languageFilter) : null;
// DEBUG
// query = Query.forHash("b4a91d8384a92269", 1178926184, languageFilter);
// add hash query
if (query != null) { if (query != null) {
List<SubtitleDescriptor> cachedResults = getCache().getSubtitleDescriptorList(query); results.put(f, getSubtitleList(query));
if (cachedResults == null) {
hashQueryList.add(query);
hashMap.put(query, file);
} else { } else {
resultMap.put(file, cachedResults); results.put(f, emptyList());
} }
} }
// prepare result map return results;
if (resultMap.get(file) == null) {
resultMap.put(file, new LinkedList<SubtitleDescriptor>());
}
} }
if (hashQueryList.size() > 0) { public Map<File, List<SubtitleDescriptor>> getSubtitleListByHash(File[] files, String language) throws Exception {
// require login return getSubtitleList(files, f -> {
login(); if (f.length() > HASH_CHUNK_SIZE) {
try {
// dispatch query for all hashes String hash = computeHash(f);
for (int bn = 0; bn < ceil((float) hashQueryList.size() / batchSize); bn++) { return Query.forHash(hash, f.length(), getLanguageFilter(language));
List<Query> batch = hashQueryList.subList(bn * batchSize, min((bn * batchSize) + batchSize, hashQueryList.size())); } catch (Exception e) {
debug.log(Level.SEVERE, "Failed to compute hash", e);
// submit query and map results to given files
for (OpenSubtitlesSubtitleDescriptor subtitle : xmlrpc.searchSubtitles(batch)) {
// get file for hash
File file = hashMap.get((batch.get(0)));
// add subtitle
if (file != null) {
resultMap.get(file).add(subtitle);
} else {
Logger.getLogger(getClass().getName()).log(Level.WARNING, "Unable to map hash to file: " + subtitle.getMovieHash());
}
}
for (Query query : batch) {
getCache().putSubtitleDescriptorList(query, resultMap.get(hashMap.get(query)));
} }
} }
return null;
});
} }
return resultMap; public Map<File, List<SubtitleDescriptor>> getSubtitleListByTag(File[] files, String language) throws Exception {
} return getSubtitleList(files, f -> {
String tag = getNameWithoutExtension(f.getName());
public synchronized Map<File, List<SubtitleDescriptor>> getSubtitleListByTag(File[] files, String languageName) throws Exception { return Query.forTag(tag, getLanguageFilter(language));
// singleton array with or empty array });
String[] languageFilter = languageName != null ? new String[] { getSubLanguageID(languageName, true) } : new String[0];
// remember tag for each file
Map<Query, File> tagMap = new HashMap<Query, File>(files.length);
Map<File, List<SubtitleDescriptor>> resultMap = new HashMap<File, List<SubtitleDescriptor>>(files.length);
// create tag query for each file
List<Query> tagQueryList = new ArrayList<Query>(files.length);
for (File file : files) {
// add tag query
String tag = getNameWithoutExtension(file.getName());
Query query = Query.forTag(tag, languageFilter);
// check tag
List<SubtitleDescriptor> cachedResults = getCache().getSubtitleDescriptorList(query);
if (cachedResults == null) {
tagQueryList.add(query);
tagMap.put(query, file);
} else {
resultMap.put(file, cachedResults);
}
// prepare result map
if (resultMap.get(file) == null) {
resultMap.put(file, new LinkedList<SubtitleDescriptor>());
}
}
if (tagQueryList.size() > 0) {
// require login
login();
// dispatch query for all hashes
for (int bn = 0; bn < ceil((float) tagQueryList.size() / batchSize); bn++) {
List<Query> batch = tagQueryList.subList(bn * batchSize, min((bn * batchSize) + batchSize, tagQueryList.size()));
// submit query and map results to given files
for (OpenSubtitlesSubtitleDescriptor subtitle : xmlrpc.searchSubtitles(batch)) {
// get file for tag
File file = tagMap.get(batch.get(0));
// add subtitle
if (file != null) {
resultMap.get(file).add(subtitle);
} else {
Logger.getLogger(getClass().getName()).log(Level.WARNING, "Unable to map release name to file: " + subtitle.getMovieReleaseName());
}
}
for (Query query : batch) {
getCache().putSubtitleDescriptorList(query, resultMap.get(tagMap.get(query)));
}
}
}
return resultMap;
} }
@Override @Override
public synchronized CheckResult checkSubtitle(File videoFile, File subtitleFile) throws Exception { public synchronized CheckResult checkSubtitle(File videoFile, File subtitleFile) throws Exception {
// subhash (md5 of subtitles), subfilename, moviehash, moviebytesize, moviefilename
SubFile sub = new SubFile();
sub.setSubHash(md5(readFile(subtitleFile)));
sub.setSubFileName(subtitleFile.getName());
sub.setMovieHash(computeHash(videoFile));
sub.setMovieByteSize(videoFile.length());
sub.setMovieFileName(videoFile.getName());
// require login // require login
login(); login();
// check if subs already exist in DB // check if subs already exist in DB
TryUploadResponse response = xmlrpc.tryUploadSubtitles(sub); SubFile subFile = getSubFile(videoFile, subtitleFile, false);
TryUploadResponse response = xmlrpc.tryUploadSubtitles(subFile);
// TryUploadResponse: false => [{HashWasAlreadyInDb=1, MovieKind=movie, IDSubtitle=3167446, MoviefilenameWasAlreadyInDb=1, ISO639=en, MovieYear=2007, SubLanguageID=eng, MovieName=Blades of Glory, MovieNameEng=, IDMovieImdb=445934}] // TryUploadResponse: false => [{HashWasAlreadyInDb=1, MovieKind=movie, IDSubtitle=3167446, MoviefilenameWasAlreadyInDb=1, ISO639=en, MovieYear=2007, SubLanguageID=eng, MovieName=Blades of Glory, MovieNameEng=, IDMovieImdb=445934}]
boolean exists = !response.isUploadRequired(); boolean exists = !response.isUploadRequired();
@ -361,7 +260,7 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS
String year = fields.get("MovieYear"); String year = fields.get("MovieYear");
identity = new Movie(name, Integer.parseInt(year), Integer.parseInt(imdb), -1); identity = new Movie(name, Integer.parseInt(year), Integer.parseInt(imdb), -1);
} catch (Exception e) { } catch (Exception e) {
Logger.getLogger(getClass().getName()).log(Level.WARNING, e.getMessage()); debug.log(Level.SEVERE, "Failed to upload subtitles", e);
} }
} }
@ -371,11 +270,13 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS
@Override @Override
public synchronized void uploadSubtitle(Object identity, Locale language, File[] videoFile, File[] subtitleFile) throws Exception { public synchronized void uploadSubtitle(Object identity, Locale language, File[] videoFile, File[] subtitleFile) throws Exception {
if (!(identity instanceof Movie && ((Movie) identity).getImdbId() > 0)) { int imdbid = -1;
try {
imdbid = ((Movie) identity).getImdbId();
} catch (Exception e) {
throw new IllegalArgumentException("Illegal Movie ID: " + identity); throw new IllegalArgumentException("Illegal Movie ID: " + identity);
} }
int imdbid = ((Movie) identity).getImdbId();
String subLanguageID = getSubLanguageID(language.getDisplayName(Locale.ENGLISH), false); String subLanguageID = getSubLanguageID(language.getDisplayName(Locale.ENGLISH), false);
BaseInfo info = new BaseInfo(); BaseInfo info = new BaseInfo();
@ -384,7 +285,7 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS
SubFile[] subFiles = new SubFile[videoFile.length]; SubFile[] subFiles = new SubFile[videoFile.length];
for (int i = 0; i < subFiles.length; i++) { for (int i = 0; i < subFiles.length; i++) {
subFiles[i] = getSubFile(info, videoFile[i], subtitleFile[i]); subFiles[i] = getSubFile(videoFile[i], subtitleFile[i], true);
} }
// require login // require login
@ -393,7 +294,7 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS
xmlrpc.uploadSubtitles(info, subFiles); xmlrpc.uploadSubtitles(info, subFiles);
} }
private SubFile getSubFile(BaseInfo info, File videoFile, File subtitleFile) throws IOException { protected SubFile getSubFile(File videoFile, File subtitleFile, boolean content) throws IOException {
// subhash (md5 of subtitles), subfilename, moviehash, moviebytesize, moviefilename // subhash (md5 of subtitles), subfilename, moviehash, moviebytesize, moviefilename
SubFile sub = new SubFile(); SubFile sub = new SubFile();
sub.setSubHash(md5(readFile(subtitleFile))); sub.setSubHash(md5(readFile(subtitleFile)));
@ -403,66 +304,32 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS
sub.setMovieFileName(videoFile.getName()); sub.setMovieFileName(videoFile.getName());
// encode subtitle contents // encode subtitle contents
if (content) {
sub.setSubContent(readFile(subtitleFile)); sub.setSubContent(readFile(subtitleFile));
}
try { try (MediaInfo mi = new MediaInfo()) {
MediaInfo mi = new MediaInfo();
mi.open(videoFile); mi.open(videoFile);
sub.setMovieFPS(mi.get(StreamKind.Video, 0, "FrameRate")); sub.setMovieFPS(mi.get(StreamKind.Video, 0, "FrameRate"));
sub.setMovieTimeMS(mi.get(StreamKind.General, 0, "Duration")); sub.setMovieTimeMS(mi.get(StreamKind.General, 0, "Duration"));
mi.close();
} catch (Throwable e) { } catch (Throwable e) {
Logger.getLogger(getClass().getName()).log(Level.WARNING, e.getMessage(), e); debug.log(Level.SEVERE, "Failed to read media info", e);
} }
return sub; return sub;
} }
@Override
public List<Movie> searchMovie(String query, Locale locale) throws Exception {
throw new UnsupportedOperationException();
}
public synchronized List<SubtitleSearchResult> searchIMDB(String query) throws Exception {
// search for movies and series
List<SubtitleSearchResult> result = getCache().getSearchResult("search", query);
if (result != null) {
return result;
}
// require login
login();
try {
// search for movies / series
result = xmlrpc.searchMoviesOnIMDB(query);
} catch (ClassCastException e) {
// unexpected xmlrpc responses (e.g. error messages instead of results) will trigger this
throw new XmlRpcException("Illegal XMLRPC response on searchMoviesOnIMDB");
}
getCache().putSearchResult("search", query, result);
return result;
}
@Override @Override
public synchronized Movie getMovieDescriptor(Movie id, Locale locale) throws Exception { public synchronized Movie getMovieDescriptor(Movie id, Locale locale) throws Exception {
if (id.getImdbId() <= 0) { if (id.getImdbId() <= 0) {
throw new IllegalArgumentException("id must not be " + id.getImdbId()); throw new IllegalArgumentException("Illegal IMDbID ID: " + id.getImdbId());
}
Movie result = getCache().getData("getMovieDescriptor", id.getImdbId(), locale, Movie.class);
if (result != null) {
return result;
} }
// require login // require login
return getLookupCache(locale).computeIf(id.getImdbId(), Cache.isAbsent(), it -> {
login(); login();
return xmlrpc.getIMDBMovieDetails(id.getImdbId());
Movie movie = xmlrpc.getIMDBMovieDetails(id.getImdbId()); });
getCache().putData("getMovieDescriptor", id.getImdbId(), locale, movie);
return movie;
} }
public Movie getMovieDescriptor(File movieFile, Locale locale) throws Exception { public Movie getMovieDescriptor(File movieFile, Locale locale) throws Exception {
@ -472,55 +339,24 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS
@Override @Override
public synchronized Map<File, Movie> getMovieDescriptors(Collection<File> movieFiles, Locale locale) throws Exception { public synchronized Map<File, Movie> getMovieDescriptors(Collection<File> movieFiles, Locale locale) throws Exception {
// create result array // create result array
Map<File, Movie> result = new HashMap<File, Movie>(); Map<File, Movie> results = new HashMap<File, Movie>();
// compute movie hashes // make sure we don't get mismatches by making sure the hash has not been confirmed numerous times
Map<String, File> hashMap = new HashMap<String, File>(movieFiles.size()); int minSeenCount = 20;
for (File file : movieFiles) { for (File f : movieFiles) {
if (file.length() > HASH_CHUNK_SIZE) { if (f.length() > HASH_CHUNK_SIZE) {
String hash = computeHash(file); String hash = computeHash(f);
Movie entry = getCache().getData("getMovieDescriptor", hash, locale, Movie.class); Movie match = getLookupCache(locale).computeIf(hash, Cache.isAbsent(), it -> {
if (entry == null) { return xmlrpc.checkMovieHash(singleton(hash), minSeenCount).get(hash);
hashMap.put(hash, file); // map file by hash });
} else if (entry.getName().length() > 0) {
result.put(file, entry);
}
}
}
if (hashMap.size() > 0) { results.put(f, match);
// require login
login();
// dispatch query for all hashes
List<String> hashes = new ArrayList<String>(hashMap.keySet());
int batchSize = 50;
for (int bn = 0; bn < ceil((float) hashes.size() / batchSize); bn++) {
List<String> batch = hashes.subList(bn * batchSize, min((bn * batchSize) + batchSize, hashes.size()));
Set<String> unmatchedHashes = new HashSet<String>(batch);
int minSeenCount = 20; // make sure we don't get mismatches by making sure the hash has not been confirmed numerous times
for (Entry<String, Movie> it : xmlrpc.checkMovieHash(batch, minSeenCount).entrySet()) {
String hash = it.getKey();
Movie movie = it.getValue();
result.put(hashMap.get(hash), movie);
getCache().putData("getMovieDescriptor", hash, locale, movie);
unmatchedHashes.remove(hash);
}
// note hashes that are not matched to any items so we can ignore them in the future
for (String hash : unmatchedHashes) {
getCache().putData("getMovieDescriptor", hash, locale, new Movie("", -1, -1, -1));
} }
} }
} return results;
return result;
} }
@Override @Override
@ -541,24 +377,16 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS
public synchronized Locale detectLanguage(byte[] data) throws Exception { public synchronized Locale detectLanguage(byte[] data) throws Exception {
if (data.length < 256) { if (data.length < 256) {
throw new IllegalArgumentException("data is not enough"); throw new IllegalArgumentException("Data is too small: " + data.length);
}
String language = getCache().getData("detectLanguage", md5(data), Locale.ROOT, String.class);
if (language != null) {
return language.isEmpty() ? null : new Locale(language);
} }
// require login // require login
List<String> languages = getCache("detect").castList(String.class).computeIf(md5(data), Cache.isAbsent(), it -> {
login(); login();
return xmlrpc.detectLanguage(data);
});
// detect language return languages.size() > 0 ? new Locale(languages.get(0)) : Locale.ROOT;
List<String> languages = xmlrpc.detectLanguage(data);
// return first language
language = languages.size() > 0 ? languages.get(0) : "";
getCache().putData("detectLanguage", md5(data), Locale.ROOT, language);
return new Locale(language);
} }
public synchronized void login() throws Exception { public synchronized void login() throws Exception {
@ -574,7 +402,7 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS
try { try {
xmlrpc.logout(); xmlrpc.logout();
} catch (Exception e) { } catch (Exception e) {
Logger.getLogger(getClass().getName()).log(Level.WARNING, "Logout failed", e); debug.log(Level.WARNING, "Failed to log out", e);
} }
} }
logoutTimer.cancel(); logoutTimer.cancel();
@ -602,32 +430,32 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS
/** /**
* SubLanguageID by English language name * SubLanguageID by English language name
*/ */
@SuppressWarnings("unchecked") protected synchronized Map<String, String> getSubLanguageMap() throws Exception {
protected synchronized Map<String, String> getSubLanguageMap() throws IOException { Map<String, String> subLanguageMap = new HashMap<String, String>();
Cache cache = Cache.getCache(getName(), CacheType.Persistent);
String cacheKey = getClass().getName() + ".subLanguageMap";
// try to get language map from cache // try to get language map from cache
Map<String, String> subLanguageMap = cache.get(cacheKey, Map.class); Cache cache = Cache.getCache(getName() + "_languages", CacheType.Persistent);
Map<?, ?> m = (Map<?, ?>) cache.computeIf("subLanguageMap", Cache.isAbsent(), it -> {
if (subLanguageMap == null) { try {
subLanguageMap = new HashMap<String, String>(); return xmlrpc.getSubLanguages();
} catch (Exception e) {
throw new IOException("Failed to retrieve subtitle language map", e);
}
});
// add additional language aliases for improved compatibility // add additional language aliases for improved compatibility
Map<String, Locale> additionalLanguageMappings = MediaDetection.releaseInfo.getLanguageMap(Locale.ENGLISH); Map<String, Locale> additionalLanguageMappings = MediaDetection.releaseInfo.getLanguageMap(Locale.ENGLISH);
// fetch language data m.forEach((k, v) -> {
try {
for (Entry<String, String> entry : xmlrpc.getSubLanguages().entrySet()) {
// map id by name // map id by name
String subLanguageID = entry.getKey().toLowerCase(); String subLanguageID = k.toString().toLowerCase();
String subLanguageName = entry.getValue().toLowerCase(); String subLanguageName = v.toString().toLowerCase();
subLanguageMap.put(subLanguageName, subLanguageID); subLanguageMap.put(subLanguageName, subLanguageID);
subLanguageMap.put(subLanguageID, subLanguageID); // add reverse mapping as well for improved compatibility subLanguageMap.put(subLanguageID, subLanguageID); // add reverse mapping as well for improved compatibility
// add additional language aliases for improved compatibility // add additional language aliases for improved compatibility
for (String key : asList(subLanguageID, subLanguageName)) { for (String key : new String[] { subLanguageID, subLanguageName }) {
Locale locale = additionalLanguageMappings.get(key); Locale locale = additionalLanguageMappings.get(key);
if (locale != null) { if (locale != null) {
for (String identifier : asList(locale.getLanguage(), locale.getISO3Language(), locale.getDisplayLanguage(Locale.ENGLISH))) { for (String identifier : asList(locale.getLanguage(), locale.getISO3Language(), locale.getDisplayLanguage(Locale.ENGLISH))) {
@ -637,20 +465,17 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS
} }
} }
} }
} });
} catch (Exception e) {
throw new IOException("Failed to retrieve subtitle language list.", e);
}
// some additional special handling // some additional special handling
subLanguageMap.put("brazilian", "pob"); subLanguageMap.put("brazilian", "pob");
subLanguageMap.put("chinese", "chi,zht,zhe"); // Chinese (Simplified) / Chinese (Traditional) / Chinese (bilingual) subLanguageMap.put("chinese", "chi,zht,zhe"); // Chinese (Simplified) / Chinese (Traditional) / Chinese (bilingual)
// cache data return subLanguageMap;
cache.put(cacheKey, subLanguageMap);
} }
return subLanguageMap; protected String[] getLanguageFilter(String language) {
return language == null || language.isEmpty() ? new String[0] : new String[] { getSubLanguageID(language, true) };
} }
protected String getSubLanguageID(String languageName, boolean allowMultiLanguageID) { protected String getSubLanguageID(String languageName, boolean allowMultiLanguageID) {
@ -663,7 +488,7 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS
subLanguageID = subLanguageID.substring(0, subLanguageID.indexOf(",")); subLanguageID = subLanguageID.substring(0, subLanguageID.indexOf(","));
} }
return subLanguageID; return subLanguageID;
} catch (IOException e) { } catch (Exception e) {
throw new IllegalStateException(e); throw new IllegalStateException(e);
} }
} }
@ -677,87 +502,20 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS
return null; return null;
} }
protected static class ResultCache { public Cache getCache(String section) {
return Cache.getCache(getName() + "_" + section, CacheType.Daily);
private final String id;
private final Cache cache;
public ResultCache(String id, Cache cache) {
this.id = id;
this.cache = cache;
} }
protected String normalize(String query) { protected TypedCache<List<SubtitleSearchResult>> getSearchCache(String method) {
return query == null ? null : query.trim().toLowerCase(); return getCache("search_" + method).castList(SubtitleSearchResult.class);
} }
public <T extends SubtitleSearchResult> List<T> putSearchResult(String method, String query, List<T> value) { protected TypedCache<List<SubtitleDescriptor>> getSubtitlesCache() {
try { return getCache("data").castList(SubtitleDescriptor.class);
cache.put(new Key(id, method, normalize(query)), value.toArray(new SubtitleSearchResult[0]));
} catch (Exception e) {
Logger.getLogger(OpenSubtitlesClient.class.getName()).log(Level.WARNING, e.getMessage());
} }
return value; protected TypedCache<Movie> getLookupCache(Locale locale) {
} return getCache("lookup_" + locale).cast(Movie.class);
@SuppressWarnings("unchecked")
public List<SubtitleSearchResult> getSearchResult(String method, String query) {
try {
SubtitleSearchResult[] array = cache.get(new Key(id, method, normalize(query)), SubtitleSearchResult[].class);
if (array != null) {
return Arrays.asList(array);
}
} catch (Exception e) {
Logger.getLogger(OpenSubtitlesClient.class.getName()).log(Level.WARNING, e.getMessage(), e);
}
return null;
}
public List<SubtitleDescriptor> putSubtitleDescriptorList(Query key, List<SubtitleDescriptor> subtitles) {
try {
cache.put(new Key(id, key), subtitles.toArray(new SubtitleDescriptor[0]));
} catch (Exception e) {
Logger.getLogger(OpenSubtitlesClient.class.getName()).log(Level.WARNING, e.getMessage());
}
return subtitles;
}
public List<SubtitleDescriptor> getSubtitleDescriptorList(Query key) {
try {
SubtitleDescriptor[] descriptors = cache.get(new Key(id, key), SubtitleDescriptor[].class);
if (descriptors != null) {
return Arrays.asList(descriptors);
}
} catch (Exception e) {
Logger.getLogger(OpenSubtitlesClient.class.getName()).log(Level.WARNING, e.getMessage(), e);
}
return null;
}
public void putData(Object category, Object key, Locale locale, Object object) {
try {
cache.put(new Key(id, category, locale, key), object);
} catch (Exception e) {
Logger.getLogger(OpenSubtitlesClient.class.getName()).log(Level.WARNING, e.getMessage());
}
}
public <T> T getData(Object category, Object key, Locale locale, Class<T> type) {
try {
T value = cache.get(new Key(id, category, locale, key), type);
if (value != null) {
return value;
}
} catch (Exception e) {
Logger.getLogger(OpenSubtitlesClient.class.getName()).log(Level.WARNING, e.getMessage(), e);
}
return null;
}
} }
protected static class OpenSubtitlesXmlRpcWithRetryAndFloodLimit extends OpenSubtitlesXmlRpc { protected static class OpenSubtitlesXmlRpcWithRetryAndFloodLimit extends OpenSubtitlesXmlRpc {

View File

@ -90,10 +90,6 @@ public class OpenSubtitlesXmlRpc {
return (Map<String, String>) invoke("ServerInfo", token); return (Map<String, String>) invoke("ServerInfo", token);
} }
public List<OpenSubtitlesSubtitleDescriptor> searchSubtitles(int imdbid, int season, int episode, String... sublanguageids) throws XmlRpcFault {
return searchSubtitles(singleton(Query.forImdbId(imdbid, season, episode, sublanguageids)));
}
public List<OpenSubtitlesSubtitleDescriptor> searchSubtitles(Collection<Query> queryList) throws XmlRpcFault { public List<OpenSubtitlesSubtitleDescriptor> searchSubtitles(Collection<Query> queryList) throws XmlRpcFault {
List<OpenSubtitlesSubtitleDescriptor> subtitles = new ArrayList<OpenSubtitlesSubtitleDescriptor>(); List<OpenSubtitlesSubtitleDescriptor> subtitles = new ArrayList<OpenSubtitlesSubtitleDescriptor>();
Map<?, ?> response = invoke("SearchSubtitles", token, queryList); Map<?, ?> response = invoke("SearchSubtitles", token, queryList);
@ -112,6 +108,8 @@ public class OpenSubtitlesXmlRpc {
} }
public List<SubtitleSearchResult> searchMoviesOnIMDB(String query) throws XmlRpcFault { public List<SubtitleSearchResult> searchMoviesOnIMDB(String query) throws XmlRpcFault {
try {
// search for movies / series
Map<?, ?> response = invoke("SearchMoviesOnIMDB", token, query); Map<?, ?> response = invoke("SearchMoviesOnIMDB", token, query);
List<Map<String, String>> movieData = (List<Map<String, String>>) response.get("data"); List<Map<String, String>> movieData = (List<Map<String, String>>) response.get("data");
@ -141,6 +139,10 @@ public class OpenSubtitlesXmlRpc {
} }
return movies; return movies;
} catch (ClassCastException e) {
// unexpected xmlrpc responses (e.g. error messages instead of results) will trigger this
throw new XmlRpcException("Illegal XMLRPC response on searchMoviesOnIMDB");
}
} }
public Movie getIMDBMovieDetails(int imdbid) throws XmlRpcFault { public Movie getIMDBMovieDetails(int imdbid) throws XmlRpcFault {

View File

@ -1,7 +1,6 @@
package net.filebot.web; package net.filebot.web;
import static java.nio.charset.StandardCharsets.*; import static java.nio.charset.StandardCharsets.*;
import static java.util.Arrays.*;
import static java.util.Collections.*; import static java.util.Collections.*;
import static java.util.stream.Collectors.*; import static java.util.stream.Collectors.*;
import static net.filebot.util.FileUtilities.*; import static net.filebot.util.FileUtilities.*;
@ -51,6 +50,10 @@ public class ShooterSubtitles implements VideoHashSubtitleService {
return ResourceManager.getIcon("search.shooter"); return ResourceManager.getIcon("search.shooter");
} }
public Cache getCache() {
return Cache.getCache(getName(), CacheType.Daily);
}
@Override @Override
public Map<File, List<SubtitleDescriptor>> getSubtitleList(File[] videoFiles, String languageName) throws Exception { public Map<File, List<SubtitleDescriptor>> getSubtitleList(File[] videoFiles, String languageName) throws Exception {
Map<File, List<SubtitleDescriptor>> result = new HashMap<File, List<SubtitleDescriptor>>(); Map<File, List<SubtitleDescriptor>> result = new HashMap<File, List<SubtitleDescriptor>>();
@ -86,24 +89,19 @@ public class ShooterSubtitles implements VideoHashSubtitleService {
param.put("format", "json"); param.put("format", "json");
param.put("lang", LANGUAGE_CHINESE.equals(languageName) ? "Chn" : "Eng"); param.put("lang", LANGUAGE_CHINESE.equals(languageName) ? "Chn" : "Eng");
Cache cache = Cache.getCache(getName(), CacheType.Daily); // use the first best option and ignore the rest
String key = endpoint.toString() + param.toString(); return getCache().castList(SubtitleDescriptor.class).computeIf(param.toString(), Cache.isAbsent(), it -> {
SubtitleDescriptor[] value = cache.get(key, SubtitleDescriptor[].class);
if (value != null) {
return asList(value);
}
ByteBuffer post = WebRequest.post(endpoint, param, null); ByteBuffer post = WebRequest.post(endpoint, param, null);
Object response = readJson(UTF_8.decode(post)); Object response = readJson(UTF_8.decode(post));
String name = getNameWithoutExtension(file.getName()); String name = getNameWithoutExtension(file.getName());
// use the first best option and ignore the rest return streamJsonObjects(response).flatMap(n -> streamJsonObjects(n, "Files")).map(f -> {
return streamJsonObjects(response).flatMap(it -> streamJsonObjects(it, "Files")).map(f -> {
String type = getString(f, "Ext"); String type = getString(f, "Ext");
String link = getString(f, "Link"); String link = getString(f, "Link");
return new ShooterSubtitleDescriptor(name, type, link, languageName); return new ShooterSubtitleDescriptor(name, type, link, languageName);
}).limit(1).collect(toList()); }).limit(1).collect(toList());
});
} }
@Override @Override

View File

@ -2,8 +2,6 @@ package net.filebot.web;
import static org.junit.Assert.*; import static org.junit.Assert.*;
import java.io.IOException;
import org.junit.Test; import org.junit.Test;
public class AcoustIDClientTest { public class AcoustIDClientTest {
@ -23,7 +21,7 @@ public class AcoustIDClientTest {
} }
@Test @Test
public void lookupChinese() throws IOException, InterruptedException { public void lookupChinese() throws Exception {
int duration = 265; int duration = 265;
String fingerprint = "AQADtJmWRFJCRkGcHXceND_Oquhz1H_xvjg8D8mYIf5-fOpKVE81NOOF5_hb1PrxJLhhRxfCLkNSOh-61Mvh3DJKOcMnG2eOPsmH_4R_I-SPJB-Df0NTB70-vPiDRtKOS3qK-Duhc8mR9-jyoOvhKhyJH-8FE5eOHHoy3BpC5Tieo_kV9LiZ41McHCF9iJmDPFTQPeiYHD5J_Cfy7DBz_IdxrXBRMTyeP9CSLeJQ8glyC18S5imeSHkSPBeavwgfHrqSD7l09Fl2mGMDVzmekAzS_Gi3HPJbIr0VvD2eJD8u-eAPxytyMccTTQ8u5YmR63iG5jm6UG3wJcmFyzFuNEclvcETfcSb4T-aDz0RWoGs5MQlg9mUBXlyodYjbId_lHke4vPxCFuO5-iNJkw-5FseJM9RckFPHJ-kH3USNL_w4i7xpT5iHvlCVOIR5viFJyFK2Yd59DzqHe8wNUfJF1rwakJTZdB_TJkk5Ntx6YOb7MHj40SZh1ERZv3wD3yaohmzC-2DyM_xND_e48-CWx-6H5FC6ApxpUf-4jpyuEo64VLCZsrwHXcWlBcaDdeWI6Zz5D9qQ0wz4WNPPPCYWWgv6OTw7Ugb_Ghihkd4gdKSqBL0hGyCH1q59GjkHaeQJ0fJLEbz2XiOmBu-RIemG55TDmGPJ3i04zouycIZuMrI4Exz480xqvnxhUS68oHKTD7-I1LGLEX7CteFmNGDD3qO6UvQRNSxKsoVXDnurEHzYWIfPKmFWJqJp3OQP4c5Ukqhx0QeY_-OXj1MLozw6AzqJBmOfNCUKslDhOHB40mGR19wHT0RplRwMsQT5VGG_Kh--D2qKEpU4quFc0Z5NIx4fJIXnHjR73COkNCaJEdF0cifE2SkPHiEJj76Y8qzHO27FE-O79hyS8R59HoQRssV6NKRx6i-o2eKP0ednGiq4Ej1oHeGZgqJnzOeCyEzvmh0XCryaLlRSUJ6YjpxHdd04-qE_kVz_EGNOsR84cuFMJNooWIgNzziKw2anUeZ5cYzBbyI3jnCPB90R0L0hFSBJxfmLLPQHF2T4Ql5nD2eWMFtwnWQL8msQF-MfEezC0x34wue48kRktORUIcbB-fhZw-87ArKI8-E5AmD-mieB7dyXKQYXMvxHDkU5T9K8sTTQ1eLPkeYG992xAweQ-eD5lmDUN_xicaX6PiNM1vxw91InGmOv7ior_AXPDueXbiOhlwkHeUV_DF-5D90Fc1zJCMzK8aT45KNV0XzYvJxJccfIZ-RLEdPKcGzDkdPonmWDMcf4WFy_MGTFVoiGp3WRwOfZMd1PDnxJ4GbCc9O5Eeih2jWo2_AKpPxKMTFHVWOhEdo_AmuD89yNJETdEdSHmGcHEezrjFoCZf0w8Lo7PC54M6DT-vwhEcPP4Hzo9uOH98TJH_TIRJzHNaS4-TBz8nRh8GZFz-eG08f9MzRrDu-iMjWC5eG_wjpoQl_zNHx3mh6BT09o1Zg93iKZj0uhSQmH8nyBHl3VHszUM9RytLwJEqKmT96NC-H41eIHzk0PUeYSx_WCE9y4qaGo7KUomk0C9eJ5L-RfUFzodSLJ3goHX3gHHkJPUMuFdfyJHB8hCeSZxH4BE1vVL6CrkmM6lEePGeK_Ei0v8KTw9GRb3gO_YfjLMgn6agYIh3xfIJ2T8jDodR2OAyso8e3bIrhJgyHHzvyKhCjpxE-MUIezeiD5suW4LzxJEfMlEJy4VFCNNeLUjqeJJ6OP8PNwsfTB9dxHUmeXAgzLuHQ4wljfDqahegj4R2JKzlO5fiDkuERJkfiKVHwxB_OHPyH6soEJk07fB9u6CyF5rmCflFwBdyiRHg4IReSBfWDX8GnHGVGNHOE-NCssEjTHFeNplsPdgmqKBea8eiD5jwuBo-SZ6gkfmhk4U6EpmKOPviJUD-S50WVgE911Id9HX3k4AmHG-9fPAneJ-hBjcWjoPzxHI9JhDSx5awQSlGPWg_8bULPHE2VE30oNNNQb7hyTD4aVUYeQUv2Ivzw1AEd7Wge9Ph-4M_xC89uJEusIVyY4cWTRC_-4-hNNI2WSciP5J3BL_Au4UyPKzgaPUN4yAuP0Dw85jqejE6CUNCRZ8OTo-mD58atGJ0jXjjjIxd09gh7hNnx4UkOnYe_4MQDTWJgaauE8Bc-KmgTDQ_6CleKD6Z8Yufxtwh1QcuCfEmGS4rwE_444TniREmOH-_wCP6MO2mDNUouPNEPTUfzqMPZI1_EB1feHDeFkjzCjMygJznuI2dYouXhS3gfhOXxi9Ap2Uj14DyeJFl6sHrwH37wKRm-8HiUJwnOC-UTNFJ1XMwz4eKDr0GbH54kVJWC78GpVfjRHD2SLcmGb0f44SAj5WgfNGOO5j36K_jUo09mNMWDbrbQOEcsLSeS50fPHL9QUceFKhOP4w_cGKGV489FPD0aJTvKP_hE6EmGpkKZH7ri4FRm44o5oZQ5-Jh-ou_RNImCw1GqTYjMo_aDRsuO__gS-CKefQlyKSf4NEdPGmGOP8eZKbjihzh1nJ_QvEa4Q6VC4UqkIz_-oEeYUH3wRR--C10JD7WOMwquhwbPQ9t6Iw1TXMNPnBciZXkC1YiYRSxq-MI35KikzEoCpcrxLNB2NDlyKcPIGH0OH3eE5wglrdB3hOGH6eKEMw725Oh39DN-NP1RMaGHhznu7Ei_ZIWei7iSCnmOZvbQM1zQ78hTo-aA6ROhJWKi49OxJzdi9fgUlL_RNMm644suXEcsJjua69BsfFpKPDrCV4fPIFe-4IknofKNo9O0Q9L4oFauoBzaiD7eG018oZSj44nW4HlCNPFioa2CPMyFUzye5vjR7mgibTitiRh5Yj7aCQAIMEQoACCQQABhCBNGCMGUEcAYYg1AQCAhDDAAAAagAQAQAAh0BmBCiPAMEHOUAAogASCBBAAklFBEEWaEIkQAAAQjQAgAgHHCIASIA0gQQwQASBggBBFKUCMAIFQSBIQERAnCECRWEACEQJIipYABnIEhACECEUAIs8wg4QRCACBmBCAEKAEIAwYpABSACAAECASCCEKgE04oYBRQChgHhABkAUEAEAIwAwAgQFAgFGMUGcIoA0AIARgAyBhAFKIKCEOIAgQZIARARhgggBGUAYiRAUAIRYQEBABAkBAECOMMIgAIhIQBCiCFCHBMEASAAEoACogoCAFGgGUGWIUAMIgRBIAzhgAFFAJEAEIEgAwZoBgAmCFmhKEMGAAUAAYsQgABgChgNAUGAASIgEAYB4gRQABBBGMGCKEAAIwggShwCChkgGIEGQaocoQBoLRBSACOHALEMQAEAkwAYoAkTDkDICEKIUMQAEQEQRRAgBCJACJCIWSAAKwBQYQABEkCBRHAAEUAoAJIgYBQTAkDAAEIIGEEMIsQx4wAwhgiEQFCGAQQA0oY4gUhCgElgEHEOECEUAYhJAQwxBgJJANIUQKEIYIoARgDhhlAHFBIIAmQQRYhoAhhCBgnLCKEKCEQIQEIR4RTCgkkCAKGCmQQQQAQgAQFgBFBCCGSAwAEIcoIoJQEDCHhEDAKKmCREswQohAgiglkACACKgOhMUAhRgwCBAipEFGMAAGEUABYhAhSAiEFoFEQASGAIAAxJQiESDEHgCEISAEQEAAIJwhAShGoBAUAUCaIUAwIgQgzghgggECAKCIcMQoJxYgSwABlhACAMgMAEkwCpCBzhgADGDACWMIEA0YbIQBiQAAgGCFAKCCEJBAAAYwhAGmiCCOOAIEcMwAIggBzjBhHmHeMSCGIEggiAA0jTCHjCABCAAUQIUQJIowQggFkKBEAGCEBBMQgBIARBkGDCCFGCMMMIIAgQBgSSAI"; String fingerprint = "AQADtJmWRFJCRkGcHXceND_Oquhz1H_xvjg8D8mYIf5-fOpKVE81NOOF5_hb1PrxJLhhRxfCLkNSOh-61Mvh3DJKOcMnG2eOPsmH_4R_I-SPJB-Df0NTB70-vPiDRtKOS3qK-Duhc8mR9-jyoOvhKhyJH-8FE5eOHHoy3BpC5Tieo_kV9LiZ41McHCF9iJmDPFTQPeiYHD5J_Cfy7DBz_IdxrXBRMTyeP9CSLeJQ8glyC18S5imeSHkSPBeavwgfHrqSD7l09Fl2mGMDVzmekAzS_Gi3HPJbIr0VvD2eJD8u-eAPxytyMccTTQ8u5YmR63iG5jm6UG3wJcmFyzFuNEclvcETfcSb4T-aDz0RWoGs5MQlg9mUBXlyodYjbId_lHke4vPxCFuO5-iNJkw-5FseJM9RckFPHJ-kH3USNL_w4i7xpT5iHvlCVOIR5viFJyFK2Yd59DzqHe8wNUfJF1rwakJTZdB_TJkk5Ntx6YOb7MHj40SZh1ERZv3wD3yaohmzC-2DyM_xND_e48-CWx-6H5FC6ApxpUf-4jpyuEo64VLCZsrwHXcWlBcaDdeWI6Zz5D9qQ0wz4WNPPPCYWWgv6OTw7Ugb_Ghihkd4gdKSqBL0hGyCH1q59GjkHaeQJ0fJLEbz2XiOmBu-RIemG55TDmGPJ3i04zouycIZuMrI4Exz480xqvnxhUS68oHKTD7-I1LGLEX7CteFmNGDD3qO6UvQRNSxKsoVXDnurEHzYWIfPKmFWJqJp3OQP4c5Ukqhx0QeY_-OXj1MLozw6AzqJBmOfNCUKslDhOHB40mGR19wHT0RplRwMsQT5VGG_Kh--D2qKEpU4quFc0Z5NIx4fJIXnHjR73COkNCaJEdF0cifE2SkPHiEJj76Y8qzHO27FE-O79hyS8R59HoQRssV6NKRx6i-o2eKP0ednGiq4Ej1oHeGZgqJnzOeCyEzvmh0XCryaLlRSUJ6YjpxHdd04-qE_kVz_EGNOsR84cuFMJNooWIgNzziKw2anUeZ5cYzBbyI3jnCPB90R0L0hFSBJxfmLLPQHF2T4Ql5nD2eWMFtwnWQL8msQF-MfEezC0x34wue48kRktORUIcbB-fhZw-87ArKI8-E5AmD-mieB7dyXKQYXMvxHDkU5T9K8sTTQ1eLPkeYG992xAweQ-eD5lmDUN_xicaX6PiNM1vxw91InGmOv7ior_AXPDueXbiOhlwkHeUV_DF-5D90Fc1zJCMzK8aT45KNV0XzYvJxJccfIZ-RLEdPKcGzDkdPonmWDMcf4WFy_MGTFVoiGp3WRwOfZMd1PDnxJ4GbCc9O5Eeih2jWo2_AKpPxKMTFHVWOhEdo_AmuD89yNJETdEdSHmGcHEezrjFoCZf0w8Lo7PC54M6DT-vwhEcPP4Hzo9uOH98TJH_TIRJzHNaS4-TBz8nRh8GZFz-eG08f9MzRrDu-iMjWC5eG_wjpoQl_zNHx3mh6BT09o1Zg93iKZj0uhSQmH8nyBHl3VHszUM9RytLwJEqKmT96NC-H41eIHzk0PUeYSx_WCE9y4qaGo7KUomk0C9eJ5L-RfUFzodSLJ3goHX3gHHkJPUMuFdfyJHB8hCeSZxH4BE1vVL6CrkmM6lEePGeK_Ei0v8KTw9GRb3gO_YfjLMgn6agYIh3xfIJ2T8jDodR2OAyso8e3bIrhJgyHHzvyKhCjpxE-MUIezeiD5suW4LzxJEfMlEJy4VFCNNeLUjqeJJ6OP8PNwsfTB9dxHUmeXAgzLuHQ4wljfDqahegj4R2JKzlO5fiDkuERJkfiKVHwxB_OHPyH6soEJk07fB9u6CyF5rmCflFwBdyiRHg4IReSBfWDX8GnHGVGNHOE-NCssEjTHFeNplsPdgmqKBea8eiD5jwuBo-SZ6gkfmhk4U6EpmKOPviJUD-S50WVgE911Id9HX3k4AmHG-9fPAneJ-hBjcWjoPzxHI9JhDSx5awQSlGPWg_8bULPHE2VE30oNNNQb7hyTD4aVUYeQUv2Ivzw1AEd7Wge9Ph-4M_xC89uJEusIVyY4cWTRC_-4-hNNI2WSciP5J3BL_Au4UyPKzgaPUN4yAuP0Dw85jqejE6CUNCRZ8OTo-mD58atGJ0jXjjjIxd09gh7hNnx4UkOnYe_4MQDTWJgaauE8Bc-KmgTDQ_6CleKD6Z8Yufxtwh1QcuCfEmGS4rwE_444TniREmOH-_wCP6MO2mDNUouPNEPTUfzqMPZI1_EB1feHDeFkjzCjMygJznuI2dYouXhS3gfhOXxi9Ap2Uj14DyeJFl6sHrwH37wKRm-8HiUJwnOC-UTNFJ1XMwz4eKDr0GbH54kVJWC78GpVfjRHD2SLcmGb0f44SAj5WgfNGOO5j36K_jUo09mNMWDbrbQOEcsLSeS50fPHL9QUceFKhOP4w_cGKGV489FPD0aJTvKP_hE6EmGpkKZH7ri4FRm44o5oZQ5-Jh-ou_RNImCw1GqTYjMo_aDRsuO__gS-CKefQlyKSf4NEdPGmGOP8eZKbjihzh1nJ_QvEa4Q6VC4UqkIz_-oEeYUH3wRR--C10JD7WOMwquhwbPQ9t6Iw1TXMNPnBciZXkC1YiYRSxq-MI35KikzEoCpcrxLNB2NDlyKcPIGH0OH3eE5wglrdB3hOGH6eKEMw725Oh39DN-NP1RMaGHhznu7Ei_ZIWei7iSCnmOZvbQM1zQ78hTo-aA6ROhJWKi49OxJzdi9fgUlL_RNMm644suXEcsJjua69BsfFpKPDrCV4fPIFe-4IknofKNo9O0Q9L4oFauoBzaiD7eG018oZSj44nW4HlCNPFioa2CPMyFUzye5vjR7mgibTitiRh5Yj7aCQAIMEQoACCQQABhCBNGCMGUEcAYYg1AQCAhDDAAAAagAQAQAAh0BmBCiPAMEHOUAAogASCBBAAklFBEEWaEIkQAAAQjQAgAgHHCIASIA0gQQwQASBggBBFKUCMAIFQSBIQERAnCECRWEACEQJIipYABnIEhACECEUAIs8wg4QRCACBmBCAEKAEIAwYpABSACAAECASCCEKgE04oYBRQChgHhABkAUEAEAIwAwAgQFAgFGMUGcIoA0AIARgAyBhAFKIKCEOIAgQZIARARhgggBGUAYiRAUAIRYQEBABAkBAECOMMIgAIhIQBCiCFCHBMEASAAEoACogoCAFGgGUGWIUAMIgRBIAzhgAFFAJEAEIEgAwZoBgAmCFmhKEMGAAUAAYsQgABgChgNAUGAASIgEAYB4gRQABBBGMGCKEAAIwggShwCChkgGIEGQaocoQBoLRBSACOHALEMQAEAkwAYoAkTDkDICEKIUMQAEQEQRRAgBCJACJCIWSAAKwBQYQABEkCBRHAAEUAoAJIgYBQTAkDAAEIIGEEMIsQx4wAwhgiEQFCGAQQA0oY4gUhCgElgEHEOECEUAYhJAQwxBgJJANIUQKEIYIoARgDhhlAHFBIIAmQQRYhoAhhCBgnLCKEKCEQIQEIR4RTCgkkCAKGCmQQQQAQgAQFgBFBCCGSAwAEIcoIoJQEDCHhEDAKKmCREswQohAgiglkACACKgOhMUAhRgwCBAipEFGMAAGEUABYhAhSAiEFoFEQASGAIAAxJQiESDEHgCEISAEQEAAIJwhAShGoBAUAUCaIUAwIgQgzghgggECAKCIcMQoJxYgSwABlhACAMgMAEkwCpCBzhgADGDACWMIEA0YbIQBiQAAgGCFAKCCEJBAAAYwhAGmiCCOOAIEcMwAIggBzjBhHmHeMSCGIEggiAA0jTCHjCABCAAUQIUQJIowQggFkKBEAGCEBBMQgBIARBkGDCCFGCMMMIIAgQBgSSAI";

View File

@ -60,7 +60,7 @@ public class OpenSubtitlesXmlRpcTest {
@Test @Test
public void getSubtitleListEnglish() throws Exception { public void getSubtitleListEnglish() throws Exception {
List<OpenSubtitlesSubtitleDescriptor> list = xmlrpc.searchSubtitles(361256, -1, -1, "eng"); List<OpenSubtitlesSubtitleDescriptor> list = xmlrpc.searchSubtitles(singleton(Query.forImdbId(361256, -1, -1, "eng")));
SubtitleDescriptor sample = list.get(0); SubtitleDescriptor sample = list.get(0);
@ -73,7 +73,7 @@ public class OpenSubtitlesXmlRpcTest {
@Test @Test
public void getSubtitleListAllLanguages() throws Exception { public void getSubtitleListAllLanguages() throws Exception {
List<OpenSubtitlesSubtitleDescriptor> list = xmlrpc.searchSubtitles(361256, -1, -1); List<OpenSubtitlesSubtitleDescriptor> list = xmlrpc.searchSubtitles(singleton(Query.forImdbId(361256, -1, -1)));
OpenSubtitlesSubtitleDescriptor sample = list.get(75); OpenSubtitlesSubtitleDescriptor sample = list.get(75);
@ -176,7 +176,7 @@ public class OpenSubtitlesXmlRpcTest {
@Test @Test
public void fetchSubtitle() throws Exception { public void fetchSubtitle() throws Exception {
List<OpenSubtitlesSubtitleDescriptor> list = xmlrpc.searchSubtitles(361256, -1, -1, "eng"); List<OpenSubtitlesSubtitleDescriptor> list = xmlrpc.searchSubtitles(singleton(Query.forImdbId(361256, -1, -1, "eng")));
// check format // check format
assertEquals("srt", list.get(0).getType()); assertEquals("srt", list.get(0).getType());
@ -190,7 +190,7 @@ public class OpenSubtitlesXmlRpcTest {
// @Test(expected = IOException.class) // @Test(expected = IOException.class)
public void fetchSubtitlesExceedLimit() throws Exception { public void fetchSubtitlesExceedLimit() throws Exception {
List<OpenSubtitlesSubtitleDescriptor> list = xmlrpc.searchSubtitles(773262, -1, -1, "eng"); List<OpenSubtitlesSubtitleDescriptor> list = xmlrpc.searchSubtitles(singleton(Query.forImdbId(773262, -1, -1, "eng")));
for (int i = 0; true; i++) { for (int i = 0; true; i++) {
System.out.format("Fetch #%d: %s%n", i, list.get(i).fetch()); System.out.format("Fetch #%d: %s%n", i, list.get(i).fetch());