From e55526c3ac8bfdf73149a16cf4b7fd60cc159fdd Mon Sep 17 00:00:00 2001 From: Reinhard Pointner Date: Mon, 11 May 2015 13:57:04 +0000 Subject: [PATCH] * implement guessMovie feature --- build-data/BuildData.groovy | 11 ++--- source/net/filebot/media/ReleaseInfo.java | 2 +- .../filebot/subtitle/SubtitleUtilities.java | 37 +++++++++++++---- .../ui/subtitle/SubtitleUploadDialog.java | 3 +- .../net/filebot/web/OpenSubtitlesClient.java | 40 ++++++++++++++----- .../net/filebot/web/OpenSubtitlesXmlRpc.java | 37 +++++++++++++++-- source/net/filebot/web/SubtitleProvider.java | 2 + .../net/filebot/web/SubtitleSearchResult.java | 32 +++++++++++---- .../filebot/web/OpenSubtitlesXmlRpcTest.java | 15 ++++++- 9 files changed, 140 insertions(+), 39 deletions(-) diff --git a/build-data/BuildData.groovy b/build-data/BuildData.groovy index 0012ee4b..244fa55b 100755 --- a/build-data/BuildData.groovy +++ b/build-data/BuildData.groovy @@ -4,7 +4,8 @@ import org.tukaani.xz.* /* ------------------------------------------------------------------------- */ -def dir_website = "../website" +def dir_root = ".." +def dir_website = "${dir_root}/website" def dir_data = "${dir_website}/data" def sortRegexList(path) { @@ -29,7 +30,7 @@ sortRegexList("${dir_data}/add-series-alias.txt") def reviews = [] -new File('reviews.csv').eachLine('UTF-8'){ +new File("${dir_root}/reviews.csv").eachLine('UTF-8'){ def s = it.split(';', 3) reviews << [user: s[0], date: s[1], text: s[2].replaceAll(/^\"|\"$/, '').replaceAll(/["]{2}/, '"') ] } @@ -58,7 +59,7 @@ def pack(file, lines) { } def rows = lines.size() def columns = lines.collect{ it.split(/\t/).length }.max() - println "$file ($rows rows, $columns columns)" + println "${file.canonicalFile} ($rows rows, $columns columns)" } @@ -119,7 +120,7 @@ new File('osdb.txt').eachLine('UTF-8'){ // 0 IDMovie, 1 IDMovieImdb, 2 MovieName, 3 MovieYear, 4 MovieKind, 5 MoviePriority if (fields.size() == 6 && fields[1] ==~ /\d+/ && fields[3] ==~ /\d{4}/) { if (fields[4] ==~ /movie|tv.series/ && isValidMovieName(fields[2]) && (fields[3] as int) >= 1970 && (fields[5] as int) >= 100) { - osdb << [fields[1] as int, fields[2], fields[3] as int, fields[4] == /movie/ ? 'm' : fields[4] == /movie/ ? 's' : '?', fields[5] as int] + osdb << [fields[1] as int, fields[2], fields[3] as int, fields[4] == /movie/ ? 'm' : fields[4] == /tv series/ ? 's' : '?', fields[5] as int] } } } @@ -137,7 +138,7 @@ pack(osdb_out, osdb*.join('\t')) // BUILD moviedb index def omdb = [] -new File('omdb.txt').eachLine('Windows-1252'){ +new File('omdbMovies.txt').eachLine('Windows-1252'){ def line = it.split(/\t/) if (line.length > 11 && line[0] ==~ /\d+/ && line[3] ==~ /\d{4}/) { def imdbid = line[1].substring(2).toInteger() diff --git a/source/net/filebot/media/ReleaseInfo.java b/source/net/filebot/media/ReleaseInfo.java index f2f391e4..7c53f894 100644 --- a/source/net/filebot/media/ReleaseInfo.java +++ b/source/net/filebot/media/ReleaseInfo.java @@ -456,7 +456,7 @@ public class ReleaseInfo { int imdbid = parseInt(row[0]); String name = row[1]; int year = parseInt(row[2]); - char kind = row[3].charAt(0); + String kind = row[3]; int score = parseInt(row[4]); result.add(new SubtitleSearchResult(imdbid, name, year, kind, score)); } diff --git a/source/net/filebot/subtitle/SubtitleUtilities.java b/source/net/filebot/subtitle/SubtitleUtilities.java index e7154117..3f237eee 100644 --- a/source/net/filebot/subtitle/SubtitleUtilities.java +++ b/source/net/filebot/subtitle/SubtitleUtilities.java @@ -17,7 +17,6 @@ import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; -import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.LinkedHashSet; @@ -66,6 +65,7 @@ public final class SubtitleUtilities { throw new InterruptedException(); // auto-detect query and search for subtitles + Collection guessSet = new LinkedHashSet(); Collection querySet = new TreeSet(String.CASE_INSENSITIVE_ORDER); List files = bySeries.getValue(); @@ -104,11 +104,11 @@ public final class SubtitleUtilities { } } - if (!searchByMovie && !searchBySeries) + if (!searchByMovie && !searchBySeries && guessSet.isEmpty()) continue; // search for subtitles online using the auto-detected or forced query information - Set subtitles = findSubtitles(service, querySet, searchByMovie, searchBySeries, languageName); + Set subtitles = findSubtitles(service, guessSet, querySet, searchByMovie, searchBySeries, languageName); // allow early abort if (Thread.interrupted()) @@ -191,18 +191,24 @@ public final class SubtitleUtilities { return subtitleByVideo; } - public static Set findSubtitles(SubtitleProvider service, Collection querySet, boolean searchByMovie, boolean searchBySeries, String languageName) throws Exception { + public static Set findSubtitles(SubtitleProvider service, Collection guessSet, Collection querySet, boolean searchByMovie, boolean searchBySeries, String languageName) throws Exception { Set subtitles = new LinkedHashSet(); // search for and automatically select movie / show entry - Set resultSet = new HashSet(); + Set resultSet = new LinkedHashSet(); + + // add known results first + resultSet.addAll(guessSet); + + resultSet.addAll(findProbableSearchResults(service, querySet, searchByMovie, searchBySeries, languageName)); + for (String query : querySet) { // search and filter by movie/series as required Stream searchResults = service.search(query).stream().filter((it) -> { return (searchByMovie && it.isMovie()) || (searchBySeries && it.isSeries()); }); - resultSet.addAll(findProbableSearchResults(query, searchResults::iterator, querySet.size() == 1 ? 4 : 2)); + resultSet.addAll(filterProbableSearchResults(query, searchResults::iterator, querySet.size() == 1 ? 4 : 2)); } // fetch subtitles for all search results @@ -213,9 +219,24 @@ public final class SubtitleUtilities { return subtitles; } - protected static Collection findProbableSearchResults(String query, Iterable searchResults, int limit) { + protected static List findProbableSearchResults(SubtitleProvider service, Collection querySet, boolean searchByMovie, boolean searchBySeries, String languageName) throws Exception { + // search for and automatically select movie / show entry + List resultSet = new ArrayList(); + + for (String query : querySet) { + // search and filter by movie/series as required + Stream searchResults = service.search(query).stream().filter((it) -> { + return (searchByMovie && it.isMovie()) || (searchBySeries && it.isSeries()); + }); + + resultSet.addAll(filterProbableSearchResults(query, searchResults::iterator, querySet.size() == 1 ? 4 : 2)); + } + return resultSet; + } + + protected static List filterProbableSearchResults(String query, Iterable searchResults, int limit) { // auto-select most probable search result - Set probableMatches = new LinkedHashSet(); + List probableMatches = new ArrayList(); // use name similarity metric SimilarityMetric metric = new MetricAvg(new SequenceMatchSimilarity(), new NameSimilarityMetric()); diff --git a/source/net/filebot/ui/subtitle/SubtitleUploadDialog.java b/source/net/filebot/ui/subtitle/SubtitleUploadDialog.java index a1aa991f..6a590d65 100644 --- a/source/net/filebot/ui/subtitle/SubtitleUploadDialog.java +++ b/source/net/filebot/ui/subtitle/SubtitleUploadDialog.java @@ -60,6 +60,7 @@ import net.filebot.util.ui.EmptySelectionModel; import net.filebot.web.Movie; import net.filebot.web.OpenSubtitlesClient; import net.filebot.web.SearchResult; +import net.filebot.web.SubtitleSearchResult; import net.filebot.web.TheTVDBSearchResult; import net.filebot.web.TheTVDBSeriesInfo; import net.filebot.web.VideoHashSubtitleService.CheckResult; @@ -163,7 +164,7 @@ public class SubtitleUploadDialog extends JDialog { File video = mapping.getVideo() != null ? mapping.getVideo() : mapping.getSubtitle(); String input = showInputDialog("Enter movie / series name:", stripReleaseInfo(FileUtilities.getName(video)), String.format("%s/%s", video.getParentFile().getName(), video.getName()), SubtitleUploadDialog.this); if (input != null && input.length() > 0) { - List options = database.searchIMDB(input); + List options = database.searchIMDB(input); if (options.size() > 0) { SelectDialog dialog = new SelectDialog(SubtitleUploadDialog.this, options); dialog.setLocation(getOffsetLocation(dialog.getOwner())); diff --git a/source/net/filebot/web/OpenSubtitlesClient.java b/source/net/filebot/web/OpenSubtitlesClient.java index 50e24d9e..92e61300 100644 --- a/source/net/filebot/web/OpenSubtitlesClient.java +++ b/source/net/filebot/web/OpenSubtitlesClient.java @@ -92,6 +92,25 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS throw new UnsupportedOperationException(); // XMLRPC::SearchMoviesOnIMDB is not allowed due to abuse } + @Override + public List guess(String tag) throws Exception { + List subtitles = getCache().getSearchResult("guess", tag); + System.out.println(tag); + System.out.println(subtitles); + if (subtitles != null) { + return subtitles; + } + + // require login + login(); + + subtitles = xmlrpc.guessMovie(singleton(tag)).getOrDefault(tag, emptyList()); + System.out.println(subtitles); + + getCache().putSearchResult("guess", tag, subtitles); + return subtitles; + } + @Override public synchronized List getSubtitleList(SubtitleSearchResult searchResult, String languageName) throws Exception { List subtitles = getCache().getSubtitleDescriptorList(searchResult, languageName); @@ -350,11 +369,11 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS throw new UnsupportedOperationException(); } - public synchronized List searchIMDB(String query) throws Exception { + public synchronized List searchIMDB(String query) throws Exception { // search for movies and series - List result = getCache().getSearchResult("search", query, null); + List result = getCache().getSearchResult("search", query); if (result != null) { - return (List) result; + return (List) result; } // require login @@ -362,15 +381,14 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS try { // search for movies / series - List resultSet = xmlrpc.searchMoviesOnIMDB(query); - result = asList(resultSet.toArray(new SearchResult[0])); + result = xmlrpc.searchMoviesOnIMDB(query); } catch (ClassCastException e) { // unexpected xmlrpc responses (e.g. error messages instead of results) will trigger this throw new XmlRpcException("Illegal XMLRPC response on searchMoviesOnIMDB"); } - getCache().putSearchResult("search", query, null, result); - return (List) result; + getCache().putSearchResult("search", query, result); + return result; } @Override @@ -608,9 +626,9 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS return query == null ? null : query.trim().toLowerCase(); } - public List putSearchResult(String method, String query, Locale locale, List value) { + public List putSearchResult(String method, String query, List value) { try { - cache.put(new Key(id, normalize(query)), value.toArray(new SearchResult[0])); + cache.put(new Key(id, method, normalize(query)), value.toArray(new SubtitleSearchResult[0])); } catch (Exception e) { Logger.getLogger(OpenSubtitlesClient.class.getName()).log(Level.WARNING, e.getMessage()); } @@ -619,9 +637,9 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS } @SuppressWarnings("unchecked") - public List getSearchResult(String method, String query, Locale locale) { + public List getSearchResult(String method, String query) { try { - SearchResult[] array = cache.get(new Key(id, normalize(query)), SearchResult[].class); + SubtitleSearchResult[] array = cache.get(new Key(id, method, normalize(query)), SubtitleSearchResult[].class); if (array != null) { return Arrays.asList(array); } diff --git a/source/net/filebot/web/OpenSubtitlesXmlRpc.java b/source/net/filebot/web/OpenSubtitlesXmlRpc.java index ad417210..665a7405 100644 --- a/source/net/filebot/web/OpenSubtitlesXmlRpc.java +++ b/source/net/filebot/web/OpenSubtitlesXmlRpc.java @@ -112,11 +112,11 @@ public class OpenSubtitlesXmlRpc { } @SuppressWarnings("unchecked") - public List searchMoviesOnIMDB(String query) throws XmlRpcFault { + public List searchMoviesOnIMDB(String query) throws XmlRpcFault { Map response = invoke("SearchMoviesOnIMDB", token, query); List> movieData = (List>) response.get("data"); - List movies = new ArrayList(); + List movies = new ArrayList(); // title pattern Pattern pattern = Pattern.compile("(.+)[(](\\d{4})([/]I+)?[)]"); @@ -135,7 +135,7 @@ public class OpenSubtitlesXmlRpc { String name = matcher.group(1).replaceAll("\"", "").trim(); int year = Integer.parseInt(matcher.group(2)); - movies.add(new Movie(name, year, Integer.parseInt(imdbid), -1)); + movies.add(new SubtitleSearchResult(Integer.parseInt(imdbid), name, year, null, -1)); } catch (Exception e) { Logger.getLogger(OpenSubtitlesXmlRpc.class.getName()).log(Level.FINE, String.format("Ignore movie [%s]: %s", movie, e.getMessage())); } @@ -233,6 +233,37 @@ public class OpenSubtitlesXmlRpc { return subHashMap; } + public Map> guessMovie(Collection tags) throws XmlRpcFault { + Map> results = new HashMap>(); + + Map response = invoke("GuessMovie", token, tags); + Object payload = response.get("data"); + + if (payload instanceof Map) { + Map>> guessMovieData = (Map>>) payload; + + for (String tag : tags) { + List value = new ArrayList<>(); + + List> matches = guessMovieData.get(tag); + if (matches != null) { + for (Map match : matches) { + String name = String.valueOf(match.get("MovieName")); + String kind = String.valueOf(match.get("MovieKind")); + int imdbid = Integer.parseInt(String.valueOf(match.get("IDMovieIMDB"))); + int year = Integer.parseInt(String.valueOf(match.get("MovieYear"))); + int score = Integer.parseInt(String.valueOf(match.get("score"))); + value.add(new SubtitleSearchResult(imdbid, name, year, kind, score)); + } + } + + results.put(tag, value); + } + } + + return results; + } + @SuppressWarnings("unchecked") public Map checkMovieHash(Collection hashes, int minSeenCount) throws XmlRpcFault { Map movieHashMap = new HashMap(); diff --git a/source/net/filebot/web/SubtitleProvider.java b/source/net/filebot/web/SubtitleProvider.java index 83cf1e7f..8f3288ae 100644 --- a/source/net/filebot/web/SubtitleProvider.java +++ b/source/net/filebot/web/SubtitleProvider.java @@ -9,6 +9,8 @@ public interface SubtitleProvider { public List search(String query) throws Exception; + public List guess(String tag) throws Exception; + public List getSubtitleList(SubtitleSearchResult searchResult, String languageName) throws Exception; public URI getSubtitleListLink(SubtitleSearchResult searchResult, String languageName); diff --git a/source/net/filebot/web/SubtitleSearchResult.java b/source/net/filebot/web/SubtitleSearchResult.java index 1c2b787e..51ea03ff 100644 --- a/source/net/filebot/web/SubtitleSearchResult.java +++ b/source/net/filebot/web/SubtitleSearchResult.java @@ -4,20 +4,36 @@ import java.util.Locale; public class SubtitleSearchResult extends Movie { - public static final char KIND_MOVIE = 'm'; - public static final char KIND_SERIES = 's'; + enum Kind { + Movie, Series, Other, Unkown; - private char kind; + public static Kind forName(String s) { + if (s == null || s.isEmpty()) + return Unkown; + else if (s.equalsIgnoreCase("m") || s.equalsIgnoreCase("movie")) + return Movie; + if (s.equalsIgnoreCase("s") || s.equalsIgnoreCase("tv series")) + return Series; + else + return Other; + } + } + + private Kind kind; private int score; - public SubtitleSearchResult(int imdbId, String name, int year, char kind, int score) { - super(name, null, year, imdbId, -1, Locale.ENGLISH); + public SubtitleSearchResult(int imdbId, String name, int year, String kind, int score) { + this(name, null, year, imdbId, -1, Locale.ENGLISH, Kind.forName(kind), score); + } + + public SubtitleSearchResult(String name, String[] aliasNames, int year, int imdbId, int tmdbId, Locale locale, Kind kind, int score) { + super(name, aliasNames, year, imdbId, tmdbId, locale); this.kind = kind; this.score = score; } - public char getKind() { + public Kind getKind() { return kind; } @@ -26,11 +42,11 @@ public class SubtitleSearchResult extends Movie { } public boolean isMovie() { - return kind == KIND_MOVIE; + return kind == Kind.Movie; } public boolean isSeries() { - return kind == KIND_SERIES; + return kind == Kind.Series; } } diff --git a/test/net/filebot/web/OpenSubtitlesXmlRpcTest.java b/test/net/filebot/web/OpenSubtitlesXmlRpcTest.java index 2d337274..7d26ec8e 100644 --- a/test/net/filebot/web/OpenSubtitlesXmlRpcTest.java +++ b/test/net/filebot/web/OpenSubtitlesXmlRpcTest.java @@ -27,9 +27,20 @@ public class OpenSubtitlesXmlRpcTest { xmlrpc.loginAnonymous(); } + @Test + public void guessMovie() throws Exception { + Map> results = xmlrpc.guessMovie(singleton("himym.s13.e12")); + SubtitleSearchResult result = results.get("himym.s13.e12").get(0); + + assertEquals(460649, result.getImdbId()); + assertEquals("How I Met Your Mother", result.getName()); + assertEquals(2005, result.getYear()); + assertEquals("Series", result.getKind().toString()); + } + @Test public void search() throws Exception { - List list = xmlrpc.searchMoviesOnIMDB("babylon 5"); + List list = xmlrpc.searchMoviesOnIMDB("babylon 5"); Movie sample = list.get(0); // check sample entry @@ -40,7 +51,7 @@ public class OpenSubtitlesXmlRpcTest { @Test(expected = IndexOutOfBoundsException.class) public void searchOST() throws Exception { - List list = xmlrpc.searchMoviesOnIMDB("Linkin.Park.New.Divide.1280-720p.Transformers.Revenge.of.the.Fallen.ost"); + List list = xmlrpc.searchMoviesOnIMDB("Linkin.Park.New.Divide.1280-720p.Transformers.Revenge.of.the.Fallen.ost"); // seek to OST entry, expect to fail for (int i = 0; !list.get(i).getName().contains("Linkin.Park"); i++)