From 38ea14d86f3eb61c39acd66c62df48b447ef7676 Mon Sep 17 00:00:00 2001 From: Reinhard Pointner Date: Tue, 14 Feb 2012 14:16:13 +0000 Subject: [PATCH] * match Movie object for nfo files directly via nfo content * enable caching for TMDb --- .../net/sourceforge/filebot/WebServices.java | 2 +- .../filebot/cli/CmdlineOperations.java | 100 ++++++++++-------- .../filebot/format/MediaBindingBean.java | 6 ++ .../filebot/media/MediaDetection.java | 11 ++ .../filebot/ui/rename/MovieHashMatcher.java | 91 ++++++++-------- .../sourceforge/filebot/web/TMDbClient.java | 39 +++++-- .../filebot/web/TheTVDBClient.java | 5 + .../sourceforge/filebot/web/WebRequest.java | 6 ++ .../net/sourceforge/tuned/FileUtilities.java | 4 +- 9 files changed, 163 insertions(+), 101 deletions(-) diff --git a/source/net/sourceforge/filebot/WebServices.java b/source/net/sourceforge/filebot/WebServices.java index 303b33b7..aebcd80f 100644 --- a/source/net/sourceforge/filebot/WebServices.java +++ b/source/net/sourceforge/filebot/WebServices.java @@ -27,7 +27,6 @@ public final class WebServices { // episode dbs public static final TVRageClient TVRage = new TVRageClient(); public static final AnidbClient AniDB = new AnidbClient(getApplicationName().toLowerCase(), 2); - public static final IMDbClient IMDb = new IMDbClient(); public static final TheTVDBClient TheTVDB = new TheTVDBClient(getApplicationProperty("thetvdb.apikey")); public static final SerienjunkiesClient Serienjunkies = new SerienjunkiesClient(getApplicationProperty("serienjunkies.apikey")); @@ -37,6 +36,7 @@ public final class WebServices { public static final SubsceneSubtitleClient Subscene = new SubsceneSubtitleClient(); // movie dbs + public static final IMDbClient IMDb = new IMDbClient(); public static final TMDbClient TMDb = new TMDbClient(getApplicationProperty("themoviedb.apikey")); diff --git a/source/net/sourceforge/filebot/cli/CmdlineOperations.java b/source/net/sourceforge/filebot/cli/CmdlineOperations.java index 7b317539..f9efb07f 100644 --- a/source/net/sourceforge/filebot/cli/CmdlineOperations.java +++ b/source/net/sourceforge/filebot/cli/CmdlineOperations.java @@ -29,6 +29,7 @@ import java.util.ListIterator; import java.util.Locale; import java.util.Map; import java.util.Map.Entry; +import java.util.NoSuchElementException; import java.util.Set; import java.util.SortedSet; import java.util.TreeMap; @@ -266,15 +267,17 @@ public class CmdlineOperations implements CmdlineInterface { // handle movie files List movieFiles = filter(files, VIDEO_FILES); + List nfoFiles = filter(files, MediaTypes.getDefaultFilter("application/nfo")); - List standaloneFiles = new ArrayList(files); - standaloneFiles.removeAll(movieFiles); + List orphanedFiles = new ArrayList(filter(files, FILES)); + orphanedFiles.removeAll(movieFiles); + orphanedFiles.removeAll(nfoFiles); Map> derivatesByMovieFile = new HashMap>(); for (File movieFile : movieFiles) { derivatesByMovieFile.put(movieFile, new ArrayList()); } - for (File file : standaloneFiles) { + for (File file : orphanedFiles) { for (File movieFile : movieFiles) { if (isDerived(file, movieFile)) { derivatesByMovieFile.get(movieFile).add(file); @@ -283,41 +286,49 @@ public class CmdlineOperations implements CmdlineInterface { } } for (List derivates : derivatesByMovieFile.values()) { - standaloneFiles.removeAll(derivates); + orphanedFiles.removeAll(derivates); } - List movieMatchFiles = new ArrayList(); - movieMatchFiles.addAll(movieFiles); - movieMatchFiles.addAll(filter(files, new ReleaseInfo().getDiskFolderFilter())); - movieMatchFiles.addAll(filter(standaloneFiles, SUBTITLE_FILES)); - - // map movies to (possibly multiple) files (in natural order) - Map> filesByMovie = new HashMap>(); - // match movie hashes online - Map movieByFile = new HashMap(); - if (query == null && movieFiles.size() > 0) { - try { - CLILogger.fine(format("Looking up movie by filehash via [%s]", service.getName())); - Map hashLookup = service.getMovieDescriptors(movieFiles, locale); - movieByFile.putAll(hashLookup); - Analytics.trackEvent(service.getName(), "HashLookup", "Movie", hashLookup.size()); // number of positive hash lookups - } catch (UnsupportedOperationException e) { - CLILogger.fine(format("%s: Hash lookup not supported", service.getName())); + final Map movieByFile = new HashMap(); + if (query == null) { + if (movieFiles.size() > 0) { + try { + CLILogger.fine(format("Looking up movie by filehash via [%s]", service.getName())); + Map hashLookup = service.getMovieDescriptors(movieFiles, locale); + movieByFile.putAll(hashLookup); + Analytics.trackEvent(service.getName(), "HashLookup", "Movie", hashLookup.size()); // number of positive hash lookups + } catch (UnsupportedOperationException e) { + CLILogger.fine(format("%s: Hash lookup not supported", service.getName())); + } } - } - - if (query != null) { + for (File nfo : nfoFiles) { + try { + movieByFile.put(nfo, grepMovie(nfo, service, locale)); + } catch (NoSuchElementException e) { + CLILogger.warning("Failed to grep IMDbID: " + nfo.getName()); + } + } + } else { CLILogger.fine(format("Looking up movie by query [%s]", query)); Movie result = (Movie) selectSearchResult(query, service.searchMovie(query, locale), strict).get(0); // force all mappings - for (File file : movieMatchFiles) { + for (File file : files) { movieByFile.put(file, result); } } + List movieMatchFiles = new ArrayList(); + movieMatchFiles.addAll(movieFiles); + movieMatchFiles.addAll(nfoFiles); + movieMatchFiles.addAll(filter(files, new ReleaseInfo().getDiskFolderFilter())); + movieMatchFiles.addAll(filter(orphanedFiles, SUBTITLE_FILES)); // run movie detection only on orphaned subtitle files + + // map movies to (possibly multiple) files (in natural order) + Map> filesByMovie = new HashMap>(); + // map all files by movie - for (File file : movieMatchFiles) { + for (final File file : movieMatchFiles) { Movie movie = movieByFile.get(file); // unknown hash, try via imdb id from nfo file @@ -345,29 +356,26 @@ public class CmdlineOperations implements CmdlineInterface { } } - // collect all File / MoviePart matches + // collect all File/MoviePart matches List> matches = new ArrayList>(); for (Entry> entry : filesByMovie.entrySet()) { - Movie movie = entry.getKey(); - - int partIndex = 0; - int partCount = entry.getValue().size(); - - // add all movie parts - for (File file : entry.getValue()) { - Movie part = movie; - if (partCount > 1) { - part = new MoviePart(movie, ++partIndex, partCount); - } - - matches.add(new Match(file, part)); - - // automatically add matches for derivates - List derivates = derivatesByMovieFile.get(file); - if (derivates != null) { - for (File derivate : derivates) { - matches.add(new Match(derivate, part)); + for (List fileSet : mapByExtension(entry.getValue()).values()) { + // resolve movie parts + for (int i = 0; i < fileSet.size(); i++) { + Movie moviePart = entry.getKey(); + if (fileSet.size() > 1) { + moviePart = new MoviePart(moviePart, i + 1, fileSet.size()); + } + + matches.add(new Match(fileSet.get(i), moviePart)); + + // automatically add matches for derivate files + List derivates = derivatesByMovieFile.get(fileSet.get(i)); + if (derivates != null) { + for (File derivate : derivates) { + matches.add(new Match(derivate, moviePart)); + } } } } diff --git a/source/net/sourceforge/filebot/format/MediaBindingBean.java b/source/net/sourceforge/filebot/format/MediaBindingBean.java index ceb45342..37f78f84 100644 --- a/source/net/sourceforge/filebot/format/MediaBindingBean.java +++ b/source/net/sourceforge/filebot/format/MediaBindingBean.java @@ -378,6 +378,12 @@ public class MediaBindingBean { } + @Define("folder") + public File getMediaParentFolder() { + return mediaFile.getParentFile(); + } + + @Define("home") public File getUserHome() throws IOException { return new File(System.getProperty("user.home")); diff --git a/source/net/sourceforge/filebot/media/MediaDetection.java b/source/net/sourceforge/filebot/media/MediaDetection.java index dc7de48a..3b521578 100644 --- a/source/net/sourceforge/filebot/media/MediaDetection.java +++ b/source/net/sourceforge/filebot/media/MediaDetection.java @@ -42,6 +42,7 @@ import net.sourceforge.filebot.web.AnidbClient.AnidbSearchResult; import net.sourceforge.filebot.web.Movie; import net.sourceforge.filebot.web.MovieIdentificationService; import net.sourceforge.filebot.web.SearchResult; +import net.sourceforge.filebot.web.TheTVDBClient.SeriesInfo; import net.sourceforge.filebot.web.TheTVDBClient.TheTVDBSearchResult; @@ -442,6 +443,16 @@ public class MediaDetection { } + public static Movie grepMovie(File nfo, MovieIdentificationService resolver, Locale locale) throws Exception { + return resolver.getMovieDescriptor(grepImdbId(new String(readFile(nfo), "UTF-8")).iterator().next(), locale); + } + + + public static SeriesInfo grepSeries(File nfo, Locale locale) throws Exception { + return WebServices.TheTVDB.getSeriesInfoByID(grepTheTvdbId(new String(readFile(nfo), "UTF-8")).iterator().next(), locale); + } + + @SuppressWarnings("unchecked") public static Comparator getLenientCollator(Locale locale) { // use maximum strength collator by default diff --git a/source/net/sourceforge/filebot/ui/rename/MovieHashMatcher.java b/source/net/sourceforge/filebot/ui/rename/MovieHashMatcher.java index 35be0578..1bef31e4 100644 --- a/source/net/sourceforge/filebot/ui/rename/MovieHashMatcher.java +++ b/source/net/sourceforge/filebot/ui/rename/MovieHashMatcher.java @@ -21,6 +21,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Map.Entry; +import java.util.NoSuchElementException; import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; @@ -30,11 +31,14 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.FutureTask; import java.util.concurrent.RunnableFuture; +import java.util.logging.Level; +import java.util.logging.Logger; import javax.swing.Action; import javax.swing.SwingUtilities; import net.sourceforge.filebot.Analytics; +import net.sourceforge.filebot.MediaTypes; import net.sourceforge.filebot.media.ReleaseInfo; import net.sourceforge.filebot.similarity.Match; import net.sourceforge.filebot.similarity.NameSimilarityMetric; @@ -60,15 +64,17 @@ class MovieHashMatcher implements AutoCompleteMatcher { public List> match(final List files, final SortOrder sortOrder, final Locale locale, final boolean autodetect, final Component parent) throws Exception { // handle movie files List movieFiles = filter(files, VIDEO_FILES); + List nfoFiles = filter(files, MediaTypes.getDefaultFilter("application/nfo")); - List standaloneFiles = new ArrayList(files); - standaloneFiles.removeAll(movieFiles); + List orphanedFiles = new ArrayList(filter(files, FILES)); + orphanedFiles.removeAll(movieFiles); + orphanedFiles.removeAll(nfoFiles); Map> derivatesByMovieFile = new HashMap>(); for (File movieFile : movieFiles) { derivatesByMovieFile.put(movieFile, new ArrayList()); } - for (File file : standaloneFiles) { + for (File file : orphanedFiles) { for (File movieFile : movieFiles) { if (isDerived(file, movieFile)) { derivatesByMovieFile.get(movieFile).add(file); @@ -77,22 +83,11 @@ class MovieHashMatcher implements AutoCompleteMatcher { } } for (List derivates : derivatesByMovieFile.values()) { - standaloneFiles.removeAll(derivates); + orphanedFiles.removeAll(derivates); } - List movieMatchFiles = new ArrayList(); - movieMatchFiles.addAll(movieFiles); - movieMatchFiles.addAll(filter(files, new ReleaseInfo().getDiskFolderFilter())); - movieMatchFiles.addAll(filter(standaloneFiles, SUBTITLE_FILES)); - - // map movies to (possibly multiple) files (in natural order) - Map> filesByMovie = new HashMap>(); - - // match remaining movies file by file in parallel - List>> grabMovieJobs = new ArrayList>>(); - // match movie hashes online - Map movieByFile = new HashMap(); + final Map movieByFile = new HashMap(); if (movieFiles.size() > 0) { try { Map hashLookup = service.getMovieDescriptors(movieFiles, locale); @@ -102,28 +97,45 @@ class MovieHashMatcher implements AutoCompleteMatcher { // ignore } } + for (File nfo : nfoFiles) { + try { + movieByFile.put(nfo, grepMovie(nfo, service, locale)); + } catch (NoSuchElementException e) { + Logger.getLogger(getClass().getName()).log(Level.WARNING, "Failed to grep IMDbID: " + nfo.getName()); + } + } + + // match remaining movies file by file in parallel + List>> grabMovieJobs = new ArrayList>>(); + + List movieMatchFiles = new ArrayList(); + movieMatchFiles.addAll(movieFiles); + movieMatchFiles.addAll(nfoFiles); + movieMatchFiles.addAll(filter(files, new ReleaseInfo().getDiskFolderFilter())); + movieMatchFiles.addAll(filter(orphanedFiles, SUBTITLE_FILES)); // run movie detection only on orphaned subtitle files // map all files by movie for (final File file : movieMatchFiles) { - final Movie movie = movieByFile.get(file); grabMovieJobs.add(new Callable>() { @Override public Entry call() throws Exception { // unknown hash, try via imdb id from nfo file - if (movie == null || !autodetect) { - Movie result = grabMovieName(file, locale, autodetect, parent, movie); + if (!movieByFile.containsKey(file) || !autodetect) { + Movie result = grabMovieName(file, locale, autodetect, parent, movieByFile.get(file)); if (result != null) { Analytics.trackEvent(service.getName(), "SearchMovie", result.toString(), 1); } return new SimpleEntry(file, result); } - - return new SimpleEntry(file, null); + return new SimpleEntry(file, movieByFile.get(file)); } }); } + // map movies to (possibly multiple) files (in natural order) + Map> filesByMovie = new HashMap>(); + ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); try { for (Future> it : executor.invokeAll(grabMovieJobs)) { @@ -134,12 +146,10 @@ class MovieHashMatcher implements AutoCompleteMatcher { // get file list for movie if (movie != null) { SortedSet movieParts = filesByMovie.get(movie); - if (movieParts == null) { movieParts = new TreeSet(); filesByMovie.put(movie, movieParts); } - movieParts.add(file); } } @@ -151,25 +161,22 @@ class MovieHashMatcher implements AutoCompleteMatcher { List> matches = new ArrayList>(); for (Entry> entry : filesByMovie.entrySet()) { - Movie movie = entry.getKey(); - - int partIndex = 0; - int partCount = entry.getValue().size(); - - // add all movie parts - for (File file : entry.getValue()) { - Movie part = movie; - if (partCount > 1) { - part = new MoviePart(movie, ++partIndex, partCount); - } - - matches.add(new Match(file, part)); - - // automatically add matches for derivates - List derivates = derivatesByMovieFile.get(file); - if (derivates != null) { - for (File derivate : derivates) { - matches.add(new Match(derivate, part)); + for (List fileSet : mapByExtension(entry.getValue()).values()) { + // resolve movie parts + for (int i = 0; i < fileSet.size(); i++) { + Movie moviePart = entry.getKey(); + if (fileSet.size() > 1) { + moviePart = new MoviePart(moviePart, i + 1, fileSet.size()); + } + + matches.add(new Match(fileSet.get(i), moviePart)); + + // automatically add matches for derivate files + List derivates = derivatesByMovieFile.get(fileSet.get(i)); + if (derivates != null) { + for (File derivate : derivates) { + matches.add(new Match(derivate, moviePart)); + } } } } diff --git a/source/net/sourceforge/filebot/web/TMDbClient.java b/source/net/sourceforge/filebot/web/TMDbClient.java index 08074088..8a879e2c 100644 --- a/source/net/sourceforge/filebot/web/TMDbClient.java +++ b/source/net/sourceforge/filebot/web/TMDbClient.java @@ -5,6 +5,7 @@ package net.sourceforge.filebot.web; import static java.util.Arrays.*; import static java.util.Collections.*; import static net.sourceforge.filebot.web.WebRequest.*; +import static net.sourceforge.tuned.FileUtilities.*; import static net.sourceforge.tuned.XPathUtilities.*; import java.io.File; @@ -28,6 +29,9 @@ import org.w3c.dom.Document; import org.w3c.dom.Node; import org.xml.sax.SAXException; +import net.sf.ehcache.Cache; +import net.sf.ehcache.CacheManager; +import net.sf.ehcache.Element; import net.sourceforge.filebot.ResourceManager; import net.sourceforge.filebot.web.TMDbClient.Artwork.ArtworkProperty; import net.sourceforge.filebot.web.TMDbClient.MovieInfo.MovieProperty; @@ -78,8 +82,8 @@ public class TMDbClient implements MovieIdentificationService { @Override public Movie getMovieDescriptor(int imdbid, Locale locale) throws Exception { - URL resource = getResource("Movie.imdbLookup", String.format("tt%07d", imdbid), locale); - Node movie = selectNode("//movie", getDocument(resource)); + Document dom = fetchResource("Movie.imdbLookup", String.format("tt%07d", imdbid), locale); + Node movie = selectNode("//movie", dom); if (movie == null) return null; @@ -98,9 +102,10 @@ public class TMDbClient implements MovieIdentificationService { protected List getMovies(String method, String parameter, Locale locale) throws IOException, SAXException { + Document dom = fetchResource(method, parameter, locale); List result = new ArrayList(); - for (Node node : selectNodes("//movie", getDocument(getResource(method, parameter, locale)))) { + for (Node node : selectNodes("//movie", dom)) { try { String name = getTextContent("name", node); @@ -120,12 +125,29 @@ public class TMDbClient implements MovieIdentificationService { } - protected URL getResource(String method, String parameter, Locale locale) throws MalformedURLException { + protected URL getResourceLocation(String method, String parameter, Locale locale) throws MalformedURLException { // e.g. http://api.themoviedb.org/2.1/Movie.search/en/xml/{apikey}/serenity return new URL("http", host, "/" + version + "/" + method + "/" + locale.getLanguage() + "/xml/" + apikey + "/" + parameter); } + protected Document fetchResource(String method, String parameter, Locale locale) throws IOException, SAXException { + URL url = getResourceLocation(method, parameter, locale); + + Cache cache = CacheManager.getInstance().getCache("web-persistent-datasource"); + Element element = cache.get(url.toString()); + if (element != null) { + return WebRequest.getDocument((String) element.getValue()); + } + + String xml = readAll(WebRequest.getReader(url.openConnection())); + Document dom = getDocument(xml); + + cache.put(new Element(url.toString(), xml)); + return dom; + } + + public MovieInfo getMovieInfo(Movie movie, Locale locale) throws Exception { if (movie.getImdbId() >= 0) { return getMovieInfoByIMDbID(movie.getImdbId(), Locale.ENGLISH); @@ -150,15 +172,14 @@ public class TMDbClient implements MovieIdentificationService { if (imdbid < 0) throw new IllegalArgumentException("Illegal IMDb ID: " + imdbid); - URL resource = getResource("Movie.imdbLookup", String.format("tt%07d", imdbid), locale); - Document dom = getDocument(resource); + // resolve imdbid to tmdbid + Document dom = fetchResource("Movie.imdbLookup", String.format("tt%07d", imdbid), locale); // get complete movie info via tmdbid lookup - resource = getResource("Movie.getInfo", selectString("//movie/id", dom), locale); - dom = getDocument(resource); + dom = fetchResource("Movie.getInfo", selectString("//movie/id", dom), locale); // select info from xml - Node node = selectNode("//movie", getDocument(resource)); + Node node = selectNode("//movie", dom); Map movieProperties = new EnumMap(MovieProperty.class); for (MovieProperty property : MovieProperty.values()) { diff --git a/source/net/sourceforge/filebot/web/TheTVDBClient.java b/source/net/sourceforge/filebot/web/TheTVDBClient.java index 8b21d18d..8d7b3ba3 100644 --- a/source/net/sourceforge/filebot/web/TheTVDBClient.java +++ b/source/net/sourceforge/filebot/web/TheTVDBClient.java @@ -382,6 +382,11 @@ public class TheTVDBClient extends AbstractEpisodeListProvider { } + public SeriesInfo getSeriesInfoByID(int thetvdbid, Locale locale) throws Exception { + return getSeriesInfo(new TheTVDBSearchResult(null, thetvdbid), locale); + } + + public SeriesInfo getSeriesInfoByIMDbID(int imdbid, Locale locale) throws Exception { return getSeriesInfo(lookupByIMDbID(imdbid, locale), locale); } diff --git a/source/net/sourceforge/filebot/web/WebRequest.java b/source/net/sourceforge/filebot/web/WebRequest.java index 921557c4..45afdfde 100644 --- a/source/net/sourceforge/filebot/web/WebRequest.java +++ b/source/net/sourceforge/filebot/web/WebRequest.java @@ -7,6 +7,7 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.OutputStream; import java.io.Reader; +import java.io.StringReader; import java.io.UnsupportedEncodingException; import java.net.HttpURLConnection; import java.net.URL; @@ -96,6 +97,11 @@ public final class WebRequest { } + public static Document getDocument(String xml) throws IOException, SAXException { + return getDocument(new InputSource(new StringReader(xml))); + } + + public static Document getDocument(InputSource source) throws IOException, SAXException { try { DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); diff --git a/source/net/sourceforge/tuned/FileUtilities.java b/source/net/sourceforge/tuned/FileUtilities.java index 6ccf0ec7..617bf5fc 100644 --- a/source/net/sourceforge/tuned/FileUtilities.java +++ b/source/net/sourceforge/tuned/FileUtilities.java @@ -253,9 +253,7 @@ public final class FileUtilities { public static boolean isDerived(String derivate, File prime) { - String withoutTypeSuffix = getNameWithoutExtension(prime.getName()); - String withoutPartSuffix = getNameWithoutExtension(withoutTypeSuffix); - String base = (withoutPartSuffix.length() > 2 ? withoutPartSuffix : withoutTypeSuffix).trim().toLowerCase(); + String base = getName(prime).trim().toLowerCase(); derivate = derivate.trim().toLowerCase(); return derivate.startsWith(base); }