From 9e6883b646a5f0000bd946f3157e440c2bfce6a2 Mon Sep 17 00:00:00 2001 From: Reinhard Pointner Date: Mon, 2 Jan 2012 09:33:50 +0000 Subject: [PATCH] * massive performance improvements * proper parallel processing in movie mode --- .../filebot/media/MediaDetection.java | 73 +++++++++++-------- .../filebot/ui/rename/MovieHashMatcher.java | 66 +++++++++++------ 2 files changed, 88 insertions(+), 51 deletions(-) diff --git a/source/net/sourceforge/filebot/media/MediaDetection.java b/source/net/sourceforge/filebot/media/MediaDetection.java index 0a93c362..ebdb7025 100644 --- a/source/net/sourceforge/filebot/media/MediaDetection.java +++ b/source/net/sourceforge/filebot/media/MediaDetection.java @@ -25,9 +25,7 @@ import java.util.Map.Entry; import java.util.Set; import java.util.SortedMap; import java.util.TreeSet; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; +import java.util.WeakHashMap; import java.util.logging.Level; import java.util.logging.Logger; import java.util.regex.Matcher; @@ -206,39 +204,27 @@ public class MediaDetection { private static List matchMovieName(final List files, final Locale locale, final boolean strict) throws Exception { // cross-reference file / folder name with movie list - final SeriesNameMatcher nameMatcher = new SeriesNameMatcher(String.CASE_INSENSITIVE_ORDER); // use simple comparator for speed (2-3x faster) - - final Map matchMap = synchronizedMap(new HashMap()); - ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); + final HighPerformanceMatcher nameMatcher = new HighPerformanceMatcher(); + final Map matchMap = new HashMap(); for (final Movie movie : releaseInfo.getMovieList()) { - executor.submit(new Runnable() { - - @Override - public void run() { - for (String name : files) { - String movieIdentifier = movie.getName(); - String commonName = nameMatcher.matchByFirstCommonWordSequence(name, movieIdentifier); - if (commonName != null && commonName.length() >= movieIdentifier.length()) { - String strictMovieIdentifier = movie.getName() + " " + movie.getYear(); - String strictCommonName = nameMatcher.matchByFirstCommonWordSequence(name, strictMovieIdentifier); - if (strictCommonName != null && strictCommonName.length() >= strictMovieIdentifier.length()) { - // prefer strict match - matchMap.put(movie, strictCommonName); - } else if (!strict) { - // make sure the common identifier is not just the year - matchMap.put(movie, commonName); - } - } + for (String name : files) { + String movieIdentifier = movie.getName(); + String commonName = nameMatcher.matchByFirstCommonWordSequence(name, movieIdentifier); + if (commonName != null && commonName.length() >= movieIdentifier.length()) { + String strictMovieIdentifier = movie.getName() + " " + movie.getYear(); + String strictCommonName = nameMatcher.matchByFirstCommonWordSequence(name, strictMovieIdentifier); + if (strictCommonName != null && strictCommonName.length() >= strictMovieIdentifier.length()) { + // prefer strict match + matchMap.put(movie, strictCommonName); + } else if (!strict) { + // make sure the common identifier is not just the year + matchMap.put(movie, commonName); } } - }); + } } - // wait for last task to finish - executor.shutdown(); - executor.awaitTermination(1, TimeUnit.MINUTES); - // sort by length of name match (descending) List results = new ArrayList(matchMap.keySet()); sort(results, new Comparator() { @@ -365,6 +351,7 @@ public class MediaDetection { } + @SuppressWarnings("unchecked") public static Comparator getLenientCollator(Locale locale) { // use maximum strength collator by default final Collator collator = Collator.getInstance(locale); @@ -373,4 +360,30 @@ public class MediaDetection { return (Comparator) collator; } + + + /* + * Heavy-duty name matcher used for matching a file to or more movies (out of a list of ~50k) + */ + private static class HighPerformanceMatcher extends SeriesNameMatcher { + + private static final Map transformCache = synchronizedMap(new WeakHashMap(65536)); + + + public HighPerformanceMatcher() { + super(String.CASE_INSENSITIVE_ORDER); // 3-4x faster than a Collator + } + + + @Override + protected String normalize(String source) { + String value = transformCache.get(source); + if (value == null) { + value = super.normalize(source); + transformCache.put(source, value); + } + return transformCache.get(source); + } + } + } diff --git a/source/net/sourceforge/filebot/ui/rename/MovieHashMatcher.java b/source/net/sourceforge/filebot/ui/rename/MovieHashMatcher.java index bd48b7cc..b1046679 100644 --- a/source/net/sourceforge/filebot/ui/rename/MovieHashMatcher.java +++ b/source/net/sourceforge/filebot/ui/rename/MovieHashMatcher.java @@ -11,6 +11,7 @@ import static net.sourceforge.tuned.ui.TunedUtilities.*; import java.awt.Component; import java.io.File; +import java.util.AbstractMap.SimpleEntry; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -26,6 +27,9 @@ import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.FutureTask; import java.util.concurrent.RunnableFuture; @@ -53,7 +57,7 @@ class MovieHashMatcher implements AutoCompleteMatcher { @Override - public List> match(final List files, Locale locale, boolean autodetect, Component parent) throws Exception { + public List> match(final List files, final Locale locale, final boolean autodetect, final Component parent) throws Exception { // handle movie files File[] movieFiles = filter(files, VIDEO_FILES).toArray(new File[0]); File[] subtitleFiles = filter(files, SUBTITLE_FILES).toArray(new File[0]); @@ -73,31 +77,51 @@ class MovieHashMatcher implements AutoCompleteMatcher { // map movies to (possibly multiple) files (in natural order) Map> filesByMovie = new HashMap>(); + // match remaining movies file by file in parallel + List>> grabMovieJobs = new ArrayList>>(); + // map all files by movie for (int i = 0; i < movieFiles.length; i++) { - Movie movie = movieByFileHash[i]; - - // unknown hash, try via imdb id from nfo file - if (movie == null || !autodetect) { - movie = grabMovieName(movieFiles[i], locale, autodetect, parent, movie); + final Movie movie = movieByFileHash[i]; + final File file = movieFiles[i]; + grabMovieJobs.add(new Callable>() { - if (movie != null) { - Analytics.trackEvent(service.getName(), "SearchMovie", movie.toString(), 1); + @Override + public Entry call() throws Exception { + // unknown hash, try via imdb id from nfo file + if (movie == null || !autodetect) { + Movie result = grabMovieName(file, locale, autodetect, parent, movie); + if (result != null) { + Analytics.trackEvent(service.getName(), "SearchMovie", result.toString(), 1); + } + return new SimpleEntry(file, result); + } + return null; + } + }); + } + + ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); + try { + for (Future> it : executor.invokeAll(grabMovieJobs)) { + // check if we managed to lookup the movie descriptor + if (it.get() != null) { + File file = it.get().getKey(); + Movie movie = it.get().getValue(); + + // get file list for movie + SortedSet movieParts = filesByMovie.get(movie); + + if (movieParts == null) { + movieParts = new TreeSet(); + filesByMovie.put(movie, movieParts); + } + + movieParts.add(file); } } - - // check if we managed to lookup the movie descriptor - if (movie != null) { - // get file list for movie - SortedSet movieParts = filesByMovie.get(movie); - - if (movieParts == null) { - movieParts = new TreeSet(); - filesByMovie.put(movie, movieParts); - } - - movieParts.add(movieFiles[i]); - } + } finally { + executor.shutdown(); } // collect all File/MoviePart matches