+ added support for fully automatic SubtitleDescriptor/File subtitle matching to CLI, i.e. match files against subtitle listings

source/net/sourceforge/filebot/cli/CmdlineOperations.java

@@ -18,6 +18,8 @@ import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
@@ -45,6 +47,7 @@ import net.sourceforge.filebot.hash.VerificationFileWriter;
 import net.sourceforge.filebot.similarity.EpisodeMetrics;
 import net.sourceforge.filebot.similarity.Match;
 import net.sourceforge.filebot.similarity.Matcher;
+import net.sourceforge.filebot.similarity.MetricCascade;
 import net.sourceforge.filebot.similarity.NameSimilarityMetric;
 import net.sourceforge.filebot.similarity.SeriesNameMatcher;
 import net.sourceforge.filebot.similarity.SimilarityMetric;
@@ -122,21 +125,9 @@ public class CmdlineOperations implements CmdlineInterface {
 	public List<File> renameSeries(Collection<File> files, String query, ExpressionFormat format, EpisodeListProvider db, Locale locale, boolean strict) throws Exception {
 		CLILogger.config(format("Rename episodes using [%s]", db.getName()));
 		List<File> mediaFiles = filter(files, VIDEO_FILES, SUBTITLE_FILES);
-		Collection<String> seriesNames;
 		
 		// auto-detect series name if not given
-		if (query == null) {
+		Collection<String> seriesNames = (query == null) ? detectQuery(mediaFiles, strict) : singleton(query);
-			seriesNames = new SeriesNameMatcher().matchAll(mediaFiles.toArray(new File[0]));
-			
-			if (seriesNames.isEmpty() || (strict && seriesNames.size() > 1)) {
-				throw new Exception("Unable to auto-select series name: " + seriesNames);
-			}
-			
-			query = seriesNames.iterator().next();
-			CLILogger.config("Auto-detected series name: " + seriesNames);
-		} else {
-			seriesNames = singleton(query);
-		}
 		
 		// fetch episode data
 		Set<Episode> episodes = fetchEpisodeSet(db, seriesNames, locale, strict);
@@ -146,11 +137,11 @@ public class CmdlineOperations implements CmdlineInterface {
 		}
 		
 		// similarity metrics for matching
-		SimilarityMetric[] sequence = strict ? StrictEpisodeMetrics.defaultSequence() : EpisodeMetrics.defaultSequence(false);
+		SimilarityMetric[] sequence = strict ? StrictEpisodeMetrics.defaultSequence(false) : EpisodeMetrics.defaultSequence(false);
 		
 		List<Match<File, Episode>> matches = new ArrayList<Match<File, Episode>>();
-		matches.addAll(match(filter(mediaFiles, VIDEO_FILES), episodes, sequence));
+		matches.addAll(matchEpisodes(filter(mediaFiles, VIDEO_FILES), episodes, sequence));
-		matches.addAll(match(filter(mediaFiles, SUBTITLE_FILES), episodes, sequence));
+		matches.addAll(matchEpisodes(filter(mediaFiles, SUBTITLE_FILES), episodes, sequence));
 		
 		if (matches.isEmpty()) {
 			throw new RuntimeException("Unable to match files to episode data");
@@ -179,6 +170,19 @@ public class CmdlineOperations implements CmdlineInterface {
 	}
 	
 
+	private List<Match<File, Episode>> matchEpisodes(Collection<File> files, Collection<Episode> episodes, SimilarityMetric[] sequence) throws Exception {
+		// always use strict fail-fast matcher
+		Matcher<File, Episode> matcher = new Matcher<File, Episode>(files, episodes, true, sequence);
+		List<Match<File, Episode>> matches = matcher.match();
+		
+		for (File failedMatch : matcher.remainingValues()) {
+			CLILogger.warning("No matching episode: " + failedMatch.getName());
+		}
+		
+		return matches;
+	}
+	
+
 	private Set<Episode> fetchEpisodeSet(final EpisodeListProvider db, final Collection<String> names, final Locale locale, final boolean strict) throws Exception {
 		List<Callable<List<Episode>>> tasks = new ArrayList<Callable<List<Episode>>>();
 		
@@ -302,132 +306,6 @@ public class CmdlineOperations implements CmdlineInterface {
 	}
 	
 
-	@Override
-	public List<File> getSubtitles(Collection<File> files, String query, String languageName, String output, String csn) throws Exception {
-		Language language = getLanguage(languageName);
-		Charset outputEncoding = (csn != null) ? Charset.forName(csn) : null;
-		
-		// match movie hashes online
-		Set<File> remainingVideos = new TreeSet<File>(filter(files, VIDEO_FILES));
-		List<File> downloadedSubtitles = new ArrayList<File>();
-		
-		if (remainingVideos.isEmpty()) {
-			throw new IllegalArgumentException("No video files: " + files);
-		}
-		
-		SubtitleFormat outputFormat = null;
-		if (output != null) {
-			outputFormat = getSubtitleFormatByName(output);
-			
-			// when rewriting subtitles to target format an encoding must be defined, default to UTF-8
-			if (outputEncoding == null) {
-				outputEncoding = Charset.forName("UTF-8");
-			}
-		}
-		
-		// lookup subtitles by hash
-		for (VideoHashSubtitleService service : WebServices.getVideoHashSubtitleServices()) {
-			if (remainingVideos.isEmpty()) {
-				break;
-			}
-			
-			CLILogger.fine("Looking up subtitles by filehash via " + service.getName());
-			
-			for (Entry<File, List<SubtitleDescriptor>> it : service.getSubtitleList(remainingVideos.toArray(new File[0]), language.getName()).entrySet()) {
-				if (it.getValue() != null && it.getValue().size() > 0) {
-					// auto-select first element if there are multiple hash matches for the same video files
-					File subtitle = fetchSubtitle(it.getValue().get(0), it.getKey(), outputFormat, outputEncoding);
-					Analytics.trackEvent(service.getName(), "DownloadSubtitle", it.getValue().get(0).getLanguageName(), 1);
-					
-					// download complete, cross this video off the list
-					remainingVideos.remove(it.getKey());
-					downloadedSubtitles.add(subtitle);
-				}
-			}
-		}
-		
-		// lookup subtitles by query and filename
-		if (query != null && remainingVideos.size() > 0) {
-			for (SubtitleProvider service : WebServices.getSubtitleProviders()) {
-				if (remainingVideos.isEmpty()) {
-					break;
-				}
-				
-				try {
-					CLILogger.fine(format("Searching for [%s] at [%s]", query, service.getName()));
-					SearchResult searchResult = selectSearchResult(query, service.search(query), false);
-					
-					CLILogger.config(format("Retrieving subtitles for [%s]", searchResult.getName()));
-					List<SubtitleDescriptor> subtitles = service.getSubtitleList(searchResult, language.getName());
-					
-					for (File video : remainingVideos.toArray(new File[0])) {
-						for (SubtitleDescriptor descriptor : subtitles) {
-							if (isDerived(descriptor.getName(), video)) {
-								File subtitle = fetchSubtitle(descriptor, video, outputFormat, outputEncoding);
-								Analytics.trackEvent(service.getName(), "DownloadSubtitle", descriptor.getLanguageName(), 1);
-								
-								// download complete, cross this video off the list
-								remainingVideos.remove(video);
-								downloadedSubtitles.add(subtitle);
-								break;
-							}
-						}
-					}
-				} catch (Exception e) {
-					CLILogger.warning(e.getMessage());
-				}
-			}
-		}
-		
-		// no subtitles for remaining video files
-		for (File video : remainingVideos) {
-			CLILogger.warning("No matching subtitles found: " + video);
-		}
-		
-		Analytics.trackEvent("CLI", "Download", "Subtitle", downloadedSubtitles.size());
-		return downloadedSubtitles;
-	}
-	
-
-	private File fetchSubtitle(SubtitleDescriptor descriptor, File movieFile, SubtitleFormat outputFormat, Charset outputEncoding) throws Exception {
-		// fetch subtitle archive
-		CLILogger.info(format("Fetching [%s.%s]", descriptor.getName(), descriptor.getType()));
-		ByteBuffer downloadedData = descriptor.fetch();
-		
-		// extract subtitles from archive
-		ArchiveType type = ArchiveType.forName(descriptor.getType());
-		MemoryFile subtitleFile;
-		
-		if (type != ArchiveType.UNDEFINED) {
-			// extract subtitle from archive
-			subtitleFile = type.fromData(downloadedData).iterator().next();
-		} else {
-			// assume that the fetched data is the subtitle
-			subtitleFile = new MemoryFile(descriptor.getName() + "." + descriptor.getType(), downloadedData);
-		}
-		
-		// subtitle filename is based on movie filename
-		String name = getName(movieFile);
-		String ext = getExtension(subtitleFile.getName());
-		ByteBuffer data = subtitleFile.getData();
-		
-		if (outputFormat != null || outputEncoding != null) {
-			if (outputFormat != null) {
-				ext = outputFormat.getFilter().extension(); // adjust extension of the output file
-			}
-			
-			CLILogger.finest(format("Export [%s] as: %s / %s", subtitleFile.getName(), outputFormat, outputEncoding.displayName(Locale.ROOT)));
-			data = exportSubtitles(subtitleFile, outputFormat, 0, outputEncoding);
-		}
-		
-		File destination = new File(movieFile.getParentFile(), name + "." + ext);
-		CLILogger.config(format("Writing [%s] to [%s]", subtitleFile.getName(), destination.getName()));
-		
-		writeFile(data, destination);
-		return destination;
-	}
-	
-
 	private List<File> renameAll(Map<File, File> renameMap) throws Exception {
 		// rename files
 		final List<Entry<File, File>> renameLog = new ArrayList<Entry<File, File>>();
@@ -484,20 +362,192 @@ public class CmdlineOperations implements CmdlineInterface {
 	}
 	
 
-	private List<Match<File, Episode>> match(Collection<File> files, Collection<Episode> episodes, SimilarityMetric[] sequence) throws Exception {
+	@Override
-		// always use strict fail-fast matcher
+	public List<File> getSubtitles(Collection<File> files, String query, String languageName, String output, String csn) throws Exception {
-		Matcher<File, Episode> matcher = new Matcher<File, Episode>(files, episodes, true, sequence);
+		final Language language = getLanguage(languageName);
-		List<Match<File, Episode>> matches = matcher.match();
 		
-		for (File failedMatch : matcher.remainingValues()) {
+		// when rewriting subtitles to target format an encoding must be defined, default to UTF-8
-			CLILogger.warning("No matching episode: " + failedMatch.getName());
+		final Charset outputEncoding = (csn != null) ? Charset.forName(csn) : (output != null) ? Charset.forName("UTF-8") : null;
+		final SubtitleFormat outputFormat = (output != null) ? getSubtitleFormatByName(output) : null;
+		
+		// try to find subtitles for each video file 
+		SubtitleCollector collector = new SubtitleCollector(filter(files, VIDEO_FILES));
+		
+		if (collector.isComplete()) {
+			throw new IllegalArgumentException("No video files: " + files);
 		}
 		
-		return matches;
+		// lookup subtitles by hash
+		for (VideoHashSubtitleService service : WebServices.getVideoHashSubtitleServices()) {
+			if (collector.isComplete()) {
+				break;
+			}
+			
+			try {
+				collector.addAll(service.getName(), lookupSubtitleByHash(service, language, collector.remainingVideos()));
+			} catch (RuntimeException e) {
+				CLILogger.warning(format("Lookup by hash failed: " + e.getMessage()));
+			}
+		}
+		
+		// lookup subtitles via text search
+		if (!collector.isComplete()) {
+			// auto-detect search query
+			Collection<String> querySet = (query == null) ? detectQuery(filter(files, VIDEO_FILES), false) : singleton(query);
+			
+			for (SubtitleProvider service : WebServices.getSubtitleProviders()) {
+				if (collector.isComplete()) {
+					break;
+				}
+				
+				try {
+					collector.addAll(service.getName(), lookupSubtitleByFileName(service, querySet, language, collector.remainingVideos()));
+				} catch (RuntimeException e) {
+					CLILogger.warning(format("Search for [%s] failed: %s", query, e.getMessage()));
+				}
+			}
+		}
+		
+		// no subtitles for remaining video files
+		for (File it : collector.remainingVideos()) {
+			CLILogger.warning("No matching subtitles found: " + it);
+		}
+		
+		// download subtitles in order
+		Map<File, Callable<File>> downloadQueue = new TreeMap<File, Callable<File>>();
+		for (final Entry<String, Map<File, SubtitleDescriptor>> source : collector.subtitlesBySource().entrySet()) {
+			for (final Entry<File, SubtitleDescriptor> descriptor : source.getValue().entrySet()) {
+				downloadQueue.put(descriptor.getKey(), new Callable<File>() {
+					
+					@Override
+					public File call() throws Exception {
+						Analytics.trackEvent(source.getKey(), "DownloadSubtitle", descriptor.getValue().getLanguageName(), 1);
+						return fetchSubtitle(descriptor.getValue(), descriptor.getKey(), outputFormat, outputEncoding);
+					}
+				});
+			}
+		}
+		
+		// parallel download
+		List<File> subtitleFiles = new ArrayList<File>();
+		ExecutorService executor = Executors.newFixedThreadPool(4);
+		
+		try {
+			for (Future<File> it : executor.invokeAll(downloadQueue.values())) {
+				subtitleFiles.add(it.get());
+			}
+		} finally {
+			executor.shutdownNow();
+		}
+		
+		Analytics.trackEvent("CLI", "Download", "Subtitle", subtitleFiles.size());
+		return subtitleFiles;
 	}
 	
 
-	private SearchResult selectSearchResult(String query, Iterable<? extends SearchResult> searchResults, boolean strict) throws IllegalArgumentException {
+	private File fetchSubtitle(SubtitleDescriptor descriptor, File movieFile, SubtitleFormat outputFormat, Charset outputEncoding) throws Exception {
+		// fetch subtitle archive
+		CLILogger.info(format("Fetching [%s]", descriptor.getPath()));
+		ByteBuffer downloadedData = descriptor.fetch();
+		
+		// extract subtitles from archive
+		ArchiveType type = ArchiveType.forName(descriptor.getType());
+		MemoryFile subtitleFile;
+		
+		if (type != ArchiveType.UNDEFINED) {
+			// extract subtitle from archive
+			subtitleFile = type.fromData(downloadedData).iterator().next();
+		} else {
+			// assume that the fetched data is the subtitle
+			subtitleFile = new MemoryFile(descriptor.getPath(), downloadedData);
+		}
+		
+		// subtitle filename is based on movie filename
+		String name = getName(movieFile);
+		String ext = getExtension(subtitleFile.getName());
+		ByteBuffer data = subtitleFile.getData();
+		
+		if (outputFormat != null || outputEncoding != null) {
+			if (outputFormat != null) {
+				ext = outputFormat.getFilter().extension(); // adjust extension of the output file
+			}
+			
+			CLILogger.finest(format("Export [%s] as: %s / %s", subtitleFile.getName(), outputFormat, outputEncoding.displayName(Locale.ROOT)));
+			data = exportSubtitles(subtitleFile, outputFormat, 0, outputEncoding);
+		}
+		
+		File destination = new File(movieFile.getParentFile(), name + "." + ext);
+		CLILogger.config(format("Writing [%s] to [%s]", subtitleFile.getName(), destination.getName()));
+		
+		writeFile(data, destination);
+		return destination;
+	}
+	
+
+	private Map<File, SubtitleDescriptor> lookupSubtitleByHash(VideoHashSubtitleService service, Language language, Collection<File> videoFiles) throws Exception {
+		Map<File, SubtitleDescriptor> subtitleByVideo = new HashMap<File, SubtitleDescriptor>(videoFiles.size());
+		CLILogger.fine("Looking up subtitles by filehash via " + service.getName());
+		
+		for (Entry<File, List<SubtitleDescriptor>> it : service.getSubtitleList(videoFiles.toArray(new File[0]), language.getName()).entrySet()) {
+			if (it.getValue() != null && it.getValue().size() > 0) {
+				CLILogger.finest(format("Matched [%s] to [%s]", it.getKey().getName(), it.getValue().get(0).getName()));
+				subtitleByVideo.put(it.getKey(), it.getValue().get(0));
+			}
+		}
+		
+		return subtitleByVideo;
+	}
+	
+
+	private Map<File, SubtitleDescriptor> lookupSubtitleByFileName(SubtitleProvider service, Collection<String> querySet, Language language, Collection<File> videoFiles) throws Exception {
+		Map<File, SubtitleDescriptor> subtitleByVideo = new HashMap<File, SubtitleDescriptor>();
+		
+		// search for and automatically select movie / show entry
+		Set<SearchResult> resultSet = new HashSet<SearchResult>();
+		for (String query : querySet) {
+			CLILogger.fine(format("Searching for [%s] at [%s]", query, service.getName()));
+			resultSet.addAll(findProbableMatches(query, service.search(query)));
+		}
+		
+		// fetch subtitles for all shows / movies and match against video files
+		if (resultSet.size() > 0) {
+			List<SubtitleDescriptor> subtitles = new ArrayList<SubtitleDescriptor>();
+			
+			for (SearchResult it : resultSet) {
+				List<SubtitleDescriptor> list = service.getSubtitleList(it, language.getName());
+				CLILogger.config(format("Found %d subtitles for [%s] at [%s]", list.size(), it.toString(), service.getName()));
+				subtitles.addAll(list);
+			}
+			
+			// first match everything as best as possible, then filter possibly bad matches
+			Matcher<File, SubtitleDescriptor> matcher = new Matcher<File, SubtitleDescriptor>(videoFiles, subtitles, false, EpisodeMetrics.defaultSequence(true));
+			SimilarityMetric sanity = new MetricCascade(StrictEpisodeMetrics.defaultSequence(true));
+			
+			for (Match<File, SubtitleDescriptor> it : matcher.match()) {
+				if (sanity.getSimilarity(it.getValue(), it.getCandidate()) >= 1) {
+					CLILogger.finest(format("Matched [%s] to [%s]", it.getValue().getName(), it.getCandidate().getName()));
+					subtitleByVideo.put(it.getValue(), it.getCandidate());
+				}
+			}
+		}
+		
+		return subtitleByVideo;
+	}
+	
+
+	private Collection<String> detectQuery(Collection<File> mediaFiles, boolean strict) {
+		Collection<String> names = new SeriesNameMatcher().matchAll(mediaFiles.toArray(new File[0]));
+		
+		if (names.isEmpty() || (strict && names.size() > 1)) {
+			throw new IllegalArgumentException("Unable to auto-select query: " + names);
+		}
+		
+		CLILogger.config("Auto-detected query: " + names);
+		return names;
+	}
+	
+
+	private Collection<SearchResult> findProbableMatches(String query, Iterable<? extends SearchResult> searchResults) {
 		// auto-select most probable search result
 		Map<String, SearchResult> probableMatches = new TreeMap<String, SearchResult>(String.CASE_INSENSITIVE_ORDER);
 		
@@ -507,18 +557,83 @@ public class CmdlineOperations implements CmdlineInterface {
 		// find probable matches using name similarity > 0.9
 		for (SearchResult result : searchResults) {
 			if (metric.getSimilarity(query, result.getName()) > 0.9) {
-				if (!probableMatches.containsKey(result.getName())) {
+				if (!probableMatches.containsKey(result.toString())) {
-					probableMatches.put(result.getName(), result);
+					probableMatches.put(result.toString(), result);
 				}
 			}
 		}
 		
+		return probableMatches.values();
+	}
+	
+
+	private SearchResult selectSearchResult(String query, Iterable<? extends SearchResult> searchResults, boolean strict) {
+		Collection<SearchResult> probableMatches = findProbableMatches(query, searchResults);
+		
 		if (probableMatches.isEmpty() || (strict && probableMatches.size() != 1)) {
-			throw new IllegalArgumentException("Failed to auto-select search result: " + probableMatches.values());
+			throw new IllegalArgumentException("Failed to auto-select search result: " + probableMatches);
 		}
 		
 		// return first and only value
-		return probableMatches.values().iterator().next();
+		return probableMatches.iterator().next();
+	}
+	
+
+	private Language getLanguage(String lang) {
+		// try to look up by language code
+		Language language = Language.getLanguage(lang);
+		
+		if (language == null) {
+			// try too look up by language name
+			language = Language.getLanguageByName(lang);
+			
+			if (language == null) {
+				// unable to lookup language
+				throw new IllegalArgumentException("Illegal language code: " + lang);
+			}
+		}
+		
+		return language;
+	}
+	
+
+	private class SubtitleCollector {
+		
+		private final Map<String, Map<File, SubtitleDescriptor>> collection = new HashMap<String, Map<File, SubtitleDescriptor>>();
+		private final Set<File> remainingVideos = new TreeSet<File>();
+		
+
+		public SubtitleCollector(Collection<File> videoFiles) {
+			remainingVideos.addAll(videoFiles);
+		}
+		
+
+		public void addAll(String source, Map<File, SubtitleDescriptor> subtitles) {
+			remainingVideos.removeAll(subtitles.keySet());
+			
+			Map<File, SubtitleDescriptor> subtitlesBySource = collection.get(source);
+			if (subtitlesBySource == null) {
+				subtitlesBySource = new TreeMap<File, SubtitleDescriptor>();
+				collection.put(source, subtitlesBySource);
+			}
+			
+			subtitlesBySource.putAll(subtitles);
+		}
+		
+
+		public Map<String, Map<File, SubtitleDescriptor>> subtitlesBySource() {
+			return collection;
+		}
+		
+
+		public Collection<File> remainingVideos() {
+			return remainingVideos;
+		}
+		
+
+		public boolean isComplete() {
+			return remainingVideos.size() == 0;
+		}
 	}
 	
 
@@ -659,22 +774,4 @@ public class CmdlineOperations implements CmdlineInterface {
 		return format.format(new MediaBindingBean(file, file));
 	}
 	
-
-	private Language getLanguage(String lang) {
-		// try to look up by language code
-		Language language = Language.getLanguage(lang);
-		
-		if (language == null) {
-			// try too look up by language name
-			language = Language.getLanguageByName(lang);
-			
-			if (language == null) {
-				// unable to lookup language
-				throw new IllegalArgumentException("Illegal language code: " + lang);
-			}
-		}
-		
-		return language;
-	}
-	
 }

source/net/sourceforge/filebot/similarity/EpisodeMetrics.java

@@ -14,7 +14,7 @@ import java.util.Map;
 import java.util.WeakHashMap;
 
 import net.sourceforge.filebot.similarity.SeasonEpisodeMatcher.SxE;
-import net.sourceforge.filebot.vfs.AbstractFile;
+import net.sourceforge.filebot.vfs.FileInfo;
 import net.sourceforge.filebot.web.Date;
 import net.sourceforge.filebot.web.Episode;
 import net.sourceforge.filebot.web.Movie;
@@ -213,12 +213,25 @@ public enum EpisodeMetrics implements SimilarityMetric {
 
 		@Override
 		protected long getLength(Object object) {
-			if (object instanceof AbstractFile) {
+			if (object instanceof FileInfo) {
-				return ((AbstractFile) object).getLength();
+				return ((FileInfo) object).getLength();
 			}
 			
 			return super.getLength(object);
 		}
+	}),
+	
+	// Match by common words at the beginning of both files
+	FileName(new FileNameMetric() {
+		
+		@Override
+		protected String getFileName(Object object) {
+			if (object instanceof File || object instanceof FileInfo) {
+				return normalizeObject(object);
+			}
+			
+			return null;
+		}
 	});
 	
 	// inner metric
@@ -242,8 +255,8 @@ public enum EpisodeMetrics implements SimilarityMetric {
 		// use name without extension
 		if (object instanceof File) {
 			name = getName((File) object);
-		} else if (object instanceof AbstractFile) {
+		} else if (object instanceof FileInfo) {
-			name = getNameWithoutExtension(((AbstractFile) object).getName());
+			name = ((FileInfo) object).getName();
 		}
 		
 		// remove checksums, any [...] or (...)
@@ -264,7 +277,7 @@ public enum EpisodeMetrics implements SimilarityMetric {
 		// 4. pass: match by generic name similarity (slow, but most matches will have been determined in second pass)
 		// 5. pass: match by generic numeric similarity
 		if (includeFileMetrics) {
-			return new SimilarityMetric[] { FileSize, EpisodeFunnel, EpisodeBalancer, SubstringFields, Name, Numeric };
+			return new SimilarityMetric[] { FileSize, new MetricCascade(FileName, EpisodeFunnel), EpisodeBalancer, SubstringFields, Name, Numeric };
 		} else {
 			return new SimilarityMetric[] { EpisodeFunnel, EpisodeBalancer, SubstringFields, Name, Numeric };
 		}

source/net/sourceforge/filebot/similarity/FileNameMetric.java

@@ -0,0 +1,35 @@
+
+package net.sourceforge.filebot.similarity;
+
+
+import static net.sourceforge.tuned.FileUtilities.*;
+
+import java.io.File;
+
+
+public class FileNameMetric implements SimilarityMetric {
+	
+	@Override
+	public float getSimilarity(Object o1, Object o2) {
+		String s1 = getFileName(o1);
+		if (s1 == null || s1.isEmpty())
+			return 0;
+		
+		String s2 = getFileName(o2);
+		if (s2 == null || s2.isEmpty())
+			return 0;
+		
+		return s1.startsWith(s2) || s2.startsWith(s1) ? 1 : 0;
+	}
+	
+
+	protected String getFileName(Object object) {
+		if (object instanceof File) {
+			// name without extension normalized to lower-case
+			return getName((File) object).trim().toLowerCase();
+		}
+		
+		return null;
+	}
+	
+}

source/net/sourceforge/filebot/similarity/FileSizeMetric.java

@@ -20,9 +20,9 @@ public class FileSizeMetric implements SimilarityMetric {
 	}
 	
 
-	protected long getLength(Object o) {
+	protected long getLength(Object object) {
-		if (o instanceof File) {
+		if (object instanceof File) {
-			return ((File) o).length();
+			return ((File) object).length();
 		}
 		
 		return -1;

source/net/sourceforge/filebot/similarity/StrictEpisodeMetrics.java

@@ -7,6 +7,7 @@ import static java.lang.Math.*;
 
 public enum StrictEpisodeMetrics implements SimilarityMetric {
 	
+	FileName(EpisodeMetrics.FileName, 1), // only allow 0 or 1
 	EpisodeIdentifier(EpisodeMetrics.EpisodeIdentifier, 1), // only allow 0 or 1
 	SubstringFields(EpisodeMetrics.SubstringFields, 2), // allow 0 or .5 or 1
 	Name(EpisodeMetrics.Name, 2); // allow 0 or .5 or 1
@@ -28,8 +29,13 @@ public enum StrictEpisodeMetrics implements SimilarityMetric {
 	}
 	
 
-	public static SimilarityMetric[] defaultSequence() {
+	public static SimilarityMetric[] defaultSequence(boolean includeFileMetrics) {
 		// use SEI for matching and SN for excluding false positives
-		return new SimilarityMetric[] { StrictEpisodeMetrics.EpisodeIdentifier, StrictEpisodeMetrics.SubstringFields, StrictEpisodeMetrics.Name };
+		if (includeFileMetrics) {
+			return new SimilarityMetric[] { FileName, EpisodeIdentifier, SubstringFields, Name };
+		} else {
+			return new SimilarityMetric[] { EpisodeIdentifier, SubstringFields, Name };
 		}
 	}
+	
+}

source/net/sourceforge/filebot/similarity/SubstringMetric.java

@@ -7,11 +7,14 @@ public class SubstringMetric implements SimilarityMetric {
 	@Override
 	public float getSimilarity(Object o1, Object o2) {
 		String s1 = normalize(o1);
-		String s2 = normalize(o2);
+		if (s1 == null || s1.isEmpty())
-		String pri = s1.length() > s2.length() ? s1 : s2;
+			return 0;
-		String sub = s1.length() > s2.length() ? s2 : s1;
 		
-		return sub.length() > 0 && pri.contains(sub) ? 1 : 0;
+		String s2 = normalize(o2);
+		if (s2 == null || s2.isEmpty())
+			return 0;
+		
+		return s1.contains(s2) || s2.contains(s1) ? 1 : 0;
 	}
 	
 

source/net/sourceforge/filebot/subtitle/SubtitleUtilities.java

@@ -86,26 +86,6 @@ public final class SubtitleUtilities {
 	}
 	
 
-	public static boolean isDerived(String subtitle, File video) {
-		return isDerived(subtitle, getName(video));
-	}
-	
-
-	public static boolean isDerived(String derivate, String base) {
-		if (derivate.equalsIgnoreCase(base))
-			return true;
-		
-		while (getExtension(derivate) != null) {
-			derivate = getNameWithoutExtension(derivate);
-			
-			if (derivate.equalsIgnoreCase(base))
-				return true;
-		}
-		
-		return false;
-	}
-	
-
 	public static SubtitleFormat getSubtitleFormat(File file) {
 		for (SubtitleFormat it : SubtitleFormat.values()) {
 			if (it.getFilter().accept(file))

source/net/sourceforge/filebot/ui/rename/FileNameFormatter.java

@@ -5,7 +5,7 @@ package net.sourceforge.filebot.ui.rename;
 import java.io.File;
 
 import net.sourceforge.filebot.similarity.Match;
-import net.sourceforge.filebot.vfs.AbstractFile;
+import net.sourceforge.filebot.vfs.FileInfo;
 import net.sourceforge.tuned.FileUtilities;
 
 
@@ -21,7 +21,7 @@ class FileNameFormatter implements MatchFormatter {
 
 	@Override
 	public boolean canFormat(Match<?, ?> match) {
-		return match.getValue() instanceof File || match.getValue() instanceof AbstractFile;
+		return match.getValue() instanceof File || match.getValue() instanceof FileInfo;
 	}
 	
 
@@ -38,9 +38,9 @@ class FileNameFormatter implements MatchFormatter {
 			return preserveExtension ? FileUtilities.getName(file) : file.getName();
 		}
 		
-		if (match.getValue() instanceof AbstractFile) {
+		if (match.getValue() instanceof FileInfo) {
-			AbstractFile file = (AbstractFile) match.getValue();
+			FileInfo file = (FileInfo) match.getValue();
-			return preserveExtension ? FileUtilities.getNameWithoutExtension(file.getName()) : file.getName();
+			return preserveExtension ? file.getName() : file.getPath();
 		}
 		
 		// cannot format value

source/net/sourceforge/filebot/ui/rename/NamesListTransferablePolicy.java

@@ -22,7 +22,7 @@ import net.sourceforge.filebot.hash.VerificationFileReader;
 import net.sourceforge.filebot.torrent.Torrent;
 import net.sourceforge.filebot.ui.transfer.ArrayTransferable;
 import net.sourceforge.filebot.ui.transfer.FileTransferablePolicy;
-import net.sourceforge.filebot.vfs.AbstractFile;
+import net.sourceforge.filebot.vfs.SimpleFileInfo;
 import net.sourceforge.filebot.web.Episode;
 import net.sourceforge.tuned.FastFile;
 
@@ -146,7 +146,7 @@ class NamesListTransferablePolicy extends FileTransferablePolicy {
 			
 			try {
 				while (parser.hasNext()) {
-					values.add(new AbstractFile(parser.next().getKey().getName(), -1));
+					values.add(new SimpleFileInfo(parser.next().getKey().getName(), -1));
 				}
 			} finally {
 				parser.close();
@@ -160,7 +160,7 @@ class NamesListTransferablePolicy extends FileTransferablePolicy {
 			Torrent torrent = new Torrent(file);
 			
 			for (Torrent.Entry entry : torrent.getFiles()) {
-				values.add(new AbstractFile(entry.getName(), entry.getLength()));
+				values.add(new SimpleFileInfo(entry.getName(), entry.getLength()));
 			}
 		}
 	}

source/net/sourceforge/filebot/ui/subtitle/SubtitlePackage.java

@@ -167,7 +167,7 @@ public class SubtitlePackage {
 			
 			if (archiveType == ArchiveType.UNDEFINED) {
 				// cannot extract files from archive
-				return singletonList(new MemoryFile(subtitle.getName() + '.' + subtitle.getType(), data));
+				return singletonList(new MemoryFile(subtitle.getPath(), data));
 			}
 			
 			// extract contents of the archive

source/net/sourceforge/filebot/vfs/AbstractFile.java

@@ -1,32 +0,0 @@
-
-package net.sourceforge.filebot.vfs;
-
-
-public class AbstractFile {
-	
-	private final String name;
-	private final long length;
-	
-
-	public AbstractFile(String name, long length) {
-		this.name = name;
-		this.length = length;
-	}
-	
-
-	public String getName() {
-		return name;
-	}
-	
-
-	public long getLength() {
-		return length;
-	}
-	
-
-	@Override
-	public String toString() {
-		return getName();
-	}
-	
-}

source/net/sourceforge/filebot/vfs/FileInfo.java

@@ -0,0 +1,18 @@
+
+package net.sourceforge.filebot.vfs;
+
+
+public interface FileInfo {
+	
+	public String getPath();
+	
+
+	public String getName();
+	
+
+	public String getType();
+	
+
+	public long getLength();
+	
+}

source/net/sourceforge/filebot/vfs/SimpleFileInfo.java

@@ -0,0 +1,66 @@
+
+package net.sourceforge.filebot.vfs;
+
+
+import static net.sourceforge.tuned.FileUtilities.*;
+
+import java.util.Arrays;
+
+
+public class SimpleFileInfo implements FileInfo {
+	
+	private final String path;
+	private final long length;
+	
+
+	public SimpleFileInfo(String path, long length) {
+		this.path = path;
+		this.length = length;
+	}
+	
+
+	@Override
+	public String getPath() {
+		return path;
+	}
+	
+
+	public String getName() {
+		return getNameWithoutExtension(path);
+	}
+	
+
+	@Override
+	public String getType() {
+		return getExtension(path);
+	}
+	
+
+	public long getLength() {
+		return length;
+	}
+	
+
+	@Override
+	public boolean equals(Object obj) {
+		if (obj instanceof FileInfo) {
+			FileInfo other = (FileInfo) obj;
+			return other.getLength() == getLength() && other.getPath().equals(getPath());
+		}
+		
+		return false;
+	}
+	
+
+	@Override
+	public int hashCode() {
+		return Arrays.hashCode(new Object[] { getPath(), getLength() });
+	}
+	
+
+	@Override
+	public String toString() {
+		return getPath();
+	}
+	
+}

source/net/sourceforge/filebot/web/OpenSubtitlesSubtitleDescriptor.java

@@ -87,6 +87,12 @@ public class OpenSubtitlesSubtitleDescriptor implements SubtitleDescriptor {
 	}
 	
 
+	@Override
+	public String getPath() {
+		return getProperty(Property.SubFileName);
+	}
+	
+
 	@Override
 	public String getName() {
 		return FileUtilities.getNameWithoutExtension(getProperty(Property.SubFileName));
@@ -105,8 +111,9 @@ public class OpenSubtitlesSubtitleDescriptor implements SubtitleDescriptor {
 	}
 	
 
-	public int getSize() {
+	@Override
-		return Integer.parseInt(getProperty(Property.SubSize));
+	public long getLength() {
+		return Long.parseLong(getProperty(Property.SubSize));
 	}
 	
 
@@ -126,7 +133,7 @@ public class OpenSubtitlesSubtitleDescriptor implements SubtitleDescriptor {
 		InputStream stream = new GZIPInputStream(resource.openStream());
 		
 		try {
-			ByteBufferOutputStream buffer = new ByteBufferOutputStream(getSize());
+			ByteBufferOutputStream buffer = new ByteBufferOutputStream(getLength());
 			
 			// read all
 			buffer.transferFully(stream);

source/net/sourceforge/filebot/web/OpenSubtitlesXmlRpc.java

@@ -141,7 +141,7 @@ public class OpenSubtitlesXmlRpc {
 				if (!matcher.find())
 					throw new IllegalArgumentException("Illegal title");
 				
-				String name = matcher.group(1).trim();
+				String name = matcher.group(1).replaceAll("\"", "").trim();
 				int year = Integer.parseInt(matcher.group(2));
 				
 				movies.add(new Movie(name, year, Integer.parseInt(imdbid)));

source/net/sourceforge/filebot/web/SublightSubtitleDescriptor.java

@@ -83,6 +83,12 @@ public class SublightSubtitleDescriptor implements SubtitleDescriptor {
 	}
 	
 
+	@Override
+	public long getLength() {
+		return subtitle.getSize();
+	}
+	
+
 	@Override
 	public ByteBuffer fetch() throws Exception {
 		byte[] archive = source.getZipArchive(subtitle);
@@ -94,7 +100,7 @@ public class SublightSubtitleDescriptor implements SubtitleDescriptor {
 			// move to subtitle entry
 			ZipEntry entry = stream.getNextEntry();
 			
-			ByteBufferOutputStream buffer = new ByteBufferOutputStream((int) entry.getSize());
+			ByteBufferOutputStream buffer = new ByteBufferOutputStream(entry.getSize());
 			
 			// read subtitle data
 			buffer.transferFully(stream);
@@ -107,6 +113,12 @@ public class SublightSubtitleDescriptor implements SubtitleDescriptor {
 	}
 	
 
+	@Override
+	public String getPath() {
+		return String.format("%s.%s", getName(), getType());
+	}
+	
+
 	@Override
 	public String toString() {
 		return String.format("%s [%s]", getName(), getLanguageName());

source/net/sourceforge/filebot/web/SubsceneSubtitleClient.java

@@ -16,6 +16,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.logging.Level;
 import java.util.logging.Logger;
+import java.util.regex.Pattern;
 
 import javax.swing.Icon;
 
@@ -60,12 +61,17 @@ public class SubsceneSubtitleClient implements SubtitleProvider {
 		List<Node> nodes = selectNodes("id('filmSearch')/A", dom);
 		List<SearchResult> searchResults = new ArrayList<SearchResult>(nodes.size());
 		
+		Pattern titleSuffixPattern = Pattern.compile("\\s-\\s([^-]+)[(](\\d{4})[)]$");
+		
 		for (Node node : nodes) {
 			String title = getTextContent(node);
 			String href = getAttribute("href", node);
 			
+			// simplified name for easy matching
+			String shortName = titleSuffixPattern.matcher(title).replaceFirst("");
+			
 			try {
-				searchResults.add(new HyperLink(title, new URL("http", host, href)));
+				searchResults.add(new SubsceneSearchResult(shortName, title, new URL("http", host, href)));
 			} catch (MalformedURLException e) {
 				Logger.getLogger(getClass().getName()).log(Level.WARNING, "Invalid href: " + href, e);
 			}
@@ -184,4 +190,28 @@ public class SubsceneSubtitleClient implements SubtitleProvider {
 		return ((HyperLink) searchResult).getURI();
 	}
 	
+
+	public static class SubsceneSearchResult extends HyperLink {
+		
+		private String shortName;
+		
+
+		public SubsceneSearchResult(String shortName, String title, URL url) {
+			super(title, url);
+			this.shortName = shortName;
+		}
+		
+
+		@Override
+		public String getName() {
+			return shortName;
+		}
+		
+
+		@Override
+		public String toString() {
+			return super.getName();
+		}
+	}
+	
 }

source/net/sourceforge/filebot/web/SubsceneSubtitleDescriptor.java

@@ -85,6 +85,18 @@ public class SubsceneSubtitleDescriptor implements SubtitleDescriptor {
 	}
 	
 
+	@Override
+	public String getPath() {
+		return String.format("%s.%s", getName(), getType());
+	}
+	
+
+	@Override
+	public long getLength() {
+		return -1;
+	}
+	
+
 	@Override
 	public String toString() {
 		return String.format("%s [%s]", getName(), getLanguageName());

source/net/sourceforge/filebot/web/SubtitleDescriptor.java

@@ -4,8 +4,10 @@ package net.sourceforge.filebot.web;
 
 import java.nio.ByteBuffer;
 
+import net.sourceforge.filebot.vfs.FileInfo;
 
-public interface SubtitleDescriptor {
+
+public interface SubtitleDescriptor extends FileInfo {
 	
 	String getName();
 	

source/net/sourceforge/tuned/ByteBufferOutputStream.java

@@ -17,6 +17,11 @@ public class ByteBufferOutputStream extends OutputStream {
 	private final float loadFactor;
 	
 
+	public ByteBufferOutputStream(long initialCapacity) {
+		this((int) initialCapacity);
+	}
+	
+
 	public ByteBufferOutputStream(int initialCapacity) {
 		this(initialCapacity, 1.0f);
 	}

source/net/sourceforge/tuned/FileUtilities.java

@@ -223,6 +223,31 @@ public final class FileUtilities {
 	}
 	
 
+	public static boolean isDerived(String derivate, File prime) {
+		String base = getName(prime).trim().toLowerCase();
+		derivate = derivate.trim().toLowerCase();
+		return derivate.startsWith(base);
+	}
+	
+
+	public static boolean isDerivedByExtension(String derivate, File prime) {
+		String base = getName(prime).trim().toLowerCase();
+		derivate = derivate.trim().toLowerCase();
+		
+		if (derivate.equals(base))
+			return true;
+		
+		while (derivate.length() > base.length() && getExtension(derivate) != null) {
+			derivate = getNameWithoutExtension(derivate);
+			
+			if (derivate.equals(base))
+				return true;
+		}
+		
+		return false;
+	}
+	
+
 	public static boolean containsOnly(Iterable<File> files, FileFilter filter) {
 		for (File file : files) {
 			if (!filter.accept(file))

test/net/sourceforge/filebot/web/SubsceneSubtitleClientTest.java

@@ -12,6 +12,8 @@ import java.util.Map;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
+import net.sourceforge.filebot.web.SubsceneSubtitleClient.SubsceneSearchResult;
+
 
 public class SubsceneSubtitleClientTest {
 	
@@ -28,8 +30,8 @@ public class SubsceneSubtitleClientTest {
 
 	@BeforeClass
 	public static void setUpBeforeClass() throws Exception {
-		twinpeaksSearchResult = new HyperLink("Twin Peaks - First Season (1990)", new URL("http://subscene.com/twin-peaks--first-season/subtitles-32482.aspx"));
+		twinpeaksSearchResult = new SubsceneSearchResult("Twin Peaks", "Twin Peaks - First Season (1990)", new URL("http://subscene.com/twin-peaks--first-season/subtitles-32482.aspx"));
-		lostSearchResult = new HyperLink("Lost - Fourth Season (2008)", new URL("http://subscene.com/Lost-Fourth-Season/subtitles-70963.aspx"));
+		lostSearchResult = new SubsceneSearchResult("Lost", "Lost - Fourth Season (2008)", new URL("http://subscene.com/Lost-Fourth-Season/subtitles-70963.aspx"));
 	}
 	
 
@@ -40,22 +42,21 @@ public class SubsceneSubtitleClientTest {
 	public void search() throws Exception {
 		List<SearchResult> results = subscene.search("twin peaks");
 		
-		HyperLink result = (HyperLink) results.get(1);
+		SubsceneSearchResult result = (SubsceneSearchResult) results.get(1);
-		
+		assertEquals(twinpeaksSearchResult.toString(), result.toString());
-		assertEquals(twinpeaksSearchResult.getName(), result.getName());
 		assertEquals(twinpeaksSearchResult.getURL().toString(), result.getURL().toString());
+		assertEquals(twinpeaksSearchResult.getName(), result.getName());
 	}
 	
 
 	@Test
 	public void searchResultPageRedirect() throws Exception {
 		List<SearchResult> results = subscene.search("firefly");
-		
 		assertEquals(2, results.size());
 		
-		HyperLink result = (HyperLink) results.get(0);
+		SubsceneSearchResult result = (SubsceneSearchResult) results.get(0);
-		
+		assertEquals("Firefly - The Complete Series (2002)", result.toString());
-		assertEquals("Firefly - The Complete Series (2002)", result.getName());
+		assertEquals("Firefly", result.getName());
 		assertEquals("http://subscene.com/Firefly-The-Complete-Series/subtitles-20008.aspx", result.getURL().toString());
 	}
 	
@@ -63,11 +64,9 @@ public class SubsceneSubtitleClientTest {
 	@Test
 	public void getSubtitleListSearchResult() throws Exception {
 		List<SubtitleDescriptor> subtitleList = subscene.getSubtitleList(twinpeaksSearchResult, "Italian");
-		
 		assertEquals(1, subtitleList.size());
 		
 		SubtitleDescriptor subtitle = subtitleList.get(0);
-		
 		assertEquals("Twin Peaks - First Season", subtitle.getName());
 		assertEquals("Italian", subtitle.getLanguageName());
 		assertEquals("zip", subtitle.getType());
@@ -104,7 +103,6 @@ public class SubsceneSubtitleClientTest {
 	public void downloadSubtitleArchive() throws Exception {
 		SearchResult selectedResult = subscene.search("firefly").get(0);
 		SubtitleDescriptor subtitleDescriptor = subscene.getSubtitleList(selectedResult, "English").get(1);
-		
 		assertEquals("Firefly - The Complete Series", subtitleDescriptor.getName());
 		
 		ByteBuffer archive = subtitleDescriptor.fetch();