+ fully-automatic subtitle matching even without hashes

source/net/sourceforge/filebot/cli/ArgumentProcessor.java

@@ -63,7 +63,7 @@ public class ArgumentProcessor {
 				Set<File> files = new LinkedHashSet<File>(args.getFiles(true));
 				
 				if (args.getSubtitles) {
-					List<File> subtitles = cli.getSubtitles(files, args.query, args.lang, args.output, args.encoding);
+					List<File> subtitles = cli.getSubtitles(files, args.query, args.lang, args.output, args.encoding, !args.nonStrict);
 					files.addAll(subtitles);
 				}
 				

source/net/sourceforge/filebot/cli/CmdlineInterface.java

@@ -12,7 +12,7 @@ public interface CmdlineInterface {
 	List<File> rename(Collection<File> files, String query, String format, String db, String lang, boolean strict) throws Exception;
 	
 
-	List<File> getSubtitles(Collection<File> files, String query, String lang, String output, String encoding) throws Exception;
+	List<File> getSubtitles(Collection<File> files, String query, String lang, String output, String encoding, boolean strict) throws Exception;
 	
 
 	boolean check(Collection<File> files) throws Exception;

source/net/sourceforge/filebot/cli/CmdlineOperations.java

@@ -19,7 +19,6 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
@@ -55,7 +54,6 @@ import net.sourceforge.filebot.similarity.StrictEpisodeMetrics;
 import net.sourceforge.filebot.subtitle.SubtitleFormat;
 import net.sourceforge.filebot.ui.Language;
 import net.sourceforge.filebot.ui.rename.HistorySpooler;
-import net.sourceforge.filebot.vfs.ArchiveType;
 import net.sourceforge.filebot.vfs.MemoryFile;
 import net.sourceforge.filebot.web.Episode;
 import net.sourceforge.filebot.web.EpisodeFormat;
@@ -363,7 +361,7 @@ public class CmdlineOperations implements CmdlineInterface {
 	
 
 	@Override
-	public List<File> getSubtitles(Collection<File> files, String query, String languageName, String output, String csn) throws Exception {
+	public List<File> getSubtitles(Collection<File> files, String query, String languageName, String output, String csn, boolean strict) throws Exception {
 		final Language language = getLanguage(languageName);
 		
 		// when rewriting subtitles to target format an encoding must be defined, default to UTF-8
@@ -384,14 +382,15 @@ public class CmdlineOperations implements CmdlineInterface {
 			}
 			
 			try {
+				CLILogger.fine("Looking up subtitles by filehash via " + service.getName());
 				collector.addAll(service.getName(), lookupSubtitleByHash(service, language, collector.remainingVideos()));
 			} catch (RuntimeException e) {
 				CLILogger.warning(format("Lookup by hash failed: " + e.getMessage()));
 			}
 		}
 		
-		// lookup subtitles via text search
+		// lookup subtitles via text search, only perform hash lookup in strict mode
-		if (!collector.isComplete()) {
+		if ((query != null || !strict) && !collector.isComplete()) {
 			// auto-detect search query
 			Collection<String> querySet = (query == null) ? detectQuery(filter(files, VIDEO_FILES), false) : singleton(query);
 			
@@ -401,6 +400,7 @@ public class CmdlineOperations implements CmdlineInterface {
 				}
 				
 				try {
+					CLILogger.fine(format("Searching for %s at [%s]", querySet.toString(), service.getName()));
 					collector.addAll(service.getName(), lookupSubtitleByFileName(service, querySet, language, collector.remainingVideos()));
 				} catch (RuntimeException e) {
 					CLILogger.warning(format("Search for [%s] failed: %s", querySet, e.getMessage()));
@@ -422,7 +422,7 @@ public class CmdlineOperations implements CmdlineInterface {
 					@Override
 					public File call() throws Exception {
 						Analytics.trackEvent(source.getKey(), "DownloadSubtitle", descriptor.getValue().getLanguageName(), 1);
-						return fetchSubtitle(descriptor.getValue(), descriptor.getKey(), outputFormat, outputEncoding);
+						return downloadSubtitle(descriptor.getValue(), descriptor.getKey(), outputFormat, outputEncoding);
 					}
 				});
 			}
@@ -430,14 +430,17 @@ public class CmdlineOperations implements CmdlineInterface {
 		
 		// parallel download
 		List<File> subtitleFiles = new ArrayList<File>();
-		ExecutorService executor = Executors.newFixedThreadPool(4);
 		
-		try {
+		if (downloadQueue.size() > 0) {
-			for (Future<File> it : executor.invokeAll(downloadQueue.values())) {
+			ExecutorService executor = Executors.newFixedThreadPool(4);
-				subtitleFiles.add(it.get());
+			
+			try {
+				for (Future<File> it : executor.invokeAll(downloadQueue.values())) {
+					subtitleFiles.add(it.get());
+				}
+			} finally {
+				executor.shutdownNow();
 			}
-		} finally {
-			executor.shutdownNow();
 		}
 		
 		Analytics.trackEvent("CLI", "Download", "Subtitle", subtitleFiles.size());
@@ -445,26 +448,13 @@ public class CmdlineOperations implements CmdlineInterface {
 	}
 	
 
-	private File fetchSubtitle(SubtitleDescriptor descriptor, File movieFile, SubtitleFormat outputFormat, Charset outputEncoding) throws Exception {
+	private File downloadSubtitle(SubtitleDescriptor descriptor, File movieFile, SubtitleFormat outputFormat, Charset outputEncoding) throws Exception {
 		// fetch subtitle archive
 		CLILogger.info(format("Fetching [%s]", descriptor.getPath()));
-		ByteBuffer downloadedData = descriptor.fetch();
+		MemoryFile subtitleFile = fetchSubtitle(descriptor);
-		
-		// extract subtitles from archive
-		ArchiveType type = ArchiveType.forName(descriptor.getType());
-		MemoryFile subtitleFile;
-		
-		if (type != ArchiveType.UNDEFINED) {
-			// extract subtitle from archive
-			subtitleFile = type.fromData(downloadedData).iterator().next();
-		} else {
-			// assume that the fetched data is the subtitle
-			subtitleFile = new MemoryFile(descriptor.getPath(), downloadedData);
-		}
 		
 		// subtitle filename is based on movie filename
-		String name = getName(movieFile);
+		String base = getName(movieFile);
-		String lang = Language.getISO3LanguageCodeByName(descriptor.getLanguageName());
 		String ext = getExtension(subtitleFile.getName());
 		ByteBuffer data = subtitleFile.getData();
 		
@@ -477,7 +467,7 @@ public class CmdlineOperations implements CmdlineInterface {
 			data = exportSubtitles(subtitleFile, outputFormat, 0, outputEncoding);
 		}
 		
-		File destination = new File(movieFile.getParentFile(), String.format("%s.%s.%s", name, lang, ext));
+		File destination = new File(movieFile.getParentFile(), formatSubtitle(base, descriptor.getLanguageName(), ext));
 		CLILogger.config(format("Writing [%s] to [%s]", subtitleFile.getName(), destination.getName()));
 		
 		writeFile(data, destination);
@@ -487,11 +477,10 @@ public class CmdlineOperations implements CmdlineInterface {
 
 	private Map<File, SubtitleDescriptor> lookupSubtitleByHash(VideoHashSubtitleService service, Language language, Collection<File> videoFiles) throws Exception {
 		Map<File, SubtitleDescriptor> subtitleByVideo = new HashMap<File, SubtitleDescriptor>(videoFiles.size());
-		CLILogger.fine("Looking up subtitles by filehash via " + service.getName());
 		
 		for (Entry<File, List<SubtitleDescriptor>> it : service.getSubtitleList(videoFiles.toArray(new File[0]), language.getName()).entrySet()) {
 			if (it.getValue() != null && it.getValue().size() > 0) {
-				CLILogger.finest(format("Matched [%s] to [%s]", it.getKey().getName(), it.getValue().get(0).getName()));
+				CLILogger.finest(format("Matched [%s] to [%s] via filehash", it.getKey().getName(), it.getValue().get(0).getName()));
 				subtitleByVideo.put(it.getKey(), it.getValue().get(0));
 			}
 		}
@@ -503,30 +492,18 @@ public class CmdlineOperations implements CmdlineInterface {
 	private Map<File, SubtitleDescriptor> lookupSubtitleByFileName(SubtitleProvider service, Collection<String> querySet, Language language, Collection<File> videoFiles) throws Exception {
 		Map<File, SubtitleDescriptor> subtitleByVideo = new HashMap<File, SubtitleDescriptor>();
 		
-		// search for and automatically select movie / show entry
+		// search for subtitles
-		Set<SearchResult> resultSet = new HashSet<SearchResult>();
+		List<SubtitleDescriptor> subtitles = findSubtitles(service, querySet, language.getName());
-		for (String query : querySet) {
-			CLILogger.fine(format("Searching for [%s] at [%s]", query, service.getName()));
-			resultSet.addAll(findProbableMatches(query, service.search(query)));
-		}
 		
-		// fetch subtitles for all shows / movies and match against video files
+		// match subtitle files to video files
-		if (resultSet.size() > 0) {
+		if (subtitles.size() > 0) {
-			List<SubtitleDescriptor> subtitles = new ArrayList<SubtitleDescriptor>();
-			
-			for (SearchResult it : resultSet) {
-				List<SubtitleDescriptor> list = service.getSubtitleList(it, language.getName());
-				CLILogger.config(format("Found %d subtitles for [%s] at [%s]", list.size(), it.toString(), service.getName()));
-				subtitles.addAll(list);
-			}
-			
 			// first match everything as best as possible, then filter possibly bad matches
 			Matcher<File, SubtitleDescriptor> matcher = new Matcher<File, SubtitleDescriptor>(videoFiles, subtitles, false, EpisodeMetrics.defaultSequence(true));
 			SimilarityMetric sanity = new MetricCascade(StrictEpisodeMetrics.defaultSequence(true));
 			
 			for (Match<File, SubtitleDescriptor> it : matcher.match()) {
 				if (sanity.getSimilarity(it.getValue(), it.getCandidate()) >= 1) {
-					CLILogger.finest(format("Matched [%s] to [%s]", it.getValue().getName(), it.getCandidate().getName()));
+					CLILogger.finest(format("Matched [%s] to [%s] via filename", it.getValue().getName(), it.getCandidate().getName()));
 					subtitleByVideo.put(it.getValue(), it.getCandidate());
 				}
 			}

source/net/sourceforge/filebot/cli/ScriptShell.lib.groovy

@@ -32,7 +32,7 @@ def rename(args) { args = _defaults(args)
 }
 
 def getSubtitles(args) { args = _defaults(args)	
-	_guarded { _cli.getSubtitles(_files(args), args.query, args.lang, args.output, args.encoding) }
+	_guarded { _cli.getSubtitles(_files(args), args.query, args.lang, args.output, args.encoding, args.strict) }
 }
 
 def check(args) {

source/net/sourceforge/filebot/subtitle/SubtitleUtilities.java

@@ -3,6 +3,7 @@ package net.sourceforge.filebot.subtitle;
 
 
 import static java.lang.Math.*;
+import static net.sourceforge.filebot.MediaTypes.*;
 import static net.sourceforge.tuned.FileUtilities.*;
 
 import java.io.File;
@@ -12,14 +13,62 @@ import java.nio.ByteBuffer;
 import java.nio.CharBuffer;
 import java.nio.charset.Charset;
 import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Set;
 
+import net.sourceforge.filebot.similarity.NameSimilarityMetric;
+import net.sourceforge.filebot.similarity.SimilarityMetric;
+import net.sourceforge.filebot.ui.Language;
+import net.sourceforge.filebot.vfs.ArchiveType;
 import net.sourceforge.filebot.vfs.MemoryFile;
+import net.sourceforge.filebot.web.SearchResult;
+import net.sourceforge.filebot.web.SubtitleDescriptor;
+import net.sourceforge.filebot.web.SubtitleProvider;
 
 
 public final class SubtitleUtilities {
 	
+	public static List<SubtitleDescriptor> findSubtitles(SubtitleProvider service, Collection<String> querySet, String languageName) throws Exception {
+		List<SubtitleDescriptor> subtitles = new ArrayList<SubtitleDescriptor>();
+		
+		// search for and automatically select movie / show entry
+		Set<SearchResult> resultSet = new HashSet<SearchResult>();
+		for (String query : querySet) {
+			resultSet.addAll(findProbableMatches(query, service.search(query), 0.9f));
+		}
+		
+		// fetch subtitles for all search results
+		for (SearchResult it : resultSet) {
+			subtitles.addAll(service.getSubtitleList(it, languageName));
+		}
+		
+		return subtitles;
+	}
+	
+
+	protected static Collection<SearchResult> findProbableMatches(String query, Iterable<? extends SearchResult> searchResults, float threshold) {
+		// auto-select most probable search result
+		Set<SearchResult> probableMatches = new LinkedHashSet<SearchResult>();
+		
+		// use name similarity metric
+		SimilarityMetric metric = new NameSimilarityMetric();
+		
+		// find probable matches using name similarity > threshold
+		for (SearchResult result : searchResults) {
+			if (metric.getSimilarity(query, result.getName()) > threshold) {
+				probableMatches.add(result);
+			}
+		}
+		
+		return probableMatches;
+	}
+	
+
 	/**
 	 * Detect charset and parse subtitle file even if extension is invalid
 	 */
@@ -111,6 +160,50 @@ public final class SubtitleUtilities {
 	}
 	
 
+	public static String formatSubtitle(String name, String languageName, String type) {
+		StringBuilder sb = new StringBuilder(name);
+		
+		if (languageName != null) {
+			String lang = Language.getISO3LanguageCodeByName(languageName);
+			
+			if (lang == null) {
+				// we probably won't get here, but just in case
+				lang = languageName.replaceAll("\\W", "");
+			}
+			
+			sb.append('.').append(lang);
+		}
+		
+		if (type != null) {
+			sb.append('.').append(type);
+		}
+		
+		return sb.toString();
+	}
+	
+
+	public static MemoryFile fetchSubtitle(SubtitleDescriptor descriptor) throws Exception {
+		ByteBuffer data = descriptor.fetch();
+		
+		// extract subtitles from archive
+		ArchiveType type = ArchiveType.forName(descriptor.getType());
+		
+		if (type != ArchiveType.UNKOWN) {
+			// extract subtitle from archive
+			Iterator<MemoryFile> it = type.fromData(data).iterator();
+			while (it.hasNext()) {
+				MemoryFile entry = it.next();
+				if (SUBTITLE_FILES.accept(entry.getName())) {
+					return entry;
+				}
+			}
+		}
+		
+		// assume that the fetched data is the subtitle
+		return new MemoryFile(descriptor.getPath(), data);
+	}
+	
+
 	/**
 	 * Dummy constructor to prevent instantiation.
 	 */

source/net/sourceforge/filebot/ui/Language.java

@@ -103,8 +103,7 @@ public class Language {
 			}
 		}
 		
-		// we won't get here, but just in case
+		return null;
-		return languageName.replaceAll("\\W", "");
 	}
 	
 

source/net/sourceforge/filebot/ui/subtitle/SubtitleDropTarget.java

@@ -32,8 +32,8 @@ import javax.swing.JDialog;
 import javax.swing.JFileChooser;
 import javax.swing.filechooser.FileNameExtensionFilter;
 
-import net.sourceforge.filebot.Analytics;
 import net.sourceforge.filebot.ResourceManager;
+import net.sourceforge.filebot.web.SubtitleProvider;
 import net.sourceforge.filebot.web.VideoHashSubtitleService;
 
 
@@ -86,7 +86,10 @@ abstract class SubtitleDropTarget extends JButton {
 	}
 	
 
-	public abstract VideoHashSubtitleService[] getServices();
+	public abstract VideoHashSubtitleService[] getVideoHashSubtitleServices();
+	
+
+	public abstract SubtitleProvider[] getSubtitleProviders();
 	
 
 	public abstract String getQueryLanguage();
@@ -98,7 +101,11 @@ abstract class SubtitleDropTarget extends JButton {
 		// initialize download parameters
 		dialog.setVideoFiles(videoFiles.toArray(new File[0]));
 		
-		for (VideoHashSubtitleService service : getServices()) {
+		for (VideoHashSubtitleService service : getVideoHashSubtitleServices()) {
+			dialog.addSubtitleService(service);
+		}
+		
+		for (SubtitleProvider service : getSubtitleProviders()) {
 			dialog.addSubtitleService(service);
 		}
 		
@@ -108,14 +115,12 @@ abstract class SubtitleDropTarget extends JButton {
 		// initialize window properties
 		dialog.setIconImage(getImage(getIcon(DropAction.Download)));
 		dialog.setDefaultCloseOperation(JDialog.DISPOSE_ON_CLOSE);
-		dialog.pack();
+		dialog.setSize(670, 575);
 		
 		// show dialog
 		dialog.setLocation(getOffsetLocation(dialog.getOwner()));
 		dialog.setVisible(true);
 		
-		// now it's up to the user
-		Analytics.trackEvent("GUI", "LookupSubtitleByHash", getQueryLanguage(), videoFiles.size());
 		return true;
 	}
 	

source/net/sourceforge/filebot/ui/subtitle/SubtitlePackage.java

@@ -165,7 +165,7 @@ public class SubtitlePackage {
 			
 			ArchiveType archiveType = ArchiveType.forName(subtitle.getType());
 			
-			if (archiveType == ArchiveType.UNDEFINED) {
+			if (archiveType == ArchiveType.UNKOWN) {
 				// cannot extract files from archive
 				return singletonList(new MemoryFile(subtitle.getPath(), data));
 			}
@@ -198,7 +198,7 @@ public class SubtitlePackage {
 					// check if file is a supported archive
 					ArchiveType type = ArchiveType.forName(FileUtilities.getExtension(file.getName()));
 					
-					if (type != ArchiveType.UNDEFINED) {
+					if (type != ArchiveType.UNKOWN) {
 						// extract nested archives recursively
 						vfs.addAll(extract(type, file.getData()));
 					}

source/net/sourceforge/filebot/ui/subtitle/SubtitlePanel.java

@@ -16,7 +16,6 @@ import java.util.List;
 
 import javax.swing.Icon;
 
-import net.sourceforge.filebot.Analytics;
 import net.sourceforge.filebot.Settings;
 import net.sourceforge.filebot.WebServices;
 import net.sourceforge.filebot.ui.AbstractSearchPanel;
@@ -51,11 +50,17 @@ public class SubtitlePanel extends AbstractSearchPanel<SubtitleProvider, Subtitl
 	private final SubtitleDropTarget dropTarget = new SubtitleDropTarget() {
 		
 		@Override
-		public VideoHashSubtitleService[] getServices() {
+		public VideoHashSubtitleService[] getVideoHashSubtitleServices() {
 			return WebServices.getVideoHashSubtitleServices();
 		}
 		
 
+		@Override
+		public SubtitleProvider[] getSubtitleProviders() {
+			return WebServices.getSubtitleProviders();
+		}
+		
+
 		@Override
 		public String getQueryLanguage() {
 			// use currently selected language for drop target
@@ -163,7 +168,6 @@ public class SubtitlePanel extends AbstractSearchPanel<SubtitleProvider, Subtitl
 				packages.add(new SubtitlePackage(request.getProvider(), subtitle));
 			}
 			
-			Analytics.trackEvent("GUI", "LookupSubtitleByName", request.getLanguageName(), 1);
 			return packages;
 		}
 		

source/net/sourceforge/filebot/ui/subtitle/VideoHashSubtitleDownloadDialog.java

@@ -4,6 +4,8 @@ package net.sourceforge.filebot.ui.subtitle;
 
 import static javax.swing.BorderFactory.*;
 import static javax.swing.JOptionPane.*;
+import static net.sourceforge.filebot.similarity.EpisodeMetrics.*;
+import static net.sourceforge.filebot.subtitle.SubtitleUtilities.*;
 import static net.sourceforge.tuned.FileUtilities.*;
 
 import java.awt.Color;
@@ -15,7 +17,6 @@ import java.beans.PropertyChangeEvent;
 import java.beans.PropertyChangeListener;
 import java.io.File;
 import java.net.URI;
-import java.nio.ByteBuffer;
 import java.util.AbstractList;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -24,7 +25,9 @@ import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.TreeSet;
 import java.util.Map.Entry;
+import java.util.concurrent.CancellationException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.logging.Level;
@@ -54,8 +57,17 @@ import javax.swing.table.DefaultTableCellRenderer;
 import net.miginfocom.swing.MigLayout;
 import net.sourceforge.filebot.Analytics;
 import net.sourceforge.filebot.ResourceManager;
+import net.sourceforge.filebot.similarity.EpisodeMetrics;
+import net.sourceforge.filebot.similarity.Match;
+import net.sourceforge.filebot.similarity.Matcher;
+import net.sourceforge.filebot.similarity.MetricCascade;
+import net.sourceforge.filebot.similarity.SeriesNameMatcher;
+import net.sourceforge.filebot.similarity.SimilarityMetric;
+import net.sourceforge.filebot.similarity.StrictEpisodeMetrics;
 import net.sourceforge.filebot.ui.Language;
+import net.sourceforge.filebot.vfs.MemoryFile;
 import net.sourceforge.filebot.web.SubtitleDescriptor;
+import net.sourceforge.filebot.web.SubtitleProvider;
 import net.sourceforge.filebot.web.VideoHashSubtitleService;
 import net.sourceforge.tuned.FileUtilities;
 import net.sourceforge.tuned.ui.AbstractBean;
@@ -66,8 +78,9 @@ import net.sourceforge.tuned.ui.RoundBorder;
 
 class VideoHashSubtitleDownloadDialog extends JDialog {
 	
-	private final JPanel servicePanel = new JPanel(new MigLayout());
+	private final JPanel hashMatcherServicePanel = createServicePanel(0xFAFAD2); // LightGoldenRodYellow
-	private final List<VideoHashSubtitleServiceBean> services = new ArrayList<VideoHashSubtitleServiceBean>();
+	private final JPanel nameMatcherServicePanel = createServicePanel(0xFFEBCD); // BlanchedAlmond
+	private final List<SubtitleServiceBean> services = new ArrayList<SubtitleServiceBean>();
 	
 	private final JTable subtitleMappingTable = createTable();
 	
@@ -81,18 +94,25 @@ class VideoHashSubtitleDownloadDialog extends JDialog {
 		JComponent content = (JComponent) getContentPane();
 		content.setLayout(new MigLayout("fill, insets dialog, nogrid", "", "[fill][pref!]"));
 		
-		servicePanel.setBorder(new RoundBorder());
-		servicePanel.setOpaque(false);
-		servicePanel.setBackground(new Color(0xFAFAD2)); // LightGoldenRodYellow
-		
 		content.add(new JScrollPane(subtitleMappingTable), "grow, wrap");
-		content.add(servicePanel, "gap after indent*2");
+		content.add(hashMatcherServicePanel, "gap after rel");
+		content.add(nameMatcherServicePanel, "gap after indent*2");
 		
 		content.add(new JButton(downloadAction), "tag ok");
 		content.add(new JButton(finishAction), "tag cancel");
 	}
 	
 
+	protected JPanel createServicePanel(int color) {
+		JPanel panel = new JPanel(new MigLayout("hidemode 3"));
+		panel.setBorder(new RoundBorder());
+		panel.setOpaque(false);
+		panel.setBackground(new Color(color));
+		panel.setVisible(false);
+		return panel;
+	}
+	
+
 	protected JTable createTable() {
 		JTable table = new JTable(new SubtitleMappingTableModel());
 		table.setDefaultRenderer(SubtitleMapping.class, new SubtitleMappingOptionRenderer());
@@ -134,25 +154,37 @@ class VideoHashSubtitleDownloadDialog extends JDialog {
 	}
 	
 
-	public void addSubtitleService(final VideoHashSubtitleService service) {
+	public void addSubtitleService(VideoHashSubtitleService service) {
-		final VideoHashSubtitleServiceBean serviceBean = new VideoHashSubtitleServiceBean(service);
+		addSubtitleService(new VideoHashSubtitleServiceBean(service), hashMatcherServicePanel);
-		final LinkButton component = new LinkButton(serviceBean.getName(), ResourceManager.getIcon("database.go"), serviceBean.getLink());
+	}
+	
+
+	public void addSubtitleService(SubtitleProvider service) {
+		addSubtitleService(new SubtitleProviderBean(service), nameMatcherServicePanel);
+	}
+	
+
+	protected void addSubtitleService(final SubtitleServiceBean service, final JPanel servicePanel) {
+		final LinkButton component = new LinkButton(service.getName(), ResourceManager.getIcon("database"), service.getLink());
+		component.setVisible(false);
 		
-		serviceBean.addPropertyChangeListener(new PropertyChangeListener() {
+		service.addPropertyChangeListener(new PropertyChangeListener() {
 			
 			@Override
 			public void propertyChange(PropertyChangeEvent evt) {
-				if (serviceBean.getState() == StateValue.STARTED) {
+				if (service.getState() == StateValue.STARTED) {
 					component.setIcon(ResourceManager.getIcon("database.go"));
 				} else {
-					component.setIcon(ResourceManager.getIcon(serviceBean.getError() == null ? "database.ok" : "database.error"));
+					component.setIcon(ResourceManager.getIcon(service.getError() == null ? "database.ok" : "database.error"));
 				}
 				
-				component.setToolTipText(serviceBean.getError() == null ? null : serviceBean.getError().getMessage());
+				servicePanel.setVisible(true);
+				component.setVisible(true);
+				component.setToolTipText(service.getError() == null ? null : service.getError().getMessage());
 			}
 		});
 		
-		services.add(serviceBean);
+		services.add(service);
 		servicePanel.add(component);
 	}
 	
@@ -160,38 +192,30 @@ class VideoHashSubtitleDownloadDialog extends JDialog {
 	public void startQuery(String languageName) {
 		final SubtitleMappingTableModel mappingModel = (SubtitleMappingTableModel) subtitleMappingTable.getModel();
 		
-		// query services sequentially
+		QueryTask queryTask = new QueryTask(services, mappingModel.getVideoFiles(), languageName) {
-		queryService = Executors.newFixedThreadPool(1);
+			
-		
+			@Override
-		for (final VideoHashSubtitleServiceBean service : services) {
+			protected void process(List<Map<File, List<SubtitleDescriptorBean>>> sequence) {
-			QueryTask task = new QueryTask(service, mappingModel.getVideoFiles(), languageName) {
+				for (Map<File, List<SubtitleDescriptorBean>> subtitles : sequence) {
-				
+					// update subtitle options
-				@Override
+					for (SubtitleMapping subtitleMapping : mappingModel) {
-				protected void done() {
+						List<SubtitleDescriptorBean> options = subtitles.get(subtitleMapping.getVideoFile());
-					try {
-						Map<File, List<SubtitleDescriptorBean>> subtitles = get();
 						
-						// update subtitle options
+						if (options != null && options.size() > 0) {
-						for (SubtitleMapping subtitleMapping : mappingModel) {
+							subtitleMapping.addOptions(options);
-							List<SubtitleDescriptorBean> options = subtitles.get(subtitleMapping.getVideoFile());
-							
-							if (options != null && options.size() > 0) {
-								subtitleMapping.addOptions(options);
-							}
 						}
-						
+					}
-						// make subtitle column visible
+					
-						Analytics.trackEvent(service.getName(), "HashLookup", "Subtitle", subtitles.size()); // number of positive hash lookups
+					// make subtitle column visible
+					if (subtitles.size() > 0) {
 						mappingModel.setOptionColumnVisible(true);
-					} catch (Exception e) {
-						Logger.getLogger(VideoHashSubtitleDownloadDialog.class.getName()).log(Level.WARNING, e.getMessage());
 					}
 				}
-			};
+			}
-			
+		};
-			// start background worker
+		
-			queryService.submit(task);
+		ExecutorService executor = Executors.newFixedThreadPool(1);
-		}
+		executor.submit(queryTask);
 	}
 	
 
@@ -244,11 +268,21 @@ class VideoHashSubtitleDownloadDialog extends JDialog {
 			// collect the subtitles that will be fetched
 			List<DownloadTask> downloadQueue = new ArrayList<DownloadTask>();
 			
-			for (SubtitleMapping mapping : mappingModel) {
+			for (final SubtitleMapping mapping : mappingModel) {
 				SubtitleDescriptorBean subtitleBean = mapping.getSelectedOption();
 				
 				if (subtitleBean != null && subtitleBean.getState() == null) {
-					downloadQueue.add(new DownloadTask(subtitleBean, mapping.getSubtitleFile()));
+					downloadQueue.add(new DownloadTask(mapping.getVideoFile(), subtitleBean) {
+						
+						@Override
+						protected void done() {
+							try {
+								mapping.setSubtitleFile(get());
+							} catch (Exception e) {
+								Logger.getLogger(VideoHashSubtitleDownloadDialog.class.getName()).log(Level.WARNING, e.getMessage());
+							}
+						}
+					});
 				}
 			}
 			
@@ -257,9 +291,12 @@ class VideoHashSubtitleDownloadDialog extends JDialog {
 			List<String> existingFiles = new ArrayList<String>();
 			
 			for (DownloadTask download : downloadQueue) {
-				if (download.getDestination().exists()) {
+				// target destination may not be known until files are extracted from archives
+				File target = download.getDestination(null);
+				
+				if (target != null && target.exists()) {
 					confirmReplaceDownloadQueue.add(download);
-					existingFiles.add(download.getDestination().getName());
+					existingFiles.add(target.getName());
 				}
 			}
 			
@@ -349,10 +386,13 @@ class VideoHashSubtitleDownloadDialog extends JDialog {
 				// download in progress
 				setText(subtitleBean.getText());
 				setIcon(ResourceManager.getIcon("action.fetch"));
-			} else {
+			} else if (mapping.getSubtitleFile() != null) {
 				// download complete
 				setText(mapping.getSubtitleFile().getName());
 				setIcon(ResourceManager.getIcon("status.ok"));
+			} else {
+				setText(null);
+				setIcon(null);
 			}
 			
 			return this;
@@ -514,7 +554,8 @@ class VideoHashSubtitleDownloadDialog extends JDialog {
 
 	private static class SubtitleMapping extends AbstractBean {
 		
-		private final File videoFile;
+		private File videoFile;
+		private File subtitleFile;
 		
 		private SubtitleDescriptorBean selectedOption;
 		private List<SubtitleDescriptorBean> options = new ArrayList<SubtitleDescriptorBean>();
@@ -531,18 +572,18 @@ class VideoHashSubtitleDownloadDialog extends JDialog {
 		
 
 		public File getSubtitleFile() {
-			if (selectedOption == null)
+			return subtitleFile;
-				throw new IllegalStateException("Selected option must not be null");
+		}
-			
+		
-			String base = FileUtilities.getName(videoFile);
+
-			String subtitleName = String.format("%s.%s.%s", base, selectedOption.getLanguage(), selectedOption.getType());
+		public void setSubtitleFile(File subtitleFile) {
-			
+			this.subtitleFile = subtitleFile;
-			return new File(videoFile.getParentFile(), subtitleName);
+			firePropertyChange("subtitleFile", null, this.subtitleFile);
 		}
 		
 
 		public boolean isEditable() {
-			return selectedOption != null && (selectedOption.getState() == null || selectedOption.getError() != null);
+			return subtitleFile == null && selectedOption != null && (selectedOption.getState() == null || selectedOption.getError() != null);
 		}
 		
 
@@ -589,21 +630,21 @@ class VideoHashSubtitleDownloadDialog extends JDialog {
 
 	private static class SubtitleDescriptorBean extends AbstractBean {
 		
-		private final SubtitleDescriptor subtitle;
+		private final SubtitleDescriptor descriptor;
-		private final VideoHashSubtitleServiceBean service;
+		private final SubtitleServiceBean service;
 		
 		private StateValue state;
 		private Exception error;
 		
 
-		public SubtitleDescriptorBean(SubtitleDescriptor subtitle, VideoHashSubtitleServiceBean source) {
+		public SubtitleDescriptorBean(SubtitleDescriptor descriptor, SubtitleServiceBean service) {
-			this.subtitle = subtitle;
+			this.descriptor = descriptor;
-			this.service = source;
+			this.service = service;
 		}
 		
 
 		public String getText() {
-			return String.format("%s.%s.%s", subtitle.getName(), getLanguage(), getType());
+			return formatSubtitle(descriptor.getName(), getLanguage(), getType());
 		}
 		
 
@@ -613,21 +654,21 @@ class VideoHashSubtitleDownloadDialog extends JDialog {
 		
 
 		public String getLanguage() {
-			return Language.getISO3LanguageCodeByName(subtitle.getLanguageName());
+			return Language.getISO3LanguageCodeByName(descriptor.getLanguageName());
 		}
 		
 
 		public String getType() {
-			return subtitle.getType();
+			return descriptor.getType();
 		}
 		
 
-		public ByteBuffer fetch() throws Exception {
+		public MemoryFile fetch() throws Exception {
 			setState(StateValue.STARTED);
 			
 			try {
-				ByteBuffer data = subtitle.fetch();
+				MemoryFile data = fetchSubtitle(descriptor);
-				Analytics.trackEvent(service.getName(), "DownloadSubtitle", subtitle.getLanguageName(), 1);
+				Analytics.trackEvent(service.getName(), "DownloadSubtitle", descriptor.getLanguageName(), 1);
 				
 				return data;
 			} catch (Exception e) {
@@ -665,60 +706,83 @@ class VideoHashSubtitleDownloadDialog extends JDialog {
 	}
 	
 
-	private static class QueryTask extends SwingWorker<Map<File, List<SubtitleDescriptorBean>>, Void> {
+	private static class QueryTask extends SwingWorker<Collection<File>, Map<File, List<SubtitleDescriptorBean>>> {
 		
-		private final VideoHashSubtitleServiceBean service;
+		private final Collection<SubtitleServiceBean> services;
 		
-		private final File[] videoFiles;
+		private final Collection<File> remainingVideos;
 		private final String languageName;
 		
 
-		public QueryTask(VideoHashSubtitleServiceBean service, Collection<File> videoFiles, String languageName) {
+		public QueryTask(Collection<SubtitleServiceBean> services, Collection<File> videoFiles, String languageName) {
-			this.service = service;
+			this.services = services;
-			this.videoFiles = videoFiles.toArray(new File[0]);
+			this.remainingVideos = new TreeSet<File>(videoFiles);
 			this.languageName = languageName;
 		}
 		
 
 		@Override
-		protected Map<File, List<SubtitleDescriptorBean>> doInBackground() throws Exception {
+		protected Collection<File> doInBackground() throws Exception {
-			Map<File, List<SubtitleDescriptorBean>> subtitleSet = new HashMap<File, List<SubtitleDescriptorBean>>();
+			for (SubtitleServiceBean service : services) {
-			
+				try {
-			for (final Entry<File, List<SubtitleDescriptor>> result : service.getSubtitleList(videoFiles, languageName).entrySet()) {
+					if (isCancelled())
-				List<SubtitleDescriptorBean> subtitles = new ArrayList<SubtitleDescriptorBean>();
+						throw new CancellationException();
-				
+					
-				// associate subtitles with services
+					Map<File, List<SubtitleDescriptorBean>> subtitleSet = new HashMap<File, List<SubtitleDescriptorBean>>();
-				for (SubtitleDescriptor subtitleDescriptor : result.getValue()) {
+					
-					subtitles.add(new SubtitleDescriptorBean(subtitleDescriptor, service));
+					for (final Entry<File, List<SubtitleDescriptor>> result : service.lookupSubtitles(remainingVideos, languageName).entrySet()) {
+						List<SubtitleDescriptorBean> subtitles = new ArrayList<SubtitleDescriptorBean>();
+						
+						// associate subtitles with services
+						for (SubtitleDescriptor subtitleDescriptor : result.getValue()) {
+							subtitles.add(new SubtitleDescriptorBean(subtitleDescriptor, service));
+						}
+						
+						subtitleSet.put(result.getKey(), subtitles);
+					}
+					
+					// only lookup subtitles for remaining videos
+					for (Entry<File, List<SubtitleDescriptorBean>> it : subtitleSet.entrySet()) {
+						if (it.getValue() != null && it.getValue().size() > 0) {
+							remainingVideos.remove(it.getKey());
+						}
+					}
+					
+					publish(subtitleSet);
+				} catch (Exception e) {
+					Logger.getLogger(VideoHashSubtitleDownloadDialog.class.getName()).log(Level.SEVERE, e.getMessage(), e);
 				}
-				
-				subtitleSet.put(result.getKey(), subtitles);
 			}
 			
-			return subtitleSet;
+			return remainingVideos;
 		}
 	}
 	
 
 	private static class DownloadTask extends SwingWorker<File, Void> {
 		
-		private final SubtitleDescriptorBean subtitle;
+		private final File video;
-		private final File destination;
+		private final SubtitleDescriptorBean descriptor;
 		
 
-		public DownloadTask(SubtitleDescriptorBean subtitle, File destination) {
+		public DownloadTask(File video, SubtitleDescriptorBean descriptor) {
-			this.subtitle = subtitle;
+			this.video = video;
-			this.destination = destination;
+			this.descriptor = descriptor;
 		}
 		
 
 		public SubtitleDescriptorBean getSubtitleBean() {
-			return subtitle;
+			return descriptor;
 		}
 		
 
-		public File getDestination() {
+		public File getDestination(MemoryFile subtitle) {
-			return destination;
+			if (descriptor.getType() == null && subtitle == null)
+				return null;
+			
+			String base = FileUtilities.getName(video);
+			String ext = descriptor.getType() != null ? descriptor.getType() : getExtension(subtitle.getName());
+			return new File(video.getParentFile(), formatSubtitle(base, descriptor.getLanguage(), ext));
 		}
 		
 
@@ -726,13 +790,14 @@ class VideoHashSubtitleDownloadDialog extends JDialog {
 		protected File doInBackground() {
 			try {
 				// fetch subtitle
-				ByteBuffer data = subtitle.fetch();
+				MemoryFile subtitle = descriptor.fetch();
 				
 				if (isCancelled())
 					return null;
 				
 				// save to file
-				writeFile(data, destination);
+				File destination = getDestination(subtitle);
+				writeFile(subtitle.getData(), destination);
 				
 				return destination;
 			} catch (Exception e) {
@@ -744,40 +809,46 @@ class VideoHashSubtitleDownloadDialog extends JDialog {
 	}
 	
 
-	private static class VideoHashSubtitleServiceBean extends AbstractBean {
+	protected static abstract class SubtitleServiceBean extends AbstractBean {
 		
-		private final VideoHashSubtitleService service;
+		private final String name;
+		private final Icon icon;
+		private final URI link;
 		
-		private StateValue state;
+		private StateValue state = StateValue.PENDING;
-		private Throwable error;
+		private Throwable error = null;
 		
 
-		public VideoHashSubtitleServiceBean(VideoHashSubtitleService service) {
+		public SubtitleServiceBean(String name, Icon icon, URI link) {
-			this.service = service;
+			this.name = name;
-			this.state = StateValue.PENDING;
+			this.icon = icon;
+			this.link = link;
 		}
 		
 
 		public String getName() {
-			return service.getName();
+			return name;
 		}
 		
 
 		public Icon getIcon() {
-			return service.getIcon();
+			return icon;
 		}
 		
 
 		public URI getLink() {
-			return service.getLink();
+			return link;
 		}
 		
 
-		public Map<File, List<SubtitleDescriptor>> getSubtitleList(File[] files, String languageName) throws Exception {
+		protected abstract Map<File, List<SubtitleDescriptor>> getSubtitleList(Collection<File> files, String languageName) throws Exception;
+		
+
+		public final Map<File, List<SubtitleDescriptor>> lookupSubtitles(Collection<File> files, String languageName) throws Exception {
 			setState(StateValue.STARTED);
 			
 			try {
-				return service.getSubtitleList(files, languageName);
+				return getSubtitleList(files, languageName);
 			} catch (Exception e) {
 				// remember error
 				error = e;
@@ -804,7 +875,74 @@ class VideoHashSubtitleDownloadDialog extends JDialog {
 		public Throwable getError() {
 			return error;
 		}
+	}
+	
+
+	protected static class VideoHashSubtitleServiceBean extends SubtitleServiceBean {
 		
+		private VideoHashSubtitleService service;
+		
+
+		public VideoHashSubtitleServiceBean(VideoHashSubtitleService service) {
+			super(service.getName(), service.getIcon(), service.getLink());
+			this.service = service;
+		}
+		
+
+		@Override
+		protected Map<File, List<SubtitleDescriptor>> getSubtitleList(Collection<File> files, String languageName) throws Exception {
+			return service.getSubtitleList(files.toArray(new File[0]), languageName);
+		}
+	}
+	
+
+	protected static class SubtitleProviderBean extends SubtitleServiceBean {
+		
+		private SubtitleProvider service;
+		
+
+		public SubtitleProviderBean(SubtitleProvider service) {
+			super(service.getName(), service.getIcon(), service.getLink());
+			this.service = service;
+		}
+		
+
+		@Override
+		protected Map<File, List<SubtitleDescriptor>> getSubtitleList(Collection<File> files, String languageName) throws Exception {
+			Map<File, List<SubtitleDescriptor>> subtitlesByFile = new HashMap<File, List<SubtitleDescriptor>>();
+			for (File file : files) {
+				subtitlesByFile.put(file, new ArrayList<SubtitleDescriptor>());
+			}
+			
+			// auto-detect query and search for subtitles
+			Collection<String> querySet = new SeriesNameMatcher().matchAll(files.toArray(new File[0]));
+			List<SubtitleDescriptor> subtitles = findSubtitles(service, querySet, languageName);
+			
+			// first match everything as best as possible, then filter possibly bad matches
+			Matcher<File, SubtitleDescriptor> matcher = new Matcher<File, SubtitleDescriptor>(files, subtitles, false, EpisodeMetrics.defaultSequence(true));
+			SimilarityMetric sanity = new MetricCascade(StrictEpisodeMetrics.defaultSequence(true));
+			
+			for (Match<File, SubtitleDescriptor> it : matcher.match()) {
+				if (sanity.getSimilarity(it.getValue(), it.getCandidate()) >= 1) {
+					subtitlesByFile.get(it.getValue()).add(it.getCandidate());
+				}
+			}
+			
+			// add other possible matches to the options
+			SimilarityMetric matchMetric = new MetricCascade(FileName, EpisodeIdentifier, Title, Name);
+			float matchSimilarity = 0.6f;
+			
+			for (File file : files) {
+				// add matching subtitles
+				for (SubtitleDescriptor it : subtitles) {
+					if (matchMetric.getSimilarity(file, it) >= matchSimilarity && !subtitlesByFile.get(file).contains(it)) {
+						subtitlesByFile.get(file).add(it);
+					}
+				}
+			}
+			
+			return subtitlesByFile;
+		}
 	}
 	
 }

source/net/sourceforge/filebot/vfs/ArchiveType.java

@@ -4,6 +4,7 @@ package net.sourceforge.filebot.vfs;
 
 import java.nio.ByteBuffer;
 import java.util.Collections;
+import java.util.EnumSet;
 
 
 public enum ArchiveType {
@@ -26,6 +27,26 @@ public enum ArchiveType {
 	
 	UNDEFINED {
 		
+		@Override
+		public Iterable<MemoryFile> fromData(ByteBuffer data) {
+			for (ArchiveType type : EnumSet.of(ZIP, RAR)) {
+				try {
+					Iterable<MemoryFile> files = type.fromData(data);
+					if (files.iterator().hasNext()) {
+						return files;
+					}
+				} catch (Exception e) {
+					// ignore
+				}
+			}
+			
+			// cannot extract data, return empty archive
+			return Collections.emptySet();
+		}
+	},
+	
+	UNKOWN {
+		
 		@Override
 		public Iterable<MemoryFile> fromData(ByteBuffer data) {
 			// cannot extract data, return empty archive
@@ -34,13 +55,16 @@ public enum ArchiveType {
 	};
 	
 	public static ArchiveType forName(String name) {
+		if (name == null)
+			return UNDEFINED;
+		
 		if ("zip".equalsIgnoreCase(name))
 			return ZIP;
 		
 		if ("rar".equalsIgnoreCase(name))
 			return RAR;
 		
-		return UNDEFINED;
+		return UNKOWN;
 	}
 	
 

source/net/sourceforge/filebot/web/OpenSubtitlesSubtitleDescriptor.java

@@ -145,6 +145,23 @@ public class OpenSubtitlesSubtitleDescriptor implements SubtitleDescriptor {
 	}
 	
 
+	@Override
+	public int hashCode() {
+		return getProperty(Property.IDSubtitle).hashCode();
+	}
+	
+
+	@Override
+	public boolean equals(Object object) {
+		if (object instanceof OpenSubtitlesSubtitleDescriptor) {
+			OpenSubtitlesSubtitleDescriptor other = (OpenSubtitlesSubtitleDescriptor) object;
+			return getProperty(Property.IDSubtitle).equals(other.getProperty(Property.IDSubtitle));
+		}
+		
+		return false;
+	}
+	
+
 	@Override
 	public String toString() {
 		return String.format("%s [%s]", getName(), getLanguageName());

source/net/sourceforge/filebot/web/SublightSubtitleDescriptor.java

@@ -119,6 +119,23 @@ public class SublightSubtitleDescriptor implements SubtitleDescriptor {
 	}
 	
 
+	@Override
+	public int hashCode() {
+		return subtitle.getSubtitleID().hashCode();
+	}
+	
+
+	@Override
+	public boolean equals(Object object) {
+		if (object instanceof SublightSubtitleDescriptor) {
+			SublightSubtitleDescriptor other = (SublightSubtitleDescriptor) object;
+			return subtitle.getSubtitleID().equals(other.subtitle.getSubtitleID());
+		}
+		
+		return false;
+	}
+	
+
 	@Override
 	public String toString() {
 		return String.format("%s [%s]", getName(), getLanguageName());

source/net/sourceforge/filebot/web/SubsceneSubtitleDescriptor.java

@@ -46,7 +46,7 @@ public class SubsceneSubtitleDescriptor implements SubtitleDescriptor {
 
 	@Override
 	public String getType() {
-		return getSubtitleInfo().get("typeId");
+		return null;
 	}
 	
 
@@ -87,7 +87,7 @@ public class SubsceneSubtitleDescriptor implements SubtitleDescriptor {
 
 	@Override
 	public String getPath() {
-		return String.format("%s.%s", getName(), getType());
+		return String.format("%s.%s", getName(), getSubtitleInfo().get("typeId"));
 	}
 	
 
@@ -97,6 +97,23 @@ public class SubsceneSubtitleDescriptor implements SubtitleDescriptor {
 	}
 	
 
+	@Override
+	public int hashCode() {
+		return subtitlePage.getPath().hashCode();
+	}
+	
+
+	@Override
+	public boolean equals(Object object) {
+		if (object instanceof SubsceneSubtitleDescriptor) {
+			SubsceneSubtitleDescriptor other = (SubsceneSubtitleDescriptor) object;
+			return subtitlePage.getPath().equals(other.getPath());
+		}
+		
+		return false;
+	}
+	
+
 	@Override
 	public String toString() {
 		return String.format("%s [%s]", getName(), getLanguageName());