mobeigi
/
filebot
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

* improved subtitle api and video hash support

This commit is contained in:
Reinhard Pointner 2009-10-20 21:16:34 +00:00
parent 0f4300b048
commit 39dd413eec
12 changed files with 269 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
lib/sublight-ws.jar
View File

Binary file not shown.

41
source/net/sourceforge/filebot/ui/panel/subtitle/SubtitleUtilities.java
View File

@ -3,16 +3,12 @@ package net.sourceforge.filebot.ui.panel.subtitle;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
@ -67,43 +63,6 @@ final class SubtitleUtilities {
}
/**
* Calculate MD5 hash.
*/
public static String md5(ByteBuffer data) {
try {
MessageDigest hash = MessageDigest.getInstance("MD5");
hash.update(data);
// return hex string
return String.format("%032x", new BigInteger(1, hash.digest()));
} catch (NoSuchAlgorithmException e) {
// will not happen
throw new UnsupportedOperationException(e);
}
}
public static byte[] read(File source) throws IOException {
InputStream in = new FileInputStream(source);
try {
byte[] data = new byte[(int) source.length()];
int position = 0;
int read = 0;
while (position < data.length && (read = in.read(data, position, data.length - position)) >= 0) {
position += read;
}
return data;
} finally {
in.close();
}
}
/**
* Write {@link ByteBuffer} to {@link File}.
*/

11
source/net/sourceforge/filebot/web/MovieIdentificationService.java Normal file
View File

@ -0,0 +1,11 @@
package net.sourceforge.filebot.web;
import java.io.File;
public interface MovieIdentificationService {
public MovieDescriptor[] getMovieDescriptors(File[] movieFiles) throws Exception;
}

105
source/net/sourceforge/filebot/web/OpenSubtitlesClient.java
View File

@ -2,7 +2,14 @@
package net.sourceforge.filebot.web;
import java.io.File;
import java.math.BigInteger;
import java.net.URI;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
@ -14,13 +21,14 @@ import java.util.logging.Logger;
import javax.swing.Icon;
import net.sourceforge.filebot.ResourceManager;
import net.sourceforge.filebot.web.OpenSubtitlesXmlRpc.Query;
import net.sourceforge.tuned.Timer;
/**
* SubtitleClient for OpenSubtitles.
*/
public class OpenSubtitlesClient implements SubtitleProvider {
public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleService, MovieIdentificationService {
private final OpenSubtitlesXmlRpc xmlrpc;
@ -56,12 +64,12 @@ public class OpenSubtitlesClient implements SubtitleProvider {
@Override
public List<SubtitleDescriptor> getSubtitleList(SearchResult searchResult, String languageName) throws Exception {
// require login
login();
// singleton array with or empty array
String[] languageFilter = languageName != null ? new String[] { getSubLanguageID(languageName) } : new String[0];
// require login
login();
@SuppressWarnings("unchecked")
List<SubtitleDescriptor> subtitles = (List) xmlrpc.searchSubtitles(((MovieDescriptor) searchResult).getImdbId(), languageFilter);
@ -69,6 +77,95 @@ public class OpenSubtitlesClient implements SubtitleProvider {
}
public Map<File, List<SubtitleDescriptor>> getSubtitleList(File[] files, String languageName) throws Exception {
// singleton array with or empty array
String[] languageFilter = languageName != null ? new String[] { getSubLanguageID(languageName) } : new String[0];
// remember hash for each file
Map<String, File> hashMap = new HashMap<String, File>(files.length);
Map<File, List<SubtitleDescriptor>> resultMap = new HashMap<File, List<SubtitleDescriptor>>(files.length);
// create hash query for each file
List<Query> queryList = new ArrayList<Query>(files.length);
for (File file : files) {
String movieHash = OpenSubtitlesHasher.computeHash(file);
// add query
queryList.add(Query.forHash(movieHash, file.length(), languageFilter));
// prepare result map
hashMap.put(movieHash, file);
resultMap.put(file, new LinkedList<SubtitleDescriptor>());
}
// require login
login();
// submit query and map results to given files
for (OpenSubtitlesSubtitleDescriptor subtitle : xmlrpc.searchSubtitles(queryList)) {
// get file for hash
File file = hashMap.get(subtitle.getMovieHash());
// add subtitle
resultMap.get(file).add(subtitle);
}
return resultMap;
}
@Override
public boolean publishSubtitle(int imdbid, String languageName, File videoFile, File subtitleFile) throws Exception {
//TODO implement upload feature
return false;
}
/**
* Calculate MD5 hash.
*/
private String md5(byte[] data) {
try {
MessageDigest hash = MessageDigest.getInstance("MD5");
hash.update(data);
// return hex string
return String.format("%032x", new BigInteger(1, hash.digest()));
} catch (NoSuchAlgorithmException e) {
throw new RuntimeException(e);
}
}
@Override
public MovieDescriptor[] getMovieDescriptors(File[] movieFiles) throws Exception {
// create result array
MovieDescriptor[] result = new MovieDescriptor[movieFiles.length];
// compute movie hashes
Map<String, Integer> indexMap = new HashMap<String, Integer>(movieFiles.length);
for (int i = 0; i < movieFiles.length; i++) {
String hash = OpenSubtitlesHasher.computeHash(movieFiles[i]);
// remember original index
indexMap.put(hash, i);
}
// require login
login();
// dispatch single query for all hashes
for (Entry<String, MovieDescriptor> entry : xmlrpc.checkMovieHash(indexMap.keySet()).entrySet()) {
int index = indexMap.get(entry.getKey());
result[index] = entry.getValue();
}
return result;
}
@Override
public URI getSubtitleListLink(SearchResult searchResult, String languageName) {
MovieDescriptor movie = (MovieDescriptor) searchResult;

2
source/net/sourceforge/filebot/web/OpenSubtitlesHasher.java
View File

@ -45,7 +45,6 @@ public final class OpenSubtitlesHasher {
public static String computeHash(InputStream stream, long length) throws IOException {
int chunkSizeForFile = (int) Math.min(HASH_CHUNK_SIZE, length);
// buffer that will contain the head and the tail chunk, chunks will overlap if length is smaller than two chunks
@ -73,7 +72,6 @@ public final class OpenSubtitlesHasher {
private static long computeHashForChunk(ByteBuffer buffer) {
LongBuffer longBuffer = buffer.order(ByteOrder.LITTLE_ENDIAN).asLongBuffer();
long hash = 0;

40
source/net/sourceforge/filebot/web/OpenSubtitlesXmlRpc.java
View File

@ -5,11 +5,11 @@ package net.sourceforge.filebot.web;
import static java.util.Collections.*;
import static net.sourceforge.tuned.StringUtilities.*;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
@ -26,7 +26,6 @@ import redstone.xmlrpc.XmlRpcFault;
import redstone.xmlrpc.util.Base64;
import net.sourceforge.filebot.web.OpenSubtitlesSubtitleDescriptor.Property;
import net.sourceforge.tuned.ByteBufferInputStream;
import net.sourceforge.tuned.ByteBufferOutputStream;
@ -111,13 +110,17 @@ public class OpenSubtitlesXmlRpc {
@SuppressWarnings("unchecked")
public List<OpenSubtitlesSubtitleDescriptor> searchSubtitles(Collection<Query> queryList) throws XmlRpcFault {
List<OpenSubtitlesSubtitleDescriptor> subtitles = new ArrayList<OpenSubtitlesSubtitleDescriptor>();
Map<?, ?> response = invoke("SearchSubtitles", token, queryList);
List<Map<String, String>> subtitleData = (List<Map<String, String>>) response.get("data");
List<OpenSubtitlesSubtitleDescriptor> subtitles = new ArrayList<OpenSubtitlesSubtitleDescriptor>();
for (Map<String, String> propertyMap : subtitleData) {
subtitles.add(new OpenSubtitlesSubtitleDescriptor(Property.asEnumMap(propertyMap)));
try {
List<Map<String, String>> subtitleData = (List<Map<String, String>>) response.get("data");
for (Map<String, String> propertyMap : subtitleData) {
subtitles.add(new OpenSubtitlesSubtitleDescriptor(Property.asEnumMap(propertyMap)));
}
} catch (ClassCastException e) {
// no subtitle have been found
}
return subtitles;
@ -179,7 +182,7 @@ public class OpenSubtitlesXmlRpc {
@SuppressWarnings("unchecked")
public List<String> detectLanguage(ByteBuffer data) throws XmlRpcFault {
public List<String> detectLanguage(byte[] data) throws XmlRpcFault {
// compress and base64 encode
String parameter = encodeData(data);
@ -211,16 +214,23 @@ public class OpenSubtitlesXmlRpc {
@SuppressWarnings("unchecked")
public Map<String, Map<Property, String>> checkMovieHash(Collection<String> hashes) throws XmlRpcFault {
public Map<String, MovieDescriptor> checkMovieHash(Collection<String> hashes) throws XmlRpcFault {
Map<?, ?> response = invoke("CheckMovieHash", token, hashes);
Map<String, ?> movieHashData = (Map<String, ?>) response.get("data");
Map<String, Map<Property, String>> movieHashMap = new HashMap<String, Map<Property, String>>();
Map<String, MovieDescriptor> movieHashMap = new HashMap<String, MovieDescriptor>();
for (Entry<String, ?> entry : movieHashData.entrySet()) {
// empty associative arrays are deserialized as array, not as map
if (entry.getValue() instanceof Map) {
movieHashMap.put(entry.getKey(), Property.asEnumMap((Map<String, String>) entry.getValue()));
Map<String, String> info = (Map<String, String>) entry.getValue();
String hash = info.get("MovieHash");
String name = info.get("MovieName");
int year = Integer.parseInt(info.get("MovieYear"));
int imdb = Integer.parseInt(info.get("MovieImdbID"));
movieHashMap.put(hash, new MovieDescriptor(name, year, imdb));
}
}
@ -281,12 +291,12 @@ public class OpenSubtitlesXmlRpc {
}
protected static String encodeData(ByteBuffer data) {
protected static String encodeData(byte[] data) {
try {
DeflaterInputStream compressedDataStream = new DeflaterInputStream(new ByteBufferInputStream(data));
DeflaterInputStream compressedDataStream = new DeflaterInputStream(new ByteArrayInputStream(data));
// compress data
ByteBufferOutputStream buffer = new ByteBufferOutputStream(data.remaining());
ByteBufferOutputStream buffer = new ByteBufferOutputStream(data.length);
buffer.transferFully(compressedDataStream);
// base64 encode
@ -416,7 +426,7 @@ public class OpenSubtitlesXmlRpc {
}
public void setSubContent(ByteBuffer data) {
public void setSubContent(byte[] data) {
put("subcontent", encodeData(data));
}
}

93
source/net/sourceforge/filebot/web/SublightSubtitleClient.java
View File

@ -10,7 +10,6 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.AbstractMap.SimpleEntry;
import java.util.Map.Entry;
import java.util.concurrent.TimeUnit;
@ -36,7 +35,7 @@ import net.sublight.webservice.SubtitlesAPI2;
import net.sublight.webservice.SubtitlesAPI2Soap;
public class SublightSubtitleClient implements SubtitleProvider {
public class SublightSubtitleClient implements SubtitleProvider, VideoHashSubtitleService {
private static final String iid = "42cc1701-3752-49e2-a148-332960073452";
@ -108,15 +107,31 @@ public class SublightSubtitleClient implements SubtitleProvider {
}
public Map<File, List<SubtitleDescriptor>> getSubtitleList(File[] files, final String languageName) throws Exception {
Map<File, List<SubtitleDescriptor>> subtitles = new HashMap<File, List<SubtitleDescriptor>>(files.length);
for (final File file : files) {
subtitles.put(file, getSubtitleList(file, languageName));
}
return subtitles;
}
public List<SubtitleDescriptor> getSubtitleList(File videoFile, String languageName) throws WebServiceException, IOException {
List<SubtitleDescriptor> subtitles = new ArrayList<SubtitleDescriptor>();
// retrieve subtitles by video hash
for (Subtitle subtitle : getSubtitleList(SublightVideoHasher.computeHash(videoFile), null, null, languageName)) {
// only keep linked subtitles
if (subtitle.isIsLinked()) {
subtitles.add(new SublightSubtitleDescriptor(subtitle, this));
try {
// retrieve subtitles by video hash
for (Subtitle subtitle : getSubtitleList(SublightVideoHasher.computeHash(videoFile), null, null, languageName)) {
// only keep linked subtitles
if (subtitle.isIsLinked()) {
subtitles.add(new SublightSubtitleDescriptor(subtitle, this));
}
}
} catch (LinkageError e) {
// MediaInfo native lib not available
throw new UnsupportedOperationException(e);
}
return subtitles;
@ -176,12 +191,50 @@ public class SublightSubtitleClient implements SubtitleProvider {
}
@SuppressWarnings("unchecked")
private static final Entry<SubtitleLanguage, String>[] aliasList = new Entry[] {
new SimpleEntry(SubtitleLanguage.PORTUGUESE_BRAZIL, "Brazilian"),
new SimpleEntry(SubtitleLanguage.BOSNIAN_LATIN, "Bosnian"),
new SimpleEntry(SubtitleLanguage.SERBIAN_LATIN, "Serbian")
};
@Override
public boolean publishSubtitle(int imdbid, String languageName, File videoFile, File subtitleFile) throws Exception {
//TODO implement upload feature
return false;
}
public void publishSubtitle(int imdbid, String videoHash, String languageName, String releaseName, byte[] data) {
// require login
login();
Subtitle subtitle = new Subtitle();
subtitle.setIMDB(String.format("http://www.imdb.com/title/tt%07d", imdbid));
subtitle.setLanguage(getSubtitleLanguage(languageName));
subtitle.setRelease(releaseName);
Holder<Boolean> result = new Holder<Boolean>();
Holder<String> subid = new Holder<String>();
Holder<String> error = new Holder<String>();
// upload subtitle
webservice.publishSubtitle2(session, subtitle, data, result, subid, null, error);
// abort if something went wrong
checkError(error);
// link subtitle to video file
webservice.addHashLink3(session, subid.value, videoHash, null, null, error);
// abort if something went wrong
checkError(error);
}
protected Map<String, SubtitleLanguage> getLanguageAliasMap() {
Map<String, SubtitleLanguage> languages = new HashMap<String, SubtitleLanguage>(4);
// insert special some additional special handling
languages.put("Brazilian", SubtitleLanguage.PORTUGUESE_BRAZIL);
languages.put("Bosnian", SubtitleLanguage.BOSNIAN_LATIN);
languages.put("Serbian", SubtitleLanguage.SERBIAN_LATIN);
return languages;
}
protected SubtitleLanguage getSubtitleLanguage(String languageName) {
@ -192,9 +245,9 @@ public class SublightSubtitleClient implements SubtitleProvider {
}
// check alias list
for (Entry<SubtitleLanguage, String> alias : aliasList) {
if (alias.getValue().equalsIgnoreCase(languageName))
return alias.getKey();
for (Entry<String, SubtitleLanguage> alias : getLanguageAliasMap().entrySet()) {
if (alias.getKey().equalsIgnoreCase(languageName))
return alias.getValue();
}
// illegal language name
@ -203,10 +256,10 @@ public class SublightSubtitleClient implements SubtitleProvider {
protected String getLanguageName(SubtitleLanguage language) {
// check alias list
for (Entry<SubtitleLanguage, String> alias : aliasList) {
if (language == alias.getKey())
return alias.getValue();
// check alias list first
for (Entry<String, SubtitleLanguage> alias : getLanguageAliasMap().entrySet()) {
if (language == alias.getValue())
return alias.getKey();
}
// use language value by default

13
source/net/sourceforge/filebot/web/SubsceneSubtitleDescriptor.java
View File

@ -36,23 +36,24 @@ public class SubsceneSubtitleDescriptor implements SubtitleDescriptor {
}
@Override
public String getLanguageName() {
return language;
}
@Override
public ByteBuffer fetch() throws Exception {
return WebRequest.fetch(downloadLink, singletonMap("Referer", referer.toString()));
}
@Override
public String getType() {
return archiveType;
}
@Override
public ByteBuffer fetch() throws Exception {
return WebRequest.fetch(downloadLink, singletonMap("Referer", referer.toString()));
}
@Override
public String toString() {
return String.format("%s [%s]", getName(), getLanguageName());

8
source/net/sourceforge/filebot/web/SubtitleSourceSubtitleDescriptor.java
View File

@ -62,14 +62,14 @@ public class SubtitleSourceSubtitleDescriptor implements SubtitleDescriptor {
@Override
public ByteBuffer fetch() throws Exception {
return WebRequest.fetch(downloadLink);
public String getType() {
return "zip";
}
@Override
public String getType() {
return "zip";
public ByteBuffer fetch() throws Exception {
return WebRequest.fetch(downloadLink);
}

17
source/net/sourceforge/filebot/web/VideoHashSubtitleService.java Normal file
View File

@ -0,0 +1,17 @@
package net.sourceforge.filebot.web;
import java.io.File;
import java.util.List;
import java.util.Map;
public interface VideoHashSubtitleService {
public Map<File, List<SubtitleDescriptor>> getSubtitleList(File[] videoFiles, String languageName) throws Exception;
public boolean publishSubtitle(int imdbid, String languageName, File videoFile, File subtitleFile) throws Exception;
}

23
source/net/sourceforge/tuned/FileUtilities.java
View File

@ -4,6 +4,9 @@ package net.sourceforge.tuned;
import java.io.File;
import java.io.FileFilter;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@ -13,6 +16,26 @@ import java.util.regex.Pattern;
public final class FileUtilities {
public static byte[] readAll(File source) throws IOException {
InputStream in = new FileInputStream(source);
try {
byte[] data = new byte[(int) source.length()];
int position = 0;
int read = 0;
while (position < data.length && (read = in.read(data, position, data.length - position)) >= 0) {
position += read;
}
return data;
} finally {
in.close();
}
}
/**
* Pattern used for matching file extensions.
*

16
test/net/sourceforge/filebot/web/OpenSubtitlesXmlRpcTest.java
View File

@ -6,7 +6,6 @@ import static java.util.Collections.*;
import static org.junit.Assert.*;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import java.util.List;
import java.util.Map;
@ -122,20 +121,21 @@ public class OpenSubtitlesXmlRpcTest {
@Test
public void checkMovieHash() throws Exception {
Map<String, Map<Property, String>> movieHashMap = xmlrpc.checkMovieHash(singleton("2bba5c34b007153b"));
Map<Property, String> movie = movieHashMap.values().iterator().next();
Map<String, MovieDescriptor> results = xmlrpc.checkMovieHash(singleton("d7aa0275cace4410"));
MovieDescriptor movie = results.get("d7aa0275cace4410");
assertEquals("\"Firefly\"", movie.get(Property.MovieName));
assertEquals("2002", movie.get(Property.MovieYear));
assertEquals("Iron Man", movie.getName());
assertEquals(2008, movie.getYear());
assertEquals(371746, movie.getImdbId());
}
@Test
public void checkMovieHashInvalid() throws Exception {
Map<String, Map<Property, String>> movieHashMap = xmlrpc.checkMovieHash(singleton("0123456789abcdef"));
Map<String, MovieDescriptor> results = xmlrpc.checkMovieHash(singleton("0123456789abcdef"));
// no movie info
assertTrue(movieHashMap.isEmpty());
assertTrue(results.isEmpty());
}
@ -143,7 +143,7 @@ public class OpenSubtitlesXmlRpcTest {
public void detectLanguage() throws Exception {
String text = "Only those that are prepared to fire should be fired at.";
List<String> languages = xmlrpc.detectLanguage(Charset.forName("utf-8").encode(text));
List<String> languages = xmlrpc.detectLanguage(text.getBytes("UTF-8"));
assertEquals("eng", languages.get(0));
assertTrue(1 == languages.size());