mobeigi
/
filebot
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

+ support IMDb as movie db

This commit is contained in:
Reinhard Pointner 2012-01-06 17:24:00 +00:00
parent 5228c76dbc
commit 81533d0a66
5 changed files with 140 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
source/net/sourceforge/filebot/WebServices.java
View File

@ -39,27 +39,27 @@ public final class WebServices {
// movie dbs
public static final TMDbClient TMDb = new TMDbClient(getApplicationProperty("themoviedb.apikey"));
public static EpisodeListProvider[] getEpisodeListProviders() {
return new EpisodeListProvider[] { TVRage, AniDB, IMDb, TheTVDB, Serienjunkies };
}
public static MovieIdentificationService[] getMovieIdentificationServices() {
return new MovieIdentificationService[] { OpenSubtitles, TMDb };
return new MovieIdentificationService[] { OpenSubtitles, IMDb, TMDb };
}
public static SubtitleProvider[] getSubtitleProviders() {
return new SubtitleProvider[] { OpenSubtitles, Sublight, Subscene };
}
public static VideoHashSubtitleService[] getVideoHashSubtitleServices() {
return new VideoHashSubtitleService[] { OpenSubtitles, Sublight };
}
public static EpisodeListProvider getEpisodeListProvider(String name) {
for (EpisodeListProvider it : WebServices.getEpisodeListProviders()) {
if (it.getName().equalsIgnoreCase(name))
@ -69,7 +69,7 @@ public final class WebServices {
return null; // default
}
public static MovieIdentificationService getMovieIdentificationService(String name) {
for (MovieIdentificationService it : getMovieIdentificationServices()) {
if (it.getName().equalsIgnoreCase(name))
@ -79,7 +79,7 @@ public final class WebServices {
return null; // default
}
/**
* Dummy constructor to prevent instantiation.
*/

114
source/net/sourceforge/filebot/web/IMDbClient.java
View File

@ -5,6 +5,7 @@ package net.sourceforge.filebot.web;
import static net.sourceforge.filebot.web.WebRequest.*;
import static net.sourceforge.tuned.XPathUtilities.*;
import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
@ -27,36 +28,34 @@ import net.sf.ehcache.CacheManager;
import net.sourceforge.filebot.ResourceManager;
public class IMDbClient extends AbstractEpisodeListProvider {
public class IMDbClient extends AbstractEpisodeListProvider implements MovieIdentificationService {
private final String host = "www.imdb.com";
@Override
public String getName() {
return "IMDb";
}
@Override
public Icon getIcon() {
return ResourceManager.getIcon("search.imdb");
}
@Override
public ResultCache getCache() {
return new ResultCache(host, CacheManager.getInstance().getCache("web-datasource"));
}
@Override
public List<SearchResult> fetchSearchResult(String query, Locale locale) throws IOException, SAXException {
URL searchUrl = new URL("http", host, "/find?s=tt&q=" + encode(query));
Document dom = getHtmlDocument(openConnection(searchUrl));
Document dom = parsePage(new URL("http", host, "/find?s=tt&q=" + encode(query)));
List<Node> nodes = selectNodes("//TABLE//A[following-sibling::SMALL[contains(.,'series')]]", dom);
List<SearchResult> results = new ArrayList<SearchResult>(nodes.size());
for (Node node : nodes) {
@ -69,25 +68,20 @@ public class IMDbClient extends AbstractEpisodeListProvider {
// we might have been redirected to the movie page
if (results.isEmpty()) {
try {
String name = normalizeName(selectString("//H1/text()", dom));
String year = new Scanner(selectString("//H1//SPAN", dom)).useDelimiter("\\D+").next();
String url = selectString("//LINK[@rel='canonical']/@href", dom);
results.add(new Movie(name, Integer.parseInt(year), getImdbId(url)));
} catch (Exception e) {
// ignore, we probably got redirected to an error page
Movie movie = scrapeMovie(dom);
if (movie != null) {
results.add(movie);
}
}
return results;
}
@Override
public List<Episode> fetchEpisodeList(SearchResult searchResult, Locale locale) throws IOException, SAXException {
Movie movie = (Movie) searchResult;
Document dom = getHtmlDocument(openConnection(getEpisodeListLink(searchResult).toURL()));
Document dom = parsePage(getEpisodeListLink(searchResult).toURL());
String seriesName = normalizeName(selectString("//H1/A", dom));
Date year = new Date(movie.getYear(), 0, 0);
@ -111,23 +105,13 @@ public class IMDbClient extends AbstractEpisodeListProvider {
return episodes;
}
protected URLConnection openConnection(URL url) throws IOException {
URLConnection connection = url.openConnection();
// IMDb refuses default user agent (Java/1.6.0_12)
connection.addRequestProperty("User-Agent", "Scraper");
return connection;
}
protected String normalizeName(String name) {
// remove quotation marks
return name.replaceAll("\"", "");
}
protected int getImdbId(String link) {
Matcher matcher = Pattern.compile("tt(\\d{7})").matcher(link);
@ -139,13 +123,13 @@ public class IMDbClient extends AbstractEpisodeListProvider {
throw new IllegalArgumentException(String.format("Cannot find imdb id: %s", link));
}
@Override
public URI getEpisodeListLink(SearchResult searchResult) {
return getEpisodeListLink(searchResult, 0);
}
@Override
public URI getEpisodeListLink(SearchResult searchResult, int season) {
try {
@ -154,4 +138,72 @@ public class IMDbClient extends AbstractEpisodeListProvider {
throw new RuntimeException(e);
}
}
@Override
public List<Movie> searchMovie(String query, Locale locale) throws Exception {
Document dom = parsePage(new URL("http", host, "/find?s=tt&q=" + encode(query)));
// select movie links followed by year in parenthesis
List<Node> nodes = selectNodes("//TABLE//A[string-length(substring-after(substring-before(following::text(),')'),'(')) = 4 and count(following-sibling::SMALL) = 0]", dom);
List<Movie> results = new ArrayList<Movie>(nodes.size());
for (Node node : nodes) {
String name = node.getTextContent().trim();
String year = node.getNextSibling().getTextContent().trim().replaceAll("[\\p{Punct}\\p{Space}]+", ""); // remove non-number characters
String href = getAttribute("href", node);
try {
results.add(new Movie(name, Integer.parseInt(year), getImdbId(href)));
} catch (NumberFormatException e) {
// ignore illegal movies (TV Shows, Videos, Video Games, etc)
}
}
// we might have been redirected to the movie page
if (results.isEmpty()) {
Movie movie = scrapeMovie(dom);
if (movie != null) {
results.add(movie);
}
}
return results;
}
protected Movie scrapeMovie(Document dom) {
try {
String name = normalizeName(selectString("//H1/text()", dom));
String year = new Scanner(selectString("//H1//SPAN", dom)).useDelimiter("\\D+").next();
String url = selectString("//LINK[@rel='canonical']/@href", dom);
return new Movie(name, Integer.parseInt(year), getImdbId(url));
} catch (Exception e) {
// ignore, we probably got redirected to an error page
return null;
}
}
@Override
public Movie getMovieDescriptor(int imdbid, Locale locale) throws Exception {
return scrapeMovie(parsePage(new URL("http", host, String.format("/title/tt%07d/", imdbid))));
}
protected Document parsePage(URL url) throws IOException, SAXException {
URLConnection connection = url.openConnection();
// IMDb refuses default user agent (Java/1.6.0_12)
connection.addRequestProperty("User-Agent", "Mozilla");
return getHtmlDocument(connection);
}
@Override
public Movie[] getMovieDescriptors(File[] movieFiles, Locale locale) throws Exception {
return new Movie[movieFiles.length]; // UNSUPPORTED OPERATION => EMPTY RESULT
}
}

6
source/net/sourceforge/filebot/web/WebRequest.java
View File

@ -98,7 +98,11 @@ public final class WebRequest {
public static Document getDocument(InputSource source) throws IOException, SAXException {
try {
return DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(source);
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
factory.setValidating(false);
factory.setFeature("http://xml.org/sax/features/namespaces", false);
factory.setFeature("http://xml.org/sax/features/validation", false);
return factory.newDocumentBuilder().parse(source);
} catch (ParserConfigurationException e) {
// will never happen
throw new RuntimeException(e);

50
test/net/sourceforge/filebot/web/IMDbClientTest.java
View File

@ -13,7 +13,7 @@ public class IMDbClientTest {
private final IMDbClient imdb = new IMDbClient();
@Test
public void search() throws Exception {
List<SearchResult> results = imdb.search("battlestar");
@ -27,7 +27,7 @@ public class IMDbClientTest {
assertEquals(8, results.size(), 0);
}
@Test
public void searchMiniSeries() throws Exception {
List<SearchResult> results = imdb.search("generation kill");
@ -39,7 +39,7 @@ public class IMDbClientTest {
assertEquals(995832, movie.getImdbId(), 0);
}
@Test
public void searchNoMatch() throws Exception {
List<SearchResult> results = imdb.search("i will not find anything for this query string");
@ -47,7 +47,7 @@ public class IMDbClientTest {
assertTrue(results.isEmpty());
}
@Test
public void searchResultPageRedirect() throws Exception {
List<SearchResult> results = imdb.search("my name is earl");
@ -61,7 +61,7 @@ public class IMDbClientTest {
assertEquals(460091, movie.getImdbId(), 0);
}
@Test
public void getEpisodeList() throws Exception {
List<Episode> list = imdb.getEpisodeList(new Movie("Buffy", 1997, 118276));
@ -87,7 +87,7 @@ public class IMDbClientTest {
assertEquals("2003-05-20", last.airdate().toString());
}
@Test
public void getEpisodeListWithUnknownSeason() throws Exception {
List<Episode> list = imdb.getEpisodeList(new Movie("Mushishi", 2005, 807832));
@ -103,7 +103,43 @@ public class IMDbClientTest {
assertEquals("1", first.getSeason().toString());
}
@Test
public void searchMovie() throws Exception {
List<Movie> results = imdb.searchMovie("Avatar", null);
assertEquals(26, results.size());
Movie movie = (Movie) results.get(0);
assertEquals("Avatar", movie.getName());
assertEquals(2009, movie.getYear());
assertEquals(499549, movie.getImdbId(), 0);
}
@Test
public void searchMovieRedirect() throws Exception {
List<Movie> results = imdb.searchMovie("battle angel alita", null);
assertEquals(1, results.size());
Movie movie = (Movie) results.get(0);
assertEquals("Battle Angel", movie.getName());
assertEquals(1993, movie.getYear());
assertEquals(107061, movie.getImdbId(), 0);
}
@Test
public void getMovieDescriptor() throws Exception {
Movie movie = imdb.getMovieDescriptor(499549, null);
assertEquals("Avatar", movie.getName());
assertEquals(2009, movie.getYear());
assertEquals(499549, movie.getImdbId(), 0);
}
@Test
public void getEpisodeListLink() throws Exception {
assertEquals("http://www.imdb.com/title/tt0407362/episodes", imdb.getEpisodeListLink(new Movie("Battlestar Galactica", 2004, 407362)).toString());

1
website/data/query-blacklist.txt
View File

@ -12,6 +12,7 @@ PROPER
READNFO
REPACK
RETAIL
HDRip
sample[s]?$
ShareReactor
ShareZONE