mobeigi
/
filebot
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

+ support IMDb as movie db

This commit is contained in:
Reinhard Pointner 2012-01-06 17:24:00 +00:00
parent 5228c76dbc
commit 81533d0a66
5 changed files with 140 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
source/net/sourceforge/filebot/WebServices.java
View File

@ -46,7 +46,7 @@ public final class WebServices {
public static MovieIdentificationService[] getMovieIdentificationServices() {
return new MovieIdentificationService[] { OpenSubtitles, TMDb };
return new MovieIdentificationService[] { OpenSubtitles, IMDb, TMDb };
}

98
source/net/sourceforge/filebot/web/IMDbClient.java
View File

@ -5,6 +5,7 @@ package net.sourceforge.filebot.web;
import static net.sourceforge.filebot.web.WebRequest.*;
import static net.sourceforge.tuned.XPathUtilities.*;
import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
@ -27,7 +28,7 @@ import net.sf.ehcache.CacheManager;
import net.sourceforge.filebot.ResourceManager;
public class IMDbClient extends AbstractEpisodeListProvider {
public class IMDbClient extends AbstractEpisodeListProvider implements MovieIdentificationService {
private final String host = "www.imdb.com";
@ -52,11 +53,9 @@ public class IMDbClient extends AbstractEpisodeListProvider {
@Override
public List<SearchResult> fetchSearchResult(String query, Locale locale) throws IOException, SAXException {
URL searchUrl = new URL("http", host, "/find?s=tt&q=" + encode(query));
Document dom = getHtmlDocument(openConnection(searchUrl));
Document dom = parsePage(new URL("http", host, "/find?s=tt&q=" + encode(query)));
List<Node> nodes = selectNodes("//TABLE//A[following-sibling::SMALL[contains(.,'series')]]", dom);
List<SearchResult> results = new ArrayList<SearchResult>(nodes.size());
for (Node node : nodes) {
@ -69,14 +68,9 @@ public class IMDbClient extends AbstractEpisodeListProvider {
// we might have been redirected to the movie page
if (results.isEmpty()) {
try {
String name = normalizeName(selectString("//H1/text()", dom));
String year = new Scanner(selectString("//H1//SPAN", dom)).useDelimiter("\\D+").next();
String url = selectString("//LINK[@rel='canonical']/@href", dom);
results.add(new Movie(name, Integer.parseInt(year), getImdbId(url)));
} catch (Exception e) {
// ignore, we probably got redirected to an error page
Movie movie = scrapeMovie(dom);
if (movie != null) {
results.add(movie);
}
}
@ -87,7 +81,7 @@ public class IMDbClient extends AbstractEpisodeListProvider {
@Override
public List<Episode> fetchEpisodeList(SearchResult searchResult, Locale locale) throws IOException, SAXException {
Movie movie = (Movie) searchResult;
Document dom = getHtmlDocument(openConnection(getEpisodeListLink(searchResult).toURL()));
Document dom = parsePage(getEpisodeListLink(searchResult).toURL());
String seriesName = normalizeName(selectString("//H1/A", dom));
Date year = new Date(movie.getYear(), 0, 0);
@ -112,16 +106,6 @@ public class IMDbClient extends AbstractEpisodeListProvider {
}
protected URLConnection openConnection(URL url) throws IOException {
URLConnection connection = url.openConnection();
// IMDb refuses default user agent (Java/1.6.0_12)
connection.addRequestProperty("User-Agent", "Scraper");
return connection;
}
protected String normalizeName(String name) {
// remove quotation marks
return name.replaceAll("\"", "");
@ -154,4 +138,72 @@ public class IMDbClient extends AbstractEpisodeListProvider {
throw new RuntimeException(e);
}
}
@Override
public List<Movie> searchMovie(String query, Locale locale) throws Exception {
Document dom = parsePage(new URL("http", host, "/find?s=tt&q=" + encode(query)));
// select movie links followed by year in parenthesis
List<Node> nodes = selectNodes("//TABLE//A[string-length(substring-after(substring-before(following::text(),')'),'(')) = 4 and count(following-sibling::SMALL) = 0]", dom);
List<Movie> results = new ArrayList<Movie>(nodes.size());
for (Node node : nodes) {
String name = node.getTextContent().trim();
String year = node.getNextSibling().getTextContent().trim().replaceAll("[\\p{Punct}\\p{Space}]+", ""); // remove non-number characters
String href = getAttribute("href", node);
try {
results.add(new Movie(name, Integer.parseInt(year), getImdbId(href)));
} catch (NumberFormatException e) {
// ignore illegal movies (TV Shows, Videos, Video Games, etc)
}
}
// we might have been redirected to the movie page
if (results.isEmpty()) {
Movie movie = scrapeMovie(dom);
if (movie != null) {
results.add(movie);
}
}
return results;
}
protected Movie scrapeMovie(Document dom) {
try {
String name = normalizeName(selectString("//H1/text()", dom));
String year = new Scanner(selectString("//H1//SPAN", dom)).useDelimiter("\\D+").next();
String url = selectString("//LINK[@rel='canonical']/@href", dom);
return new Movie(name, Integer.parseInt(year), getImdbId(url));
} catch (Exception e) {
// ignore, we probably got redirected to an error page
return null;
}
}
@Override
public Movie getMovieDescriptor(int imdbid, Locale locale) throws Exception {
return scrapeMovie(parsePage(new URL("http", host, String.format("/title/tt%07d/", imdbid))));
}
protected Document parsePage(URL url) throws IOException, SAXException {
URLConnection connection = url.openConnection();
// IMDb refuses default user agent (Java/1.6.0_12)
connection.addRequestProperty("User-Agent", "Mozilla");
return getHtmlDocument(connection);
}
@Override
public Movie[] getMovieDescriptors(File[] movieFiles, Locale locale) throws Exception {
return new Movie[movieFiles.length]; // UNSUPPORTED OPERATION => EMPTY RESULT
}
}

6
source/net/sourceforge/filebot/web/WebRequest.java
View File

@ -98,7 +98,11 @@ public final class WebRequest {
public static Document getDocument(InputSource source) throws IOException, SAXException {
try {
return DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(source);
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
factory.setValidating(false);
factory.setFeature("http://xml.org/sax/features/namespaces", false);
factory.setFeature("http://xml.org/sax/features/validation", false);
return factory.newDocumentBuilder().parse(source);
} catch (ParserConfigurationException e) {
// will never happen
throw new RuntimeException(e);

36
test/net/sourceforge/filebot/web/IMDbClientTest.java
View File

@ -104,6 +104,42 @@ public class IMDbClientTest {
}
@Test
public void searchMovie() throws Exception {
List<Movie> results = imdb.searchMovie("Avatar", null);
assertEquals(26, results.size());
Movie movie = (Movie) results.get(0);
assertEquals("Avatar", movie.getName());
assertEquals(2009, movie.getYear());
assertEquals(499549, movie.getImdbId(), 0);
}
@Test
public void searchMovieRedirect() throws Exception {
List<Movie> results = imdb.searchMovie("battle angel alita", null);
assertEquals(1, results.size());
Movie movie = (Movie) results.get(0);
assertEquals("Battle Angel", movie.getName());
assertEquals(1993, movie.getYear());
assertEquals(107061, movie.getImdbId(), 0);
}
@Test
public void getMovieDescriptor() throws Exception {
Movie movie = imdb.getMovieDescriptor(499549, null);
assertEquals("Avatar", movie.getName());
assertEquals(2009, movie.getYear());
assertEquals(499549, movie.getImdbId(), 0);
}
@Test
public void getEpisodeListLink() throws Exception {
assertEquals("http://www.imdb.com/title/tt0407362/episodes", imdb.getEpisodeListLink(new Movie("Battlestar Galactica", 2004, 407362)).toString());

1
website/data/query-blacklist.txt
View File

@ -12,6 +12,7 @@ PROPER
READNFO
REPACK
RETAIL
HDRip
sample[s]?$
ShareReactor
ShareZONE