* support and include TheMovieDB alternative_titles data in search and index
This commit is contained in:
Reinhard Pointner
parent
62e3995dc2
commit
f5b4dbee19
|
|
@ -117,16 +117,20 @@ def tmdb_txt = new File('tmdb.txt')
|
|||
def tmdb_index = csv(tmdb_txt, '\t', 1, [0..-1])
|
||||
|
||||
def tmdb = omdb.findResults{ m ->
|
||||
if (tmdb_index.containsKey(m[0])) {
|
||||
def sync = System.currentTimeMillis()
|
||||
if (tmdb_index.containsKey(m[0]) && (sync - tmdb_index[m[0]][0].toLong()) < (360 * 24 * 60 * 60 * 1000L) ) {
|
||||
return tmdb_index[m[0]]
|
||||
}
|
||||
|
||||
def sync = System.currentTimeMillis()
|
||||
def row = [sync, m[0].pad(7), 0, m[2], m[1]]
|
||||
try {
|
||||
def info = net.sourceforge.filebot.WebServices.TMDb.getMovieInfo("tt${m[0]}", Locale.ENGLISH, false)
|
||||
def names = [info.name, info.originalName, m[1]]
|
||||
row = [sync, m[0].pad(7), info.id.pad(7), info.released?.year ?: m[2]] + names.findResults{ it ?: '' }
|
||||
def info = net.sourceforge.filebot.WebServices.TMDb.getMovieInfo("tt${m[0]}", Locale.ENGLISH, true, false)
|
||||
def names = [info.name, info.originalName] + info.alternativeTitles
|
||||
if (info.released != null) {
|
||||
row = [sync, m[0].pad(7), info.id.pad(7), info.released.year] + names
|
||||
} else {
|
||||
println "Illegal movie: ${info.name} | ${m}"
|
||||
}
|
||||
} catch(FileNotFoundException e) {
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -51,7 +51,7 @@
|
|||
-->
|
||||
<cache name="web-datasource-lv3"
|
||||
maxElementsInMemory="200"
|
||||
maxElementsOnDisk="95000"
|
||||
maxElementsOnDisk="200000"
|
||||
eternal="false"
|
||||
timeToIdleSeconds="5256000"
|
||||
timeToLiveSeconds="5256000"
|
||||
|
|
@ -65,7 +65,7 @@
|
|||
-->
|
||||
<cache name="web-persistent-datasource"
|
||||
maxElementsInMemory="50"
|
||||
maxElementsOnDisk="5000"
|
||||
maxElementsOnDisk="20000"
|
||||
eternal="false"
|
||||
timeToIdleSeconds="10512000"
|
||||
timeToLiveSeconds="10512000"
|
||||
|
|
|
|||
|
|
@ -61,7 +61,6 @@ import net.sourceforge.filebot.web.Episode;
|
|||
import net.sourceforge.filebot.web.Movie;
|
||||
import net.sourceforge.filebot.web.MovieIdentificationService;
|
||||
import net.sourceforge.filebot.web.SearchResult;
|
||||
import net.sourceforge.filebot.web.TMDbClient.MovieInfo;
|
||||
import net.sourceforge.filebot.web.TheTVDBClient.SeriesInfo;
|
||||
import net.sourceforge.filebot.web.TheTVDBSearchResult;
|
||||
|
||||
|
|
@ -1087,14 +1086,6 @@ public class MediaDetection {
|
|||
return WebServices.TheTVDB.getSeriesInfoByID(grepTheTvdbId(new String(readFile(nfo), "UTF-8")).iterator().next(), locale);
|
||||
}
|
||||
|
||||
public static Movie tmdb2imdb(Movie m) throws IOException {
|
||||
if (m.getTmdbId() <= 0 && m.getImdbId() <= 0)
|
||||
throw new IllegalArgumentException();
|
||||
|
||||
MovieInfo info = WebServices.TMDb.getMovieInfo(m, Locale.ENGLISH);
|
||||
return new Movie(info.getName(), info.getReleased().getYear(), info.getImdbId(), info.getId());
|
||||
}
|
||||
|
||||
public static List<SearchResult> getProbableMatches(String query, Collection<SearchResult> options) {
|
||||
// auto-select most probable search result
|
||||
List<SearchResult> probableMatches = new LinkedList<SearchResult>();
|
||||
|
|
|
|||
|
|
@ -52,6 +52,7 @@ import net.miginfocom.swing.MigLayout;
|
|||
import net.sourceforge.filebot.Analytics;
|
||||
import net.sourceforge.filebot.Language;
|
||||
import net.sourceforge.filebot.ResourceManager;
|
||||
import net.sourceforge.filebot.WebServices;
|
||||
import net.sourceforge.filebot.media.MediaDetection;
|
||||
import net.sourceforge.filebot.ui.LanguageComboBox;
|
||||
import net.sourceforge.filebot.ui.SelectDialog;
|
||||
|
|
@ -668,7 +669,7 @@ public class SubtitleUploadDialog extends JDialog {
|
|||
Collection<Movie> identity = MediaDetection.detectMovie(mapping.getVideo(), database, database, Locale.ENGLISH, true);
|
||||
for (Movie it : identity) {
|
||||
if (it.getImdbId() <= 0 && it.getTmdbId() > 0) {
|
||||
it = MediaDetection.tmdb2imdb(it);
|
||||
it = WebServices.TMDb.getMovieDescriptor(it.getTmdbId(), Locale.ENGLISH, false);
|
||||
}
|
||||
if (it != null && it.getImdbId() > 0) {
|
||||
mapping.setIdentity(it);
|
||||
|
|
|
|||
|
|
@ -234,6 +234,6 @@ public class IMDbClient implements MovieIdentificationService {
|
|||
actors.add(new Person(writer, null, "Writer"));
|
||||
}
|
||||
|
||||
return new MovieInfo(fields, genres, new ArrayList<String>(0), actors, new ArrayList<Trailer>(0));
|
||||
return new MovieInfo(fields, new ArrayList<String>(0), genres, new ArrayList<String>(0), actors, new ArrayList<Trailer>(0));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -395,6 +395,10 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS
|
|||
}
|
||||
|
||||
public Locale detectLanguage(byte[] data) throws Exception {
|
||||
if (data.length < 256) {
|
||||
throw new IllegalArgumentException("data is not enough");
|
||||
}
|
||||
|
||||
// require login
|
||||
login();
|
||||
|
||||
|
|
|
|||
|
|
@ -17,10 +17,12 @@ import java.util.ArrayList;
|
|||
import java.util.Collection;
|
||||
import java.util.EnumMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Scanner;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
|
|
@ -65,8 +67,9 @@ public class TMDbClient implements MovieIdentificationService {
|
|||
List<Movie> result = new ArrayList<Movie>();
|
||||
|
||||
for (JSONObject it : jsonList(response.get("results"))) {
|
||||
if (it == null)
|
||||
if (it == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// e.g.
|
||||
// {"id":16320,"title":"冲出宁静号","release_date":"2005-09-30","original_title":"Serenity"}
|
||||
|
|
@ -85,7 +88,28 @@ public class TMDbClient implements MovieIdentificationService {
|
|||
} catch (Exception e) {
|
||||
throw new IllegalArgumentException("Missing data: release date");
|
||||
}
|
||||
result.add(new Movie(title, title.equals(originalTitle) ? new String[] {} : new String[] { originalTitle }, year, -1, id));
|
||||
|
||||
Set<String> alternativeTitles = new LinkedHashSet<String>();
|
||||
if (originalTitle != null) {
|
||||
alternativeTitles.add(originalTitle);
|
||||
}
|
||||
|
||||
try {
|
||||
String countryCode = locale.getCountry().isEmpty() ? "US" : locale.getCountry();
|
||||
JSONObject titles = request("movie/" + id + "/alternative_titles", null, null, REQUEST_LIMIT);
|
||||
for (JSONObject node : jsonList(titles.get("titles"))) {
|
||||
if (countryCode.equals(node.get("iso_3166_1"))) {
|
||||
alternativeTitles.add((String) node.get("title"));
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
Logger.getLogger(TMDbClient.class.getName()).log(Level.WARNING, String.format("Unable to retrieve alternative titles [%s]: %s", title, e.getMessage()));
|
||||
}
|
||||
|
||||
// make sure main title is not in the set of alternative titles
|
||||
alternativeTitles.remove(title);
|
||||
|
||||
result.add(new Movie(title, alternativeTitles.toArray(new String[0]), year, -1, id));
|
||||
} catch (Exception e) {
|
||||
// only print 'missing release date' warnings for matching movie titles
|
||||
if (query.equalsIgnoreCase(title) || query.equalsIgnoreCase(originalTitle)) {
|
||||
|
|
@ -102,9 +126,13 @@ public class TMDbClient implements MovieIdentificationService {
|
|||
|
||||
@Override
|
||||
public Movie getMovieDescriptor(int imdbid, Locale locale) throws IOException {
|
||||
String id = String.format("tt%07d", imdbid);
|
||||
return getMovieDescriptor(imdbid, locale, true);
|
||||
}
|
||||
|
||||
public Movie getMovieDescriptor(int imdbtmdbid, Locale locale, boolean byIMDB) throws IOException {
|
||||
String id = byIMDB ? String.format("tt%07d", imdbtmdbid) : String.valueOf(imdbtmdbid);
|
||||
try {
|
||||
MovieInfo info = getMovieInfo(id, locale, false);
|
||||
MovieInfo info = getMovieInfo(id, locale, false, false);
|
||||
return new Movie(info.getName(), info.getReleased().getYear(), info.getImdbId(), info.getId());
|
||||
} catch (FileNotFoundException e) {
|
||||
Logger.getLogger(getClass().getName()).log(Level.WARNING, "Movie not found: " + id);
|
||||
|
|
@ -122,20 +150,20 @@ public class TMDbClient implements MovieIdentificationService {
|
|||
|
||||
public MovieInfo getMovieInfo(Movie movie, Locale locale) throws IOException {
|
||||
if (movie.getTmdbId() >= 0) {
|
||||
return getMovieInfo(String.valueOf(movie.getTmdbId()), locale, true);
|
||||
return getMovieInfo(String.valueOf(movie.getTmdbId()), locale, true, true);
|
||||
} else if (movie.getImdbId() >= 0) {
|
||||
return getMovieInfo(String.format("tt%07d", movie.getImdbId()), locale, true);
|
||||
return getMovieInfo(String.format("tt%07d", movie.getImdbId()), locale, true, true);
|
||||
} else {
|
||||
for (Movie result : searchMovie(movie.getName(), locale)) {
|
||||
if (movie.getName().equalsIgnoreCase(result.getName()) && movie.getYear() == result.getYear()) {
|
||||
return getMovieInfo(String.valueOf(result.getTmdbId()), locale, true);
|
||||
return getMovieInfo(String.valueOf(result.getTmdbId()), locale, true, true);
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public MovieInfo getMovieInfo(String id, Locale locale, boolean extendedInfo) throws IOException {
|
||||
public MovieInfo getMovieInfo(String id, Locale locale, boolean includeAlternativeTitles, boolean includeExtendedInfo) throws IOException {
|
||||
JSONObject response = request("movie/" + id, null, locale, REQUEST_LIMIT);
|
||||
|
||||
Map<MovieProperty, String> fields = new EnumMap<MovieProperty, String>(MovieProperty.class);
|
||||
|
|
@ -163,17 +191,29 @@ public class TMDbClient implements MovieIdentificationService {
|
|||
spokenLanguages.add((String) it.get("iso_639_1"));
|
||||
}
|
||||
|
||||
if (extendedInfo) {
|
||||
List<String> alternativeTitles = new ArrayList<String>();
|
||||
if (includeAlternativeTitles) {
|
||||
String countryCode = locale.getCountry().isEmpty() ? "US" : locale.getCountry();
|
||||
JSONObject titles = request("movie/" + fields.get(MovieProperty.id) + "/alternative_titles", null, null, REQUEST_LIMIT);
|
||||
for (JSONObject it : jsonList(titles.get("titles"))) {
|
||||
if (countryCode.equals(it.get("iso_3166_1"))) {
|
||||
alternativeTitles.add((String) it.get("title"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (includeExtendedInfo) {
|
||||
String countryCode = locale.getCountry().isEmpty() ? "US" : locale.getCountry();
|
||||
JSONObject releases = request("movie/" + fields.get(MovieProperty.id) + "/releases", null, null, REQUEST_LIMIT);
|
||||
for (JSONObject it : jsonList(releases.get("countries"))) {
|
||||
if ("US".equals(it.get("iso_3166_1"))) {
|
||||
if (countryCode.equals(it.get("iso_3166_1"))) {
|
||||
fields.put(MovieProperty.certification, (String) it.get("certification"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
List<Person> cast = new ArrayList<Person>();
|
||||
if (extendedInfo) {
|
||||
if (includeExtendedInfo) {
|
||||
JSONObject castResponse = request("movie/" + fields.get(MovieProperty.id) + "/casts", null, null, REQUEST_LIMIT);
|
||||
for (String section : new String[] { "cast", "crew" }) {
|
||||
for (JSONObject it : jsonList(castResponse.get(section))) {
|
||||
|
|
@ -190,7 +230,7 @@ public class TMDbClient implements MovieIdentificationService {
|
|||
}
|
||||
|
||||
List<Trailer> trailers = new ArrayList<Trailer>();
|
||||
if (extendedInfo) {
|
||||
if (includeExtendedInfo) {
|
||||
JSONObject trailerResponse = request("movie/" + fields.get(MovieProperty.id) + "/trailers", null, null, REQUEST_LIMIT);
|
||||
for (String section : new String[] { "quicktime", "youtube" }) {
|
||||
for (JSONObject it : jsonList(trailerResponse.get(section))) {
|
||||
|
|
@ -207,7 +247,7 @@ public class TMDbClient implements MovieIdentificationService {
|
|||
}
|
||||
}
|
||||
|
||||
return new MovieInfo(fields, genres, spokenLanguages, cast, trailers);
|
||||
return new MovieInfo(fields, alternativeTitles, genres, spokenLanguages, cast, trailers);
|
||||
}
|
||||
|
||||
public List<Artwork> getArtwork(String id) throws IOException {
|
||||
|
|
@ -300,6 +340,7 @@ public class TMDbClient implements MovieIdentificationService {
|
|||
|
||||
protected Map<MovieProperty, String> fields;
|
||||
|
||||
protected String[] alternativeTitles;
|
||||
protected String[] genres;
|
||||
protected String[] spokenLanguages;
|
||||
|
||||
|
|
@ -310,8 +351,9 @@ public class TMDbClient implements MovieIdentificationService {
|
|||
// used by serializer
|
||||
}
|
||||
|
||||
protected MovieInfo(Map<MovieProperty, String> fields, List<String> genres, List<String> spokenLanguages, List<Person> people, List<Trailer> trailers) {
|
||||
protected MovieInfo(Map<MovieProperty, String> fields, List<String> alternativeTitles, List<String> genres, List<String> spokenLanguages, List<Person> people, List<Trailer> trailers) {
|
||||
this.fields = new EnumMap<MovieProperty, String>(fields);
|
||||
this.alternativeTitles = alternativeTitles.toArray(new String[0]);
|
||||
this.genres = genres.toArray(new String[0]);
|
||||
this.spokenLanguages = spokenLanguages.toArray(new String[0]);
|
||||
this.people = people.toArray(new Person[0]);
|
||||
|
|
@ -476,6 +518,10 @@ public class TMDbClient implements MovieIdentificationService {
|
|||
return unmodifiableList(asList(trailers));
|
||||
}
|
||||
|
||||
public List<String> getAlternativeTitles() {
|
||||
return unmodifiableList(asList(alternativeTitles));
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return fields.toString();
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@
|
|||
^AUDIO_TS$
|
||||
^bad$
|
||||
^BDMV$
|
||||
^Best$
|
||||
^Big$
|
||||
^clean$
|
||||
^cleaned$
|
||||
|
|
@ -111,6 +112,7 @@
|
|||
^temporary$
|
||||
^test$
|
||||
^testdata$
|
||||
^The.Best$
|
||||
^tmp$
|
||||
^to$
|
||||
^Torrent$
|
||||
|
|
@ -233,6 +235,7 @@ LMAO
|
|||
lol
|
||||
Los.Sustitutos
|
||||
M.HD
|
||||
MegaPack
|
||||
mkvonly
|
||||
Movie.Pack
|
||||
mpg$
|
||||
|
|
|
|||
Loading…
Reference in New Issue