From 4dc5e26ae90e0657c318a5a408bc33777a2f62d4 Mon Sep 17 00:00:00 2001 From: Reinhard Pointner Date: Sun, 17 Mar 2013 14:19:11 +0000 Subject: [PATCH] * fix cache misconfig to stop anidb abuse * build our own anidb index file like with thetvdb --- source/ehcache.xml | 16 +++++++- .../net/sourceforge/filebot/WebServices.java | 16 +++++++- .../filebot/media/ReleaseInfo.java | 40 ++++++++++++++++++- .../filebot/media/ReleaseInfo.properties | 17 ++++---- .../sourceforge/filebot/web/AnidbClient.java | 6 +-- website/data/.htaccess | 1 + 6 files changed, 82 insertions(+), 14 deletions(-) diff --git a/source/ehcache.xml b/source/ehcache.xml index dc021626..5ebea897 100644 --- a/source/ehcache.xml +++ b/source/ehcache.xml @@ -19,7 +19,7 @@ /> + + + diff --git a/source/net/sourceforge/filebot/WebServices.java b/source/net/sourceforge/filebot/WebServices.java index ff175ece..bf351d67 100644 --- a/source/net/sourceforge/filebot/WebServices.java +++ b/source/net/sourceforge/filebot/WebServices.java @@ -49,7 +49,7 @@ public final class WebServices { // episode dbs public static final TVRageClient TVRage = new TVRageClient(); - public static final AnidbClient AniDB = new AnidbClient(getApplicationName().toLowerCase(), 3); + public static final AnidbClient AniDB = new AnidbClientWithLocalSearch(getApplicationName().toLowerCase(), 4); public static final SerienjunkiesClient Serienjunkies = new SerienjunkiesClient(getApplicationProperty("serienjunkies.apikey")); // extended TheTVDB module with local search @@ -202,6 +202,20 @@ public final class WebServices { } + public static class AnidbClientWithLocalSearch extends AnidbClient { + + public AnidbClientWithLocalSearch(String client, int clientver) { + super(client, clientver); + } + + + @Override + public List getAnimeTitles() throws Exception { + return asList(MediaDetection.releaseInfo.getAnidbIndex()); + } + } + + /** * Dummy constructor to prevent instantiation. */ diff --git a/source/net/sourceforge/filebot/media/ReleaseInfo.java b/source/net/sourceforge/filebot/media/ReleaseInfo.java index 346f14fd..ada22cf2 100644 --- a/source/net/sourceforge/filebot/media/ReleaseInfo.java +++ b/source/net/sourceforge/filebot/media/ReleaseInfo.java @@ -35,6 +35,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.zip.GZIPInputStream; +import net.sourceforge.filebot.web.AnidbClient.AnidbSearchResult; import net.sourceforge.filebot.web.CachedResource; import net.sourceforge.filebot.web.Movie; import net.sourceforge.filebot.web.TheTVDBClient.TheTVDBSearchResult; @@ -243,7 +244,12 @@ public class ReleaseInfo { public TheTVDBSearchResult[] getTheTVDBIndex() throws IOException { - return theTVDBIndexResource.get(); + return tvdbIndexResource.get(); + } + + + public AnidbSearchResult[] getAnidbIndex() throws IOException { + return anidbIndexResource.get(); } @@ -275,7 +281,8 @@ public class ReleaseInfo { protected final CachedResource movieListResource = new MovieResource(getBundle(getClass().getName()).getString("url.movie-list")); protected final CachedResource seriesListResource = new SeriesListResource(getBundle(getClass().getName()).getString("url.series-list")); protected final CachedResource seriesDirectMappingsResource = new PatternResource(getBundle(getClass().getName()).getString("url.series-mappings")); - protected final CachedResource theTVDBIndexResource = new TheTVDBIndexResource(getBundle(getClass().getName()).getString("url.thetvdb-index")); + protected final CachedResource tvdbIndexResource = new TheTVDBIndexResource(getBundle(getClass().getName()).getString("url.thetvdb-index")); + protected final CachedResource anidbIndexResource = new AnidbIndexResource(getBundle(getClass().getName()).getString("url.anidb-index")); protected static class PatternResource extends CachedResource { @@ -353,6 +360,35 @@ public class ReleaseInfo { } + protected static class AnidbIndexResource extends CachedResource { + + public AnidbIndexResource(String resource) { + super(resource, AnidbSearchResult[].class, 7 * 24 * 60 * 60 * 1000); // check for updates once a week + } + + + @Override + public AnidbSearchResult[] process(ByteBuffer data) throws IOException { + Scanner scanner = new Scanner(new GZIPInputStream(new ByteBufferInputStream(data)), "UTF-8").useDelimiter("\t|\n"); + + List anime = new ArrayList(); + while (scanner.hasNext() && scanner.hasNextInt()) { + int aid = scanner.nextInt(); + String primaryTitle = scanner.next().trim(); + String englishTitle = scanner.next().trim(); + + if (englishTitle.isEmpty() || englishTitle.equals(primaryTitle)) { + anime.add(new AnidbSearchResult(aid, primaryTitle, null)); + } else { + anime.add(new AnidbSearchResult(aid, primaryTitle, singletonMap("en", englishTitle))); + } + } + + return anime.toArray(new AnidbSearchResult[0]); + } + } + + protected static class FolderEntryFilter implements FileFilter { private final Pattern entryPattern; diff --git a/source/net/sourceforge/filebot/media/ReleaseInfo.properties b/source/net/sourceforge/filebot/media/ReleaseInfo.properties index e50daf54..8e28847d 100644 --- a/source/net/sourceforge/filebot/media/ReleaseInfo.properties +++ b/source/net/sourceforge/filebot/media/ReleaseInfo.properties @@ -5,25 +5,28 @@ pattern.video.source: CAMRip|CAM|PDVD|TS|TELESYNC|PDVD|PPV|PPVRip|Screener|SCR|S pattern.video.format: DivX|Xvid|AVC|x264|h264|3ivx|mpeg|mpeg4|mp3|aac|ac3|dd20|dd51|2ch|6ch|DTS|WS|HR|7p|720p|18p|1080p|NTSC # known release group names -url.release-groups: http://www.filebot.net/data/release-groups.txt +url.release-groups: http://filebot.net/data/release-groups.txt # blacklisted terms that will be ignored -url.query-blacklist: http://www.filebot.net/data/query-blacklist.txt +url.query-blacklist: http://filebot.net/data/query-blacklist.txt # clutter files that will be ignored -url.exclude-blacklist: http://www.filebot.net/data/exclude-blacklist.txt +url.exclude-blacklist: http://filebot.net/data/exclude-blacklist.txt # list of patterns directly matching files to series names -url.series-mappings: http://www.filebot.net/data/series-mappings.txt +url.series-mappings: http://filebot.net/data/series-mappings.txt # list of all movies (id, name, year) -url.movie-list: http://www.filebot.net/data/movies.txt.gz +url.movie-list: http://filebot.net/data/movies.txt.gz # list of tv show and anime names -url.series-list: http://www.filebot.net/data/series.list.gz +url.series-list: http://filebot.net/data/series.list.gz # TheTVDB index -url.thetvdb-index: http://www.filebot.net/data/thetvdb.txt.gz +url.thetvdb-index: http://filebot.net/data/thetvdb.txt.gz + +# AniDB index +url.anidb-index: http://filebot.net/data/anidb.txt.gz # disk folder matcher pattern.diskfolder.entry: BDMV|HVDVD_TS|VIDEO_TS|AUDIO_TS|VCD|movie.nfo diff --git a/source/net/sourceforge/filebot/web/AnidbClient.java b/source/net/sourceforge/filebot/web/AnidbClient.java index 5274f0ba..19b9d6b6 100644 --- a/source/net/sourceforge/filebot/web/AnidbClient.java +++ b/source/net/sourceforge/filebot/web/AnidbClient.java @@ -70,7 +70,7 @@ public class AnidbClient extends AbstractEpisodeListProvider { @Override public ResultCache getCache() { - return new ResultCache(host, Cache.getCache("web-datasource")); + return new ResultCache(host, Cache.getCache("web-datasource-lv2")); } @@ -87,7 +87,7 @@ public class AnidbClient extends AbstractEpisodeListProvider { @Override protected Set getFields(AnidbSearchResult anime) { - return set(anime.getPrimaryTitle(), anime.getOfficialTitle("en"), anime.getOfficialTitle(locale.getLanguage())); + return set(anime.getPrimaryTitle(), anime.getOfficialTitle("en")); } }; @@ -154,7 +154,7 @@ public class AnidbClient extends AbstractEpisodeListProvider { public synchronized List getAnimeTitles() throws Exception { - URL url = new URL("http", host, "/api/animetitles.dat.gz"); + URL url = new URL("http", host, "/api/anime-titles.dat.gz"); ResultCache cache = getCache(); @SuppressWarnings("unchecked") diff --git a/website/data/.htaccess b/website/data/.htaccess index 57aa4bc8..1b68a8df 100644 --- a/website/data/.htaccess +++ b/website/data/.htaccess @@ -3,3 +3,4 @@ options +indexes redirect 301 /data/movies.txt.gz http://sourceforge.net/projects/filebot/files/data/movies.txt.gz/download redirect 301 /data/series.list.gz http://sourceforge.net/projects/filebot/files/data/series.list.gz/download redirect 301 /data/thetvdb.txt.gz http://sourceforge.net/projects/filebot/files/data/thetvdb.txt.gz/download +redirect 301 /data/anidb.txt.gz http://sourceforge.net/projects/filebot/files/data/anidb.txt.gz/download