* fix cache misconfig to stop anidb abuse
* build our own anidb index file like with thetvdb
This commit is contained in:
Reinhard Pointner
parent
5dbeb12e88
commit
4dc5e26ae9
|
|
@ -19,7 +19,7 @@
|
|||
/>
|
||||
|
||||
<!--
|
||||
Long-lived (2 days) persistent disk cache for web responses
|
||||
Short-lived (2 days) persistent disk cache for web responses
|
||||
-->
|
||||
<cache name="web-datasource"
|
||||
maxElementsInMemory="200"
|
||||
|
|
@ -32,6 +32,20 @@
|
|||
memoryStoreEvictionPolicy="LRU"
|
||||
/>
|
||||
|
||||
<!--
|
||||
Long-lived (2 weeks) persistent disk cache for web responses
|
||||
-->
|
||||
<cache name="web-datasource-lv2"
|
||||
maxElementsInMemory="200"
|
||||
maxElementsOnDisk="10000"
|
||||
eternal="false"
|
||||
timeToIdleSeconds="1209600"
|
||||
timeToLiveSeconds="1209600"
|
||||
overflowToDisk="true"
|
||||
diskPersistent="true"
|
||||
memoryStoreEvictionPolicy="LRU"
|
||||
/>
|
||||
|
||||
<!--
|
||||
Very long-lived cache (2 months) anime/series lists, movie index, etc
|
||||
-->
|
||||
|
|
|
|||
|
|
@ -49,7 +49,7 @@ public final class WebServices {
|
|||
|
||||
// episode dbs
|
||||
public static final TVRageClient TVRage = new TVRageClient();
|
||||
public static final AnidbClient AniDB = new AnidbClient(getApplicationName().toLowerCase(), 3);
|
||||
public static final AnidbClient AniDB = new AnidbClientWithLocalSearch(getApplicationName().toLowerCase(), 4);
|
||||
public static final SerienjunkiesClient Serienjunkies = new SerienjunkiesClient(getApplicationProperty("serienjunkies.apikey"));
|
||||
|
||||
// extended TheTVDB module with local search
|
||||
|
|
@ -202,6 +202,20 @@ public final class WebServices {
|
|||
}
|
||||
|
||||
|
||||
public static class AnidbClientWithLocalSearch extends AnidbClient {
|
||||
|
||||
public AnidbClientWithLocalSearch(String client, int clientver) {
|
||||
super(client, clientver);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public List<AnidbSearchResult> getAnimeTitles() throws Exception {
|
||||
return asList(MediaDetection.releaseInfo.getAnidbIndex());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Dummy constructor to prevent instantiation.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ import java.util.regex.Matcher;
|
|||
import java.util.regex.Pattern;
|
||||
import java.util.zip.GZIPInputStream;
|
||||
|
||||
import net.sourceforge.filebot.web.AnidbClient.AnidbSearchResult;
|
||||
import net.sourceforge.filebot.web.CachedResource;
|
||||
import net.sourceforge.filebot.web.Movie;
|
||||
import net.sourceforge.filebot.web.TheTVDBClient.TheTVDBSearchResult;
|
||||
|
|
@ -243,7 +244,12 @@ public class ReleaseInfo {
|
|||
|
||||
|
||||
public TheTVDBSearchResult[] getTheTVDBIndex() throws IOException {
|
||||
return theTVDBIndexResource.get();
|
||||
return tvdbIndexResource.get();
|
||||
}
|
||||
|
||||
|
||||
public AnidbSearchResult[] getAnidbIndex() throws IOException {
|
||||
return anidbIndexResource.get();
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -275,7 +281,8 @@ public class ReleaseInfo {
|
|||
protected final CachedResource<Movie[]> movieListResource = new MovieResource(getBundle(getClass().getName()).getString("url.movie-list"));
|
||||
protected final CachedResource<String[]> seriesListResource = new SeriesListResource(getBundle(getClass().getName()).getString("url.series-list"));
|
||||
protected final CachedResource<String[]> seriesDirectMappingsResource = new PatternResource(getBundle(getClass().getName()).getString("url.series-mappings"));
|
||||
protected final CachedResource<TheTVDBSearchResult[]> theTVDBIndexResource = new TheTVDBIndexResource(getBundle(getClass().getName()).getString("url.thetvdb-index"));
|
||||
protected final CachedResource<TheTVDBSearchResult[]> tvdbIndexResource = new TheTVDBIndexResource(getBundle(getClass().getName()).getString("url.thetvdb-index"));
|
||||
protected final CachedResource<AnidbSearchResult[]> anidbIndexResource = new AnidbIndexResource(getBundle(getClass().getName()).getString("url.anidb-index"));
|
||||
|
||||
|
||||
protected static class PatternResource extends CachedResource<String[]> {
|
||||
|
|
@ -353,6 +360,35 @@ public class ReleaseInfo {
|
|||
}
|
||||
|
||||
|
||||
protected static class AnidbIndexResource extends CachedResource<AnidbSearchResult[]> {
|
||||
|
||||
public AnidbIndexResource(String resource) {
|
||||
super(resource, AnidbSearchResult[].class, 7 * 24 * 60 * 60 * 1000); // check for updates once a week
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public AnidbSearchResult[] process(ByteBuffer data) throws IOException {
|
||||
Scanner scanner = new Scanner(new GZIPInputStream(new ByteBufferInputStream(data)), "UTF-8").useDelimiter("\t|\n");
|
||||
|
||||
List<AnidbSearchResult> anime = new ArrayList<AnidbSearchResult>();
|
||||
while (scanner.hasNext() && scanner.hasNextInt()) {
|
||||
int aid = scanner.nextInt();
|
||||
String primaryTitle = scanner.next().trim();
|
||||
String englishTitle = scanner.next().trim();
|
||||
|
||||
if (englishTitle.isEmpty() || englishTitle.equals(primaryTitle)) {
|
||||
anime.add(new AnidbSearchResult(aid, primaryTitle, null));
|
||||
} else {
|
||||
anime.add(new AnidbSearchResult(aid, primaryTitle, singletonMap("en", englishTitle)));
|
||||
}
|
||||
}
|
||||
|
||||
return anime.toArray(new AnidbSearchResult[0]);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
protected static class FolderEntryFilter implements FileFilter {
|
||||
|
||||
private final Pattern entryPattern;
|
||||
|
|
|
|||
|
|
@ -5,25 +5,28 @@ pattern.video.source: CAMRip|CAM|PDVD|TS|TELESYNC|PDVD|PPV|PPVRip|Screener|SCR|S
|
|||
pattern.video.format: DivX|Xvid|AVC|x264|h264|3ivx|mpeg|mpeg4|mp3|aac|ac3|dd20|dd51|2ch|6ch|DTS|WS|HR|7p|720p|18p|1080p|NTSC
|
||||
|
||||
# known release group names
|
||||
url.release-groups: http://www.filebot.net/data/release-groups.txt
|
||||
url.release-groups: http://filebot.net/data/release-groups.txt
|
||||
|
||||
# blacklisted terms that will be ignored
|
||||
url.query-blacklist: http://www.filebot.net/data/query-blacklist.txt
|
||||
url.query-blacklist: http://filebot.net/data/query-blacklist.txt
|
||||
|
||||
# clutter files that will be ignored
|
||||
url.exclude-blacklist: http://www.filebot.net/data/exclude-blacklist.txt
|
||||
url.exclude-blacklist: http://filebot.net/data/exclude-blacklist.txt
|
||||
|
||||
# list of patterns directly matching files to series names
|
||||
url.series-mappings: http://www.filebot.net/data/series-mappings.txt
|
||||
url.series-mappings: http://filebot.net/data/series-mappings.txt
|
||||
|
||||
# list of all movies (id, name, year)
|
||||
url.movie-list: http://www.filebot.net/data/movies.txt.gz
|
||||
url.movie-list: http://filebot.net/data/movies.txt.gz
|
||||
|
||||
# list of tv show and anime names
|
||||
url.series-list: http://www.filebot.net/data/series.list.gz
|
||||
url.series-list: http://filebot.net/data/series.list.gz
|
||||
|
||||
# TheTVDB index
|
||||
url.thetvdb-index: http://www.filebot.net/data/thetvdb.txt.gz
|
||||
url.thetvdb-index: http://filebot.net/data/thetvdb.txt.gz
|
||||
|
||||
# AniDB index
|
||||
url.anidb-index: http://filebot.net/data/anidb.txt.gz
|
||||
|
||||
# disk folder matcher
|
||||
pattern.diskfolder.entry: BDMV|HVDVD_TS|VIDEO_TS|AUDIO_TS|VCD|movie.nfo
|
||||
|
|
|
|||
|
|
@ -70,7 +70,7 @@ public class AnidbClient extends AbstractEpisodeListProvider {
|
|||
|
||||
@Override
|
||||
public ResultCache getCache() {
|
||||
return new ResultCache(host, Cache.getCache("web-datasource"));
|
||||
return new ResultCache(host, Cache.getCache("web-datasource-lv2"));
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -87,7 +87,7 @@ public class AnidbClient extends AbstractEpisodeListProvider {
|
|||
|
||||
@Override
|
||||
protected Set<String> getFields(AnidbSearchResult anime) {
|
||||
return set(anime.getPrimaryTitle(), anime.getOfficialTitle("en"), anime.getOfficialTitle(locale.getLanguage()));
|
||||
return set(anime.getPrimaryTitle(), anime.getOfficialTitle("en"));
|
||||
}
|
||||
};
|
||||
|
||||
|
|
@ -154,7 +154,7 @@ public class AnidbClient extends AbstractEpisodeListProvider {
|
|||
|
||||
|
||||
public synchronized List<AnidbSearchResult> getAnimeTitles() throws Exception {
|
||||
URL url = new URL("http", host, "/api/animetitles.dat.gz");
|
||||
URL url = new URL("http", host, "/api/anime-titles.dat.gz");
|
||||
ResultCache cache = getCache();
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
|
|
|
|||
|
|
@ -3,3 +3,4 @@ options +indexes
|
|||
redirect 301 /data/movies.txt.gz http://sourceforge.net/projects/filebot/files/data/movies.txt.gz/download
|
||||
redirect 301 /data/series.list.gz http://sourceforge.net/projects/filebot/files/data/series.list.gz/download
|
||||
redirect 301 /data/thetvdb.txt.gz http://sourceforge.net/projects/filebot/files/data/thetvdb.txt.gz/download
|
||||
redirect 301 /data/anidb.txt.gz http://sourceforge.net/projects/filebot/files/data/anidb.txt.gz/download
|
||||
|
|
|
|||
Loading…
Reference in New Issue