* fix cache misconfig to stop anidb abuse

* build our own anidb index file like with thetvdb

source/ehcache.xml

@@ -19,7 +19,7 @@
 	/>
 	
 	<!--
-	Long-lived (2 days) persistent disk cache for web responses
+	Short-lived (2 days) persistent disk cache for web responses
 	-->
 	<cache name="web-datasource"
 		maxElementsInMemory="200"
@@ -32,6 +32,20 @@
 		memoryStoreEvictionPolicy="LRU"
 	/>
 	
+	<!--
+	Long-lived (2 weeks) persistent disk cache for web responses
+	-->
+	<cache name="web-datasource-lv2"
+		maxElementsInMemory="200"
+		maxElementsOnDisk="10000"
+		eternal="false"
+		timeToIdleSeconds="1209600"
+		timeToLiveSeconds="1209600"
+		overflowToDisk="true"
+		diskPersistent="true"
+		memoryStoreEvictionPolicy="LRU"
+	/>
+	
 	<!--
 	Very long-lived cache (2 months) anime/series lists, movie index, etc
 	-->

source/net/sourceforge/filebot/WebServices.java

@@ -49,7 +49,7 @@ public final class WebServices {
 	
 	// episode dbs
 	public static final TVRageClient TVRage = new TVRageClient();
-	public static final AnidbClient AniDB = new AnidbClient(getApplicationName().toLowerCase(), 3);
+	public static final AnidbClient AniDB = new AnidbClientWithLocalSearch(getApplicationName().toLowerCase(), 4);
 	public static final SerienjunkiesClient Serienjunkies = new SerienjunkiesClient(getApplicationProperty("serienjunkies.apikey"));
 	
 	// extended TheTVDB module with local search
@@ -202,6 +202,20 @@ public final class WebServices {
 	}
 	
 	
+	public static class AnidbClientWithLocalSearch extends AnidbClient {
+		
+		public AnidbClientWithLocalSearch(String client, int clientver) {
+			super(client, clientver);
+		}
+		
+		
+		@Override
+		public List<AnidbSearchResult> getAnimeTitles() throws Exception {
+			return asList(MediaDetection.releaseInfo.getAnidbIndex());
+		}
+	}
+	
+	
 	/**
 	 * Dummy constructor to prevent instantiation.
 	 */

source/net/sourceforge/filebot/media/ReleaseInfo.java

@@ -35,6 +35,7 @@ import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.zip.GZIPInputStream;
 
+import net.sourceforge.filebot.web.AnidbClient.AnidbSearchResult;
 import net.sourceforge.filebot.web.CachedResource;
 import net.sourceforge.filebot.web.Movie;
 import net.sourceforge.filebot.web.TheTVDBClient.TheTVDBSearchResult;
@@ -243,7 +244,12 @@ public class ReleaseInfo {
 	
 	
 	public TheTVDBSearchResult[] getTheTVDBIndex() throws IOException {
-		return theTVDBIndexResource.get();
+		return tvdbIndexResource.get();
+	}
+	
+	
+	public AnidbSearchResult[] getAnidbIndex() throws IOException {
+		return anidbIndexResource.get();
 	}
 	
 	
@@ -275,7 +281,8 @@ public class ReleaseInfo {
 	protected final CachedResource<Movie[]> movieListResource = new MovieResource(getBundle(getClass().getName()).getString("url.movie-list"));
 	protected final CachedResource<String[]> seriesListResource = new SeriesListResource(getBundle(getClass().getName()).getString("url.series-list"));
 	protected final CachedResource<String[]> seriesDirectMappingsResource = new PatternResource(getBundle(getClass().getName()).getString("url.series-mappings"));
-	protected final CachedResource<TheTVDBSearchResult[]> theTVDBIndexResource = new TheTVDBIndexResource(getBundle(getClass().getName()).getString("url.thetvdb-index"));
+	protected final CachedResource<TheTVDBSearchResult[]> tvdbIndexResource = new TheTVDBIndexResource(getBundle(getClass().getName()).getString("url.thetvdb-index"));
+	protected final CachedResource<AnidbSearchResult[]> anidbIndexResource = new AnidbIndexResource(getBundle(getClass().getName()).getString("url.anidb-index"));
 	
 	
 	protected static class PatternResource extends CachedResource<String[]> {
@@ -353,6 +360,35 @@ public class ReleaseInfo {
 	}
 	
 	
+	protected static class AnidbIndexResource extends CachedResource<AnidbSearchResult[]> {
+		
+		public AnidbIndexResource(String resource) {
+			super(resource, AnidbSearchResult[].class, 7 * 24 * 60 * 60 * 1000); // check for updates once a week
+		}
+		
+		
+		@Override
+		public AnidbSearchResult[] process(ByteBuffer data) throws IOException {
+			Scanner scanner = new Scanner(new GZIPInputStream(new ByteBufferInputStream(data)), "UTF-8").useDelimiter("\t|\n");
+			
+			List<AnidbSearchResult> anime = new ArrayList<AnidbSearchResult>();
+			while (scanner.hasNext() && scanner.hasNextInt()) {
+				int aid = scanner.nextInt();
+				String primaryTitle = scanner.next().trim();
+				String englishTitle = scanner.next().trim();
+				
+				if (englishTitle.isEmpty() || englishTitle.equals(primaryTitle)) {
+					anime.add(new AnidbSearchResult(aid, primaryTitle, null));
+				} else {
+					anime.add(new AnidbSearchResult(aid, primaryTitle, singletonMap("en", englishTitle)));
+				}
+			}
+			
+			return anime.toArray(new AnidbSearchResult[0]);
+		}
+	}
+	
+	
 	protected static class FolderEntryFilter implements FileFilter {
 		
 		private final Pattern entryPattern;

source/net/sourceforge/filebot/media/ReleaseInfo.properties

@@ -5,25 +5,28 @@ pattern.video.source: CAMRip|CAM|PDVD|TS|TELESYNC|PDVD|PPV|PPVRip|Screener|SCR|S
 pattern.video.format: DivX|Xvid|AVC|x264|h264|3ivx|mpeg|mpeg4|mp3|aac|ac3|dd20|dd51|2ch|6ch|DTS|WS|HR|7p|720p|18p|1080p|NTSC
 
 # known release group names
-url.release-groups: http://www.filebot.net/data/release-groups.txt
+url.release-groups: http://filebot.net/data/release-groups.txt
 
 # blacklisted terms that will be ignored
-url.query-blacklist: http://www.filebot.net/data/query-blacklist.txt
+url.query-blacklist: http://filebot.net/data/query-blacklist.txt
 
 # clutter files that will be ignored 
-url.exclude-blacklist: http://www.filebot.net/data/exclude-blacklist.txt
+url.exclude-blacklist: http://filebot.net/data/exclude-blacklist.txt
 
 # list of patterns directly matching files to series names 
-url.series-mappings: http://www.filebot.net/data/series-mappings.txt
+url.series-mappings: http://filebot.net/data/series-mappings.txt
 
 # list of all movies (id, name, year)
-url.movie-list: http://www.filebot.net/data/movies.txt.gz
+url.movie-list: http://filebot.net/data/movies.txt.gz
 
 # list of tv show and anime names
-url.series-list: http://www.filebot.net/data/series.list.gz
+url.series-list: http://filebot.net/data/series.list.gz
 
 # TheTVDB index
-url.thetvdb-index: http://www.filebot.net/data/thetvdb.txt.gz
+url.thetvdb-index: http://filebot.net/data/thetvdb.txt.gz
+
+# AniDB index
+url.anidb-index: http://filebot.net/data/anidb.txt.gz
 
 # disk folder matcher
 pattern.diskfolder.entry: BDMV|HVDVD_TS|VIDEO_TS|AUDIO_TS|VCD|movie.nfo

source/net/sourceforge/filebot/web/AnidbClient.java

@@ -70,7 +70,7 @@ public class AnidbClient extends AbstractEpisodeListProvider {
 	
 	@Override
 	public ResultCache getCache() {
-		return new ResultCache(host, Cache.getCache("web-datasource"));
+		return new ResultCache(host, Cache.getCache("web-datasource-lv2"));
 	}
 	
 	
@@ -87,7 +87,7 @@ public class AnidbClient extends AbstractEpisodeListProvider {
 			
 			@Override
 			protected Set<String> getFields(AnidbSearchResult anime) {
-				return set(anime.getPrimaryTitle(), anime.getOfficialTitle("en"), anime.getOfficialTitle(locale.getLanguage()));
+				return set(anime.getPrimaryTitle(), anime.getOfficialTitle("en"));
 			}
 		};
 		
@@ -154,7 +154,7 @@ public class AnidbClient extends AbstractEpisodeListProvider {
 	
 	
 	public synchronized List<AnidbSearchResult> getAnimeTitles() throws Exception {
-		URL url = new URL("http", host, "/api/animetitles.dat.gz");
+		URL url = new URL("http", host, "/api/anime-titles.dat.gz");
 		ResultCache cache = getCache();
 		
 		@SuppressWarnings("unchecked")

website/data/.htaccess

@@ -3,3 +3,4 @@ options +indexes
 redirect 301 /data/movies.txt.gz http://sourceforge.net/projects/filebot/files/data/movies.txt.gz/download
 redirect 301 /data/series.list.gz http://sourceforge.net/projects/filebot/files/data/series.list.gz/download
 redirect 301 /data/thetvdb.txt.gz http://sourceforge.net/projects/filebot/files/data/thetvdb.txt.gz/download
+redirect 301 /data/anidb.txt.gz http://sourceforge.net/projects/filebot/files/data/anidb.txt.gz/download