Experiment with new CachedResource framework

source/net/filebot/Cache.java

@@ -2,6 +2,7 @@ package net.filebot;
 
 import static java.nio.charset.StandardCharsets.*;
 import static net.filebot.Logging.*;
+import static net.filebot.web.CachedResource2.*;
 
 import java.io.Serializable;
 import java.net.URL;
@@ -10,9 +11,13 @@ import java.util.Arrays;
 import java.util.function.Predicate;
 
 import net.filebot.web.CachedResource2;
+import net.filebot.web.CachedResource2.Source;
 import net.filebot.web.FloodLimit;
+import net.filebot.web.Resource;
 import net.sf.ehcache.Element;
 
+import org.w3c.dom.Document;
+
 public class Cache {
 
 	public static Cache getCache(String name, CacheType type) {
@@ -131,8 +136,12 @@ public class Cache {
 		}
 	}
 
-	public CachedResource2<String, String> resource(String url, Duration expirationTime, FloodLimit limit) {
+	public Resource<Document> xml(String key, Source<String> source, Duration expirationTime) {
-		return new CachedResource2<String, String>(url, URL::new, CachedResource2.fetchIfModified(limit), CachedResource2.decode(UTF_8), expirationTime, this);
+		return new CachedResource2<String, Document>(key, source, fetchIfModified(), validateXml(getText(UTF_8)), getXml(String.class::cast), DEFAULT_RETRY_LIMIT, DEFAULT_RETRY_DELAY, expirationTime, this);
+	}
+
+	public Resource<String> resource(String url, Duration expirationTime, FloodLimit limit) {
+		return new CachedResource2<String, String>(url, URL::new, withPermit(fetchIfModified(), r -> limit.acquirePermit() != null), getText(UTF_8), String.class::cast, DEFAULT_RETRY_LIMIT, DEFAULT_RETRY_DELAY, expirationTime, this);
 	}
 
 }

source/net/filebot/web/CachedResource2.java

@@ -12,7 +12,9 @@ import java.util.concurrent.Callable;
 
 import net.filebot.Cache;
 
-public class CachedResource2<K, R> {
+import org.w3c.dom.Document;
+
+public class CachedResource2<K, R> implements Resource<R> {
 
 	public static final int DEFAULT_RETRY_LIMIT = 2;
 	public static final Duration DEFAULT_RETRY_DELAY = Duration.ofSeconds(2);
@@ -21,7 +23,8 @@ public class CachedResource2<K, R> {
 
 	protected final Source<K> source;
 	protected final Fetch fetch;
-	protected final Parse<R> parse;
+	protected final Transform<ByteBuffer, ? extends Object> parse;
+	protected final Transform<? super Object, R> cast;
 
 	protected final Duration expirationTime;
 
@@ -30,24 +33,21 @@ public class CachedResource2<K, R> {
 
 	protected final Cache cache;
 
-	public CachedResource2(K key, Source<K> source, Fetch fetch, Parse<R> parse, Duration expirationTime, Cache cache) {
+	public CachedResource2(K key, Source<K> source, Fetch fetch, Transform<ByteBuffer, ? extends Object> parse, Transform<? super Object, R> cast, int retryCountLimit, Duration retryWaitTime, Duration expirationTime, Cache cache) {
-		this(key, source, fetch, parse, DEFAULT_RETRY_LIMIT, DEFAULT_RETRY_DELAY, expirationTime, cache);
-	}
-
-	public CachedResource2(K key, Source<K> source, Fetch fetch, Parse<R> parse, int retryCountLimit, Duration retryWaitTime, Duration expirationTime, Cache cache) {
 		this.key = key;
 		this.source = source;
 		this.fetch = fetch;
 		this.parse = parse;
+		this.cast = cast;
 		this.expirationTime = expirationTime;
 		this.retryCountLimit = retryCountLimit;
 		this.retryWaitTime = retryWaitTime.toMillis();
 		this.cache = cache;
 	}
 
-	@SuppressWarnings("unchecked")
+	@Override
 	public synchronized R get() throws Exception {
-		return (R) cache.computeIfStale(key, expirationTime, element -> {
+		Object value = cache.computeIfStale(key, expirationTime, element -> {
 			URL resource = source.source(key);
 			long lastModified = element == null ? 0 : element.getLatestOfCreationAndUpdateTime();
 
@@ -61,7 +61,7 @@ public class CachedResource2<K, R> {
 					return element.getObjectValue();
 				}
 
-				return parse.parse(data);
+				return parse.transform(data);
 			} catch (IOException e) {
 				debug.fine(format("Fetch failed => %s", e));
 
@@ -72,6 +72,8 @@ public class CachedResource2<K, R> {
 				return element.getObjectKey();
 			}
 		});
+
+		return cast.transform(value);
 	}
 
 	protected <T> T retry(Callable<T> callable, int retryCount, long retryWaitTime) throws Exception {
@@ -101,23 +103,51 @@ public class CachedResource2<K, R> {
 	}
 
 	@FunctionalInterface
-	public interface Parse<R> {
+	public interface Transform<T, R> {
-		R parse(ByteBuffer bytes) throws Exception;
+		R transform(T object) throws Exception;
 	}
 
-	public static Parse<String> decode(Charset charset) {
+	@FunctionalInterface
-		return (bb) -> charset.decode(bb).toString();
+	public interface Permit<P> {
+		boolean acquirePermit(URL resource) throws Exception;
 	}
 
-	public static Fetch fetchIfModified(FloodLimit limit) {
+	public static Transform<ByteBuffer, String> getText(Charset charset) {
+		return (data) -> charset.decode(data).toString();
+	}
+
+	public static <T> Transform<T, String> validateXml(Transform<T, String> parse) {
+		return (object) -> {
+			String xml = parse.transform(object);
+			WebRequest.validateXml(xml);
+			return xml;
+		};
+	}
+
+	public static <T> Transform<T, Document> getXml(Transform<T, String> parse) {
+		return (object) -> {
+			return WebRequest.getDocument(parse.transform(object));
+		};
+	}
+
+	public static Fetch fetchIfModified() {
 		return (url, lastModified) -> {
 			try {
-				limit.acquirePermit();
 				return WebRequest.fetchIfModified(url, lastModified);
 			} catch (FileNotFoundException e) {
+				debug.warning(format("Resource not found: %s => %s", url, e));
 				return ByteBuffer.allocate(0);
 			}
 		};
 	}
 
+	public static Fetch withPermit(Fetch fetch, Permit<?> permit) {
+		return (url, lastModified) -> {
+			if (permit.acquirePermit(url)) {
+				return fetch.fetch(url, lastModified);
+			}
+			return null;
+		};
+	}
+
 }

source/net/filebot/web/Resource.java

@@ -0,0 +1,8 @@
+package net.filebot.web;
+
+@FunctionalInterface
+public interface Resource<R> {
+
+	R get() throws Exception;
+
+}

source/net/filebot/web/TheTVDBClient.java

@@ -2,17 +2,17 @@ package net.filebot.web;
 
 import static java.util.Arrays.*;
 import static java.util.Collections.*;
+import static java.util.stream.Collectors.*;
 import static net.filebot.util.StringUtilities.*;
 import static net.filebot.util.XPathUtilities.*;
 import static net.filebot.web.EpisodeUtilities.*;
 import static net.filebot.web.WebRequest.*;
 
-import java.io.FileNotFoundException;
-import java.io.IOException;
 import java.io.Serializable;
 import java.net.MalformedURLException;
 import java.net.URI;
 import java.net.URL;
+import java.time.Duration;
 import java.util.ArrayList;
 import java.util.EnumMap;
 import java.util.EnumSet;
@@ -40,7 +40,7 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
 
 	private final String host = "www.thetvdb.com";
 
-	private final Map<MirrorType, String> mirrors = new EnumMap<MirrorType, String>(MirrorType.class);
+	private final Map<MirrorType, String> mirrors = MirrorType.newMap();
 
 	private final String apikey;
 
@@ -106,7 +106,7 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
 	@Override
 	public List<SearchResult> fetchSearchResult(String query, Locale locale) throws Exception {
 		// perform online search
-		Document dom = getXmlResource(MirrorType.SEARCH, "/api/GetSeries.php?seriesname=" + encode(query, true) + "&language=" + getLanguageCode(locale));
+		Document dom = getXmlResource(MirrorType.SEARCH, "GetSeries.php?seriesname=" + encode(query, true) + "&language=" + getLanguageCode(locale));
 
 		List<Node> nodes = selectNodes("Data/Series", dom);
 		Map<Integer, TheTVDBSearchResult> resultSet = new LinkedHashMap<Integer, TheTVDBSearchResult>();
@@ -140,7 +140,7 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
 	@Override
 	protected SeriesData fetchSeriesData(SearchResult searchResult, SortOrder sortOrder, Locale locale) throws Exception {
 		TheTVDBSearchResult series = (TheTVDBSearchResult) searchResult;
-		Document dom = getXmlResource(MirrorType.XML, "/api/" + apikey + "/series/" + series.getSeriesId() + "/all/" + getLanguageCode(locale) + ".xml");
+		Document dom = getXmlResource(MirrorType.XML, "series/" + series.getSeriesId() + "/all/" + getLanguageCode(locale) + ".xml");
 
 		// parse series info
 		Node seriesNode = selectNode("Data/Series", dom);
@@ -163,9 +163,9 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
 		seriesInfo.setGenres(getListContent("Genre", "\\|", seriesNode));
 		seriesInfo.setStartDate(SimpleDate.parse(getTextContent("FirstAired", seriesNode)));
 
-		seriesInfo.setBannerUrl(getResourceURL(MirrorType.BANNER, "/banners/" + getTextContent("banner", seriesNode)));
+		seriesInfo.setBannerUrl(getResource(MirrorType.BANNER, getTextContent("banner", seriesNode)));
-		seriesInfo.setFanartUrl(getResourceURL(MirrorType.BANNER, "/banners/" + getTextContent("fanart", seriesNode)));
+		seriesInfo.setFanartUrl(getResource(MirrorType.BANNER, getTextContent("fanart", seriesNode)));
-		seriesInfo.setPosterUrl(getResourceURL(MirrorType.BANNER, "/banners/" + getTextContent("poster", seriesNode)));
+		seriesInfo.setPosterUrl(getResource(MirrorType.BANNER, getTextContent("poster", seriesNode)));
 
 		// parse episode data
 		List<Node> nodes = selectNodes("Data/Episode", dom);
@@ -237,7 +237,7 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
 			return cachedItem;
 		}
 
-		Document dom = getXmlResource(MirrorType.XML, "/api/" + apikey + "/series/" + id + "/all/" + getLanguageCode(locale) + ".xml");
+		Document dom = getXmlResource(MirrorType.XML, "series/" + id + "/all/" + getLanguageCode(locale) + ".xml");
 		String name = selectString("//SeriesName", dom);
 
 		TheTVDBSearchResult series = new TheTVDBSearchResult(name, id);
@@ -255,7 +255,7 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
 			return cachedItem;
 		}
 
-		Document dom = getXmlResource(null, "/api/GetSeriesByRemoteID.php?imdbid=" + imdbid + "&language=" + getLanguageCode(locale));
+		Document dom = getXmlResource(MirrorType.SEARCH, "GetSeriesByRemoteID.php?imdbid=" + imdbid + "&language=" + getLanguageCode(locale));
 
 		String id = selectString("//seriesid", dom);
 		String name = selectString("//SeriesName", dom);
@@ -268,113 +268,87 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
 		return series;
 	}
 
-	protected String getMirror(MirrorType mirrorType) throws IOException {
+	protected String getMirror(MirrorType mirrorType) throws Exception {
+		// use default server
+		if (mirrorType == MirrorType.NULL) {
+			return "http://thetvdb.com";
+		}
+
 		synchronized (mirrors) {
+			// initialize mirrors
 			if (mirrors.isEmpty()) {
-				// try cache first
+				Document dom = getXmlResource(MirrorType.NULL, "mirrors.xml");
-				try {
-					@SuppressWarnings("unchecked")
-					Map<MirrorType, String> cachedMirrors = getCache().getData("mirrors", null, null, Map.class);
-					if (cachedMirrors != null) {
-						mirrors.putAll(cachedMirrors);
-						return mirrors.get(mirrorType);
-					}
-				} catch (Exception e) {
-					Logger.getLogger(getClass().getName()).log(Level.SEVERE, e.getMessage(), e);
-				}
 
-				// initialize mirrors
+				// collect all mirror data
-				Document dom = getXmlResource(null, "/api/" + apikey + "/mirrors.xml");
+				Map<MirrorType, List<String>> mirrorLists = selectNodes("Mirrors/Mirror", dom).stream().flatMap(node -> {
-
-				// all mirrors by type
-				Map<MirrorType, List<String>> mirrorListMap = new EnumMap<MirrorType, List<String>>(MirrorType.class);
-
-				// initialize mirror list per type
-				for (MirrorType type : MirrorType.values()) {
-					mirrorListMap.put(type, new ArrayList<String>(5));
-				}
-
-				// traverse all mirrors
-				for (Node node : selectNodes("Mirrors/Mirror", dom)) {
-					// mirror data
 					String mirror = getTextContent("mirrorpath", node);
 					int typeMask = Integer.parseInt(getTextContent("typemask", node));
 
-					// add mirror to the according type lists
+					return MirrorType.fromTypeMask(typeMask).stream().collect(toMap(m -> m, m -> mirror)).entrySet().stream();
-					for (MirrorType type : MirrorType.fromTypeMask(typeMask)) {
+				}).collect(groupingBy(Entry::getKey, MirrorType::newMap, mapping(Entry::getValue, toList())));
-						mirrorListMap.get(type).add(mirror);
-					}
-				}
 
-				// put random entry from each type list into mirrors
+				// select random mirror for each type
 				Random random = new Random();
 
-				for (MirrorType type : MirrorType.values()) {
+				mirrorLists.forEach((type, options) -> {
-					List<String> list = mirrorListMap.get(type);
+					String selection = options.get(random.nextInt(options.size()));
-
+					mirrors.put(type, selection);
-					if (!list.isEmpty()) {
+				});
-						mirrors.put(type, list.get(random.nextInt(list.size())));
-					}
-				}
-
-				getCache().putData("mirrors", null, null, mirrors);
 			}
 
+			// return selected mirror
 			return mirrors.get(mirrorType);
 		}
 	}
 
-	protected Document getXmlResource(final MirrorType mirrorType, final String path) throws IOException {
+	protected Document getXmlResource(MirrorType mirror, String path) throws Exception {
-		CachedXmlResource resource = new CachedXmlResource(path) {
+		Cache cache = Cache.getCache(getName(), CacheType.Monthly);
-
+		Duration expirationTime = Duration.ofDays(1);
-			@Override
+		Resource<Document> xml = cache.xml(path, s -> getResource(mirror, s), expirationTime);
-			protected URL getResourceLocation(String path) throws IOException {
+		return xml.get();
-				return getResourceURL(mirrorType, path);
-			};
-		};
-
-		// fetch data or retrieve from cache
-		try {
-			return resource.getDocument();
-		} catch (FileNotFoundException e) {
-			throw new FileNotFoundException("Resource not found: " + getResourceURL(mirrorType, path)); // simplify error message
-		}
 	}
 
-	protected URL getResourceURL(MirrorType mirrorType, String path) throws IOException {
+	protected URL getResource(MirrorType mirror, String path) throws Exception {
-		if (mirrorType != null) {
+		StringBuilder url = new StringBuilder(getMirror(mirror)).append('/').append(mirror.prefix()).append('/');
-			// use mirror
+		if (mirror.keyRequired()) {
-			String mirror = getMirror(mirrorType);
+			url.append(apikey).append('/');
-			if (mirror != null && mirror.length() > 0) {
-				return new URL(mirror + path);
-			}
 		}
-
+		return new URL(url.append(path).toString());
-		// use default server
-		return new URL("http", "thetvdb.com", path);
 	}
 
 	protected static enum MirrorType {
-		XML(1), BANNER(2), ZIP(4), SEARCH(1);
 
-		private final int bitMask;
+		NULL(0), SEARCH(1), XML(1), BANNER(2);
+
+		final int bitMask;
 
 		private MirrorType(int bitMask) {
 			this.bitMask = bitMask;
 		}
 
-		public static EnumSet<MirrorType> fromTypeMask(int typeMask) {
+		public String prefix() {
-			// initialize enum set with all types
+			return this != BANNER ? "api" : "banners";
-			EnumSet<MirrorType> enumSet = EnumSet.allOf(MirrorType.class);
+		}
-			for (MirrorType type : values()) {
+
-				if ((typeMask & type.bitMask) == 0) {
+		public boolean keyRequired() {
-					// remove types that are not set
+			return this != BANNER && this != SEARCH;
-					enumSet.remove(type);
+		}
-				}
+
-			}
+		public static EnumSet<MirrorType> fromTypeMask(int mask) {
-			return enumSet;
+			// convert bit mask to enumset
+			return EnumSet.of(SEARCH, XML, BANNER).stream().filter(m -> {
+				return (mask & m.bitMask) != 0;
+			}).collect(toCollection(MirrorType::newSet));
 		};
 
+		public static EnumSet<MirrorType> newSet() {
+			return EnumSet.noneOf(MirrorType.class);
+		}
+
+		public static <T> EnumMap<MirrorType, T> newMap() {
+			return new EnumMap<MirrorType, T>(MirrorType.class);
+		}
+
 	}
 
 	public SeriesInfo getSeriesInfoByIMDbID(int imdbid, Locale locale) throws Exception {
@@ -421,7 +395,7 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
 			return asList(cachedList);
 		}
 
-		Document dom = getXmlResource(MirrorType.XML, "/api/" + apikey + "/series/" + series.getId() + "/banners.xml");
+		Document dom = getXmlResource(MirrorType.XML, "series/" + series.getId() + "/banners.xml");
 
 		List<Node> nodes = selectNodes("//Banner", dom);
 		List<BannerDescriptor> banners = new ArrayList<BannerDescriptor>();
@@ -431,7 +405,7 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
 				Map<BannerProperty, String> item = new EnumMap<BannerProperty, String>(BannerProperty.class);
 
 				// insert banner mirror
-				item.put(BannerProperty.BannerMirror, getResourceURL(MirrorType.BANNER, "/banners/").toString());
+				item.put(BannerProperty.BannerMirror, getResource(MirrorType.BANNER, "").toString());
 
 				// copy values from xml
 				for (BannerProperty key : BannerProperty.values()) {

source/net/filebot/web/WebRequest.java

@@ -31,6 +31,7 @@ import java.util.zip.InflaterInputStream;
 
 import javax.xml.parsers.DocumentBuilderFactory;
 import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.parsers.SAXParserFactory;
 import javax.xml.transform.OutputKeys;
 import javax.xml.transform.Transformer;
 import javax.xml.transform.TransformerException;
@@ -43,6 +44,8 @@ import net.filebot.util.ByteBufferOutputStream;
 import org.w3c.dom.Document;
 import org.xml.sax.InputSource;
 import org.xml.sax.SAXException;
+import org.xml.sax.XMLReader;
+import org.xml.sax.helpers.DefaultHandler;
 
 public final class WebRequest {
 
@@ -150,8 +153,9 @@ public final class WebRequest {
 		}
 
 		// no data, e.g. If-Modified-Since requests
-		if (contentLength < 0 && buffer.getByteBuffer().remaining() == 0)
+		if (contentLength < 0 && buffer.getByteBuffer().remaining() == 0) {
 			return null;
+		}
 
 		return buffer.getByteBuffer();
 	}
@@ -285,10 +289,20 @@ public final class WebRequest {
 		return buffer.toString();
 	}
 
-	/**
+	public static void validateXml(String xml) throws SAXException, ParserConfigurationException, IOException {
-	 * Dummy constructor to prevent instantiation.
+		SAXParserFactory sax = SAXParserFactory.newInstance();
-	 */
+		sax.setValidating(false);
+		sax.setNamespaceAware(false);
+
+		XMLReader reader = sax.newSAXParser().getXMLReader();
+
+		// throw exception on error
+		reader.setErrorHandler(new DefaultHandler());
+		reader.parse(new InputSource(new StringReader(xml)));
+	}
+
 	private WebRequest() {
 		throw new UnsupportedOperationException();
 	}
+
 }

test/net/filebot/web/TheTVDBClientTest.java

@@ -103,23 +103,13 @@ public class TheTVDBClientTest {
 		assertEquals("http://www.thetvdb.com/?tab=seasonall&id=78874", thetvdb.getEpisodeListLink(new TheTVDBSearchResult("Firefly", 78874)).toString());
 	}
 
-	@Test
-	public void getMirror() throws Exception {
-		assertNotNull(thetvdb.getMirror(MirrorType.XML));
-		assertNotNull(thetvdb.getMirror(MirrorType.BANNER));
-		assertNotNull(thetvdb.getMirror(MirrorType.ZIP));
-	}
-
 	@Test
 	public void resolveTypeMask() {
 		// no flags set
-		assertEquals(EnumSet.noneOf(MirrorType.class), MirrorType.fromTypeMask(0));
+		assertEquals(MirrorType.newSet(), MirrorType.fromTypeMask(0));
-
-		// xml and zip flags set
-		assertEquals(EnumSet.of(MirrorType.ZIP, MirrorType.XML, MirrorType.SEARCH), MirrorType.fromTypeMask(5));
 
 		// all flags set
-		assertEquals(EnumSet.allOf(MirrorType.class), MirrorType.fromTypeMask(7));
+		assertEquals(EnumSet.of(MirrorType.SEARCH, MirrorType.XML, MirrorType.BANNER), MirrorType.fromTypeMask(7));
 	}
 
 	@Test
@@ -145,9 +135,9 @@ public class TheTVDBClientTest {
 		assertEquals("2007-09-24", it.getFirstAired().toString());
 		assertEquals("Action", it.getGenres().get(0));
 		assertEquals("tt0934814", it.getImdbId());
-		assertEquals("English", it.getLanguage());
+		assertEquals("en", it.getLanguage());
-		assertEquals(310, it.getOverview().length());
+		assertEquals(987, it.getOverview().length());
-		assertEquals("60", it.getRuntime());
+		assertEquals("45", it.getRuntime().toString());
 		assertEquals("Chuck", it.getName());
 	}
 
@@ -174,7 +164,7 @@ public class TheTVDBClientTest {
 
 		assertEquals("fanart", banners.get(0).getBannerType());
 		assertEquals("1280x720", banners.get(0).getBannerType2());
-		assertEquals(486993, WebRequest.fetch(banners.get(0).getUrl()).remaining(), 0);
+		assertEquals(460058, WebRequest.fetch(banners.get(0).getUrl()).remaining());
 	}
 
 }