* optimize TheTVDB network requests (and make sure gzipped compression is used at all times)
This commit is contained in:
Reinhard Pointner
parent
1145a1f16b
commit
f6e4f1bb8f
|
|
@ -1,5 +1,6 @@
|
|||
package net.sourceforge.filebot.web;
|
||||
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.io.Serializable;
|
||||
import java.net.URL;
|
||||
|
|
@ -124,6 +125,9 @@ public abstract class AbstractCachedResource<R, T extends Serializable> {
|
|||
Thread.sleep(retryWaitTime);
|
||||
}
|
||||
return fetchData(url, lastModified);
|
||||
} catch (FileNotFoundException e) {
|
||||
// if the resource doesn't exist no need for retries
|
||||
throw e;
|
||||
} catch (IOException e) {
|
||||
if (i >= 0 && i >= retries) {
|
||||
throw e;
|
||||
|
|
|
|||
|
|
@ -3,17 +3,14 @@ package net.sourceforge.filebot.web;
|
|||
import static java.util.Arrays.*;
|
||||
import static net.sourceforge.filebot.web.EpisodeUtilities.*;
|
||||
import static net.sourceforge.filebot.web.WebRequest.*;
|
||||
import static net.sourceforge.tuned.FileUtilities.*;
|
||||
import static net.sourceforge.tuned.XPathUtilities.*;
|
||||
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStreamReader;
|
||||
import java.io.Serializable;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URI;
|
||||
import java.net.URL;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.ArrayList;
|
||||
import java.util.EnumMap;
|
||||
import java.util.EnumSet;
|
||||
|
|
@ -25,8 +22,6 @@ import java.util.Map.Entry;
|
|||
import java.util.Random;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipInputStream;
|
||||
|
||||
import javax.swing.Icon;
|
||||
|
||||
|
|
@ -34,7 +29,6 @@ import net.sourceforge.filebot.Cache;
|
|||
import net.sourceforge.filebot.ResourceManager;
|
||||
import net.sourceforge.filebot.web.TheTVDBClient.BannerDescriptor.BannerProperty;
|
||||
import net.sourceforge.filebot.web.TheTVDBClient.SeriesInfo.SeriesProperty;
|
||||
import net.sourceforge.tuned.ByteBufferInputStream;
|
||||
import net.sourceforge.tuned.FileUtilities;
|
||||
|
||||
import org.w3c.dom.Document;
|
||||
|
|
@ -124,13 +118,13 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
|
|||
@Override
|
||||
public List<Episode> fetchEpisodeList(SearchResult searchResult, SortOrder sortOrder, Locale locale) throws Exception {
|
||||
TheTVDBSearchResult series = (TheTVDBSearchResult) searchResult;
|
||||
Document seriesRecord = getSeriesRecord(series, getLanguageCode(locale));
|
||||
Document dom = getXmlResource(MirrorType.XML, "/api/" + apikey + "/series/" + series.getSeriesId() + "/all/" + locale.getLanguage() + ".xml");
|
||||
|
||||
// we could get the series name from the search result, but the language may not match the given parameter
|
||||
String seriesName = selectString("Data/Series/SeriesName", seriesRecord);
|
||||
Date seriesStartDate = Date.parse(selectString("Data/Series/FirstAired", seriesRecord), "yyyy-MM-dd");
|
||||
String seriesName = selectString("Data/Series/SeriesName", dom);
|
||||
Date seriesStartDate = Date.parse(selectString("Data/Series/FirstAired", dom), "yyyy-MM-dd");
|
||||
|
||||
List<Node> nodes = selectNodes("Data/Episode", seriesRecord);
|
||||
List<Node> nodes = selectNodes("Data/Episode", dom);
|
||||
|
||||
List<Episode> episodes = new ArrayList<Episode>(nodes.size());
|
||||
List<Episode> specials = new ArrayList<Episode>(5);
|
||||
|
|
@ -185,54 +179,6 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
|
|||
return episodes;
|
||||
}
|
||||
|
||||
public Document getSeriesRecord(final TheTVDBSearchResult searchResult, final String languageCode) throws Exception {
|
||||
final String path = "/api/" + apikey + "/series/" + searchResult.getSeriesId() + "/all/" + languageCode + ".zip";
|
||||
final MirrorType mirror = MirrorType.ZIP;
|
||||
|
||||
CachedXmlResource record = new CachedXmlResource(path) {
|
||||
@Override
|
||||
protected URL getResourceLocation(String resource) throws IOException {
|
||||
return getResourceURL(mirror, path);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String fetchData(URL url, long lastModified) throws IOException {
|
||||
try {
|
||||
ByteBuffer data = WebRequest.fetchIfModified(url, lastModified);
|
||||
if (data == null)
|
||||
return null; // not modified
|
||||
|
||||
ZipInputStream zipInputStream = new ZipInputStream(new ByteBufferInputStream(data));
|
||||
ZipEntry zipEntry;
|
||||
|
||||
try {
|
||||
String seriesRecordName = languageCode + ".xml";
|
||||
|
||||
while ((zipEntry = zipInputStream.getNextEntry()) != null) {
|
||||
if (seriesRecordName.equals(zipEntry.getName())) {
|
||||
return readAll(new InputStreamReader(zipInputStream, "UTF-8"));
|
||||
}
|
||||
}
|
||||
|
||||
// zip file must contain the series record
|
||||
throw new FileNotFoundException(String.format("Archive must contain %s: %s", seriesRecordName, getResourceURL(mirror, path)));
|
||||
} finally {
|
||||
zipInputStream.close();
|
||||
}
|
||||
} catch (FileNotFoundException e) {
|
||||
throw new FileNotFoundException(String.format("Series record not found: %s [%s]: %s", searchResult.getName(), languageCode, getResourceURL(mirror, path)));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String process(String data) throws Exception {
|
||||
return data;
|
||||
}
|
||||
};
|
||||
|
||||
return record.getDocument();
|
||||
}
|
||||
|
||||
public TheTVDBSearchResult lookupByID(int id, Locale locale) throws Exception {
|
||||
TheTVDBSearchResult cachedItem = getCache().getData("lookupByID", id, locale, TheTVDBSearchResult.class);
|
||||
if (cachedItem != null) {
|
||||
|
|
@ -342,7 +288,11 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
|
|||
};
|
||||
|
||||
// fetch data or retrieve from cache
|
||||
try {
|
||||
return resource.getDocument();
|
||||
} catch (FileNotFoundException e) {
|
||||
throw new FileNotFoundException("Resource not found: " + getResourceURL(mirrorType, path)); // simplify error message
|
||||
}
|
||||
}
|
||||
|
||||
protected URL getResourceURL(MirrorType mirrorType, String path) throws IOException {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,5 @@
|
|||
|
||||
package net.sourceforge.filebot.web;
|
||||
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
|
|
@ -42,19 +40,16 @@ import org.w3c.dom.Document;
|
|||
import org.xml.sax.InputSource;
|
||||
import org.xml.sax.SAXException;
|
||||
|
||||
|
||||
public final class WebRequest {
|
||||
|
||||
public static Document getHtmlDocument(URL url) throws IOException, SAXException {
|
||||
return getHtmlDocument(url.openConnection());
|
||||
}
|
||||
|
||||
|
||||
public static Document getHtmlDocument(URLConnection connection) throws IOException, SAXException {
|
||||
return getHtmlDocument(getReader(connection));
|
||||
}
|
||||
|
||||
|
||||
public static Reader getReader(URLConnection connection) throws IOException {
|
||||
try {
|
||||
connection.addRequestProperty("Accept-Encoding", "gzip,deflate");
|
||||
|
|
@ -77,7 +72,6 @@ public final class WebRequest {
|
|||
return new InputStreamReader(inputStream, charset);
|
||||
}
|
||||
|
||||
|
||||
public static Document getHtmlDocument(Reader reader) throws SAXException, IOException {
|
||||
DOMParser parser = new DOMParser();
|
||||
parser.setFeature("http://xml.org/sax/features/namespaces", false);
|
||||
|
|
@ -86,27 +80,22 @@ public final class WebRequest {
|
|||
return parser.getDocument();
|
||||
}
|
||||
|
||||
|
||||
public static Document getHtmlDocument(String html) throws SAXException, IOException {
|
||||
return getHtmlDocument(new StringReader(html));
|
||||
}
|
||||
|
||||
|
||||
public static Document getDocument(URL url) throws IOException, SAXException {
|
||||
return getDocument(url.openConnection());
|
||||
}
|
||||
|
||||
|
||||
public static Document getDocument(URLConnection connection) throws IOException, SAXException {
|
||||
return getDocument(new InputSource(getReader(connection)));
|
||||
}
|
||||
|
||||
|
||||
public static Document getDocument(String xml) throws IOException, SAXException {
|
||||
return getDocument(new InputSource(new StringReader(xml)));
|
||||
}
|
||||
|
||||
|
||||
public static Document getDocument(InputSource source) throws IOException, SAXException {
|
||||
try {
|
||||
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
|
||||
|
|
@ -120,23 +109,27 @@ public final class WebRequest {
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
public static ByteBuffer fetch(URL resource) throws IOException {
|
||||
return fetch(resource, 0, null);
|
||||
}
|
||||
|
||||
|
||||
public static ByteBuffer fetchIfModified(URL resource, long ifModifiedSince) throws IOException {
|
||||
return fetch(resource, ifModifiedSince, null);
|
||||
}
|
||||
|
||||
|
||||
public static ByteBuffer fetch(URL url, long ifModifiedSince, Map<String, String> requestParameters) throws IOException {
|
||||
URLConnection connection = url.openConnection();
|
||||
if (ifModifiedSince > 0) {
|
||||
connection.setIfModifiedSince(ifModifiedSince);
|
||||
}
|
||||
|
||||
try {
|
||||
connection.addRequestProperty("Accept-Encoding", "gzip");
|
||||
connection.addRequestProperty("Accept-Charset", "UTF-8");
|
||||
} catch (IllegalStateException e) {
|
||||
// too bad, can't request gzipped data
|
||||
}
|
||||
|
||||
if (requestParameters != null) {
|
||||
for (Entry<String, String> parameter : requestParameters.entrySet()) {
|
||||
connection.addRequestProperty(parameter.getKey(), parameter.getValue());
|
||||
|
|
@ -144,10 +137,14 @@ public final class WebRequest {
|
|||
}
|
||||
|
||||
int contentLength = connection.getContentLength();
|
||||
String encoding = connection.getContentEncoding();
|
||||
|
||||
InputStream in = connection.getInputStream();
|
||||
ByteBufferOutputStream buffer = new ByteBufferOutputStream(contentLength >= 0 ? contentLength : 4 * 1024);
|
||||
if ("gzip".equalsIgnoreCase(encoding)) {
|
||||
in = new GZIPInputStream(in);
|
||||
}
|
||||
|
||||
ByteBufferOutputStream buffer = new ByteBufferOutputStream(contentLength >= 0 ? contentLength : 4 * 1024);
|
||||
try {
|
||||
// read all
|
||||
buffer.transferFully(in);
|
||||
|
|
@ -168,12 +165,10 @@ public final class WebRequest {
|
|||
return buffer.getByteBuffer();
|
||||
}
|
||||
|
||||
|
||||
public static ByteBuffer post(HttpURLConnection connection, Map<String, ?> parameters) throws IOException {
|
||||
return post(connection, encodeParameters(parameters, true).getBytes("UTF-8"), "application/x-www-form-urlencoded");
|
||||
}
|
||||
|
||||
|
||||
public static ByteBuffer post(HttpURLConnection connection, byte[] postData, String contentType) throws IOException {
|
||||
connection.addRequestProperty("Content-Length", String.valueOf(postData.length));
|
||||
connection.addRequestProperty("Content-Type", contentType);
|
||||
|
|
@ -213,7 +208,6 @@ public final class WebRequest {
|
|||
return buffer.getByteBuffer();
|
||||
}
|
||||
|
||||
|
||||
private static Charset getCharset(String contentType) {
|
||||
if (contentType != null) {
|
||||
// e.g. Content-Type: text/html; charset=iso-8859-1
|
||||
|
|
@ -237,7 +231,6 @@ public final class WebRequest {
|
|||
return Charset.forName("UTF-8");
|
||||
}
|
||||
|
||||
|
||||
public static String encodeParameters(Map<String, ?> parameters, boolean unicode) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
|
||||
|
|
@ -256,7 +249,6 @@ public final class WebRequest {
|
|||
return sb.toString();
|
||||
}
|
||||
|
||||
|
||||
public static String encode(String string, boolean unicode) {
|
||||
try {
|
||||
return URLEncoder.encode(string, unicode ? "UTF-8" : "ISO-8859-1");
|
||||
|
|
@ -265,7 +257,6 @@ public final class WebRequest {
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
public static SSLSocketFactory createIgnoreCertificateSocketFactory() {
|
||||
// create a trust manager that does not validate certificate chains
|
||||
TrustManager trustAnyCertificate = new X509TrustManager() {
|
||||
|
|
@ -275,12 +266,10 @@ public final class WebRequest {
|
|||
return null;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void checkClientTrusted(X509Certificate[] certs, String authType) {
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void checkServerTrusted(X509Certificate[] certs, String authType) {
|
||||
}
|
||||
|
|
@ -295,7 +284,6 @@ public final class WebRequest {
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Dummy constructor to prevent instantiation.
|
||||
*/
|
||||
|
|
|
|||
Loading…
Reference in New Issue