From 335c8576884e2e52f515338f0305d1381bce4bb6 Mon Sep 17 00:00:00 2001 From: Reinhard Pointner Date: Sun, 6 Mar 2016 13:57:16 +0000 Subject: [PATCH] Fix json-io parse issues --- source/net/filebot/util/JsonUtilities.java | 48 ++++++++++-- source/net/filebot/web/AcoustIDClient.java | 47 ++++++------ source/net/filebot/web/FanartTVClient.java | 59 +++++++-------- source/net/filebot/web/OMDbClient.java | 13 +--- source/net/filebot/web/TVMazeClient.java | 86 +++++++--------------- test/net/filebot/web/TVMazeClientTest.java | 10 +++ 6 files changed, 130 insertions(+), 133 deletions(-) diff --git a/source/net/filebot/util/JsonUtilities.java b/source/net/filebot/util/JsonUtilities.java index 32e16083..557dfeaa 100644 --- a/source/net/filebot/util/JsonUtilities.java +++ b/source/net/filebot/util/JsonUtilities.java @@ -1,13 +1,19 @@ package net.filebot.util; +import static java.util.Arrays.*; import static java.util.Collections.*; +import static net.filebot.Logging.*; import java.util.Map; +import java.util.function.Function; +import com.cedarsoftware.util.io.JsonObject; import com.cedarsoftware.util.io.JsonReader; public class JsonUtilities { + public static final Object[] EMPTY_ARRAY = new Object[0]; + public static Map readJson(CharSequence json) { return (Map) JsonReader.jsonToJava(json.toString(), singletonMap(JsonReader.USE_MAPS, true)); } @@ -16,30 +22,48 @@ public class JsonUtilities { if (node instanceof Map) { return (Map) node; } - return null; + return EMPTY_MAP; } public static Object[] asArray(Object node) { if (node instanceof Object[]) { return (Object[]) node; } - return null; + if (node instanceof JsonObject) { + JsonObject jsonObject = (JsonObject) node; + if (jsonObject.isArray()) { + return jsonObject.getArray(); + } + } + return EMPTY_ARRAY; + } + + public static Map[] asMapArray(Object node) { + return stream(asArray(node)).map(JsonUtilities::asMap).filter(m -> m.size() > 0).toArray(Map[]::new); } public static Object[] getArray(Object node, String key) { - return asArray(((Map) node).get(key)); + return asArray(asMap(node).get(key)); + } + + public static Map getMap(Object node, String key) { + return asMap(asMap(node).get(key)); + } + + public static Map[] getMapArray(Object node, String key) { + return asMapArray(asMap(node).get(key)); } public static Map getFirstMap(Object node, String key) { Object[] values = getArray(node, key); - if (values != null && values.length > 0) { - return (Map) values[0]; + if (values.length > 0) { + return asMap(values[0]); } - return null; + return EMPTY_MAP; } public static String getString(Object node, String key) { - Object value = ((Map) node).get(key); + Object value = asMap(node).get(key); if (value != null) { return value.toString(); } @@ -47,9 +71,17 @@ public class JsonUtilities { } public static Integer getInteger(Object node, String key) { + return getStringValue(node, key, Integer::parseInt); + } + + public static V getStringValue(Object node, String key, Function converter) { String value = getString(node, key); if (value != null && value.length() > 0) { - return new Integer(value.toString()); + try { + return converter.apply(getString(node, key)); + } catch (Exception e) { + debug.warning(format("Bad %s value: %s => %s", key, value, e)); + } } return null; } diff --git a/source/net/filebot/web/AcoustIDClient.java b/source/net/filebot/web/AcoustIDClient.java index 843a87eb..1aff6d8e 100644 --- a/source/net/filebot/web/AcoustIDClient.java +++ b/source/net/filebot/web/AcoustIDClient.java @@ -1,5 +1,7 @@ package net.filebot.web; +import static java.nio.charset.StandardCharsets.*; +import static java.util.Arrays.*; import static net.filebot.util.JsonUtilities.*; import static net.filebot.web.WebRequest.*; @@ -9,7 +11,6 @@ import java.io.InputStreamReader; import java.lang.ProcessBuilder.Redirect; import java.lang.reflect.Field; import java.net.URL; -import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Collection; import java.util.Comparator; @@ -18,11 +19,11 @@ import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Scanner; import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; -import java.util.stream.Stream; import javax.swing.Icon; @@ -99,7 +100,7 @@ public class AcoustIDClient implements MusicIdentificationService { requestParam.put("Accept-Encoding", "gzip"); // submit - response = Charset.forName("UTF-8").decode(post(url, postParam, requestParam)).toString(); + response = UTF_8.decode(post(url, postParam, requestParam)).toString(); cache.put(cacheKey, response); return response; @@ -114,20 +115,18 @@ public class AcoustIDClient implements MusicIdentificationService { for (Object result : getArray(data, "results")) { // pick most likely matching recording - return Stream.of(getArray(result, "recordings")).sorted((Object o1, Object o2) -> { - Integer i1 = getInteger(o1, "duration"); - Integer i2 = getInteger(o2, "duration"); + return stream(getMapArray(result, "recordings")).sorted((Map r1, Map r2) -> { + Integer i1 = getInteger(r1, "duration"); + Integer i2 = getInteger(r2, "duration"); return Double.compare(i1 == null ? Double.NaN : Math.abs(i1 - targetDuration), i2 == null ? Double.NaN : Math.abs(i2 - targetDuration)); - }).map((Object o) -> { - Map recording = (Map) o; + }).map((Map recording) -> { try { Map releaseGroup = getFirstMap(recording, "releasegroups"); - if (releaseGroup == null) { - return null; - } + String artist = getString(getFirstMap(recording, "artists"), "name"); + String title = getString(recording, "title"); - String artist = (String) getFirstMap(recording, "artists").get("name"); - String title = (String) recording.get("title"); + if (artist == null || title == null || releaseGroup.isEmpty()) + return null; AudioTrack audioTrack = new AudioTrack(artist, title, null); audioTrack.mbid = getString(result, "id"); @@ -136,15 +135,15 @@ public class AcoustIDClient implements MusicIdentificationService { Object[] secondaryTypes = getArray(releaseGroup, "secondarytypes"); Object[] releases = getArray(releaseGroup, "releases"); - if (releases == null || secondaryTypes != null || (!"Album".equals(type))) { + if (releases.length == 0 || secondaryTypes.length > 0 || (!"Album".equals(type))) { return audioTrack; // default to simple music info if album data is undesirable } - for (Object it : releases) { + for (Map release : asMapArray(releases)) { AudioTrack thisRelease = audioTrack.clone(); - Map release = (Map) it; - Map date = (Map) release.get("date"); + try { + Map date = getMap(release, "date"); thisRelease.albumReleaseDate = new SimpleDate(getInteger(date, "year"), getInteger(date, "month"), getInteger(date, "day")); } catch (Exception e) { thisRelease.albumReleaseDate = null; @@ -163,16 +162,16 @@ public class AcoustIDClient implements MusicIdentificationService { thisRelease.trackCount = getInteger(medium, "track_count"); try { - thisRelease.album = release.get("title").toString(); + thisRelease.album = getString(release, "title"); } catch (Exception e) { - thisRelease.album = (String) releaseGroup.get("title"); + thisRelease.album = getString(releaseGroup, "title"); } try { - thisRelease.albumArtist = (String) getFirstMap(releaseGroup, "artists").get("name"); + thisRelease.albumArtist = getString(getFirstMap(releaseGroup, "artists"), "name"); } catch (Exception e) { thisRelease.albumArtist = null; } - thisRelease.trackTitle = (String) track.get("title"); + thisRelease.trackTitle = getString(track, "title"); if (!"Various Artists".equalsIgnoreCase(thisRelease.albumArtist) && (thisRelease.album == null || !thisRelease.album.contains("Greatest Hits"))) { // full info audio track @@ -183,10 +182,10 @@ public class AcoustIDClient implements MusicIdentificationService { // default to simple music info if extended info is not available return audioTrack; } catch (Exception e) { - Logger.getLogger(AcoustIDClient.class.getName()).log(Level.WARNING, e.toString(), e); + Logger.getLogger(AcoustIDClient.class.getName()).log(Level.WARNING, e.getMessage(), e); return null; } - }).filter(o -> o != null).sorted(new MostFieldsNotNull()).findFirst().get(); + }).filter(Objects::nonNull).sorted(new MostFieldsNotNull()).findFirst().get(); } return null; @@ -211,7 +210,7 @@ public class AcoustIDClient implements MusicIdentificationService { throw new IOException("Failed to exec fpcalc: " + e.getMessage()); } - Scanner scanner = new Scanner(new InputStreamReader(process.getInputStream(), "UTF-8")); + Scanner scanner = new Scanner(new InputStreamReader(process.getInputStream(), UTF_8)); LinkedList> results = new LinkedList>(); try { diff --git a/source/net/filebot/web/FanartTVClient.java b/source/net/filebot/web/FanartTVClient.java index 6222e88d..cac6b79a 100644 --- a/source/net/filebot/web/FanartTVClient.java +++ b/source/net/filebot/web/FanartTVClient.java @@ -1,12 +1,13 @@ package net.filebot.web; import static java.nio.charset.StandardCharsets.*; +import static java.util.Arrays.*; +import static java.util.Collections.*; import static net.filebot.util.JsonUtilities.*; import java.io.Serializable; import java.net.URL; import java.nio.ByteBuffer; -import java.util.ArrayList; import java.util.Arrays; import java.util.EnumMap; import java.util.List; @@ -41,33 +42,21 @@ public class FanartTVClient { @Override public FanartDescriptor[] process(ByteBuffer data) throws Exception { - List fanart = new ArrayList(); - - readJson(UTF_8.decode(data)).forEach((k, v) -> { - Object[] array = asArray(v); - if (array == null) - return; - - for (Object it : array) { - Map item = asMap(it); - if (item == null) - return; - + return readJson(UTF_8.decode(data)).entrySet().stream().flatMap(it -> { + return stream(asMapArray(it.getValue())).map(item -> { Map fields = new EnumMap(FanartProperty.class); - fields.put(FanartProperty.type, k.toString()); + fields.put(FanartProperty.type, it.getKey().toString()); + for (FanartProperty prop : FanartProperty.values()) { Object value = item.get(prop.name()); if (value != null) { fields.put(prop, value.toString()); } } - if (fields.size() > 1) { - fanart.add(new FanartDescriptor(fields)); - } - } - }); - return fanart.toArray(new FanartDescriptor[0]); + return new FanartDescriptor(fields); + }).filter(art -> art.getProperties().size() > 1); + }).toArray(FanartDescriptor[]::new); } @Override @@ -90,31 +79,35 @@ public class FanartTVClient { type, id, url, lang, likes, season, disc, disc_type } - protected Map fields; + protected Map properties; protected FanartDescriptor() { // used by serializer } protected FanartDescriptor(Map fields) { - this.fields = new EnumMap(fields); + this.properties = new EnumMap(fields); + } + + public Map getProperties() { + return unmodifiableMap(properties); } public String get(Object key) { - return fields.get(FanartProperty.valueOf(key.toString())); + return properties.get(FanartProperty.valueOf(key.toString())); } public String get(FanartProperty key) { - return fields.get(key); + return properties.get(key); } public String getType() { - return fields.get(FanartProperty.type); + return properties.get(FanartProperty.type); } public Integer getId() { try { - return new Integer(fields.get(FanartProperty.id)); + return new Integer(properties.get(FanartProperty.id)); } catch (Exception e) { return null; } @@ -122,7 +115,7 @@ public class FanartTVClient { public URL getUrl() { try { - return new URL(fields.get(FanartProperty.url)); + return new URL(properties.get(FanartProperty.url)); } catch (Exception e) { return null; } @@ -130,7 +123,7 @@ public class FanartTVClient { public Integer getLikes() { try { - return new Integer(fields.get(FanartProperty.likes)); + return new Integer(properties.get(FanartProperty.likes)); } catch (Exception e) { return null; } @@ -138,7 +131,7 @@ public class FanartTVClient { public Locale getLanguage() { try { - return new Locale(fields.get(FanartProperty.lang)); + return new Locale(properties.get(FanartProperty.lang)); } catch (Exception e) { return null; } @@ -146,7 +139,7 @@ public class FanartTVClient { public Integer getSeason() { try { - return new Integer(fields.get(FanartProperty.season)); + return new Integer(properties.get(FanartProperty.season)); } catch (Exception e) { return null; } @@ -154,19 +147,19 @@ public class FanartTVClient { public Integer getDiskNumber() { try { - return new Integer(fields.get(FanartProperty.disc)); + return new Integer(properties.get(FanartProperty.disc)); } catch (Exception e) { return null; } } public String getDiskType() { - return fields.get(FanartProperty.disc_type); + return properties.get(FanartProperty.disc_type); } @Override public String toString() { - return fields.toString(); + return properties.toString(); } } diff --git a/source/net/filebot/web/OMDbClient.java b/source/net/filebot/web/OMDbClient.java index 07595961..e4ea294a 100644 --- a/source/net/filebot/web/OMDbClient.java +++ b/source/net/filebot/web/OMDbClient.java @@ -2,6 +2,7 @@ package net.filebot.web; import static java.util.Collections.*; import static java.util.stream.Collectors.*; +import static net.filebot.util.JsonUtilities.*; import static net.filebot.util.StringUtilities.*; import static net.filebot.web.WebRequest.*; @@ -40,9 +41,6 @@ import net.filebot.web.TMDbClient.Person; import net.sf.ehcache.Cache; import net.sf.ehcache.CacheManager; -import com.cedarsoftware.util.io.JsonObject; -import com.cedarsoftware.util.io.JsonReader; - public class OMDbClient implements MovieIdentificationService { private static final FloodLimit REQUEST_LIMIT = new FloodLimit(20, 10, TimeUnit.SECONDS); @@ -71,11 +69,6 @@ public class OMDbClient implements MovieIdentificationService { throw new IllegalArgumentException(String.format("Cannot find imdb id: %s", link)); } - private static Object[] array(Object node, String key) { - Object value = ((Map) node).get(key); - return value == null ? new Object[0] : ((JsonObject) value).getArray(); - } - @Override public List searchMovie(String query, Locale locale) throws IOException { // query by name with year filter if possible @@ -97,7 +90,7 @@ public class OMDbClient implements MovieIdentificationService { Map response = request(param, REQUEST_LIMIT); List result = new ArrayList(); - for (Object it : array(response, "Search")) { + for (Object it : getArray(response, "Search")) { Map info = getInfoMap(it); if ("movie".equals(info.get("Type"))) { result.add(getMovie(info)); @@ -178,7 +171,7 @@ public class OMDbClient implements MovieIdentificationService { } }; - return JsonReader.jsonToMaps(json.get()); + return readJson(json.get()); } public Map getMovieInfo(Integer i, String t, String y, boolean tomatoes) throws IOException { diff --git a/source/net/filebot/web/TVMazeClient.java b/source/net/filebot/web/TVMazeClient.java index 2fad348b..28d979ed 100644 --- a/source/net/filebot/web/TVMazeClient.java +++ b/source/net/filebot/web/TVMazeClient.java @@ -1,27 +1,23 @@ package net.filebot.web; +import static java.util.Arrays.*; +import static java.util.stream.Collectors.*; +import static net.filebot.util.JsonUtilities.*; import static net.filebot.web.WebRequest.*; import java.io.IOException; import java.net.URI; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Objects; -import java.util.function.Function; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; import javax.swing.Icon; import net.filebot.Cache; import net.filebot.ResourceManager; -import com.cedarsoftware.util.io.JsonObject; - public class TVMazeClient extends AbstractEpisodeListProvider { @Override @@ -62,33 +58,27 @@ public class TVMazeClient extends AbstractEpisodeListProvider { @Override public List fetchSearchResult(String query, Locale locale) throws IOException { // e.g. http://api.tvmaze.com/search/shows?q=girls - JsonObject response = request("search/shows?q=" + encode(query, true)); + Object response = request("search/shows?q=" + encode(query, true)); - List results = new ArrayList(); - if (response.isArray()) { - for (Object result : response.getArray()) { - Map show = (Map) ((Map) result).get("show"); + // TODO: FUTURE WORK: consider adding TVmaze aka titles for each result, e.g. http://api.tvmaze.com/shows/1/akas + return stream(asMapArray(response)).map(it -> { + Object show = it.get("show"); + Integer id = getInteger(show, "id"); + String name = getString(show, "name"); - Integer id = getValue(show, "id", Integer::new); - String name = getValue(show, "name", String::new); - - // FUTURE WORK: consider adding TVmaze aka titles for each result, e.g. http://api.tvmaze.com/shows/1/akas - results.add(new TVMazeSearchResult(id, name)); - } - } - - return results; + return new TVMazeSearchResult(id, name); + }).collect(toList()); } protected SeriesInfo fetchSeriesInfo(TVMazeSearchResult show, SortOrder sortOrder, Locale locale) throws IOException { // e.g. http://api.tvmaze.com/shows/1 - JsonObject response = request("shows/" + show.getId()); + Object response = request("shows/" + show.getId()); - String status = getValue(response, "status", String::new); - SimpleDate premiered = getValue(response, "premiered", SimpleDate::parse); - Integer runtime = getValue(response, "runtime", Integer::new); - JsonObject genres = (JsonObject) response.get("genres"); - JsonObject rating = (JsonObject) response.get("rating"); + String status = getStringValue(response, "status", String::new); + SimpleDate premiered = getStringValue(response, "premiered", SimpleDate::parse); + Integer runtime = getStringValue(response, "runtime", Integer::new); + Object[] genres = getArray(response, "genres"); + Double rating = getStringValue(getMap(response, "rating"), "average", Double::new); SeriesInfo seriesInfo = new SeriesInfo(getName(), sortOrder, locale, show.getId()); seriesInfo.setName(show.getName()); @@ -96,13 +86,8 @@ public class TVMazeClient extends AbstractEpisodeListProvider { seriesInfo.setStatus(status); seriesInfo.setRuntime(runtime); seriesInfo.setStartDate(premiered); - - if (genres != null && genres.isArray()) { - seriesInfo.setGenres(Arrays.stream(genres.getArray()).map(Objects::toString).collect(Collectors.toList())); - } - if (rating != null && !rating.isEmpty()) { - seriesInfo.setRating(getValue(rating, "average", Double::new)); - } + seriesInfo.setRating(rating); + seriesInfo.setGenres(stream(genres).map(Objects::toString).collect(toList())); return seriesInfo; } @@ -112,21 +97,18 @@ public class TVMazeClient extends AbstractEpisodeListProvider { TVMazeSearchResult show = (TVMazeSearchResult) searchResult; SeriesInfo seriesInfo = fetchSeriesInfo(show, sortOrder, locale); - - // e.g. http://api.tvmaze.com/shows/1/episodes - JsonObject response = request("shows/" + show.getId() + "/episodes"); - List episodes = new ArrayList(25); - if (response.isArray()) { - for (Object element : response.getArray()) { - JsonObject episode = (JsonObject) element; + // e.g. http://api.tvmaze.com/shows/1/episodes + Object response = request("shows/" + show.getId() + "/episodes"); - String episodeTitle = getValue(episode, "name", String::new); - Integer seasonNumber = getValue(episode, "season", Integer::new); - Integer episodeNumber = getValue(episode, "number", Integer::new); - SimpleDate airdate = getValue(episode, "airdate", SimpleDate::parse); + for (Map episode : asMapArray(response)) { + String episodeTitle = getString(episode, "name"); + Integer seasonNumber = getInteger(episode, "season"); + Integer episodeNumber = getInteger(episode, "number"); + SimpleDate airdate = getStringValue(episode, "airdate", SimpleDate::parse); + if (episodeNumber != null && episodeTitle != null) { episodes.add(new Episode(seriesInfo.getName(), seasonNumber, episodeNumber, episodeTitle, null, null, airdate, new SeriesInfo(seriesInfo))); } } @@ -134,19 +116,7 @@ public class TVMazeClient extends AbstractEpisodeListProvider { return new SeriesData(seriesInfo, episodes); } - private V getValue(Map json, String key, Function converter) { - try { - Object value = json.get(key); - if (value != null) { - return converter.apply(value.toString()); - } - } catch (Exception e) { - Logger.getLogger(TVMazeClient.class.getName()).log(Level.WARNING, "Illegal " + key + " value: " + json); - } - return null; - } - - public JsonObject request(String resource) throws IOException { + public Object request(String resource) throws IOException { return new CachedJsonResource("http://api.tvmaze.com/" + resource).getJSON(); } diff --git a/test/net/filebot/web/TVMazeClientTest.java b/test/net/filebot/web/TVMazeClientTest.java index a13a4cfc..dd69685b 100644 --- a/test/net/filebot/web/TVMazeClientTest.java +++ b/test/net/filebot/web/TVMazeClientTest.java @@ -5,6 +5,10 @@ import static org.junit.Assert.*; import java.util.List; import java.util.Locale; +import net.sf.ehcache.CacheManager; + +import org.junit.AfterClass; +import org.junit.BeforeClass; import org.junit.Test; public class TVMazeClientTest { @@ -64,4 +68,10 @@ public class TVMazeClientTest { assertEquals("http://www.tvmaze.com/shows/427", client.getEpisodeListLink(buffySearchResult).toString()); } + @BeforeClass + @AfterClass + public static void clearCache() { + CacheManager.getInstance().clearAll(); + } + }