mobeigi
/
filebot
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix json-io parse issues

This commit is contained in:
Reinhard Pointner 2016-03-06 13:57:16 +00:00
parent ed455635bd
commit 335c857688
6 changed files with 130 additions and 133 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
source/net/filebot/util/JsonUtilities.java
View File

@ -1,13 +1,19 @@
package net.filebot.util;
import static java.util.Arrays.*;
import static java.util.Collections.*;
import static net.filebot.Logging.*;
import java.util.Map;
import java.util.function.Function;
import com.cedarsoftware.util.io.JsonObject;
import com.cedarsoftware.util.io.JsonReader;
public class JsonUtilities {
public static final Object[] EMPTY_ARRAY = new Object[0];
public static Map<?, ?> readJson(CharSequence json) {
return (Map<?, ?>) JsonReader.jsonToJava(json.toString(), singletonMap(JsonReader.USE_MAPS, true));
}
@ -16,30 +22,48 @@ public class JsonUtilities {
if (node instanceof Map) {
return (Map<?, ?>) node;
}
return null;
return EMPTY_MAP;
}
public static Object[] asArray(Object node) {
if (node instanceof Object[]) {
return (Object[]) node;
}
return null;
if (node instanceof JsonObject) {
JsonObject<?, ?> jsonObject = (JsonObject<?, ?>) node;
if (jsonObject.isArray()) {
return jsonObject.getArray();
}
}
return EMPTY_ARRAY;
}
public static Map<?, ?>[] asMapArray(Object node) {
return stream(asArray(node)).map(JsonUtilities::asMap).filter(m -> m.size() > 0).toArray(Map[]::new);
}
public static Object[] getArray(Object node, String key) {
return asArray(((Map<?, ?>) node).get(key));
return asArray(asMap(node).get(key));
}
public static Map<?, ?> getMap(Object node, String key) {
return asMap(asMap(node).get(key));
}
public static Map<?, ?>[] getMapArray(Object node, String key) {
return asMapArray(asMap(node).get(key));
}
public static Map<?, ?> getFirstMap(Object node, String key) {
Object[] values = getArray(node, key);
if (values != null && values.length > 0) {
return (Map<?, ?>) values[0];
if (values.length > 0) {
return asMap(values[0]);
}
return null;
return EMPTY_MAP;
}
public static String getString(Object node, String key) {
Object value = ((Map<?, ?>) node).get(key);
Object value = asMap(node).get(key);
if (value != null) {
return value.toString();
}
@ -47,9 +71,17 @@ public class JsonUtilities {
}
public static Integer getInteger(Object node, String key) {
return getStringValue(node, key, Integer::parseInt);
}
public static <V> V getStringValue(Object node, String key, Function<String, V> converter) {
String value = getString(node, key);
if (value != null && value.length() > 0) {
return new Integer(value.toString());
try {
return converter.apply(getString(node, key));
} catch (Exception e) {
debug.warning(format("Bad %s value: %s => %s", key, value, e));
}
}
return null;
}

47
source/net/filebot/web/AcoustIDClient.java
View File

@ -1,5 +1,7 @@
package net.filebot.web;
import static java.nio.charset.StandardCharsets.*;
import static java.util.Arrays.*;
import static net.filebot.util.JsonUtilities.*;
import static net.filebot.web.WebRequest.*;
@ -9,7 +11,6 @@ import java.io.InputStreamReader;
import java.lang.ProcessBuilder.Redirect;
import java.lang.reflect.Field;
import java.net.URL;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
@ -18,11 +19,11 @@ import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Scanner;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Stream;
import javax.swing.Icon;
@ -99,7 +100,7 @@ public class AcoustIDClient implements MusicIdentificationService {
requestParam.put("Accept-Encoding", "gzip");
// submit
response = Charset.forName("UTF-8").decode(post(url, postParam, requestParam)).toString();
response = UTF_8.decode(post(url, postParam, requestParam)).toString();
cache.put(cacheKey, response);
return response;
@ -114,20 +115,18 @@ public class AcoustIDClient implements MusicIdentificationService {
for (Object result : getArray(data, "results")) {
// pick most likely matching recording
return Stream.of(getArray(result, "recordings")).sorted((Object o1, Object o2) -> {
Integer i1 = getInteger(o1, "duration");
Integer i2 = getInteger(o2, "duration");
return stream(getMapArray(result, "recordings")).sorted((Map<?, ?> r1, Map<?, ?> r2) -> {
Integer i1 = getInteger(r1, "duration");
Integer i2 = getInteger(r2, "duration");
return Double.compare(i1 == null ? Double.NaN : Math.abs(i1 - targetDuration), i2 == null ? Double.NaN : Math.abs(i2 - targetDuration));
}).map((Object o) -> {
Map<?, ?> recording = (Map<?, ?>) o;
}).map((Map<?, ?> recording) -> {
try {
Map<?, ?> releaseGroup = getFirstMap(recording, "releasegroups");
if (releaseGroup == null) {
return null;
}
String artist = getString(getFirstMap(recording, "artists"), "name");
String title = getString(recording, "title");
String artist = (String) getFirstMap(recording, "artists").get("name");
String title = (String) recording.get("title");
if (artist == null || title == null || releaseGroup.isEmpty())
return null;
AudioTrack audioTrack = new AudioTrack(artist, title, null);
audioTrack.mbid = getString(result, "id");
@ -136,15 +135,15 @@ public class AcoustIDClient implements MusicIdentificationService {
Object[] secondaryTypes = getArray(releaseGroup, "secondarytypes");
Object[] releases = getArray(releaseGroup, "releases");
if (releases == null || secondaryTypes != null || (!"Album".equals(type))) {
if (releases.length == 0 || secondaryTypes.length > 0 || (!"Album".equals(type))) {
return audioTrack; // default to simple music info if album data is undesirable
}
for (Object it : releases) {
for (Map<?, ?> release : asMapArray(releases)) {
AudioTrack thisRelease = audioTrack.clone();
Map<?, ?> release = (Map<?, ?>) it;
Map<?, ?> date = (Map<?, ?>) release.get("date");
try {
Map<?, ?> date = getMap(release, "date");
thisRelease.albumReleaseDate = new SimpleDate(getInteger(date, "year"), getInteger(date, "month"), getInteger(date, "day"));
} catch (Exception e) {
thisRelease.albumReleaseDate = null;
@ -163,16 +162,16 @@ public class AcoustIDClient implements MusicIdentificationService {
thisRelease.trackCount = getInteger(medium, "track_count");
try {
thisRelease.album = release.get("title").toString();
thisRelease.album = getString(release, "title");
} catch (Exception e) {
thisRelease.album = (String) releaseGroup.get("title");
thisRelease.album = getString(releaseGroup, "title");
}
try {
thisRelease.albumArtist = (String) getFirstMap(releaseGroup, "artists").get("name");
thisRelease.albumArtist = getString(getFirstMap(releaseGroup, "artists"), "name");
} catch (Exception e) {
thisRelease.albumArtist = null;
}
thisRelease.trackTitle = (String) track.get("title");
thisRelease.trackTitle = getString(track, "title");
if (!"Various Artists".equalsIgnoreCase(thisRelease.albumArtist) && (thisRelease.album == null || !thisRelease.album.contains("Greatest Hits"))) {
// full info audio track
@ -183,10 +182,10 @@ public class AcoustIDClient implements MusicIdentificationService {
// default to simple music info if extended info is not available
return audioTrack;
} catch (Exception e) {
Logger.getLogger(AcoustIDClient.class.getName()).log(Level.WARNING, e.toString(), e);
Logger.getLogger(AcoustIDClient.class.getName()).log(Level.WARNING, e.getMessage(), e);
return null;
}
}).filter(o -> o != null).sorted(new MostFieldsNotNull()).findFirst().get();
}).filter(Objects::nonNull).sorted(new MostFieldsNotNull()).findFirst().get();
}
return null;
@ -211,7 +210,7 @@ public class AcoustIDClient implements MusicIdentificationService {
throw new IOException("Failed to exec fpcalc: " + e.getMessage());
}
Scanner scanner = new Scanner(new InputStreamReader(process.getInputStream(), "UTF-8"));
Scanner scanner = new Scanner(new InputStreamReader(process.getInputStream(), UTF_8));
LinkedList<Map<String, String>> results = new LinkedList<Map<String, String>>();
try {

59
source/net/filebot/web/FanartTVClient.java
View File

@ -1,12 +1,13 @@
package net.filebot.web;
import static java.nio.charset.StandardCharsets.*;
import static java.util.Arrays.*;
import static java.util.Collections.*;
import static net.filebot.util.JsonUtilities.*;
import java.io.Serializable;
import java.net.URL;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.EnumMap;
import java.util.List;
@ -41,33 +42,21 @@ public class FanartTVClient {
@Override
public FanartDescriptor[] process(ByteBuffer data) throws Exception {
List<FanartDescriptor> fanart = new ArrayList<FanartDescriptor>();
readJson(UTF_8.decode(data)).forEach((k, v) -> {
Object[] array = asArray(v);
if (array == null)
return;
for (Object it : array) {
Map<?, ?> item = asMap(it);
if (item == null)
return;
return readJson(UTF_8.decode(data)).entrySet().stream().flatMap(it -> {
return stream(asMapArray(it.getValue())).map(item -> {
Map<FanartProperty, String> fields = new EnumMap<FanartProperty, String>(FanartProperty.class);
fields.put(FanartProperty.type, k.toString());
fields.put(FanartProperty.type, it.getKey().toString());
for (FanartProperty prop : FanartProperty.values()) {
Object value = item.get(prop.name());
if (value != null) {
fields.put(prop, value.toString());
}
}
if (fields.size() > 1) {
fanart.add(new FanartDescriptor(fields));
}
}
});
return fanart.toArray(new FanartDescriptor[0]);
return new FanartDescriptor(fields);
}).filter(art -> art.getProperties().size() > 1);
}).toArray(FanartDescriptor[]::new);
}
@Override
@ -90,31 +79,35 @@ public class FanartTVClient {
type, id, url, lang, likes, season, disc, disc_type
}
protected Map<FanartProperty, String> fields;
protected Map<FanartProperty, String> properties;
protected FanartDescriptor() {
// used by serializer
}
protected FanartDescriptor(Map<FanartProperty, String> fields) {
this.fields = new EnumMap<FanartProperty, String>(fields);
this.properties = new EnumMap<FanartProperty, String>(fields);
}
public Map<FanartProperty, String> getProperties() {
return unmodifiableMap(properties);
}
public String get(Object key) {
return fields.get(FanartProperty.valueOf(key.toString()));
return properties.get(FanartProperty.valueOf(key.toString()));
}
public String get(FanartProperty key) {
return fields.get(key);
return properties.get(key);
}
public String getType() {
return fields.get(FanartProperty.type);
return properties.get(FanartProperty.type);
}
public Integer getId() {
try {
return new Integer(fields.get(FanartProperty.id));
return new Integer(properties.get(FanartProperty.id));
} catch (Exception e) {
return null;
}
@ -122,7 +115,7 @@ public class FanartTVClient {
public URL getUrl() {
try {
return new URL(fields.get(FanartProperty.url));
return new URL(properties.get(FanartProperty.url));
} catch (Exception e) {
return null;
}
@ -130,7 +123,7 @@ public class FanartTVClient {
public Integer getLikes() {
try {
return new Integer(fields.get(FanartProperty.likes));
return new Integer(properties.get(FanartProperty.likes));
} catch (Exception e) {
return null;
}
@ -138,7 +131,7 @@ public class FanartTVClient {
public Locale getLanguage() {
try {
return new Locale(fields.get(FanartProperty.lang));
return new Locale(properties.get(FanartProperty.lang));
} catch (Exception e) {
return null;
}
@ -146,7 +139,7 @@ public class FanartTVClient {
public Integer getSeason() {
try {
return new Integer(fields.get(FanartProperty.season));
return new Integer(properties.get(FanartProperty.season));
} catch (Exception e) {
return null;
}
@ -154,19 +147,19 @@ public class FanartTVClient {
public Integer getDiskNumber() {
try {
return new Integer(fields.get(FanartProperty.disc));
return new Integer(properties.get(FanartProperty.disc));
} catch (Exception e) {
return null;
}
}
public String getDiskType() {
return fields.get(FanartProperty.disc_type);
return properties.get(FanartProperty.disc_type);
}
@Override
public String toString() {
return fields.toString();
return properties.toString();
}
}

13
source/net/filebot/web/OMDbClient.java
View File

@ -2,6 +2,7 @@ package net.filebot.web;
import static java.util.Collections.*;
import static java.util.stream.Collectors.*;
import static net.filebot.util.JsonUtilities.*;
import static net.filebot.util.StringUtilities.*;
import static net.filebot.web.WebRequest.*;
@ -40,9 +41,6 @@ import net.filebot.web.TMDbClient.Person;
import net.sf.ehcache.Cache;
import net.sf.ehcache.CacheManager;
import com.cedarsoftware.util.io.JsonObject;
import com.cedarsoftware.util.io.JsonReader;
public class OMDbClient implements MovieIdentificationService {
private static final FloodLimit REQUEST_LIMIT = new FloodLimit(20, 10, TimeUnit.SECONDS);
@ -71,11 +69,6 @@ public class OMDbClient implements MovieIdentificationService {
throw new IllegalArgumentException(String.format("Cannot find imdb id: %s", link));
}
private static Object[] array(Object node, String key) {
Object value = ((Map<?, ?>) node).get(key);
return value == null ? new Object[0] : ((JsonObject<?, ?>) value).getArray();
}
@Override
public List<Movie> searchMovie(String query, Locale locale) throws IOException {
// query by name with year filter if possible
@ -97,7 +90,7 @@ public class OMDbClient implements MovieIdentificationService {
Map<?, ?> response = request(param, REQUEST_LIMIT);
List<Movie> result = new ArrayList<Movie>();
for (Object it : array(response, "Search")) {
for (Object it : getArray(response, "Search")) {
Map<String, String> info = getInfoMap(it);
if ("movie".equals(info.get("Type"))) {
result.add(getMovie(info));
@ -178,7 +171,7 @@ public class OMDbClient implements MovieIdentificationService {
}
};
return JsonReader.jsonToMaps(json.get());
return readJson(json.get());
}
public Map<String, String> getMovieInfo(Integer i, String t, String y, boolean tomatoes) throws IOException {

86
source/net/filebot/web/TVMazeClient.java
View File

@ -1,27 +1,23 @@
package net.filebot.web;
import static java.util.Arrays.*;
import static java.util.stream.Collectors.*;
import static net.filebot.util.JsonUtilities.*;
import static net.filebot.web.WebRequest.*;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.function.Function;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Collectors;
import javax.swing.Icon;
import net.filebot.Cache;
import net.filebot.ResourceManager;
import com.cedarsoftware.util.io.JsonObject;
public class TVMazeClient extends AbstractEpisodeListProvider {
@Override
@ -62,33 +58,27 @@ public class TVMazeClient extends AbstractEpisodeListProvider {
@Override
public List<SearchResult> fetchSearchResult(String query, Locale locale) throws IOException {
// e.g. http://api.tvmaze.com/search/shows?q=girls
JsonObject<?, ?> response = request("search/shows?q=" + encode(query, true));
Object response = request("search/shows?q=" + encode(query, true));
List<SearchResult> results = new ArrayList<SearchResult>();
if (response.isArray()) {
for (Object result : response.getArray()) {
Map<?, ?> show = (Map<?, ?>) ((Map<?, ?>) result).get("show");
// TODO: FUTURE WORK: consider adding TVmaze aka titles for each result, e.g. http://api.tvmaze.com/shows/1/akas
return stream(asMapArray(response)).map(it -> {
Object show = it.get("show");
Integer id = getInteger(show, "id");
String name = getString(show, "name");
Integer id = getValue(show, "id", Integer::new);
String name = getValue(show, "name", String::new);
// FUTURE WORK: consider adding TVmaze aka titles for each result, e.g. http://api.tvmaze.com/shows/1/akas
results.add(new TVMazeSearchResult(id, name));
}
}
return results;
return new TVMazeSearchResult(id, name);
}).collect(toList());
}
protected SeriesInfo fetchSeriesInfo(TVMazeSearchResult show, SortOrder sortOrder, Locale locale) throws IOException {
// e.g. http://api.tvmaze.com/shows/1
JsonObject<?, ?> response = request("shows/" + show.getId());
Object response = request("shows/" + show.getId());
String status = getValue(response, "status", String::new);
SimpleDate premiered = getValue(response, "premiered", SimpleDate::parse);
Integer runtime = getValue(response, "runtime", Integer::new);
JsonObject<?, ?> genres = (JsonObject<?, ?>) response.get("genres");
JsonObject<?, ?> rating = (JsonObject<?, ?>) response.get("rating");
String status = getStringValue(response, "status", String::new);
SimpleDate premiered = getStringValue(response, "premiered", SimpleDate::parse);
Integer runtime = getStringValue(response, "runtime", Integer::new);
Object[] genres = getArray(response, "genres");
Double rating = getStringValue(getMap(response, "rating"), "average", Double::new);
SeriesInfo seriesInfo = new SeriesInfo(getName(), sortOrder, locale, show.getId());
seriesInfo.setName(show.getName());
@ -96,13 +86,8 @@ public class TVMazeClient extends AbstractEpisodeListProvider {
seriesInfo.setStatus(status);
seriesInfo.setRuntime(runtime);
seriesInfo.setStartDate(premiered);
if (genres != null && genres.isArray()) {
seriesInfo.setGenres(Arrays.stream(genres.getArray()).map(Objects::toString).collect(Collectors.toList()));
}
if (rating != null && !rating.isEmpty()) {
seriesInfo.setRating(getValue(rating, "average", Double::new));
}
seriesInfo.setRating(rating);
seriesInfo.setGenres(stream(genres).map(Objects::toString).collect(toList()));
return seriesInfo;
}
@ -112,21 +97,18 @@ public class TVMazeClient extends AbstractEpisodeListProvider {
TVMazeSearchResult show = (TVMazeSearchResult) searchResult;
SeriesInfo seriesInfo = fetchSeriesInfo(show, sortOrder, locale);
// e.g. http://api.tvmaze.com/shows/1/episodes
JsonObject<?, ?> response = request("shows/" + show.getId() + "/episodes");
List<Episode> episodes = new ArrayList<Episode>(25);
if (response.isArray()) {
for (Object element : response.getArray()) {
JsonObject<?, ?> episode = (JsonObject<?, ?>) element;
// e.g. http://api.tvmaze.com/shows/1/episodes
Object response = request("shows/" + show.getId() + "/episodes");
String episodeTitle = getValue(episode, "name", String::new);
Integer seasonNumber = getValue(episode, "season", Integer::new);
Integer episodeNumber = getValue(episode, "number", Integer::new);
SimpleDate airdate = getValue(episode, "airdate", SimpleDate::parse);
for (Map<?, ?> episode : asMapArray(response)) {
String episodeTitle = getString(episode, "name");
Integer seasonNumber = getInteger(episode, "season");
Integer episodeNumber = getInteger(episode, "number");
SimpleDate airdate = getStringValue(episode, "airdate", SimpleDate::parse);
if (episodeNumber != null && episodeTitle != null) {
episodes.add(new Episode(seriesInfo.getName(), seasonNumber, episodeNumber, episodeTitle, null, null, airdate, new SeriesInfo(seriesInfo)));
}
}
@ -134,19 +116,7 @@ public class TVMazeClient extends AbstractEpisodeListProvider {
return new SeriesData(seriesInfo, episodes);
}
private <V> V getValue(Map<?, ?> json, String key, Function<String, V> converter) {
try {
Object value = json.get(key);
if (value != null) {
return converter.apply(value.toString());
}
} catch (Exception e) {
Logger.getLogger(TVMazeClient.class.getName()).log(Level.WARNING, "Illegal " + key + " value: " + json);
}
return null;
}
public JsonObject<?, ?> request(String resource) throws IOException {
public Object request(String resource) throws IOException {
return new CachedJsonResource("http://api.tvmaze.com/" + resource).getJSON();
}

10
test/net/filebot/web/TVMazeClientTest.java
View File

@ -5,6 +5,10 @@ import static org.junit.Assert.*;
import java.util.List;
import java.util.Locale;
import net.sf.ehcache.CacheManager;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
public class TVMazeClientTest {
@ -64,4 +68,10 @@ public class TVMazeClientTest {
assertEquals("http://www.tvmaze.com/shows/427", client.getEpisodeListLink(buffySearchResult).toString());
}
@BeforeClass
@AfterClass
public static void clearCache() {
CacheManager.getInstance().clearAll();
}
}