From 9e13e1928e3709ab5c76c05c287c7a1d94adbc25 Mon Sep 17 00:00:00 2001 From: Reinhard Pointner Date: Thu, 23 Feb 2017 20:10:12 +0800 Subject: [PATCH] Streamify -mediainfo and -list commands to make them more responsive when used on the command-line on large datasets --- source/net/filebot/cli/ArgumentBean.java | 5 ++ source/net/filebot/cli/ArgumentProcessor.java | 22 ++----- source/net/filebot/cli/CmdlineInterface.java | 6 +- source/net/filebot/cli/CmdlineOperations.java | 65 +++++++++++-------- .../net/filebot/cli/ScriptShellBaseClass.java | 2 +- 5 files changed, 56 insertions(+), 44 deletions(-) diff --git a/source/net/filebot/cli/ArgumentBean.java b/source/net/filebot/cli/ArgumentBean.java index ef7ba4f0..c1226710 100644 --- a/source/net/filebot/cli/ArgumentBean.java +++ b/source/net/filebot/cli/ArgumentBean.java @@ -37,6 +37,7 @@ import net.filebot.hash.HashType; import net.filebot.subtitle.SubtitleFormat; import net.filebot.subtitle.SubtitleNaming; import net.filebot.web.Datasource; +import net.filebot.web.EpisodeListProvider; import net.filebot.web.SortOrder; public class ArgumentBean { @@ -227,6 +228,10 @@ public class ArgumentBean { return db == null ? null : WebServices.getService(db); } + public EpisodeListProvider getEpisodeListProvider() { + return db == null ? WebServices.TheTVDB : WebServices.getEpisodeListProvider(db); // default to TheTVDB if --db is not set + } + public String getSearchQuery() { return query == null || query.isEmpty() ? null : query; } diff --git a/source/net/filebot/cli/ArgumentProcessor.java b/source/net/filebot/cli/ArgumentProcessor.java index 4172474d..7401b38d 100644 --- a/source/net/filebot/cli/ArgumentProcessor.java +++ b/source/net/filebot/cli/ArgumentProcessor.java @@ -7,9 +7,9 @@ import static net.filebot.util.FileUtilities.*; import java.io.File; import java.util.LinkedHashSet; -import java.util.List; import java.util.Set; import java.util.logging.Level; +import java.util.stream.Stream; import javax.script.Bindings; import javax.script.SimpleBindings; @@ -55,7 +55,7 @@ public class ArgumentProcessor { // print episode info if (args.list) { - return print(cli.fetchEpisodeList(args.getDatasource(), args.getSearchQuery(), args.getExpressionFormat(), args.getExpressionFilter(), args.getSortOrder(), args.getLanguage().getLocale(), args.isStrict())); + return print(cli.fetchEpisodeList(args.getEpisodeListProvider(), args.getSearchQuery(), args.getExpressionFormat(), args.getExpressionFilter(), args.getSortOrder(), args.getLanguage().getLocale(), args.isStrict())); } // print media info @@ -97,19 +97,11 @@ public class ArgumentProcessor { return 0; } - private int print(List values) { - int lines = 0; - - for (int i = 0; i < values.size(); i++) { - try { - System.out.println(values.get(i)); - lines++; - } catch (Exception e) { - debug.warning(e::getMessage); - } - } - - return lines == 0 ? 1 : 0; + private int print(Stream values) { + return values.mapToInt(v -> { + System.out.println(v); + return 1; + }).sum() == 0 ? 1 : 0; } public void runScript(CmdlineInterface cli, ArgumentBean args) throws Throwable { diff --git a/source/net/filebot/cli/CmdlineInterface.java b/source/net/filebot/cli/CmdlineInterface.java index bd3e22cd..4faca6cf 100644 --- a/source/net/filebot/cli/CmdlineInterface.java +++ b/source/net/filebot/cli/CmdlineInterface.java @@ -7,6 +7,7 @@ import java.util.Collection; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.stream.Stream; import net.filebot.Language; import net.filebot.RenameAction; @@ -16,6 +17,7 @@ import net.filebot.hash.HashType; import net.filebot.subtitle.SubtitleFormat; import net.filebot.subtitle.SubtitleNaming; import net.filebot.web.Datasource; +import net.filebot.web.EpisodeListProvider; import net.filebot.web.SortOrder; public interface CmdlineInterface { @@ -34,9 +36,9 @@ public interface CmdlineInterface { File compute(Collection files, File output, HashType hash, Charset encoding) throws Exception; - List fetchEpisodeList(Datasource db, String query, ExpressionFormat format, ExpressionFilter filter, SortOrder order, Locale locale, boolean strict) throws Exception; + Stream fetchEpisodeList(EpisodeListProvider db, String query, ExpressionFormat format, ExpressionFilter filter, SortOrder order, Locale locale, boolean strict) throws Exception; - List getMediaInfo(Collection files, FileFilter filter, ExpressionFormat format) throws Exception; + Stream getMediaInfo(Collection files, FileFilter filter, ExpressionFormat format) throws Exception; List extract(Collection files, File output, ConflictAction conflict, FileFilter filter, boolean forceExtractAll) throws Exception; diff --git a/source/net/filebot/cli/CmdlineOperations.java b/source/net/filebot/cli/CmdlineOperations.java index c404fa49..55e23184 100644 --- a/source/net/filebot/cli/CmdlineOperations.java +++ b/source/net/filebot/cli/CmdlineOperations.java @@ -62,7 +62,6 @@ import net.filebot.subtitle.SubtitleFormat; import net.filebot.subtitle.SubtitleNaming; import net.filebot.util.EntryList; import net.filebot.util.FileUtilities.ParentFilter; -import net.filebot.util.FunctionList; import net.filebot.vfs.FileInfo; import net.filebot.vfs.MemoryFile; import net.filebot.vfs.SimpleFileInfo; @@ -986,55 +985,69 @@ public class CmdlineOperations implements CmdlineInterface { } @Override - public List fetchEpisodeList(Datasource db, String query, ExpressionFormat format, ExpressionFilter filter, SortOrder order, Locale locale, boolean strict) throws Exception { - if (query == null) { - throw new IllegalArgumentException("query is not defined"); + public Stream fetchEpisodeList(EpisodeListProvider db, String query, ExpressionFormat format, ExpressionFilter filter, SortOrder order, Locale locale, boolean strict) throws Exception { + // default episode format + if (format == null) { + return fetchEpisodeList(db, query, new ExpressionFormat("{episode}"), filter, order, locale, strict); } - // find series on the web and fetch episode list - EpisodeListProvider service = db instanceof EpisodeListProvider ? (EpisodeListProvider) db : TheTVDB; + // sanity check + if (query == null) { + throw new CmdlineException(String.format("%s: query parameter is required", db.getName())); + } // search and select search result - List results = service.search(query, locale); + List results = db.search(query, locale); + + // sanity check if (results.isEmpty()) { - throw new CmdlineException(String.format("%s: no results", service.getName())); + throw new CmdlineException(String.format("%s: no results", db.getName())); } List options = selectSearchResult(query, results, false, false, false, strict ? 1 : 5); - List episodes = new ArrayList(); // fetch episodes + List episodes = new ArrayList(); for (SearchResult option : options) { - episodes.addAll(service.getEpisodeList(option, order, locale)); + episodes.addAll(db.getEpisodeList(option, order, locale)); } // apply filter episodes = applyExpressionFilter(episodes, filter); + // lazy format Map context = new EntryList(null, episodes); - // lazy format - return new FunctionList(episodes, e -> { - return format != null ? format.format(new MediaBindingBean(e, null, context)) : EpisodeFormat.SeasonEpisode.format(e); - }); + return episodes.stream().map(episode -> { + try { + return format.format(new MediaBindingBean(episode, null, context)); + } catch (Exception e) { + debug.warning(e::getMessage); + } + return null; + }).filter(Objects::nonNull); } @Override - public List getMediaInfo(Collection files, FileFilter filter, ExpressionFormat format) throws Exception { - List selection = filter(files, FILES); - - // apply custom filter - if (filter != null) { - selection = filter(selection, filter); + public Stream getMediaInfo(Collection files, FileFilter filter, ExpressionFormat format) throws Exception { + // use default file filter + if (filter == null) { + return getMediaInfo(files, FILES, format); } - // default expression format if not set - ExpressionFormat formatter = format != null ? format : new ExpressionFormat("{fn} [{resolution} {vc} {channels} {ac} {minutes}m]"); + // use default expression format if not set + if (format == null) { + return getMediaInfo(files, filter, new ExpressionFormat("{fn} [{resolution} {vc} {channels} {ac} {minutes}m]")); + } - // lazy format - return new FunctionList(selection, f -> { - return formatter.format(new MediaBindingBean(xattr.getMetaInfo(f), f)); - }); + return files.stream().filter(filter::accept).map(f -> { + try { + return format.format(new MediaBindingBean(xattr.getMetaInfo(f), f)); + } catch (Exception e) { + debug.warning(e::getMessage); + } + return null; + }).filter(Objects::nonNull); } @Override diff --git a/source/net/filebot/cli/ScriptShellBaseClass.java b/source/net/filebot/cli/ScriptShellBaseClass.java index e94a5a4e..f6543c86 100644 --- a/source/net/filebot/cli/ScriptShellBaseClass.java +++ b/source/net/filebot/cli/ScriptShellBaseClass.java @@ -424,7 +424,7 @@ public abstract class ScriptShellBaseClass extends Script { ArgumentBean args = getArgumentBean(parameters); try { - return getCLI().fetchEpisodeList(args.getDatasource(), args.getSearchQuery(), args.getExpressionFormat(), args.getExpressionFilter(), args.getSortOrder(), args.getLanguage().getLocale(), args.isStrict()); + return getCLI().fetchEpisodeList(args.getEpisodeListProvider(), args.getSearchQuery(), args.getExpressionFormat(), args.getExpressionFilter(), args.getSortOrder(), args.getLanguage().getLocale(), args.isStrict()).collect(toList()); } catch (Exception e) { printException(e); }