From d25b30e8af4aa731b20be81e87e06b687c7c87f7 Mon Sep 17 00:00:00 2001 From: Reinhard Pointner Date: Sun, 3 Apr 2016 18:14:27 +0000 Subject: [PATCH] Make sure to ignore unexpected BOMs @see https://www.filebot.net/forums/viewtopic.php?f=8&t=3582 --- .../net/filebot/cli/ScriptShellMethods.java | 6 +- .../format/ExpressionFormatFunctions.java | 13 ++--- source/net/filebot/util/FileSet.java | 13 ++--- source/net/filebot/util/FileUtilities.java | 57 +++---------------- 4 files changed, 25 insertions(+), 64 deletions(-) diff --git a/source/net/filebot/cli/ScriptShellMethods.java b/source/net/filebot/cli/ScriptShellMethods.java index 563fb494..7fd1dbba 100644 --- a/source/net/filebot/cli/ScriptShellMethods.java +++ b/source/net/filebot/cli/ScriptShellMethods.java @@ -256,7 +256,11 @@ public class ScriptShellMethods { } public static void createFileIfNotExists(File self) throws IOException { - FileUtilities.createFileIfNotExists(self); + if (!self.isFile()) { + // create parent folder structure if necessary & create file + Files.createDirectories(self.getParentFile().toPath()); + Files.createFile(self.toPath()); + } } public static File relativize(File self, File other) throws IOException { diff --git a/source/net/filebot/format/ExpressionFormatFunctions.java b/source/net/filebot/format/ExpressionFormatFunctions.java index d0764d5b..c7aaf90d 100644 --- a/source/net/filebot/format/ExpressionFormatFunctions.java +++ b/source/net/filebot/format/ExpressionFormatFunctions.java @@ -1,12 +1,11 @@ package net.filebot.format; -import static java.nio.charset.StandardCharsets.*; import static java.util.stream.Collectors.*; +import static net.filebot.util.FileUtilities.*; import static net.filebot.util.RegularExpressions.*; +import java.io.File; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -53,20 +52,20 @@ public class ExpressionFormatFunctions { public static Map csv(String path) throws IOException { Map map = new LinkedHashMap(); - for (String line : Files.readAllLines(Paths.get(path), UTF_8)) { + streamLines(new File(path)).forEach(line -> { for (Pattern delim : new Pattern[] { TAB, SEMICOLON }) { String[] field = delim.split(line, 2); if (field.length >= 2) { - map.put(field[0], field[1]); + map.put(field[0].trim(), field[1].trim()); break; } } - } + }); return map; } public static List readLines(String path) throws IOException { - return Files.readAllLines(Paths.get(path), UTF_8); + return streamLines(new File(path)).collect(toList()); } } diff --git a/source/net/filebot/util/FileSet.java b/source/net/filebot/util/FileSet.java index 07919d38..509ef10f 100644 --- a/source/net/filebot/util/FileSet.java +++ b/source/net/filebot/util/FileSet.java @@ -3,6 +3,7 @@ package net.filebot.util; import static java.nio.charset.StandardCharsets.*; import static java.util.Collections.*; import static java.util.stream.Collectors.*; +import static net.filebot.util.FileUtilities.*; import java.io.File; import java.io.IOException; @@ -10,7 +11,9 @@ import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; import java.util.AbstractSet; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -152,16 +155,12 @@ public class FileSet extends AbstractSet { files.clear(); } - public void feed(Stream stream) { - stream.forEach(it -> add(getPath(it))); - } - public void load(File f) throws IOException { - feed(Files.lines(f.toPath(), UTF_8)); + streamLines(f).forEach(this::add); } - public void store(File f) throws IOException { - Files.write(f.toPath(), stream().map(it -> (CharSequence) it.toString())::iterator, UTF_8); + public void append(File f, Collection... paths) throws IOException { + Files.write(f.toPath(), Stream.of(paths).flatMap(Collection::stream).map(this::getPath).filter(it -> !contains(it)).map(Path::toString).collect(toList()), UTF_8, StandardOpenOption.APPEND); } } diff --git a/source/net/filebot/util/FileUtilities.java b/source/net/filebot/util/FileUtilities.java index 5c97bd30..e10f6519 100644 --- a/source/net/filebot/util/FileUtilities.java +++ b/source/net/filebot/util/FileUtilities.java @@ -3,16 +3,17 @@ package net.filebot.util; import static java.nio.charset.StandardCharsets.*; import static java.util.Arrays.*; import static java.util.Collections.*; +import static java.util.stream.Collectors.*; import static net.filebot.Logging.*; import static net.filebot.util.RegularExpressions.*; import java.io.BufferedInputStream; +import java.io.BufferedReader; import java.io.File; import java.io.FileFilter; import java.io.FileInputStream; import java.io.FilenameFilter; import java.io.IOException; -import java.io.InputStream; import java.io.InputStreamReader; import java.io.Reader; import java.math.BigInteger; @@ -39,12 +40,12 @@ import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Scanner; import java.util.SortedMap; import java.util.TreeMap; import java.util.TreeSet; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Stream; import org.apache.commons.io.FileUtils; @@ -166,45 +167,16 @@ public final class FileUtilities { return Files.createDirectories(folder.toPath()).toFile(); } - public static void createFileIfNotExists(File file) throws IOException { - if (!file.isFile()) { - // create parent folder structure if necessary & create file - Files.createDirectories(file.getParentFile().toPath()); - Files.createFile(file.toPath()); - } + public static byte[] readFile(File source) throws IOException { + return Files.readAllBytes(source.toPath()); } - public static byte[] readFile(File source) throws IOException { - long size = source.length(); - if (size < 0 || size > Integer.MAX_VALUE) { - throw new IllegalArgumentException("Unable to read file: " + source); - } - - try (InputStream in = new FileInputStream(source)) { - byte[] data = new byte[(int) size]; - int position = 0; - int read = 0; - - while (position < data.length && (read = in.read(data, position, data.length - position)) >= 0) { - position += read; - } - - return data; - } + public static Stream streamLines(File file) throws IOException { + return new BufferedReader(new UnicodeReader(new BufferedInputStream(new FileInputStream(file)), false, UTF_8), BUFFER_SIZE).lines(); } public static String readTextFile(File file) throws IOException { - try (Reader reader = new UnicodeReader(new BufferedInputStream(new FileInputStream(file), BUFFER_SIZE), false, StandardCharsets.UTF_8)) { - StringBuilder text = new StringBuilder(); - char[] buffer = new char[BUFFER_SIZE]; - - int read = 0; - while ((read = reader.read(buffer)) >= 0) { - text.append(buffer, 0, read); - } - - return text.toString(); - } + return streamLines(file).collect(joining(System.lineSeparator())); } public static File writeFile(ByteBuffer data, File destination) throws IOException { @@ -214,19 +186,6 @@ public final class FileUtilities { return destination; } - public static List readCSV(InputStream source, String charsetName, String pattern) { - try (Scanner scanner = new Scanner(source, charsetName)) { - Pattern separator = Pattern.compile(pattern); - List rows = new ArrayList(65536); - - while (scanner.hasNextLine()) { - rows.add(separator.split(scanner.nextLine())); - } - - return rows; - } - } - public static Reader createTextReader(File file) throws IOException { CharsetDetector detector = new CharsetDetector(); detector.setDeclaredEncoding("UTF-8"); // small boost for UTF-8 as default encoding