Make sure to ignore unexpected BOMs

@see https://www.filebot.net/forums/viewtopic.php?f=8&t=3582

source/net/filebot/cli/ScriptShellMethods.java

@@ -256,7 +256,11 @@ public class ScriptShellMethods {
 	}
 
 	public static void createFileIfNotExists(File self) throws IOException {
-		FileUtilities.createFileIfNotExists(self);
+		if (!self.isFile()) {
+			// create parent folder structure if necessary & create file
+			Files.createDirectories(self.getParentFile().toPath());
+			Files.createFile(self.toPath());
+		}
 	}
 
 	public static File relativize(File self, File other) throws IOException {

source/net/filebot/format/ExpressionFormatFunctions.java

@@ -1,12 +1,11 @@
 package net.filebot.format;
 
-import static java.nio.charset.StandardCharsets.*;
 import static java.util.stream.Collectors.*;
+import static net.filebot.util.FileUtilities.*;
 import static net.filebot.util.RegularExpressions.*;
 
+import java.io.File;
 import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Paths;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -53,20 +52,20 @@ public class ExpressionFormatFunctions {
 
 	public static Map<String, String> csv(String path) throws IOException {
 		Map<String, String> map = new LinkedHashMap<String, String>();
-		for (String line : Files.readAllLines(Paths.get(path), UTF_8)) {
+		streamLines(new File(path)).forEach(line -> {
 			for (Pattern delim : new Pattern[] { TAB, SEMICOLON }) {
 				String[] field = delim.split(line, 2);
 				if (field.length >= 2) {
-					map.put(field[0], field[1]);
+					map.put(field[0].trim(), field[1].trim());
 					break;
 				}
 			}
-		}
+		});
 		return map;
 	}
 
 	public static List<String> readLines(String path) throws IOException {
-		return Files.readAllLines(Paths.get(path), UTF_8);
+		return streamLines(new File(path)).collect(toList());
 	}
 
 }

source/net/filebot/util/FileSet.java

@@ -3,6 +3,7 @@ package net.filebot.util;
 import static java.nio.charset.StandardCharsets.*;
 import static java.util.Collections.*;
 import static java.util.stream.Collectors.*;
+import static net.filebot.util.FileUtilities.*;
 
 import java.io.File;
 import java.io.IOException;
@@ -10,7 +11,9 @@ import java.net.URI;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
 import java.util.AbstractSet;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -152,16 +155,12 @@ public class FileSet extends AbstractSet<Path> {
 		files.clear();
 	}
 
-	public void feed(Stream<? extends Object> stream) {
-		stream.forEach(it -> add(getPath(it)));
-	}
-
 	public void load(File f) throws IOException {
-		feed(Files.lines(f.toPath(), UTF_8));
+		streamLines(f).forEach(this::add);
 	}
 
-	public void store(File f) throws IOException {
+	public void append(File f, Collection<?>... paths) throws IOException {
-		Files.write(f.toPath(), stream().map(it -> (CharSequence) it.toString())::iterator, UTF_8);
+		Files.write(f.toPath(), Stream.of(paths).flatMap(Collection::stream).map(this::getPath).filter(it -> !contains(it)).map(Path::toString).collect(toList()), UTF_8, StandardOpenOption.APPEND);
 	}
 
 }

source/net/filebot/util/FileUtilities.java

@@ -3,16 +3,17 @@ package net.filebot.util;
 import static java.nio.charset.StandardCharsets.*;
 import static java.util.Arrays.*;
 import static java.util.Collections.*;
+import static java.util.stream.Collectors.*;
 import static net.filebot.Logging.*;
 import static net.filebot.util.RegularExpressions.*;
 
 import java.io.BufferedInputStream;
+import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileFilter;
 import java.io.FileInputStream;
 import java.io.FilenameFilter;
 import java.io.IOException;
-import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.Reader;
 import java.math.BigInteger;
@@ -39,12 +40,12 @@ import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
-import java.util.Scanner;
 import java.util.SortedMap;
 import java.util.TreeMap;
 import java.util.TreeSet;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.stream.Stream;
 
 import org.apache.commons.io.FileUtils;
 
@@ -166,45 +167,16 @@ public final class FileUtilities {
 		return Files.createDirectories(folder.toPath()).toFile();
 	}
 
-	public static void createFileIfNotExists(File file) throws IOException {
+	public static byte[] readFile(File source) throws IOException {
-		if (!file.isFile()) {
+		return Files.readAllBytes(source.toPath());
-			// create parent folder structure if necessary & create file
-			Files.createDirectories(file.getParentFile().toPath());
-			Files.createFile(file.toPath());
-		}
 	}
 
-	public static byte[] readFile(File source) throws IOException {
+	public static Stream<String> streamLines(File file) throws IOException {
-		long size = source.length();
+		return new BufferedReader(new UnicodeReader(new BufferedInputStream(new FileInputStream(file)), false, UTF_8), BUFFER_SIZE).lines();
-		if (size < 0 || size > Integer.MAX_VALUE) {
-			throw new IllegalArgumentException("Unable to read file: " + source);
-		}
-
-		try (InputStream in = new FileInputStream(source)) {
-			byte[] data = new byte[(int) size];
-			int position = 0;
-			int read = 0;
-
-			while (position < data.length && (read = in.read(data, position, data.length - position)) >= 0) {
-				position += read;
-			}
-
-			return data;
-		}
 	}
 
 	public static String readTextFile(File file) throws IOException {
-		try (Reader reader = new UnicodeReader(new BufferedInputStream(new FileInputStream(file), BUFFER_SIZE), false, StandardCharsets.UTF_8)) {
+		return streamLines(file).collect(joining(System.lineSeparator()));
-			StringBuilder text = new StringBuilder();
-			char[] buffer = new char[BUFFER_SIZE];
-
-			int read = 0;
-			while ((read = reader.read(buffer)) >= 0) {
-				text.append(buffer, 0, read);
-			}
-
-			return text.toString();
-		}
 	}
 
 	public static File writeFile(ByteBuffer data, File destination) throws IOException {
@@ -214,19 +186,6 @@ public final class FileUtilities {
 		return destination;
 	}
 
-	public static List<String[]> readCSV(InputStream source, String charsetName, String pattern) {
-		try (Scanner scanner = new Scanner(source, charsetName)) {
-			Pattern separator = Pattern.compile(pattern);
-			List<String[]> rows = new ArrayList<String[]>(65536);
-
-			while (scanner.hasNextLine()) {
-				rows.add(separator.split(scanner.nextLine()));
-			}
-
-			return rows;
-		}
-	}
-
 	public static Reader createTextReader(File file) throws IOException {
 		CharsetDetector detector = new CharsetDetector();
 		detector.setDeclaredEncoding("UTF-8"); // small boost for UTF-8 as default encoding