Cache xattr values so that metadata works as expected at least for the current session even if xattr is not supported by the filesystem and thus metadata can't be persisted

This commit is contained in:
Reinhard Pointner 2016-03-27 16:56:54 +00:00
parent b9fdfcbe10
commit dc22249794
17 changed files with 119 additions and 121 deletions

View File

@ -14,7 +14,7 @@ public enum CacheType {
Daily(Duration.ofHours(18), true), Daily(Duration.ofHours(18), true),
Ephemeral(Duration.ofHours(2), false); Ephemeral(Duration.ofHours(4), false);
final long timeToLiveSeconds; final long timeToLiveSeconds;
final boolean diskPersistent; final boolean diskPersistent;
@ -27,7 +27,7 @@ public enum CacheType {
@SuppressWarnings("deprecation") @SuppressWarnings("deprecation")
CacheConfiguration getConfiguration(String name) { CacheConfiguration getConfiguration(String name) {
// Strategy.LOCALTEMPSWAP is not restartable so we can't but use the deprecated disk persistent code (see http://stackoverflow.com/a/24623527/1514467) // Strategy.LOCALTEMPSWAP is not restartable so we can't but use the deprecated disk persistent code (see http://stackoverflow.com/a/24623527/1514467)
return new CacheConfiguration().name(name).maxEntriesLocalHeap(diskPersistent ? 200 : 0).maxEntriesLocalDisk(0).eternal(false).timeToLiveSeconds(timeToLiveSeconds).timeToIdleSeconds(timeToLiveSeconds).overflowToDisk(diskPersistent).diskPersistent(diskPersistent); return new CacheConfiguration().name(name).maxEntriesLocalHeap(diskPersistent ? 200 : 20_000).maxEntriesLocalDisk(0).eternal(false).timeToLiveSeconds(timeToLiveSeconds).timeToIdleSeconds(timeToLiveSeconds).overflowToDisk(diskPersistent).diskPersistent(diskPersistent);
} }
} }

View File

@ -660,14 +660,14 @@ public class CmdlineOperations implements CmdlineInterface {
} }
// write metadata into xattr if xattr is enabled // write metadata into xattr if xattr is enabled
if (matches != null && renameLog.size() > 0 && (useExtendedFileAttributes() || useCreationDate()) && renameAction != StandardRenameAction.TEST) { if (matches != null && renameLog.size() > 0 && renameAction != StandardRenameAction.TEST) {
for (Match<File, ?> match : matches) { for (Match<File, ?> match : matches) {
File source = match.getValue(); File source = match.getValue();
Object infoObject = match.getCandidate(); Object infoObject = match.getCandidate();
if (infoObject != null) { if (infoObject != null) {
File destination = renameLog.get(source); File destination = renameLog.get(source);
if (destination != null && destination.isFile()) { if (destination != null && destination.isFile()) {
xattr.storeMetaInfo(destination, infoObject, source.getName()); xattr.setMetaInfo(destination, infoObject, source.getName());
} }
} }
} }
@ -1076,7 +1076,7 @@ public class CmdlineOperations implements CmdlineInterface {
List<String> output = new ArrayList<String>(); List<String> output = new ArrayList<String>();
for (File file : filter(files, fileFilter)) { for (File file : filter(files, fileFilter)) {
String line = formatter.format(new MediaBindingBean(xattr.readMetaInfo(file), file, null)); String line = formatter.format(new MediaBindingBean(xattr.getMetaInfo(file), file, null));
output.add(line); output.add(line);
} }
return output; return output;

View File

@ -227,7 +227,7 @@ public abstract class ScriptShellBaseClass extends Script {
public Movie detectMovie(File file, boolean strict) { public Movie detectMovie(File file, boolean strict) {
// 1. xattr // 1. xattr
Object metaObject = xattr.readMetaInfo(file); Object metaObject = xattr.getMetaInfo(file);
if (metaObject instanceof Movie) { if (metaObject instanceof Movie) {
return (Movie) metaObject; return (Movie) metaObject;
} }

View File

@ -4,6 +4,7 @@ import static java.nio.charset.StandardCharsets.*;
import static java.util.Arrays.*; import static java.util.Arrays.*;
import static java.util.Collections.*; import static java.util.Collections.*;
import static net.filebot.MediaTypes.*; import static net.filebot.MediaTypes.*;
import static net.filebot.media.XattrMetaInfo.*;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
@ -28,7 +29,6 @@ import groovy.lang.Closure;
import net.filebot.MediaTypes; import net.filebot.MediaTypes;
import net.filebot.MetaAttributeView; import net.filebot.MetaAttributeView;
import net.filebot.media.MediaDetection; import net.filebot.media.MediaDetection;
import net.filebot.media.MetaAttributes;
import net.filebot.similarity.NameSimilarityMetric; import net.filebot.similarity.NameSimilarityMetric;
import net.filebot.similarity.Normalization; import net.filebot.similarity.Normalization;
import net.filebot.similarity.SimilarityMetric; import net.filebot.similarity.SimilarityMetric;
@ -394,7 +394,7 @@ public class ScriptShellMethods {
public static Object getMetadata(File self) { public static Object getMetadata(File self) {
try { try {
return new MetaAttributes(self).getObject(); return xattr.getMetaInfo(self);
} catch (Exception e) { } catch (Exception e) {
return null; return null;
} }

View File

@ -23,7 +23,7 @@ public class ExpressionFileFilter implements FileFilter {
@Override @Override
public boolean accept(File f) { public boolean accept(File f) {
try { try {
return filter.matches(new MediaBindingBean(xattr.readMetaInfo(f), f, null)); return filter.matches(new MediaBindingBean(xattr.getMetaInfo(f), f, null));
} catch (Exception e) { } catch (Exception e) {
debug.warning(format("Expression failed: %s", e)); debug.warning(format("Expression failed: %s", e));
return error; return error;

View File

@ -8,6 +8,7 @@ import static net.filebot.format.Define.*;
import static net.filebot.format.ExpressionFormatMethods.*; import static net.filebot.format.ExpressionFormatMethods.*;
import static net.filebot.hash.VerificationUtilities.*; import static net.filebot.hash.VerificationUtilities.*;
import static net.filebot.media.MediaDetection.*; import static net.filebot.media.MediaDetection.*;
import static net.filebot.media.XattrMetaInfo.*;
import static net.filebot.similarity.Normalization.*; import static net.filebot.similarity.Normalization.*;
import static net.filebot.subtitle.SubtitleUtilities.*; import static net.filebot.subtitle.SubtitleUtilities.*;
import static net.filebot.util.FileUtilities.*; import static net.filebot.util.FileUtilities.*;
@ -44,7 +45,6 @@ import net.filebot.Settings;
import net.filebot.Settings.ApplicationFolder; import net.filebot.Settings.ApplicationFolder;
import net.filebot.WebServices; import net.filebot.WebServices;
import net.filebot.hash.HashType; import net.filebot.hash.HashType;
import net.filebot.media.MetaAttributes;
import net.filebot.mediainfo.MediaInfo; import net.filebot.mediainfo.MediaInfo;
import net.filebot.mediainfo.MediaInfo.StreamKind; import net.filebot.mediainfo.MediaInfo.StreamKind;
import net.filebot.mediainfo.MediaInfoException; import net.filebot.mediainfo.MediaInfoException;
@ -411,7 +411,7 @@ public class MediaBindingBean {
@Define("xattr") @Define("xattr")
public Object getMetaAttributesObject() throws Exception { public Object getMetaAttributesObject() throws Exception {
return new MetaAttributes(getMediaFile()).getObject(); return xattr.getMetaInfo(getMediaFile());
} }
@Define("crc32") @Define("crc32")
@ -1054,11 +1054,7 @@ public class MediaBindingBean {
} }
private String getOriginalFileName(File file) { private String getOriginalFileName(File file) {
try { return getNameWithoutExtension(xattr.getOriginalName(file));
return getNameWithoutExtension(new MetaAttributes(file).getOriginalName());
} catch (Throwable e) {
return null;
}
} }
private List<String> getKeywords() { private List<String> getKeywords() {

View File

@ -300,7 +300,7 @@ public class MediaDetection {
// try xattr metadata if enabled // try xattr metadata if enabled
for (File it : files) { for (File it : files) {
Object metaObject = xattr.readMetaInfo(it); Object metaObject = xattr.getMetaInfo(it);
if (metaObject instanceof Episode) { if (metaObject instanceof Episode) {
unids.add(((Episode) metaObject).getSeriesName()); unids.add(((Episode) metaObject).getSeriesName());
} }
@ -575,7 +575,7 @@ public class MediaDetection {
List<Movie> options = new ArrayList<Movie>(); List<Movie> options = new ArrayList<Movie>();
// try xattr metadata if enabled // try xattr metadata if enabled
Object metaObject = xattr.readMetaInfo(movieFile); Object metaObject = xattr.getMetaInfo(movieFile);
if (metaObject instanceof Movie) { if (metaObject instanceof Movie) {
options.add((Movie) metaObject); options.add((Movie) metaObject);
} }

View File

@ -13,8 +13,8 @@ import net.filebot.MetaAttributeView;
public class MetaAttributes { public class MetaAttributes {
private static final String FILENAME_KEY = "net.filebot.filename"; public static final String FILENAME_KEY = "net.filebot.filename";
private static final String METADATA_KEY = "net.filebot.metadata"; public static final String METADATA_KEY = "net.filebot.metadata";
private final BasicFileAttributeView fileAttributeView; private final BasicFileAttributeView fileAttributeView;
private final MetaAttributeView metaAttributeView; private final MetaAttributeView metaAttributeView;

View File

@ -4,9 +4,11 @@ import static net.filebot.Logging.*;
import static net.filebot.Settings.*; import static net.filebot.Settings.*;
import java.io.File; import java.io.File;
import java.io.IOException;
import java.util.Locale; import java.util.Locale;
import net.filebot.Cache;
import net.filebot.Cache.Compute;
import net.filebot.CacheType;
import net.filebot.WebServices; import net.filebot.WebServices;
import net.filebot.web.Episode; import net.filebot.web.Episode;
import net.filebot.web.Movie; import net.filebot.web.Movie;
@ -19,6 +21,9 @@ public class XattrMetaInfo {
private final boolean useExtendedFileAttributes; private final boolean useExtendedFileAttributes;
private final boolean useCreationDate; private final boolean useCreationDate;
private final Cache xattrMetaInfoCache = Cache.getCache(MetaAttributes.METADATA_KEY, CacheType.Ephemeral);
private final Cache xattrOriginalNameCache = Cache.getCache(MetaAttributes.FILENAME_KEY, CacheType.Ephemeral);
public XattrMetaInfo(boolean useExtendedFileAttributes, boolean useCreationDate) { public XattrMetaInfo(boolean useExtendedFileAttributes, boolean useCreationDate) {
this.useExtendedFileAttributes = useExtendedFileAttributes; this.useExtendedFileAttributes = useExtendedFileAttributes;
this.useCreationDate = useCreationDate; this.useCreationDate = useCreationDate;
@ -46,60 +51,80 @@ public class XattrMetaInfo {
return -1; return -1;
} }
public Object readMetaInfo(File file) { public synchronized Object getMetaInfo(File file) {
if (useExtendedFileAttributes) { return getCachedValue(xattrMetaInfoCache, file, key -> {
try { Object metadata = new MetaAttributes(file).getObject();
MetaAttributes attr = new MetaAttributes(file); return isMetaInfo(metadata) ? metadata : null;
Object metadata = attr.getObject(); });
if (isMetaInfo(metadata)) {
return metadata;
} }
public synchronized String getOriginalName(File file) {
return (String) getCachedValue(xattrOriginalNameCache, file, key -> {
return new MetaAttributes(file).getOriginalName();
});
}
private Object getCachedValue(Cache cache, File file, Compute<?> compute) {
// try in-memory cache of previously stored xattr metadata
try {
return cache.computeIfAbsent(file, key -> {
if (useExtendedFileAttributes) {
return compute.apply(key);
}
return null;
});
} catch (Throwable e) { } catch (Throwable e) {
debug.warning("Failed to read xattr: " + e.getMessage()); debug.warning("Failed to read xattr: " + e.getMessage());
} }
}
return null; return null;
} }
public void storeMetaInfo(File file, Object model, String original) { public synchronized void setMetaInfo(File file, Object model, String original) {
// only for Episode / Movie objects // only for Episode / Movie objects
if ((useExtendedFileAttributes || useCreationDate) && isMetaInfo(model) && file.isFile()) { if (!isMetaInfo(model) || !file.isFile()) {
try { return;
MetaAttributes attr = new MetaAttributes(file); }
// set creation date to episode / movie release date // set creation date to episode / movie release date
if (useCreationDate) { if (useCreationDate) {
try { try {
long t = getTimeStamp(model); long t = getTimeStamp(model);
if (t > 0) { if (t > 0) {
attr.setCreationDate(t); new MetaAttributes(file).setCreationDate(t);
} }
} catch (Throwable e) { } catch (Throwable e) {
if (e.getCause() instanceof IOException) {
e = e.getCause();
}
debug.warning("Failed to set creation date: " + e.getMessage()); debug.warning("Failed to set creation date: " + e.getMessage());
} }
} }
// store original name and model as xattr // store metadata object and original name as xattr
if (useExtendedFileAttributes) { if (useExtendedFileAttributes) {
try { try {
MetaAttributes attr = new MetaAttributes(file);
if (isMetaInfo(model)) { if (isMetaInfo(model)) {
xattrMetaInfoCache.put(file, model);
attr.setObject(model); attr.setObject(model);
} }
if (attr.getOriginalName() == null && original != null && original.length() > 0) { if (original != null && original.length() > 0 && getOriginalName(file) == null) {
xattrOriginalNameCache.put(file, original);
attr.setOriginalName(original); attr.setOriginalName(original);
} }
} catch (Throwable e) { } catch (Throwable e) {
if (e.getCause() instanceof IOException) {
e = e.getCause();
}
debug.warning("Failed to set xattr: " + e.getMessage()); debug.warning("Failed to set xattr: " + e.getMessage());
} }
} }
} catch (Throwable t) { }
debug.warning("Failed to store xattr: " + t.getMessage());
public synchronized void clear(File file) {
// clear in-memory cache
xattrMetaInfoCache.remove(file);
xattrOriginalNameCache.remove(file);
if (useExtendedFileAttributes) {
try {
new MetaAttributes(file).clear();
} catch (Throwable e) {
debug.warning("Failed to clear xattr: " + e.getMessage());
} }
} }
} }

View File

@ -26,7 +26,7 @@ public class XattrMetaInfoProvider implements Datasource {
Map<File, Object> result = new LinkedHashMap<File, Object>(); Map<File, Object> result = new LinkedHashMap<File, Object>();
for (File f : files) { for (File f : files) {
Object metaObject = xattr.readMetaInfo(f); Object metaObject = xattr.getMetaInfo(f);
if (metaObject != null) { if (metaObject != null) {
result.put(f, metaObject); result.put(f, metaObject);
} }

View File

@ -679,7 +679,7 @@ public enum EpisodeMetrics implements SimilarityMetric {
// deserialize MetaAttributes if enabled and available // deserialize MetaAttributes if enabled and available
if (object instanceof File) { if (object instanceof File) {
Object metaObject = xattr.readMetaInfo((File) object); Object metaObject = xattr.getMetaInfo((File) object);
if (metaObject != null) { if (metaObject != null) {
return super.getProperties(metaObject); return super.getProperties(metaObject);
} }

View File

@ -3,6 +3,7 @@ package net.filebot.subtitle;
import static java.util.Collections.*; import static java.util.Collections.*;
import static net.filebot.Logging.*; import static net.filebot.Logging.*;
import static net.filebot.media.MediaDetection.*; import static net.filebot.media.MediaDetection.*;
import static net.filebot.media.XattrMetaInfo.*;
import static net.filebot.similarity.EpisodeMetrics.*; import static net.filebot.similarity.EpisodeMetrics.*;
import static net.filebot.util.FileUtilities.*; import static net.filebot.util.FileUtilities.*;
@ -14,7 +15,6 @@ import java.util.WeakHashMap;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import net.filebot.media.MetaAttributes;
import net.filebot.mediainfo.MediaInfo; import net.filebot.mediainfo.MediaInfo;
import net.filebot.mediainfo.MediaInfo.StreamKind; import net.filebot.mediainfo.MediaInfo.StreamKind;
import net.filebot.similarity.CrossPropertyMetric; import net.filebot.similarity.CrossPropertyMetric;
@ -124,26 +124,20 @@ public enum SubtitleMetrics implements SimilarityMetric {
return (float) match.length() / Math.max(s1.length(), s2.length()) > 0.8 ? 1 : 0; return (float) match.length() / Math.max(s1.length(), s2.length()) > 0.8 ? 1 : 0;
} }
private final Map<File, String> xattrCache = new WeakHashMap<File, String>(64);
@Override @Override
public String normalize(Object obj) { public String normalize(Object object) {
if (obj instanceof File) { if (object instanceof File) {
synchronized (xattrCache) { File file = (File) object;
return xattrCache.computeIfAbsent((File) obj, (f) -> { String name = xattr.getOriginalName(file);
try { if (name == null) {
String originalName = new MetaAttributes(f).getOriginalName(); name = file.getName();
return super.normalize(getNameWithoutExtension(originalName));
} catch (Exception e) {
return super.normalize(getNameWithoutExtension(f.getName()));
} }
}); return super.normalize(getNameWithoutExtension(name));
} } else if (object instanceof OpenSubtitlesSubtitleDescriptor) {
} else if (obj instanceof OpenSubtitlesSubtitleDescriptor) { String name = ((OpenSubtitlesSubtitleDescriptor) object).getName();
String name = ((OpenSubtitlesSubtitleDescriptor) obj).getName();
return super.normalize(name); return super.normalize(name);
} }
return super.normalize(obj); return super.normalize(object);
} }
}), }),

View File

@ -1,7 +1,7 @@
package net.filebot.ui.filter; package net.filebot.ui.filter;
import static net.filebot.Logging.*;
import static net.filebot.MediaTypes.*; import static net.filebot.MediaTypes.*;
import static net.filebot.media.XattrMetaInfo.*;
import static net.filebot.util.FileUtilities.*; import static net.filebot.util.FileUtilities.*;
import java.awt.Color; import java.awt.Color;
@ -16,7 +16,6 @@ import javax.swing.ListSelectionModel;
import javax.swing.table.AbstractTableModel; import javax.swing.table.AbstractTableModel;
import javax.swing.table.TableModel; import javax.swing.table.TableModel;
import net.filebot.media.MetaAttributes;
import net.filebot.util.FileUtilities; import net.filebot.util.FileUtilities;
import net.filebot.util.ui.LoadingOverlayPane; import net.filebot.util.ui.LoadingOverlayPane;
import net.filebot.web.Episode; import net.filebot.web.Episode;
@ -58,37 +57,22 @@ class AttributeTool extends Tool<TableModel> {
} }
for (File file : filter(FileUtilities.listFiles(root), VIDEO_FILES, SUBTITLE_FILES)) { for (File file : filter(FileUtilities.listFiles(root), VIDEO_FILES, SUBTITLE_FILES)) {
try { Object metaObject = xattr.getMetaInfo(file);
MetaAttributes attributes = new MetaAttributes(file); String originalName = xattr.getOriginalName(file);
String metaId = null;
Object metaObject = null;
String originalName = null;
try {
originalName = attributes.getOriginalName();
metaObject = attributes.getObject();
if (metaObject instanceof Episode) { if (metaObject instanceof Episode) {
SeriesInfo seriesInfo = ((Episode) metaObject).getSeriesInfo(); SeriesInfo seriesInfo = ((Episode) metaObject).getSeriesInfo();
if (seriesInfo != null) { if (seriesInfo != null) {
metaId = String.format("%s::%d", seriesInfo.getDatabase(), seriesInfo.getId()); model.addRow(String.format("%s::%d", seriesInfo.getDatabase(), seriesInfo.getId()), metaObject, originalName, file);
} }
} else if (metaObject instanceof Movie) { } else if (metaObject instanceof Movie) {
Movie movie = (Movie) metaObject; Movie movie = (Movie) metaObject;
if (movie.getTmdbId() > 0) { if (movie.getTmdbId() > 0) {
metaId = String.format("%s::%d", "TheMovieDB", movie.getTmdbId()); model.addRow(String.format("%s::%d", "TheMovieDB", movie.getTmdbId()), metaObject, originalName, file);
} else if (movie.getImdbId() > 0) { } else if (movie.getImdbId() > 0) {
metaId = String.format("%s::%d", "OMDb", movie.getImdbId()); model.addRow(String.format("%s::%d", "OMDb", movie.getImdbId()), metaObject, originalName, file);
} }
} }
} catch (Exception e) {
// ignore
}
model.addRow(metaId, metaObject, originalName, file);
} catch (Exception e) {
debug.warning("Failed to read xattr: " + e);
}
} }
return model; return model;

View File

@ -378,7 +378,7 @@ class BindingDialog extends JDialog {
mediaFileTextField.setText(file.get(0).getAbsolutePath()); mediaFileTextField.setText(file.get(0).getAbsolutePath());
// set info object from xattr if possible // set info object from xattr if possible
Object object = xattr.readMetaInfo(file.get(0)); Object object = xattr.getMetaInfo(file.get(0));
if (object != null && infoObjectFormat.format(object) != null) { if (object != null && infoObjectFormat.format(object) != null) {
setInfoObject(object); setInfoObject(object);
} }

View File

@ -8,6 +8,7 @@ import static javax.swing.JOptionPane.*;
import static net.filebot.Logging.*; import static net.filebot.Logging.*;
import static net.filebot.Settings.*; import static net.filebot.Settings.*;
import static net.filebot.UserFiles.*; import static net.filebot.UserFiles.*;
import static net.filebot.media.XattrMetaInfo.*;
import static net.filebot.util.ui.SwingUI.*; import static net.filebot.util.ui.SwingUI.*;
import java.awt.Color; import java.awt.Color;
@ -73,7 +74,6 @@ import net.filebot.History.Sequence;
import net.filebot.ResourceManager; import net.filebot.ResourceManager;
import net.filebot.StandardRenameAction; import net.filebot.StandardRenameAction;
import net.filebot.mac.MacAppUtilities; import net.filebot.mac.MacAppUtilities;
import net.filebot.media.MetaAttributes;
import net.filebot.ui.transfer.FileExportHandler; import net.filebot.ui.transfer.FileExportHandler;
import net.filebot.ui.transfer.FileTransferablePolicy; import net.filebot.ui.transfer.FileTransferablePolicy;
import net.filebot.ui.transfer.LoadAction; import net.filebot.ui.transfer.LoadAction;
@ -539,9 +539,8 @@ class HistoryDialog extends JDialog {
File original = StandardRenameAction.revert(entry.getKey(), entry.getValue()); File original = StandardRenameAction.revert(entry.getKey(), entry.getValue());
count++; count++;
if (original.isFile() && useExtendedFileAttributes()) { // clear xattr that may or may not exist
new MetaAttributes(original).clear(); xattr.clear(original);
}
} }
} catch (Exception e) { } catch (Exception e) {
log.log(Level.WARNING, "Failed to revert files: " + e.getMessage(), e); log.log(Level.WARNING, "Failed to revert files: " + e.getMessage(), e);

View File

@ -124,7 +124,7 @@ class RenameAction extends AbstractAction {
if (renameMap.containsKey(file) && meta != null) { if (renameMap.containsKey(file) && meta != null) {
File destination = resolveDestination(file, renameMap.get(file), false); File destination = resolveDestination(file, renameMap.get(file), false);
if (destination.isFile()) { if (destination.isFile()) {
xattr.storeMetaInfo(destination, meta, file.getName()); xattr.setMetaInfo(destination, meta, file.getName());
} }
} }
} }

View File

@ -333,7 +333,7 @@ public class RenamePanel extends JComponent {
List<Object> objects = new ArrayList<Object>(files.size()); List<Object> objects = new ArrayList<Object>(files.size());
List<File> objectsTail = new ArrayList<File>(); List<File> objectsTail = new ArrayList<File>();
for (File file : files) { for (File file : files) {
Object metaObject = xattr.readMetaInfo(file); Object metaObject = xattr.getMetaInfo(file);
if (metaObject != null) { if (metaObject != null) {
objects.add(metaObject); // upper list is based on xattr metadata objects.add(metaObject); // upper list is based on xattr metadata
} else { } else {