From dc222497943d6c74770ba644904dbce288029e71 Mon Sep 17 00:00:00 2001 From: Reinhard Pointner Date: Sun, 27 Mar 2016 16:56:54 +0000 Subject: [PATCH] Cache xattr values so that metadata works as expected at least for the current session even if xattr is not supported by the filesystem and thus metadata can't be persisted --- source/net/filebot/CacheType.java | 4 +- source/net/filebot/cli/CmdlineOperations.java | 6 +- .../net/filebot/cli/ScriptShellBaseClass.java | 2 +- .../net/filebot/cli/ScriptShellMethods.java | 4 +- .../filebot/format/ExpressionFileFilter.java | 2 +- .../net/filebot/format/MediaBindingBean.java | 10 +- source/net/filebot/media/MediaDetection.java | 4 +- source/net/filebot/media/MetaAttributes.java | 4 +- source/net/filebot/media/XattrMetaInfo.java | 115 +++++++++++------- .../filebot/media/XattrMetaInfoProvider.java | 2 +- .../filebot/similarity/EpisodeMetrics.java | 2 +- .../net/filebot/subtitle/SubtitleMetrics.java | 28 ++--- .../net/filebot/ui/filter/AttributeTool.java | 44 +++---- .../net/filebot/ui/rename/BindingDialog.java | 2 +- .../net/filebot/ui/rename/HistoryDialog.java | 7 +- .../net/filebot/ui/rename/RenameAction.java | 2 +- source/net/filebot/ui/rename/RenamePanel.java | 2 +- 17 files changed, 119 insertions(+), 121 deletions(-) diff --git a/source/net/filebot/CacheType.java b/source/net/filebot/CacheType.java index cbdef806..228f132c 100644 --- a/source/net/filebot/CacheType.java +++ b/source/net/filebot/CacheType.java @@ -14,7 +14,7 @@ public enum CacheType { Daily(Duration.ofHours(18), true), - Ephemeral(Duration.ofHours(2), false); + Ephemeral(Duration.ofHours(4), false); final long timeToLiveSeconds; final boolean diskPersistent; @@ -27,7 +27,7 @@ public enum CacheType { @SuppressWarnings("deprecation") CacheConfiguration getConfiguration(String name) { // Strategy.LOCALTEMPSWAP is not restartable so we can't but use the deprecated disk persistent code (see http://stackoverflow.com/a/24623527/1514467) - return new CacheConfiguration().name(name).maxEntriesLocalHeap(diskPersistent ? 200 : 0).maxEntriesLocalDisk(0).eternal(false).timeToLiveSeconds(timeToLiveSeconds).timeToIdleSeconds(timeToLiveSeconds).overflowToDisk(diskPersistent).diskPersistent(diskPersistent); + return new CacheConfiguration().name(name).maxEntriesLocalHeap(diskPersistent ? 200 : 20_000).maxEntriesLocalDisk(0).eternal(false).timeToLiveSeconds(timeToLiveSeconds).timeToIdleSeconds(timeToLiveSeconds).overflowToDisk(diskPersistent).diskPersistent(diskPersistent); } } diff --git a/source/net/filebot/cli/CmdlineOperations.java b/source/net/filebot/cli/CmdlineOperations.java index 5626175c..6b344214 100644 --- a/source/net/filebot/cli/CmdlineOperations.java +++ b/source/net/filebot/cli/CmdlineOperations.java @@ -660,14 +660,14 @@ public class CmdlineOperations implements CmdlineInterface { } // write metadata into xattr if xattr is enabled - if (matches != null && renameLog.size() > 0 && (useExtendedFileAttributes() || useCreationDate()) && renameAction != StandardRenameAction.TEST) { + if (matches != null && renameLog.size() > 0 && renameAction != StandardRenameAction.TEST) { for (Match match : matches) { File source = match.getValue(); Object infoObject = match.getCandidate(); if (infoObject != null) { File destination = renameLog.get(source); if (destination != null && destination.isFile()) { - xattr.storeMetaInfo(destination, infoObject, source.getName()); + xattr.setMetaInfo(destination, infoObject, source.getName()); } } } @@ -1076,7 +1076,7 @@ public class CmdlineOperations implements CmdlineInterface { List output = new ArrayList(); for (File file : filter(files, fileFilter)) { - String line = formatter.format(new MediaBindingBean(xattr.readMetaInfo(file), file, null)); + String line = formatter.format(new MediaBindingBean(xattr.getMetaInfo(file), file, null)); output.add(line); } return output; diff --git a/source/net/filebot/cli/ScriptShellBaseClass.java b/source/net/filebot/cli/ScriptShellBaseClass.java index 3af6e418..63727e7a 100644 --- a/source/net/filebot/cli/ScriptShellBaseClass.java +++ b/source/net/filebot/cli/ScriptShellBaseClass.java @@ -227,7 +227,7 @@ public abstract class ScriptShellBaseClass extends Script { public Movie detectMovie(File file, boolean strict) { // 1. xattr - Object metaObject = xattr.readMetaInfo(file); + Object metaObject = xattr.getMetaInfo(file); if (metaObject instanceof Movie) { return (Movie) metaObject; } diff --git a/source/net/filebot/cli/ScriptShellMethods.java b/source/net/filebot/cli/ScriptShellMethods.java index f14c31c5..4fb19e41 100644 --- a/source/net/filebot/cli/ScriptShellMethods.java +++ b/source/net/filebot/cli/ScriptShellMethods.java @@ -4,6 +4,7 @@ import static java.nio.charset.StandardCharsets.*; import static java.util.Arrays.*; import static java.util.Collections.*; import static net.filebot.MediaTypes.*; +import static net.filebot.media.XattrMetaInfo.*; import java.io.File; import java.io.IOException; @@ -28,7 +29,6 @@ import groovy.lang.Closure; import net.filebot.MediaTypes; import net.filebot.MetaAttributeView; import net.filebot.media.MediaDetection; -import net.filebot.media.MetaAttributes; import net.filebot.similarity.NameSimilarityMetric; import net.filebot.similarity.Normalization; import net.filebot.similarity.SimilarityMetric; @@ -394,7 +394,7 @@ public class ScriptShellMethods { public static Object getMetadata(File self) { try { - return new MetaAttributes(self).getObject(); + return xattr.getMetaInfo(self); } catch (Exception e) { return null; } diff --git a/source/net/filebot/format/ExpressionFileFilter.java b/source/net/filebot/format/ExpressionFileFilter.java index b9d1d29d..fd46ac73 100644 --- a/source/net/filebot/format/ExpressionFileFilter.java +++ b/source/net/filebot/format/ExpressionFileFilter.java @@ -23,7 +23,7 @@ public class ExpressionFileFilter implements FileFilter { @Override public boolean accept(File f) { try { - return filter.matches(new MediaBindingBean(xattr.readMetaInfo(f), f, null)); + return filter.matches(new MediaBindingBean(xattr.getMetaInfo(f), f, null)); } catch (Exception e) { debug.warning(format("Expression failed: %s", e)); return error; diff --git a/source/net/filebot/format/MediaBindingBean.java b/source/net/filebot/format/MediaBindingBean.java index 09a62fd4..b4c676d1 100644 --- a/source/net/filebot/format/MediaBindingBean.java +++ b/source/net/filebot/format/MediaBindingBean.java @@ -8,6 +8,7 @@ import static net.filebot.format.Define.*; import static net.filebot.format.ExpressionFormatMethods.*; import static net.filebot.hash.VerificationUtilities.*; import static net.filebot.media.MediaDetection.*; +import static net.filebot.media.XattrMetaInfo.*; import static net.filebot.similarity.Normalization.*; import static net.filebot.subtitle.SubtitleUtilities.*; import static net.filebot.util.FileUtilities.*; @@ -44,7 +45,6 @@ import net.filebot.Settings; import net.filebot.Settings.ApplicationFolder; import net.filebot.WebServices; import net.filebot.hash.HashType; -import net.filebot.media.MetaAttributes; import net.filebot.mediainfo.MediaInfo; import net.filebot.mediainfo.MediaInfo.StreamKind; import net.filebot.mediainfo.MediaInfoException; @@ -411,7 +411,7 @@ public class MediaBindingBean { @Define("xattr") public Object getMetaAttributesObject() throws Exception { - return new MetaAttributes(getMediaFile()).getObject(); + return xattr.getMetaInfo(getMediaFile()); } @Define("crc32") @@ -1054,11 +1054,7 @@ public class MediaBindingBean { } private String getOriginalFileName(File file) { - try { - return getNameWithoutExtension(new MetaAttributes(file).getOriginalName()); - } catch (Throwable e) { - return null; - } + return getNameWithoutExtension(xattr.getOriginalName(file)); } private List getKeywords() { diff --git a/source/net/filebot/media/MediaDetection.java b/source/net/filebot/media/MediaDetection.java index ca8a6b66..ed8debae 100644 --- a/source/net/filebot/media/MediaDetection.java +++ b/source/net/filebot/media/MediaDetection.java @@ -300,7 +300,7 @@ public class MediaDetection { // try xattr metadata if enabled for (File it : files) { - Object metaObject = xattr.readMetaInfo(it); + Object metaObject = xattr.getMetaInfo(it); if (metaObject instanceof Episode) { unids.add(((Episode) metaObject).getSeriesName()); } @@ -575,7 +575,7 @@ public class MediaDetection { List options = new ArrayList(); // try xattr metadata if enabled - Object metaObject = xattr.readMetaInfo(movieFile); + Object metaObject = xattr.getMetaInfo(movieFile); if (metaObject instanceof Movie) { options.add((Movie) metaObject); } diff --git a/source/net/filebot/media/MetaAttributes.java b/source/net/filebot/media/MetaAttributes.java index 7c3f88b6..57632e4b 100644 --- a/source/net/filebot/media/MetaAttributes.java +++ b/source/net/filebot/media/MetaAttributes.java @@ -13,8 +13,8 @@ import net.filebot.MetaAttributeView; public class MetaAttributes { - private static final String FILENAME_KEY = "net.filebot.filename"; - private static final String METADATA_KEY = "net.filebot.metadata"; + public static final String FILENAME_KEY = "net.filebot.filename"; + public static final String METADATA_KEY = "net.filebot.metadata"; private final BasicFileAttributeView fileAttributeView; private final MetaAttributeView metaAttributeView; diff --git a/source/net/filebot/media/XattrMetaInfo.java b/source/net/filebot/media/XattrMetaInfo.java index 50141a71..eff7fee7 100644 --- a/source/net/filebot/media/XattrMetaInfo.java +++ b/source/net/filebot/media/XattrMetaInfo.java @@ -4,9 +4,11 @@ import static net.filebot.Logging.*; import static net.filebot.Settings.*; import java.io.File; -import java.io.IOException; import java.util.Locale; +import net.filebot.Cache; +import net.filebot.Cache.Compute; +import net.filebot.CacheType; import net.filebot.WebServices; import net.filebot.web.Episode; import net.filebot.web.Movie; @@ -19,6 +21,9 @@ public class XattrMetaInfo { private final boolean useExtendedFileAttributes; private final boolean useCreationDate; + private final Cache xattrMetaInfoCache = Cache.getCache(MetaAttributes.METADATA_KEY, CacheType.Ephemeral); + private final Cache xattrOriginalNameCache = Cache.getCache(MetaAttributes.FILENAME_KEY, CacheType.Ephemeral); + public XattrMetaInfo(boolean useExtendedFileAttributes, boolean useCreationDate) { this.useExtendedFileAttributes = useExtendedFileAttributes; this.useCreationDate = useCreationDate; @@ -46,60 +51,80 @@ public class XattrMetaInfo { return -1; } - public Object readMetaInfo(File file) { - if (useExtendedFileAttributes) { - try { - MetaAttributes attr = new MetaAttributes(file); - Object metadata = attr.getObject(); - if (isMetaInfo(metadata)) { - return metadata; + public synchronized Object getMetaInfo(File file) { + return getCachedValue(xattrMetaInfoCache, file, key -> { + Object metadata = new MetaAttributes(file).getObject(); + return isMetaInfo(metadata) ? metadata : null; + }); + } + + public synchronized String getOriginalName(File file) { + return (String) getCachedValue(xattrOriginalNameCache, file, key -> { + return new MetaAttributes(file).getOriginalName(); + }); + } + + private Object getCachedValue(Cache cache, File file, Compute compute) { + // try in-memory cache of previously stored xattr metadata + try { + return cache.computeIfAbsent(file, key -> { + if (useExtendedFileAttributes) { + return compute.apply(key); } - } catch (Throwable e) { - debug.warning("Failed to read xattr: " + e.getMessage()); - } + return null; + }); + } catch (Throwable e) { + debug.warning("Failed to read xattr: " + e.getMessage()); } return null; } - public void storeMetaInfo(File file, Object model, String original) { + public synchronized void setMetaInfo(File file, Object model, String original) { // only for Episode / Movie objects - if ((useExtendedFileAttributes || useCreationDate) && isMetaInfo(model) && file.isFile()) { + if (!isMetaInfo(model) || !file.isFile()) { + return; + } + + // set creation date to episode / movie release date + if (useCreationDate) { + try { + long t = getTimeStamp(model); + if (t > 0) { + new MetaAttributes(file).setCreationDate(t); + } + } catch (Throwable e) { + debug.warning("Failed to set creation date: " + e.getMessage()); + } + } + + // store metadata object and original name as xattr + if (useExtendedFileAttributes) { try { MetaAttributes attr = new MetaAttributes(file); - - // set creation date to episode / movie release date - if (useCreationDate) { - try { - long t = getTimeStamp(model); - if (t > 0) { - attr.setCreationDate(t); - } - } catch (Throwable e) { - if (e.getCause() instanceof IOException) { - e = e.getCause(); - } - debug.warning("Failed to set creation date: " + e.getMessage()); - } + if (isMetaInfo(model)) { + xattrMetaInfoCache.put(file, model); + attr.setObject(model); } - - // store original name and model as xattr - if (useExtendedFileAttributes) { - try { - if (isMetaInfo(model)) { - attr.setObject(model); - } - if (attr.getOriginalName() == null && original != null && original.length() > 0) { - attr.setOriginalName(original); - } - } catch (Throwable e) { - if (e.getCause() instanceof IOException) { - e = e.getCause(); - } - debug.warning("Failed to set xattr: " + e.getMessage()); - } + if (original != null && original.length() > 0 && getOriginalName(file) == null) { + xattrOriginalNameCache.put(file, original); + attr.setOriginalName(original); } - } catch (Throwable t) { - debug.warning("Failed to store xattr: " + t.getMessage()); + } catch (Throwable e) { + debug.warning("Failed to set xattr: " + e.getMessage()); + } + } + } + + public synchronized void clear(File file) { + // clear in-memory cache + xattrMetaInfoCache.remove(file); + xattrOriginalNameCache.remove(file); + + if (useExtendedFileAttributes) { + try { + new MetaAttributes(file).clear(); + } catch (Throwable e) { + debug.warning("Failed to clear xattr: " + e.getMessage()); } } } diff --git a/source/net/filebot/media/XattrMetaInfoProvider.java b/source/net/filebot/media/XattrMetaInfoProvider.java index 0240523a..95d2bd3a 100644 --- a/source/net/filebot/media/XattrMetaInfoProvider.java +++ b/source/net/filebot/media/XattrMetaInfoProvider.java @@ -26,7 +26,7 @@ public class XattrMetaInfoProvider implements Datasource { Map result = new LinkedHashMap(); for (File f : files) { - Object metaObject = xattr.readMetaInfo(f); + Object metaObject = xattr.getMetaInfo(f); if (metaObject != null) { result.put(f, metaObject); } diff --git a/source/net/filebot/similarity/EpisodeMetrics.java b/source/net/filebot/similarity/EpisodeMetrics.java index 66f4cdb9..96aca53d 100644 --- a/source/net/filebot/similarity/EpisodeMetrics.java +++ b/source/net/filebot/similarity/EpisodeMetrics.java @@ -679,7 +679,7 @@ public enum EpisodeMetrics implements SimilarityMetric { // deserialize MetaAttributes if enabled and available if (object instanceof File) { - Object metaObject = xattr.readMetaInfo((File) object); + Object metaObject = xattr.getMetaInfo((File) object); if (metaObject != null) { return super.getProperties(metaObject); } diff --git a/source/net/filebot/subtitle/SubtitleMetrics.java b/source/net/filebot/subtitle/SubtitleMetrics.java index d082cda2..51b2dbd0 100644 --- a/source/net/filebot/subtitle/SubtitleMetrics.java +++ b/source/net/filebot/subtitle/SubtitleMetrics.java @@ -3,6 +3,7 @@ package net.filebot.subtitle; import static java.util.Collections.*; import static net.filebot.Logging.*; import static net.filebot.media.MediaDetection.*; +import static net.filebot.media.XattrMetaInfo.*; import static net.filebot.similarity.EpisodeMetrics.*; import static net.filebot.util.FileUtilities.*; @@ -14,7 +15,6 @@ import java.util.WeakHashMap; import java.util.regex.Matcher; import java.util.regex.Pattern; -import net.filebot.media.MetaAttributes; import net.filebot.mediainfo.MediaInfo; import net.filebot.mediainfo.MediaInfo.StreamKind; import net.filebot.similarity.CrossPropertyMetric; @@ -124,26 +124,20 @@ public enum SubtitleMetrics implements SimilarityMetric { return (float) match.length() / Math.max(s1.length(), s2.length()) > 0.8 ? 1 : 0; } - private final Map xattrCache = new WeakHashMap(64); - @Override - public String normalize(Object obj) { - if (obj instanceof File) { - synchronized (xattrCache) { - return xattrCache.computeIfAbsent((File) obj, (f) -> { - try { - String originalName = new MetaAttributes(f).getOriginalName(); - return super.normalize(getNameWithoutExtension(originalName)); - } catch (Exception e) { - return super.normalize(getNameWithoutExtension(f.getName())); - } - }); + public String normalize(Object object) { + if (object instanceof File) { + File file = (File) object; + String name = xattr.getOriginalName(file); + if (name == null) { + name = file.getName(); } - } else if (obj instanceof OpenSubtitlesSubtitleDescriptor) { - String name = ((OpenSubtitlesSubtitleDescriptor) obj).getName(); + return super.normalize(getNameWithoutExtension(name)); + } else if (object instanceof OpenSubtitlesSubtitleDescriptor) { + String name = ((OpenSubtitlesSubtitleDescriptor) object).getName(); return super.normalize(name); } - return super.normalize(obj); + return super.normalize(object); } }), diff --git a/source/net/filebot/ui/filter/AttributeTool.java b/source/net/filebot/ui/filter/AttributeTool.java index 3b2148a8..800f6d3b 100644 --- a/source/net/filebot/ui/filter/AttributeTool.java +++ b/source/net/filebot/ui/filter/AttributeTool.java @@ -1,7 +1,7 @@ package net.filebot.ui.filter; -import static net.filebot.Logging.*; import static net.filebot.MediaTypes.*; +import static net.filebot.media.XattrMetaInfo.*; import static net.filebot.util.FileUtilities.*; import java.awt.Color; @@ -16,7 +16,6 @@ import javax.swing.ListSelectionModel; import javax.swing.table.AbstractTableModel; import javax.swing.table.TableModel; -import net.filebot.media.MetaAttributes; import net.filebot.util.FileUtilities; import net.filebot.util.ui.LoadingOverlayPane; import net.filebot.web.Episode; @@ -58,36 +57,21 @@ class AttributeTool extends Tool { } for (File file : filter(FileUtilities.listFiles(root), VIDEO_FILES, SUBTITLE_FILES)) { - try { - MetaAttributes attributes = new MetaAttributes(file); - String metaId = null; - Object metaObject = null; - String originalName = null; + Object metaObject = xattr.getMetaInfo(file); + String originalName = xattr.getOriginalName(file); - try { - originalName = attributes.getOriginalName(); - metaObject = attributes.getObject(); - - if (metaObject instanceof Episode) { - SeriesInfo seriesInfo = ((Episode) metaObject).getSeriesInfo(); - if (seriesInfo != null) { - metaId = String.format("%s::%d", seriesInfo.getDatabase(), seriesInfo.getId()); - } - } else if (metaObject instanceof Movie) { - Movie movie = (Movie) metaObject; - if (movie.getTmdbId() > 0) { - metaId = String.format("%s::%d", "TheMovieDB", movie.getTmdbId()); - } else if (movie.getImdbId() > 0) { - metaId = String.format("%s::%d", "OMDb", movie.getImdbId()); - } - } - } catch (Exception e) { - // ignore + if (metaObject instanceof Episode) { + SeriesInfo seriesInfo = ((Episode) metaObject).getSeriesInfo(); + if (seriesInfo != null) { + model.addRow(String.format("%s::%d", seriesInfo.getDatabase(), seriesInfo.getId()), metaObject, originalName, file); + } + } else if (metaObject instanceof Movie) { + Movie movie = (Movie) metaObject; + if (movie.getTmdbId() > 0) { + model.addRow(String.format("%s::%d", "TheMovieDB", movie.getTmdbId()), metaObject, originalName, file); + } else if (movie.getImdbId() > 0) { + model.addRow(String.format("%s::%d", "OMDb", movie.getImdbId()), metaObject, originalName, file); } - - model.addRow(metaId, metaObject, originalName, file); - } catch (Exception e) { - debug.warning("Failed to read xattr: " + e); } } diff --git a/source/net/filebot/ui/rename/BindingDialog.java b/source/net/filebot/ui/rename/BindingDialog.java index 3eead0d8..9bd66555 100644 --- a/source/net/filebot/ui/rename/BindingDialog.java +++ b/source/net/filebot/ui/rename/BindingDialog.java @@ -378,7 +378,7 @@ class BindingDialog extends JDialog { mediaFileTextField.setText(file.get(0).getAbsolutePath()); // set info object from xattr if possible - Object object = xattr.readMetaInfo(file.get(0)); + Object object = xattr.getMetaInfo(file.get(0)); if (object != null && infoObjectFormat.format(object) != null) { setInfoObject(object); } diff --git a/source/net/filebot/ui/rename/HistoryDialog.java b/source/net/filebot/ui/rename/HistoryDialog.java index 0f79bf64..5930a6e0 100644 --- a/source/net/filebot/ui/rename/HistoryDialog.java +++ b/source/net/filebot/ui/rename/HistoryDialog.java @@ -8,6 +8,7 @@ import static javax.swing.JOptionPane.*; import static net.filebot.Logging.*; import static net.filebot.Settings.*; import static net.filebot.UserFiles.*; +import static net.filebot.media.XattrMetaInfo.*; import static net.filebot.util.ui.SwingUI.*; import java.awt.Color; @@ -73,7 +74,6 @@ import net.filebot.History.Sequence; import net.filebot.ResourceManager; import net.filebot.StandardRenameAction; import net.filebot.mac.MacAppUtilities; -import net.filebot.media.MetaAttributes; import net.filebot.ui.transfer.FileExportHandler; import net.filebot.ui.transfer.FileTransferablePolicy; import net.filebot.ui.transfer.LoadAction; @@ -539,9 +539,8 @@ class HistoryDialog extends JDialog { File original = StandardRenameAction.revert(entry.getKey(), entry.getValue()); count++; - if (original.isFile() && useExtendedFileAttributes()) { - new MetaAttributes(original).clear(); - } + // clear xattr that may or may not exist + xattr.clear(original); } } catch (Exception e) { log.log(Level.WARNING, "Failed to revert files: " + e.getMessage(), e); diff --git a/source/net/filebot/ui/rename/RenameAction.java b/source/net/filebot/ui/rename/RenameAction.java index 040c07cc..03c5e77f 100644 --- a/source/net/filebot/ui/rename/RenameAction.java +++ b/source/net/filebot/ui/rename/RenameAction.java @@ -124,7 +124,7 @@ class RenameAction extends AbstractAction { if (renameMap.containsKey(file) && meta != null) { File destination = resolveDestination(file, renameMap.get(file), false); if (destination.isFile()) { - xattr.storeMetaInfo(destination, meta, file.getName()); + xattr.setMetaInfo(destination, meta, file.getName()); } } } diff --git a/source/net/filebot/ui/rename/RenamePanel.java b/source/net/filebot/ui/rename/RenamePanel.java index ceeeb715..a3602579 100644 --- a/source/net/filebot/ui/rename/RenamePanel.java +++ b/source/net/filebot/ui/rename/RenamePanel.java @@ -333,7 +333,7 @@ public class RenamePanel extends JComponent { List objects = new ArrayList(files.size()); List objectsTail = new ArrayList(); for (File file : files) { - Object metaObject = xattr.readMetaInfo(file); + Object metaObject = xattr.getMetaInfo(file); if (metaObject != null) { objects.add(metaObject); // upper list is based on xattr metadata } else {