forked from Eauldane/SnowcloakClient
Initial
This commit is contained in:
859
MareSynchronos/FileCache/CacheMonitor.cs
Normal file
859
MareSynchronos/FileCache/CacheMonitor.cs
Normal file
@@ -0,0 +1,859 @@
|
||||
using MareSynchronos.Interop.Ipc;
|
||||
using MareSynchronos.MareConfiguration;
|
||||
using MareSynchronos.Services;
|
||||
using MareSynchronos.Services.Mediator;
|
||||
using MareSynchronos.Utils;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Immutable;
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
namespace MareSynchronos.FileCache;
|
||||
|
||||
public sealed class CacheMonitor : DisposableMediatorSubscriberBase
|
||||
{
|
||||
private readonly MareConfigService _configService;
|
||||
private readonly DalamudUtilService _dalamudUtil;
|
||||
private readonly FileCompactor _fileCompactor;
|
||||
private readonly FileCacheManager _fileDbManager;
|
||||
private readonly IpcManager _ipcManager;
|
||||
private readonly PerformanceCollectorService _performanceCollector;
|
||||
private long _currentFileProgress = 0;
|
||||
private CancellationTokenSource _scanCancellationTokenSource = new();
|
||||
private readonly CancellationTokenSource _periodicCalculationTokenSource = new();
|
||||
public static readonly IImmutableList<string> AllowedFileExtensions = [".mdl", ".tex", ".mtrl", ".tmb", ".pap", ".avfx", ".atex", ".sklb", ".eid", ".phyb", ".pbd", ".scd", ".skp", ".shpk"];
|
||||
|
||||
public CacheMonitor(ILogger<CacheMonitor> logger, IpcManager ipcManager, MareConfigService configService,
|
||||
FileCacheManager fileDbManager, MareMediator mediator, PerformanceCollectorService performanceCollector, DalamudUtilService dalamudUtil,
|
||||
FileCompactor fileCompactor) : base(logger, mediator)
|
||||
{
|
||||
_ipcManager = ipcManager;
|
||||
_configService = configService;
|
||||
_fileDbManager = fileDbManager;
|
||||
_performanceCollector = performanceCollector;
|
||||
_dalamudUtil = dalamudUtil;
|
||||
_fileCompactor = fileCompactor;
|
||||
Mediator.Subscribe<PenumbraInitializedMessage>(this, (_) =>
|
||||
{
|
||||
StartPenumbraWatcher(_ipcManager.Penumbra.ModDirectory);
|
||||
StartMareWatcher(configService.Current.CacheFolder);
|
||||
StartSubstWatcher(_fileDbManager.SubstFolder);
|
||||
InvokeScan();
|
||||
});
|
||||
Mediator.Subscribe<HaltScanMessage>(this, (msg) => HaltScan(msg.Source));
|
||||
Mediator.Subscribe<ResumeScanMessage>(this, (msg) => ResumeScan(msg.Source));
|
||||
Mediator.Subscribe<DalamudLoginMessage>(this, (_) =>
|
||||
{
|
||||
StartMareWatcher(configService.Current.CacheFolder);
|
||||
StartSubstWatcher(_fileDbManager.SubstFolder);
|
||||
StartPenumbraWatcher(_ipcManager.Penumbra.ModDirectory);
|
||||
InvokeScan();
|
||||
});
|
||||
Mediator.Subscribe<PenumbraDirectoryChangedMessage>(this, (msg) =>
|
||||
{
|
||||
StartPenumbraWatcher(msg.ModDirectory);
|
||||
InvokeScan();
|
||||
});
|
||||
if (_ipcManager.Penumbra.APIAvailable && !string.IsNullOrEmpty(_ipcManager.Penumbra.ModDirectory))
|
||||
{
|
||||
StartPenumbraWatcher(_ipcManager.Penumbra.ModDirectory);
|
||||
}
|
||||
if (configService.Current.HasValidSetup())
|
||||
{
|
||||
StartMareWatcher(configService.Current.CacheFolder);
|
||||
StartSubstWatcher(_fileDbManager.SubstFolder);
|
||||
InvokeScan();
|
||||
}
|
||||
|
||||
var token = _periodicCalculationTokenSource.Token;
|
||||
_ = Task.Run(async () =>
|
||||
{
|
||||
Logger.LogInformation("Starting Periodic Storage Directory Calculation Task");
|
||||
var token = _periodicCalculationTokenSource.Token;
|
||||
while (!token.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
while (_dalamudUtil.IsOnFrameworkThread && !token.IsCancellationRequested)
|
||||
{
|
||||
await Task.Delay(1).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
RecalculateFileCacheSize(token);
|
||||
}
|
||||
catch
|
||||
{
|
||||
// ignore
|
||||
}
|
||||
await Task.Delay(TimeSpan.FromMinutes(1), token).ConfigureAwait(false);
|
||||
}
|
||||
}, token);
|
||||
}
|
||||
|
||||
public long CurrentFileProgress => _currentFileProgress;
|
||||
public long FileCacheSize { get; set; }
|
||||
public long FileCacheDriveFree { get; set; }
|
||||
public ConcurrentDictionary<string, StrongBox<int>> HaltScanLocks { get; set; } = new(StringComparer.Ordinal);
|
||||
public bool IsScanRunning => CurrentFileProgress > 0 || TotalFiles > 0;
|
||||
public long TotalFiles { get; private set; }
|
||||
public long TotalFilesStorage { get; private set; }
|
||||
|
||||
public void HaltScan(string source)
|
||||
{
|
||||
HaltScanLocks.TryAdd(source, new(0));
|
||||
Interlocked.Increment(ref HaltScanLocks[source].Value);
|
||||
}
|
||||
|
||||
record WatcherChange(WatcherChangeTypes ChangeType, string? OldPath = null);
|
||||
private readonly Dictionary<string, WatcherChange> _watcherChanges = new Dictionary<string, WatcherChange>(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly Dictionary<string, WatcherChange> _mareChanges = new Dictionary<string, WatcherChange>(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly Dictionary<string, WatcherChange> _substChanges = new Dictionary<string, WatcherChange>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public void StopMonitoring()
|
||||
{
|
||||
Logger.LogInformation("Stopping monitoring of Penumbra and Mare storage folders");
|
||||
MareWatcher?.Dispose();
|
||||
SubstWatcher?.Dispose();
|
||||
PenumbraWatcher?.Dispose();
|
||||
MareWatcher = null;
|
||||
SubstWatcher = null;
|
||||
PenumbraWatcher = null;
|
||||
}
|
||||
|
||||
public bool StorageisNTFS { get; private set; } = false;
|
||||
|
||||
public void StartMareWatcher(string? marePath)
|
||||
{
|
||||
MareWatcher?.Dispose();
|
||||
if (string.IsNullOrEmpty(marePath) || !Directory.Exists(marePath))
|
||||
{
|
||||
MareWatcher = null;
|
||||
Logger.LogWarning("Mare file path is not set, cannot start the FSW for Mare.");
|
||||
return;
|
||||
}
|
||||
|
||||
DriveInfo di = new(new DirectoryInfo(_configService.Current.CacheFolder).Root.FullName);
|
||||
StorageisNTFS = string.Equals("NTFS", di.DriveFormat, StringComparison.OrdinalIgnoreCase);
|
||||
Logger.LogInformation("Mare Storage is on NTFS drive: {isNtfs}", StorageisNTFS);
|
||||
|
||||
Logger.LogDebug("Initializing Mare FSW on {path}", marePath);
|
||||
MareWatcher = new()
|
||||
{
|
||||
Path = marePath,
|
||||
InternalBufferSize = 8388608,
|
||||
NotifyFilter = NotifyFilters.CreationTime
|
||||
| NotifyFilters.LastWrite
|
||||
| NotifyFilters.FileName
|
||||
| NotifyFilters.DirectoryName
|
||||
| NotifyFilters.Size,
|
||||
Filter = "*.*",
|
||||
IncludeSubdirectories = false,
|
||||
};
|
||||
|
||||
MareWatcher.Deleted += MareWatcher_FileChanged;
|
||||
MareWatcher.Created += MareWatcher_FileChanged;
|
||||
MareWatcher.EnableRaisingEvents = true;
|
||||
}
|
||||
|
||||
public void StartSubstWatcher(string? substPath)
|
||||
{
|
||||
SubstWatcher?.Dispose();
|
||||
if (string.IsNullOrEmpty(substPath))
|
||||
{
|
||||
SubstWatcher = null;
|
||||
Logger.LogWarning("Mare file path is not set, cannot start the FSW for Mare.");
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (!Directory.Exists(substPath))
|
||||
Directory.CreateDirectory(substPath);
|
||||
}
|
||||
catch
|
||||
{
|
||||
Logger.LogWarning("Could not create subst directory at {path}.", substPath);
|
||||
return;
|
||||
}
|
||||
|
||||
Logger.LogDebug("Initializing Subst FSW on {path}", substPath);
|
||||
SubstWatcher = new()
|
||||
{
|
||||
Path = substPath,
|
||||
InternalBufferSize = 8388608,
|
||||
NotifyFilter = NotifyFilters.CreationTime
|
||||
| NotifyFilters.LastWrite
|
||||
| NotifyFilters.FileName
|
||||
| NotifyFilters.DirectoryName
|
||||
| NotifyFilters.Size,
|
||||
Filter = "*.*",
|
||||
IncludeSubdirectories = false,
|
||||
};
|
||||
|
||||
SubstWatcher.Deleted += SubstWatcher_FileChanged;
|
||||
SubstWatcher.Created += SubstWatcher_FileChanged;
|
||||
SubstWatcher.EnableRaisingEvents = true;
|
||||
}
|
||||
|
||||
private void MareWatcher_FileChanged(object sender, FileSystemEventArgs e)
|
||||
{
|
||||
Logger.LogTrace("Mare FSW: FileChanged: {change} => {path}", e.ChangeType, e.FullPath);
|
||||
|
||||
if (!AllowedFileExtensions.Any(ext => e.FullPath.EndsWith(ext, StringComparison.OrdinalIgnoreCase))) return;
|
||||
|
||||
lock (_mareChanges)
|
||||
{
|
||||
_mareChanges[e.FullPath] = new(e.ChangeType);
|
||||
}
|
||||
|
||||
_ = MareWatcherExecution();
|
||||
}
|
||||
|
||||
private void SubstWatcher_FileChanged(object sender, FileSystemEventArgs e)
|
||||
{
|
||||
Logger.LogTrace("Subst FSW: FileChanged: {change} => {path}", e.ChangeType, e.FullPath);
|
||||
|
||||
if (!AllowedFileExtensions.Any(ext => e.FullPath.EndsWith(ext, StringComparison.OrdinalIgnoreCase))) return;
|
||||
|
||||
lock (_substChanges)
|
||||
{
|
||||
_substChanges[e.FullPath] = new(e.ChangeType);
|
||||
}
|
||||
|
||||
_ = SubstWatcherExecution();
|
||||
}
|
||||
|
||||
public void StartPenumbraWatcher(string? penumbraPath)
|
||||
{
|
||||
PenumbraWatcher?.Dispose();
|
||||
if (string.IsNullOrEmpty(penumbraPath))
|
||||
{
|
||||
PenumbraWatcher = null;
|
||||
Logger.LogWarning("Penumbra is not connected or the path is not set, cannot start FSW for Penumbra.");
|
||||
return;
|
||||
}
|
||||
|
||||
Logger.LogDebug("Initializing Penumbra FSW on {path}", penumbraPath);
|
||||
PenumbraWatcher = new()
|
||||
{
|
||||
Path = penumbraPath,
|
||||
InternalBufferSize = 8388608,
|
||||
NotifyFilter = NotifyFilters.CreationTime
|
||||
| NotifyFilters.LastWrite
|
||||
| NotifyFilters.FileName
|
||||
| NotifyFilters.DirectoryName
|
||||
| NotifyFilters.Size,
|
||||
Filter = "*.*",
|
||||
IncludeSubdirectories = true
|
||||
};
|
||||
|
||||
PenumbraWatcher.Deleted += Fs_Changed;
|
||||
PenumbraWatcher.Created += Fs_Changed;
|
||||
PenumbraWatcher.Changed += Fs_Changed;
|
||||
PenumbraWatcher.Renamed += Fs_Renamed;
|
||||
PenumbraWatcher.EnableRaisingEvents = true;
|
||||
}
|
||||
|
||||
private void Fs_Changed(object sender, FileSystemEventArgs e)
|
||||
{
|
||||
if (Directory.Exists(e.FullPath)) return;
|
||||
if (!AllowedFileExtensions.Any(ext => e.FullPath.EndsWith(ext, StringComparison.OrdinalIgnoreCase))) return;
|
||||
|
||||
if (e.ChangeType is not (WatcherChangeTypes.Changed or WatcherChangeTypes.Deleted or WatcherChangeTypes.Created))
|
||||
return;
|
||||
|
||||
lock (_watcherChanges)
|
||||
{
|
||||
_watcherChanges[e.FullPath] = new(e.ChangeType);
|
||||
}
|
||||
|
||||
Logger.LogTrace("FSW {event}: {path}", e.ChangeType, e.FullPath);
|
||||
|
||||
_ = PenumbraWatcherExecution();
|
||||
}
|
||||
|
||||
private void Fs_Renamed(object sender, RenamedEventArgs e)
|
||||
{
|
||||
if (Directory.Exists(e.FullPath))
|
||||
{
|
||||
var directoryFiles = Directory.GetFiles(e.FullPath, "*.*", SearchOption.AllDirectories);
|
||||
lock (_watcherChanges)
|
||||
{
|
||||
foreach (var file in directoryFiles)
|
||||
{
|
||||
if (!AllowedFileExtensions.Any(ext => file.EndsWith(ext, StringComparison.OrdinalIgnoreCase))) continue;
|
||||
var oldPath = file.Replace(e.FullPath, e.OldFullPath, StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
_watcherChanges.Remove(oldPath);
|
||||
_watcherChanges[file] = new(WatcherChangeTypes.Renamed, oldPath);
|
||||
Logger.LogTrace("FSW Renamed: {path} -> {new}", oldPath, file);
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if (!AllowedFileExtensions.Any(ext => e.FullPath.EndsWith(ext, StringComparison.OrdinalIgnoreCase))) return;
|
||||
|
||||
lock (_watcherChanges)
|
||||
{
|
||||
_watcherChanges.Remove(e.OldFullPath);
|
||||
_watcherChanges[e.FullPath] = new(WatcherChangeTypes.Renamed, e.OldFullPath);
|
||||
}
|
||||
|
||||
Logger.LogTrace("FSW Renamed: {path} -> {new}", e.OldFullPath, e.FullPath);
|
||||
}
|
||||
|
||||
_ = PenumbraWatcherExecution();
|
||||
}
|
||||
|
||||
private CancellationTokenSource _penumbraFswCts = new();
|
||||
private CancellationTokenSource _mareFswCts = new();
|
||||
private CancellationTokenSource _substFswCts = new();
|
||||
public FileSystemWatcher? PenumbraWatcher { get; private set; }
|
||||
public FileSystemWatcher? MareWatcher { get; private set; }
|
||||
public FileSystemWatcher? SubstWatcher { get; private set; }
|
||||
|
||||
private async Task MareWatcherExecution()
|
||||
{
|
||||
_mareFswCts = _mareFswCts.CancelRecreate();
|
||||
var token = _mareFswCts.Token;
|
||||
var delay = TimeSpan.FromSeconds(5);
|
||||
Dictionary<string, WatcherChange> changes;
|
||||
lock (_mareChanges)
|
||||
changes = _mareChanges.ToDictionary(t => t.Key, t => t.Value, StringComparer.Ordinal);
|
||||
try
|
||||
{
|
||||
do
|
||||
{
|
||||
await Task.Delay(delay, token).ConfigureAwait(false);
|
||||
} while (HaltScanLocks.Any(f => f.Value.Value > 0));
|
||||
}
|
||||
catch (TaskCanceledException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
lock (_mareChanges)
|
||||
{
|
||||
foreach (var key in changes.Keys)
|
||||
{
|
||||
_mareChanges.Remove(key);
|
||||
}
|
||||
}
|
||||
|
||||
HandleChanges(changes);
|
||||
}
|
||||
|
||||
private async Task SubstWatcherExecution()
|
||||
{
|
||||
_substFswCts = _substFswCts.CancelRecreate();
|
||||
var token = _substFswCts.Token;
|
||||
var delay = TimeSpan.FromSeconds(5);
|
||||
Dictionary<string, WatcherChange> changes;
|
||||
lock (_substChanges)
|
||||
changes = _substChanges.ToDictionary(t => t.Key, t => t.Value, StringComparer.Ordinal);
|
||||
try
|
||||
{
|
||||
do
|
||||
{
|
||||
await Task.Delay(delay, token).ConfigureAwait(false);
|
||||
} while (HaltScanLocks.Any(f => f.Value.Value > 0));
|
||||
}
|
||||
catch (TaskCanceledException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
lock (_substChanges)
|
||||
{
|
||||
foreach (var key in changes.Keys)
|
||||
{
|
||||
_substChanges.Remove(key);
|
||||
}
|
||||
}
|
||||
|
||||
HandleChanges(changes);
|
||||
}
|
||||
|
||||
public void ClearSubstStorage()
|
||||
{
|
||||
var substDir = _fileDbManager.SubstFolder;
|
||||
var allSubstFiles = Directory.GetFiles(substDir, "*.*", SearchOption.TopDirectoryOnly)
|
||||
.Where(f =>
|
||||
{
|
||||
var val = f.Split('\\')[^1];
|
||||
return val.Length == 40 || (val.Split('.').FirstOrDefault()?.Length ?? 0) == 40
|
||||
|| val.EndsWith(".tmp", StringComparison.OrdinalIgnoreCase);
|
||||
});
|
||||
if (SubstWatcher != null)
|
||||
SubstWatcher.EnableRaisingEvents = false;
|
||||
|
||||
Dictionary<string, WatcherChange> changes = _substChanges.ToDictionary(t => t.Key, t => new WatcherChange(WatcherChangeTypes.Deleted, t.Key), StringComparer.Ordinal);
|
||||
|
||||
foreach (var file in allSubstFiles)
|
||||
{
|
||||
try
|
||||
{
|
||||
File.Delete(file);
|
||||
}
|
||||
catch { }
|
||||
}
|
||||
|
||||
HandleChanges(changes);
|
||||
|
||||
if (SubstWatcher != null)
|
||||
SubstWatcher.EnableRaisingEvents = true;
|
||||
}
|
||||
|
||||
public void DeleteSubstOriginals()
|
||||
{
|
||||
var cacheDir = _configService.Current.CacheFolder;
|
||||
var substDir = _fileDbManager.SubstFolder;
|
||||
var allSubstFiles = Directory.GetFiles(substDir, "*.*", SearchOption.TopDirectoryOnly)
|
||||
.Where(f =>
|
||||
{
|
||||
var val = f.Split('\\')[^1];
|
||||
return val.Length == 40 || (val.Split('.').FirstOrDefault()?.Length ?? 0) == 40
|
||||
|| val.EndsWith(".tmp", StringComparison.OrdinalIgnoreCase);
|
||||
});
|
||||
|
||||
foreach (var substFile in allSubstFiles)
|
||||
{
|
||||
var cacheFile = Path.Join(cacheDir, Path.GetFileName(substFile));
|
||||
try
|
||||
{
|
||||
if (File.Exists(cacheFile))
|
||||
File.Delete(cacheFile);
|
||||
}
|
||||
catch { }
|
||||
}
|
||||
}
|
||||
|
||||
private void HandleChanges(Dictionary<string, WatcherChange> changes)
|
||||
{
|
||||
lock (_fileDbManager)
|
||||
{
|
||||
var deletedEntries = changes.Where(c => c.Value.ChangeType == WatcherChangeTypes.Deleted).Select(c => c.Key);
|
||||
var renamedEntries = changes.Where(c => c.Value.ChangeType == WatcherChangeTypes.Renamed);
|
||||
var remainingEntries = changes.Where(c => c.Value.ChangeType != WatcherChangeTypes.Deleted).Select(c => c.Key);
|
||||
|
||||
foreach (var entry in deletedEntries)
|
||||
{
|
||||
Logger.LogDebug("FSW Change: Deletion - {val}", entry);
|
||||
}
|
||||
|
||||
foreach (var entry in renamedEntries)
|
||||
{
|
||||
Logger.LogDebug("FSW Change: Renamed - {oldVal} => {val}", entry.Value.OldPath, entry.Key);
|
||||
}
|
||||
|
||||
foreach (var entry in remainingEntries)
|
||||
{
|
||||
Logger.LogDebug("FSW Change: Creation or Change - {val}", entry);
|
||||
}
|
||||
|
||||
var allChanges = deletedEntries
|
||||
.Concat(renamedEntries.Select(c => c.Value.OldPath!))
|
||||
.Concat(renamedEntries.Select(c => c.Key))
|
||||
.Concat(remainingEntries)
|
||||
.ToArray();
|
||||
|
||||
_ = _fileDbManager.GetFileCachesByPaths(allChanges);
|
||||
|
||||
_fileDbManager.WriteOutFullCsv();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task PenumbraWatcherExecution()
|
||||
{
|
||||
_penumbraFswCts = _penumbraFswCts.CancelRecreate();
|
||||
var token = _penumbraFswCts.Token;
|
||||
Dictionary<string, WatcherChange> changes;
|
||||
lock (_watcherChanges)
|
||||
changes = _watcherChanges.ToDictionary(t => t.Key, t => t.Value, StringComparer.Ordinal);
|
||||
var delay = TimeSpan.FromSeconds(10);
|
||||
try
|
||||
{
|
||||
do
|
||||
{
|
||||
await Task.Delay(delay, token).ConfigureAwait(false);
|
||||
} while (HaltScanLocks.Any(f => f.Value.Value > 0));
|
||||
}
|
||||
catch (TaskCanceledException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
lock (_watcherChanges)
|
||||
{
|
||||
foreach (var key in changes.Keys)
|
||||
{
|
||||
_watcherChanges.Remove(key);
|
||||
}
|
||||
}
|
||||
|
||||
HandleChanges(changes);
|
||||
}
|
||||
|
||||
public void InvokeScan()
|
||||
{
|
||||
TotalFiles = 0;
|
||||
_currentFileProgress = 0;
|
||||
_scanCancellationTokenSource = _scanCancellationTokenSource?.CancelRecreate() ?? new CancellationTokenSource();
|
||||
var token = _scanCancellationTokenSource.Token;
|
||||
_ = Task.Run(async () =>
|
||||
{
|
||||
Logger.LogDebug("Starting Full File Scan");
|
||||
TotalFiles = 0;
|
||||
_currentFileProgress = 0;
|
||||
while (_dalamudUtil.IsOnFrameworkThread)
|
||||
{
|
||||
Logger.LogWarning("Scanner is on framework, waiting for leaving thread before continuing");
|
||||
await Task.Delay(250, token).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
Thread scanThread = new(() =>
|
||||
{
|
||||
try
|
||||
{
|
||||
_performanceCollector.LogPerformance(this, $"FullFileScan", () => FullFileScan(token));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.LogError(ex, "Error during Full File Scan");
|
||||
}
|
||||
})
|
||||
{
|
||||
Priority = ThreadPriority.Lowest,
|
||||
IsBackground = true
|
||||
};
|
||||
scanThread.Start();
|
||||
while (scanThread.IsAlive)
|
||||
{
|
||||
await Task.Delay(250).ConfigureAwait(false);
|
||||
}
|
||||
TotalFiles = 0;
|
||||
_currentFileProgress = 0;
|
||||
}, token);
|
||||
}
|
||||
|
||||
public void RecalculateFileCacheSize(CancellationToken token)
|
||||
{
|
||||
if (string.IsNullOrEmpty(_configService.Current.CacheFolder) || !Directory.Exists(_configService.Current.CacheFolder))
|
||||
{
|
||||
FileCacheSize = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
FileCacheSize = -1;
|
||||
DriveInfo di = new(new DirectoryInfo(_configService.Current.CacheFolder).Root.FullName);
|
||||
try
|
||||
{
|
||||
FileCacheDriveFree = di.AvailableFreeSpace;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.LogWarning(ex, "Could not determine drive size for Storage Folder {folder}", _configService.Current.CacheFolder);
|
||||
}
|
||||
|
||||
var files = Directory.EnumerateFiles(_configService.Current.CacheFolder)
|
||||
.Concat(Directory.EnumerateFiles(_fileDbManager.SubstFolder))
|
||||
.Select(f => new FileInfo(f))
|
||||
.OrderBy(f => f.LastAccessTime).ToList();
|
||||
FileCacheSize = files
|
||||
.Sum(f =>
|
||||
{
|
||||
token.ThrowIfCancellationRequested();
|
||||
|
||||
try
|
||||
{
|
||||
return _fileCompactor.GetFileSizeOnDisk(f, StorageisNTFS);
|
||||
}
|
||||
catch
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
});
|
||||
|
||||
var maxCacheInBytes = (long)(_configService.Current.MaxLocalCacheInGiB * 1024d * 1024d * 1024d);
|
||||
|
||||
if (FileCacheSize < maxCacheInBytes) return;
|
||||
|
||||
var substDir = _fileDbManager.SubstFolder;
|
||||
|
||||
var maxCacheBuffer = maxCacheInBytes * 0.05d;
|
||||
while (FileCacheSize > maxCacheInBytes - (long)maxCacheBuffer)
|
||||
{
|
||||
var oldestFile = files[0];
|
||||
FileCacheSize -= _fileCompactor.GetFileSizeOnDisk(oldestFile);
|
||||
File.Delete(oldestFile.FullName);
|
||||
files.Remove(oldestFile);
|
||||
}
|
||||
}
|
||||
|
||||
public void ResetLocks()
|
||||
{
|
||||
HaltScanLocks.Clear();
|
||||
}
|
||||
|
||||
public void ResumeScan(string source)
|
||||
{
|
||||
HaltScanLocks.TryAdd(source, new(0));
|
||||
Interlocked.Decrement(ref HaltScanLocks[source].Value);
|
||||
}
|
||||
|
||||
protected override void Dispose(bool disposing)
|
||||
{
|
||||
base.Dispose(disposing);
|
||||
_scanCancellationTokenSource?.Cancel();
|
||||
PenumbraWatcher?.Dispose();
|
||||
MareWatcher?.Dispose();
|
||||
SubstWatcher?.Dispose();
|
||||
_penumbraFswCts?.CancelDispose();
|
||||
_mareFswCts?.CancelDispose();
|
||||
_substFswCts?.CancelDispose();
|
||||
_periodicCalculationTokenSource?.CancelDispose();
|
||||
}
|
||||
|
||||
private void FullFileScan(CancellationToken ct)
|
||||
{
|
||||
TotalFiles = 1;
|
||||
var penumbraDir = _ipcManager.Penumbra.ModDirectory;
|
||||
bool penDirExists = true;
|
||||
bool cacheDirExists = true;
|
||||
var substDir = _fileDbManager.SubstFolder;
|
||||
if (string.IsNullOrEmpty(penumbraDir) || !Directory.Exists(penumbraDir))
|
||||
{
|
||||
penDirExists = false;
|
||||
Logger.LogWarning("Penumbra directory is not set or does not exist.");
|
||||
}
|
||||
if (string.IsNullOrEmpty(_configService.Current.CacheFolder) || !Directory.Exists(_configService.Current.CacheFolder))
|
||||
{
|
||||
cacheDirExists = false;
|
||||
Logger.LogWarning("Elezen Cache directory is not set or does not exist.");
|
||||
}
|
||||
if (!penDirExists || !cacheDirExists)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (!Directory.Exists(substDir))
|
||||
Directory.CreateDirectory(substDir);
|
||||
}
|
||||
catch
|
||||
{
|
||||
Logger.LogWarning("Could not create subst directory at {path}.", substDir);
|
||||
}
|
||||
|
||||
var previousThreadPriority = Thread.CurrentThread.Priority;
|
||||
Thread.CurrentThread.Priority = ThreadPriority.Lowest;
|
||||
Logger.LogDebug("Getting files from {penumbra} and {storage}", penumbraDir, _configService.Current.CacheFolder);
|
||||
|
||||
Dictionary<string, string[]> penumbraFiles = new(StringComparer.Ordinal);
|
||||
foreach (var folder in Directory.EnumerateDirectories(penumbraDir!))
|
||||
{
|
||||
try
|
||||
{
|
||||
penumbraFiles[folder] =
|
||||
[
|
||||
.. Directory.GetFiles(folder, "*.*", SearchOption.AllDirectories)
|
||||
.AsParallel()
|
||||
.Where(f => AllowedFileExtensions.Any(e => f.EndsWith(e, StringComparison.OrdinalIgnoreCase))
|
||||
&& !f.Contains(@"\bg\", StringComparison.OrdinalIgnoreCase)
|
||||
&& !f.Contains(@"\bgcommon\", StringComparison.OrdinalIgnoreCase)
|
||||
&& !f.Contains(@"\ui\", StringComparison.OrdinalIgnoreCase)),
|
||||
];
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.LogWarning(ex, "Could not enumerate path {path}", folder);
|
||||
}
|
||||
Thread.Sleep(50);
|
||||
if (ct.IsCancellationRequested) return;
|
||||
}
|
||||
|
||||
var allCacheFiles = Directory.GetFiles(_configService.Current.CacheFolder, "*.*", SearchOption.TopDirectoryOnly)
|
||||
.Concat(Directory.GetFiles(substDir, "*.*", SearchOption.TopDirectoryOnly))
|
||||
.AsParallel()
|
||||
.Where(f =>
|
||||
{
|
||||
var val = f.Split('\\')[^1];
|
||||
return val.Length == 40 || (val.Split('.').FirstOrDefault()?.Length ?? 0) == 40;
|
||||
});
|
||||
|
||||
if (ct.IsCancellationRequested) return;
|
||||
|
||||
var allScannedFiles = (penumbraFiles.SelectMany(k => k.Value))
|
||||
.Concat(allCacheFiles)
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.ToDictionary(t => t.ToLowerInvariant(), t => false, StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
TotalFiles = allScannedFiles.Count;
|
||||
Thread.CurrentThread.Priority = previousThreadPriority;
|
||||
|
||||
Thread.Sleep(TimeSpan.FromSeconds(2));
|
||||
|
||||
if (ct.IsCancellationRequested) return;
|
||||
|
||||
// scan files from database
|
||||
var threadCount = Math.Clamp((int)(Environment.ProcessorCount / 2.0f), 2, 8);
|
||||
|
||||
List<FileCacheEntity> entitiesToRemove = [];
|
||||
List<FileCacheEntity> entitiesToUpdate = [];
|
||||
Lock sync = new();
|
||||
Thread[] workerThreads = new Thread[threadCount];
|
||||
|
||||
ConcurrentQueue<FileCacheEntity> fileCaches = new(_fileDbManager.GetAllFileCaches());
|
||||
|
||||
TotalFilesStorage = fileCaches.Count;
|
||||
|
||||
for (int i = 0; i < threadCount; i++)
|
||||
{
|
||||
Logger.LogTrace("Creating Thread {i}", i);
|
||||
workerThreads[i] = new((tcounter) =>
|
||||
{
|
||||
var threadNr = (int)tcounter!;
|
||||
Logger.LogTrace("Spawning Worker Thread {i}", threadNr);
|
||||
while (!ct.IsCancellationRequested && fileCaches.TryDequeue(out var workload))
|
||||
{
|
||||
try
|
||||
{
|
||||
if (ct.IsCancellationRequested) return;
|
||||
|
||||
if (!_ipcManager.Penumbra.APIAvailable)
|
||||
{
|
||||
Logger.LogWarning("Penumbra not available");
|
||||
return;
|
||||
}
|
||||
|
||||
var validatedCacheResult = _fileDbManager.ValidateFileCacheEntity(workload);
|
||||
if (validatedCacheResult.State != FileState.RequireDeletion)
|
||||
{
|
||||
lock (sync) { allScannedFiles[validatedCacheResult.FileCache.ResolvedFilepath] = true; }
|
||||
}
|
||||
if (validatedCacheResult.State == FileState.RequireUpdate)
|
||||
{
|
||||
Logger.LogTrace("To update: {path}", validatedCacheResult.FileCache.ResolvedFilepath);
|
||||
lock (sync) { entitiesToUpdate.Add(validatedCacheResult.FileCache); }
|
||||
}
|
||||
else if (validatedCacheResult.State == FileState.RequireDeletion)
|
||||
{
|
||||
Logger.LogTrace("To delete: {path}", validatedCacheResult.FileCache.ResolvedFilepath);
|
||||
lock (sync) { entitiesToRemove.Add(validatedCacheResult.FileCache); }
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.LogWarning(ex, "Failed validating {path}", workload.ResolvedFilepath);
|
||||
}
|
||||
Interlocked.Increment(ref _currentFileProgress);
|
||||
}
|
||||
|
||||
Logger.LogTrace("Ending Worker Thread {i}", threadNr);
|
||||
})
|
||||
{
|
||||
Priority = ThreadPriority.Lowest,
|
||||
IsBackground = true
|
||||
};
|
||||
workerThreads[i].Start(i);
|
||||
}
|
||||
|
||||
while (!ct.IsCancellationRequested && workerThreads.Any(u => u.IsAlive))
|
||||
{
|
||||
Thread.Sleep(1000);
|
||||
}
|
||||
|
||||
if (ct.IsCancellationRequested) return;
|
||||
|
||||
Logger.LogTrace("Threads exited");
|
||||
|
||||
if (!_ipcManager.Penumbra.APIAvailable)
|
||||
{
|
||||
Logger.LogWarning("Penumbra not available");
|
||||
return;
|
||||
}
|
||||
|
||||
if (entitiesToUpdate.Any() || entitiesToRemove.Any())
|
||||
{
|
||||
foreach (var entity in entitiesToUpdate)
|
||||
{
|
||||
_fileDbManager.UpdateHashedFile(entity);
|
||||
}
|
||||
|
||||
foreach (var entity in entitiesToRemove)
|
||||
{
|
||||
_fileDbManager.RemoveHashedFile(entity.Hash, entity.PrefixedFilePath);
|
||||
}
|
||||
|
||||
_fileDbManager.WriteOutFullCsv();
|
||||
}
|
||||
|
||||
Logger.LogTrace("Scanner validated existing db files");
|
||||
|
||||
if (!_ipcManager.Penumbra.APIAvailable)
|
||||
{
|
||||
Logger.LogWarning("Penumbra not available");
|
||||
return;
|
||||
}
|
||||
|
||||
if (ct.IsCancellationRequested) return;
|
||||
|
||||
// scan new files
|
||||
if (allScannedFiles.Any(c => !c.Value))
|
||||
{
|
||||
Parallel.ForEach(allScannedFiles.Where(c => !c.Value).Select(c => c.Key),
|
||||
new ParallelOptions()
|
||||
{
|
||||
MaxDegreeOfParallelism = threadCount,
|
||||
CancellationToken = ct
|
||||
}, (cachePath) =>
|
||||
{
|
||||
if (ct.IsCancellationRequested) return;
|
||||
|
||||
if (!_ipcManager.Penumbra.APIAvailable)
|
||||
{
|
||||
Logger.LogWarning("Penumbra not available");
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var entry = _fileDbManager.CreateFileEntry(cachePath);
|
||||
if (entry == null)
|
||||
{
|
||||
if (cachePath.StartsWith(substDir, StringComparison.Ordinal))
|
||||
_ = _fileDbManager.CreateSubstEntry(cachePath);
|
||||
else
|
||||
_ = _fileDbManager.CreateCacheEntry(cachePath);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.LogWarning(ex, "Failed adding {file}", cachePath);
|
||||
}
|
||||
|
||||
Interlocked.Increment(ref _currentFileProgress);
|
||||
});
|
||||
|
||||
Logger.LogTrace("Scanner added {notScanned} new files to db", allScannedFiles.Count(c => !c.Value));
|
||||
}
|
||||
|
||||
Logger.LogDebug("Scan complete");
|
||||
TotalFiles = 0;
|
||||
_currentFileProgress = 0;
|
||||
entitiesToRemove.Clear();
|
||||
allScannedFiles.Clear();
|
||||
|
||||
if (!_configService.Current.InitialScanComplete)
|
||||
{
|
||||
_configService.Current.InitialScanComplete = true;
|
||||
_configService.Save();
|
||||
StartMareWatcher(_configService.Current.CacheFolder);
|
||||
StartSubstWatcher(_fileDbManager.SubstFolder);
|
||||
StartPenumbraWatcher(penumbraDir);
|
||||
}
|
||||
}
|
||||
}
|
||||
30
MareSynchronos/FileCache/FileCacheEntity.cs
Normal file
30
MareSynchronos/FileCache/FileCacheEntity.cs
Normal file
@@ -0,0 +1,30 @@
|
||||
#nullable disable
|
||||
|
||||
namespace MareSynchronos.FileCache;
|
||||
|
||||
public class FileCacheEntity
|
||||
{
|
||||
public FileCacheEntity(string hash, string path, string lastModifiedDateTicks, long? size = null, long? compressedSize = null)
|
||||
{
|
||||
Size = size;
|
||||
CompressedSize = compressedSize;
|
||||
Hash = hash;
|
||||
PrefixedFilePath = path;
|
||||
LastModifiedDateTicks = lastModifiedDateTicks;
|
||||
}
|
||||
|
||||
public long? CompressedSize { get; set; }
|
||||
public string CsvEntry => $"{Hash}{FileCacheManager.CsvSplit}{PrefixedFilePath}{FileCacheManager.CsvSplit}{LastModifiedDateTicks}|{Size ?? -1}|{CompressedSize ?? -1}";
|
||||
public string Hash { get; set; }
|
||||
public bool IsCacheEntry => PrefixedFilePath.StartsWith(FileCacheManager.CachePrefix, StringComparison.OrdinalIgnoreCase);
|
||||
public bool IsSubstEntry => PrefixedFilePath.StartsWith(FileCacheManager.SubstPrefix, StringComparison.OrdinalIgnoreCase);
|
||||
public string LastModifiedDateTicks { get; set; }
|
||||
public string PrefixedFilePath { get; init; }
|
||||
public string ResolvedFilepath { get; private set; } = string.Empty;
|
||||
public long? Size { get; set; }
|
||||
|
||||
public void SetResolvedFilePath(string filePath)
|
||||
{
|
||||
ResolvedFilepath = filePath.ToLowerInvariant().Replace("\\\\", "\\", StringComparison.Ordinal);
|
||||
}
|
||||
}
|
||||
556
MareSynchronos/FileCache/FileCacheManager.cs
Normal file
556
MareSynchronos/FileCache/FileCacheManager.cs
Normal file
@@ -0,0 +1,556 @@
|
||||
using Dalamud.Utility;
|
||||
using K4os.Compression.LZ4.Streams;
|
||||
using MareSynchronos.Interop.Ipc;
|
||||
using MareSynchronos.MareConfiguration;
|
||||
using MareSynchronos.Services.Mediator;
|
||||
using MareSynchronos.Utils;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Globalization;
|
||||
using System.Text;
|
||||
|
||||
namespace MareSynchronos.FileCache;
|
||||
|
||||
public sealed class FileCacheManager : IHostedService
|
||||
{
|
||||
public const string CachePrefix = "{cache}";
|
||||
public const string CsvSplit = "|";
|
||||
public const string PenumbraPrefix = "{penumbra}";
|
||||
public const string SubstPrefix = "{subst}";
|
||||
public const string SubstPath = "subst";
|
||||
public string CacheFolder => _configService.Current.CacheFolder;
|
||||
public string SubstFolder => CacheFolder.IsNullOrEmpty() ? string.Empty : CacheFolder.ToLowerInvariant().TrimEnd('\\') + "\\" + SubstPath;
|
||||
private readonly MareConfigService _configService;
|
||||
private readonly MareMediator _mareMediator;
|
||||
private readonly string _csvPath;
|
||||
private readonly ConcurrentDictionary<string, List<FileCacheEntity>> _fileCaches = new(StringComparer.Ordinal);
|
||||
private readonly SemaphoreSlim _getCachesByPathsSemaphore = new(1, 1);
|
||||
private readonly Lock _fileWriteLock = new();
|
||||
private readonly IpcManager _ipcManager;
|
||||
private readonly ILogger<FileCacheManager> _logger;
|
||||
|
||||
public FileCacheManager(ILogger<FileCacheManager> logger, IpcManager ipcManager, MareConfigService configService, MareMediator mareMediator)
|
||||
{
|
||||
_logger = logger;
|
||||
_ipcManager = ipcManager;
|
||||
_configService = configService;
|
||||
_mareMediator = mareMediator;
|
||||
_csvPath = Path.Combine(configService.ConfigurationDirectory, "FileCache.csv");
|
||||
}
|
||||
|
||||
private string CsvBakPath => _csvPath + ".bak";
|
||||
|
||||
public FileCacheEntity? CreateCacheEntry(string path, string? hash = null)
|
||||
{
|
||||
FileInfo fi = new(path);
|
||||
if (!fi.Exists) return null;
|
||||
_logger.LogTrace("Creating cache entry for {path}", path);
|
||||
var fullName = fi.FullName.ToLowerInvariant();
|
||||
if (!fullName.Contains(_configService.Current.CacheFolder.ToLowerInvariant(), StringComparison.Ordinal)) return null;
|
||||
string prefixedPath = fullName.Replace(_configService.Current.CacheFolder.ToLowerInvariant(), CachePrefix + "\\", StringComparison.Ordinal).Replace("\\\\", "\\", StringComparison.Ordinal);
|
||||
if (hash != null)
|
||||
return CreateFileCacheEntity(fi, prefixedPath, hash);
|
||||
else
|
||||
return CreateFileCacheEntity(fi, prefixedPath);
|
||||
}
|
||||
|
||||
public FileCacheEntity? CreateSubstEntry(string path)
|
||||
{
|
||||
FileInfo fi = new(path);
|
||||
if (!fi.Exists) return null;
|
||||
_logger.LogTrace("Creating substitute entry for {path}", path);
|
||||
var fullName = fi.FullName.ToLowerInvariant();
|
||||
if (!fullName.Contains(SubstFolder, StringComparison.Ordinal)) return null;
|
||||
string prefixedPath = fullName.Replace(SubstFolder, SubstPrefix + "\\", StringComparison.Ordinal).Replace("\\\\", "\\", StringComparison.Ordinal);
|
||||
var fakeHash = Path.GetFileNameWithoutExtension(fi.FullName).ToUpperInvariant();
|
||||
var result = CreateFileCacheEntity(fi, prefixedPath, fakeHash);
|
||||
return result;
|
||||
}
|
||||
|
||||
public FileCacheEntity? CreateFileEntry(string path)
|
||||
{
|
||||
FileInfo fi = new(path);
|
||||
if (!fi.Exists) return null;
|
||||
_logger.LogTrace("Creating file entry for {path}", path);
|
||||
var fullName = fi.FullName.ToLowerInvariant();
|
||||
if (!fullName.Contains(_ipcManager.Penumbra.ModDirectory!.ToLowerInvariant(), StringComparison.Ordinal)) return null;
|
||||
string prefixedPath = fullName.Replace(_ipcManager.Penumbra.ModDirectory!.ToLowerInvariant(), PenumbraPrefix + "\\", StringComparison.Ordinal).Replace("\\\\", "\\", StringComparison.Ordinal);
|
||||
return CreateFileCacheEntity(fi, prefixedPath);
|
||||
}
|
||||
|
||||
public List<FileCacheEntity> GetAllFileCaches() => _fileCaches.Values.SelectMany(v => v).ToList();
|
||||
|
||||
public List<FileCacheEntity> GetAllFileCachesByHash(string hash, bool ignoreCacheEntries = false, bool validate = true)
|
||||
{
|
||||
List<FileCacheEntity> output = [];
|
||||
if (_fileCaches.TryGetValue(hash, out var fileCacheEntities))
|
||||
{
|
||||
foreach (var fileCache in fileCacheEntities.Where(c => ignoreCacheEntries ? (!c.IsCacheEntry && !c.IsSubstEntry) : true).ToList())
|
||||
{
|
||||
if (!validate) output.Add(fileCache);
|
||||
else
|
||||
{
|
||||
var validated = GetValidatedFileCache(fileCache);
|
||||
if (validated != null) output.Add(validated);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
public Task<List<FileCacheEntity>> ValidateLocalIntegrity(IProgress<(int, int, FileCacheEntity)> progress, CancellationToken cancellationToken)
|
||||
{
|
||||
_mareMediator.Publish(new HaltScanMessage(nameof(ValidateLocalIntegrity)));
|
||||
_logger.LogInformation("Validating local storage");
|
||||
var cacheEntries = _fileCaches.SelectMany(v => v.Value).Where(v => v.IsCacheEntry).ToList();
|
||||
List<FileCacheEntity> brokenEntities = [];
|
||||
int i = 0;
|
||||
foreach (var fileCache in cacheEntries)
|
||||
{
|
||||
if (cancellationToken.IsCancellationRequested) break;
|
||||
if (fileCache.IsSubstEntry) continue;
|
||||
|
||||
_logger.LogInformation("Validating {file}", fileCache.ResolvedFilepath);
|
||||
|
||||
progress.Report((i, cacheEntries.Count, fileCache));
|
||||
i++;
|
||||
if (!File.Exists(fileCache.ResolvedFilepath))
|
||||
{
|
||||
brokenEntities.Add(fileCache);
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var computedHash = Crypto.GetFileHash(fileCache.ResolvedFilepath);
|
||||
if (!string.Equals(computedHash, fileCache.Hash, StringComparison.Ordinal))
|
||||
{
|
||||
_logger.LogInformation("Failed to validate {file}, got hash {hash}, expected hash {expectedHash}", fileCache.ResolvedFilepath, computedHash, fileCache.Hash);
|
||||
brokenEntities.Add(fileCache);
|
||||
}
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogWarning(e, "Error during validation of {file}", fileCache.ResolvedFilepath);
|
||||
brokenEntities.Add(fileCache);
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var brokenEntity in brokenEntities)
|
||||
{
|
||||
RemoveHashedFile(brokenEntity.Hash, brokenEntity.PrefixedFilePath);
|
||||
|
||||
try
|
||||
{
|
||||
File.Delete(brokenEntity.ResolvedFilepath);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Could not delete {file}", brokenEntity.ResolvedFilepath);
|
||||
}
|
||||
}
|
||||
|
||||
_mareMediator.Publish(new ResumeScanMessage(nameof(ValidateLocalIntegrity)));
|
||||
return Task.FromResult(brokenEntities);
|
||||
}
|
||||
|
||||
public string GetCacheFilePath(string hash, string extension)
|
||||
{
|
||||
return Path.Combine(_configService.Current.CacheFolder, hash + "." + extension);
|
||||
}
|
||||
|
||||
public string GetSubstFilePath(string hash, string extension)
|
||||
{
|
||||
return Path.Combine(SubstFolder, hash + "." + extension);
|
||||
}
|
||||
|
||||
public async Task<(string, byte[])> GetCompressedFileData(string fileHash, CancellationToken uploadToken)
|
||||
{
|
||||
var fileCache = GetFileCacheByHash(fileHash)!;
|
||||
using var fs = File.OpenRead(fileCache.ResolvedFilepath);
|
||||
var ms = new MemoryStream(64 * 1024);
|
||||
using var encstream = LZ4Stream.Encode(ms, new LZ4EncoderSettings(){CompressionLevel=K4os.Compression.LZ4.LZ4Level.L09_HC});
|
||||
await fs.CopyToAsync(encstream, uploadToken).ConfigureAwait(false);
|
||||
encstream.Close();
|
||||
fileCache.CompressedSize = encstream.Length;
|
||||
return (fileHash, ms.ToArray());
|
||||
}
|
||||
|
||||
public FileCacheEntity? GetFileCacheByHash(string hash, bool preferSubst = false)
|
||||
{
|
||||
var caches = GetFileCachesByHash(hash);
|
||||
if (preferSubst && caches.Subst != null)
|
||||
return caches.Subst;
|
||||
return caches.Penumbra ?? caches.Cache;
|
||||
}
|
||||
|
||||
public (FileCacheEntity? Penumbra, FileCacheEntity? Cache, FileCacheEntity? Subst) GetFileCachesByHash(string hash)
|
||||
{
|
||||
(FileCacheEntity? Penumbra, FileCacheEntity? Cache, FileCacheEntity? Subst) result = (null, null, null);
|
||||
if (_fileCaches.TryGetValue(hash, out var hashes))
|
||||
{
|
||||
result.Penumbra = hashes.Where(p => p.PrefixedFilePath.StartsWith(PenumbraPrefix, StringComparison.Ordinal)).Select(GetValidatedFileCache).FirstOrDefault();
|
||||
result.Cache = hashes.Where(p => p.PrefixedFilePath.StartsWith(CachePrefix, StringComparison.Ordinal)).Select(GetValidatedFileCache).FirstOrDefault();
|
||||
result.Subst = hashes.Where(p => p.PrefixedFilePath.StartsWith(SubstPrefix, StringComparison.Ordinal)).Select(GetValidatedFileCache).FirstOrDefault();
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private FileCacheEntity? GetFileCacheByPath(string path)
|
||||
{
|
||||
var cleanedPath = path.Replace("/", "\\", StringComparison.OrdinalIgnoreCase).ToLowerInvariant()
|
||||
.Replace(_ipcManager.Penumbra.ModDirectory!.ToLowerInvariant(), "", StringComparison.OrdinalIgnoreCase);
|
||||
var entry = _fileCaches.SelectMany(v => v.Value).FirstOrDefault(f => f.ResolvedFilepath.EndsWith(cleanedPath, StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
if (entry == null)
|
||||
{
|
||||
_logger.LogDebug("Found no entries for {path}", cleanedPath);
|
||||
return CreateFileEntry(path);
|
||||
}
|
||||
|
||||
var validatedCacheEntry = GetValidatedFileCache(entry);
|
||||
|
||||
return validatedCacheEntry;
|
||||
}
|
||||
|
||||
public Dictionary<string, FileCacheEntity?> GetFileCachesByPaths(string[] paths)
|
||||
{
|
||||
_getCachesByPathsSemaphore.Wait();
|
||||
|
||||
try
|
||||
{
|
||||
var cleanedPaths = paths.Distinct(StringComparer.OrdinalIgnoreCase).ToDictionary(p => p,
|
||||
p => p.Replace("/", "\\", StringComparison.OrdinalIgnoreCase)
|
||||
.Replace(_ipcManager.Penumbra.ModDirectory!, _ipcManager.Penumbra.ModDirectory!.EndsWith('\\') ? PenumbraPrefix + '\\' : PenumbraPrefix, StringComparison.OrdinalIgnoreCase)
|
||||
.Replace(SubstFolder, SubstPrefix, StringComparison.OrdinalIgnoreCase)
|
||||
.Replace(_configService.Current.CacheFolder, _configService.Current.CacheFolder.EndsWith('\\') ? CachePrefix + '\\' : CachePrefix, StringComparison.OrdinalIgnoreCase)
|
||||
.Replace("\\\\", "\\", StringComparison.Ordinal),
|
||||
StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
Dictionary<string, FileCacheEntity?> result = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
var dict = _fileCaches.SelectMany(f => f.Value)
|
||||
.ToDictionary(d => d.PrefixedFilePath, d => d, StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (var entry in cleanedPaths)
|
||||
{
|
||||
//_logger.LogDebug("Checking {path}", entry.Value);
|
||||
|
||||
if (dict.TryGetValue(entry.Value, out var entity))
|
||||
{
|
||||
var validatedCache = GetValidatedFileCache(entity);
|
||||
result.Add(entry.Key, validatedCache);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (entry.Value.StartsWith(PenumbraPrefix, StringComparison.Ordinal))
|
||||
result.Add(entry.Key, CreateFileEntry(entry.Key));
|
||||
else if (entry.Value.StartsWith(SubstPrefix, StringComparison.Ordinal))
|
||||
result.Add(entry.Key, CreateSubstEntry(entry.Key));
|
||||
else if (entry.Value.StartsWith(CachePrefix, StringComparison.Ordinal))
|
||||
result.Add(entry.Key, CreateCacheEntry(entry.Key));
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_getCachesByPathsSemaphore.Release();
|
||||
}
|
||||
}
|
||||
|
||||
public void RemoveHashedFile(string hash, string prefixedFilePath)
|
||||
{
|
||||
if (_fileCaches.TryGetValue(hash, out var caches))
|
||||
{
|
||||
var removedCount = caches?.RemoveAll(c => string.Equals(c.PrefixedFilePath, prefixedFilePath, StringComparison.Ordinal));
|
||||
_logger.LogTrace("Removed from DB: {count} file(s) with hash {hash} and file cache {path}", removedCount, hash, prefixedFilePath);
|
||||
|
||||
if (caches?.Count == 0)
|
||||
{
|
||||
_fileCaches.Remove(hash, out var _);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void UpdateHashedFile(FileCacheEntity fileCache, bool computeProperties = true)
|
||||
{
|
||||
_logger.LogTrace("Updating hash for {path}", fileCache.ResolvedFilepath);
|
||||
var oldHash = fileCache.Hash;
|
||||
var prefixedPath = fileCache.PrefixedFilePath;
|
||||
if (computeProperties)
|
||||
{
|
||||
var fi = new FileInfo(fileCache.ResolvedFilepath);
|
||||
fileCache.Size = fi.Length;
|
||||
fileCache.CompressedSize = null;
|
||||
fileCache.Hash = Crypto.GetFileHash(fileCache.ResolvedFilepath);
|
||||
fileCache.LastModifiedDateTicks = fi.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture);
|
||||
}
|
||||
RemoveHashedFile(oldHash, prefixedPath);
|
||||
AddHashedFile(fileCache);
|
||||
}
|
||||
|
||||
public (FileState State, FileCacheEntity FileCache) ValidateFileCacheEntity(FileCacheEntity fileCache)
|
||||
{
|
||||
fileCache = ReplacePathPrefixes(fileCache);
|
||||
FileInfo fi = new(fileCache.ResolvedFilepath);
|
||||
if (!fi.Exists)
|
||||
{
|
||||
return (FileState.RequireDeletion, fileCache);
|
||||
}
|
||||
if (!string.Equals(fi.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture), fileCache.LastModifiedDateTicks, StringComparison.Ordinal))
|
||||
{
|
||||
return (FileState.RequireUpdate, fileCache);
|
||||
}
|
||||
|
||||
return (FileState.Valid, fileCache);
|
||||
}
|
||||
|
||||
public void WriteOutFullCsv()
|
||||
{
|
||||
lock (_fileWriteLock)
|
||||
{
|
||||
StringBuilder sb = new();
|
||||
foreach (var entry in _fileCaches.SelectMany(k => k.Value).OrderBy(f => f.PrefixedFilePath, StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
sb.AppendLine(entry.CsvEntry);
|
||||
}
|
||||
|
||||
if (File.Exists(_csvPath))
|
||||
{
|
||||
File.Copy(_csvPath, CsvBakPath, overwrite: true);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
File.WriteAllText(_csvPath, sb.ToString());
|
||||
File.Delete(CsvBakPath);
|
||||
}
|
||||
catch
|
||||
{
|
||||
File.WriteAllText(CsvBakPath, sb.ToString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
internal FileCacheEntity MigrateFileHashToExtension(FileCacheEntity fileCache, string ext)
|
||||
{
|
||||
try
|
||||
{
|
||||
RemoveHashedFile(fileCache.Hash, fileCache.PrefixedFilePath);
|
||||
var extensionPath = fileCache.ResolvedFilepath.ToUpper(CultureInfo.InvariantCulture) + "." + ext;
|
||||
File.Move(fileCache.ResolvedFilepath, extensionPath, overwrite: true);
|
||||
var newHashedEntity = new FileCacheEntity(fileCache.Hash, fileCache.PrefixedFilePath + "." + ext, DateTime.UtcNow.Ticks.ToString(CultureInfo.InvariantCulture));
|
||||
newHashedEntity.SetResolvedFilePath(extensionPath);
|
||||
AddHashedFile(newHashedEntity);
|
||||
_logger.LogTrace("Migrated from {oldPath} to {newPath}", fileCache.ResolvedFilepath, newHashedEntity.ResolvedFilepath);
|
||||
return newHashedEntity;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
AddHashedFile(fileCache);
|
||||
_logger.LogWarning(ex, "Failed to migrate entity {entity}", fileCache.PrefixedFilePath);
|
||||
return fileCache;
|
||||
}
|
||||
}
|
||||
|
||||
private void AddHashedFile(FileCacheEntity fileCache)
|
||||
{
|
||||
if (!_fileCaches.TryGetValue(fileCache.Hash, out var entries) || entries is null)
|
||||
{
|
||||
_fileCaches[fileCache.Hash] = entries = [];
|
||||
}
|
||||
|
||||
if (!entries.Exists(u => string.Equals(u.PrefixedFilePath, fileCache.PrefixedFilePath, StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
//_logger.LogTrace("Adding to DB: {hash} => {path}", fileCache.Hash, fileCache.PrefixedFilePath);
|
||||
entries.Add(fileCache);
|
||||
}
|
||||
}
|
||||
|
||||
private FileCacheEntity? CreateFileCacheEntity(FileInfo fileInfo, string prefixedPath, string? hash = null)
|
||||
{
|
||||
hash ??= Crypto.GetFileHash(fileInfo.FullName);
|
||||
var entity = new FileCacheEntity(hash, prefixedPath, fileInfo.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture), fileInfo.Length);
|
||||
entity = ReplacePathPrefixes(entity);
|
||||
AddHashedFile(entity);
|
||||
lock (_fileWriteLock)
|
||||
{
|
||||
File.AppendAllLines(_csvPath, new[] { entity.CsvEntry });
|
||||
}
|
||||
var result = GetFileCacheByPath(fileInfo.FullName);
|
||||
_logger.LogTrace("Creating cache entity for {name} success: {success}", fileInfo.FullName, (result != null));
|
||||
return result;
|
||||
}
|
||||
|
||||
private FileCacheEntity? GetValidatedFileCache(FileCacheEntity fileCache)
|
||||
{
|
||||
var resultingFileCache = ReplacePathPrefixes(fileCache);
|
||||
//_logger.LogTrace("Validating {path}", fileCache.PrefixedFilePath);
|
||||
resultingFileCache = Validate(resultingFileCache);
|
||||
return resultingFileCache;
|
||||
}
|
||||
|
||||
private FileCacheEntity ReplacePathPrefixes(FileCacheEntity fileCache)
|
||||
{
|
||||
if (fileCache.PrefixedFilePath.StartsWith(PenumbraPrefix, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
fileCache.SetResolvedFilePath(fileCache.PrefixedFilePath.Replace(PenumbraPrefix, _ipcManager.Penumbra.ModDirectory, StringComparison.Ordinal));
|
||||
}
|
||||
else if (fileCache.PrefixedFilePath.StartsWith(SubstPrefix, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
fileCache.SetResolvedFilePath(fileCache.PrefixedFilePath.Replace(SubstPrefix, SubstFolder, StringComparison.Ordinal));
|
||||
}
|
||||
else if (fileCache.PrefixedFilePath.StartsWith(CachePrefix, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
fileCache.SetResolvedFilePath(fileCache.PrefixedFilePath.Replace(CachePrefix, _configService.Current.CacheFolder, StringComparison.Ordinal));
|
||||
}
|
||||
|
||||
return fileCache;
|
||||
}
|
||||
|
||||
private FileCacheEntity? Validate(FileCacheEntity fileCache)
|
||||
{
|
||||
var file = new FileInfo(fileCache.ResolvedFilepath);
|
||||
if (!file.Exists)
|
||||
{
|
||||
RemoveHashedFile(fileCache.Hash, fileCache.PrefixedFilePath);
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!string.Equals(file.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture), fileCache.LastModifiedDateTicks, StringComparison.Ordinal))
|
||||
{
|
||||
UpdateHashedFile(fileCache);
|
||||
}
|
||||
|
||||
return fileCache;
|
||||
}
|
||||
|
||||
public Task StartAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
_logger.LogInformation("Starting FileCacheManager");
|
||||
lock (_fileWriteLock)
|
||||
{
|
||||
try
|
||||
{
|
||||
_logger.LogInformation("Checking for {bakPath}", CsvBakPath);
|
||||
|
||||
if (File.Exists(CsvBakPath))
|
||||
{
|
||||
_logger.LogInformation("{bakPath} found, moving to {csvPath}", CsvBakPath, _csvPath);
|
||||
|
||||
File.Move(CsvBakPath, _csvPath, overwrite: true);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to move BAK to ORG, deleting BAK");
|
||||
try
|
||||
{
|
||||
if (File.Exists(CsvBakPath))
|
||||
File.Delete(CsvBakPath);
|
||||
}
|
||||
catch (Exception ex1)
|
||||
{
|
||||
_logger.LogWarning(ex1, "Could not delete bak file");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (File.Exists(_csvPath))
|
||||
{
|
||||
if (!_ipcManager.Penumbra.APIAvailable || string.IsNullOrEmpty(_ipcManager.Penumbra.ModDirectory))
|
||||
{
|
||||
_mareMediator.Publish(new NotificationMessage("Penumbra not connected",
|
||||
"Could not load local file cache data. Penumbra is not connected or not properly set up. Please enable and/or configure Penumbra properly to use Elezen. After, reload Elezen in the Plugin installer.",
|
||||
MareConfiguration.Models.NotificationType.Error));
|
||||
}
|
||||
|
||||
_logger.LogInformation("{csvPath} found, parsing", _csvPath);
|
||||
|
||||
bool success = false;
|
||||
string[] entries = [];
|
||||
int attempts = 0;
|
||||
while (!success && attempts < 10)
|
||||
{
|
||||
try
|
||||
{
|
||||
_logger.LogInformation("Attempting to read {csvPath}", _csvPath);
|
||||
entries = File.ReadAllLines(_csvPath);
|
||||
success = true;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
attempts++;
|
||||
_logger.LogWarning(ex, "Could not open {file}, trying again", _csvPath);
|
||||
Thread.Sleep(100);
|
||||
}
|
||||
}
|
||||
|
||||
if (!entries.Any())
|
||||
{
|
||||
_logger.LogWarning("Could not load entries from {path}, continuing with empty file cache", _csvPath);
|
||||
}
|
||||
|
||||
_logger.LogInformation("Found {amount} files in {path}", entries.Length, _csvPath);
|
||||
|
||||
Dictionary<string, bool> processedFiles = new(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var entry in entries)
|
||||
{
|
||||
var splittedEntry = entry.Split(CsvSplit, StringSplitOptions.None);
|
||||
try
|
||||
{
|
||||
var hash = splittedEntry[0];
|
||||
if (hash.Length != 40) throw new InvalidOperationException("Expected Hash length of 40, received " + hash.Length);
|
||||
var path = splittedEntry[1];
|
||||
var time = splittedEntry[2];
|
||||
|
||||
if (processedFiles.ContainsKey(path))
|
||||
{
|
||||
_logger.LogWarning("Already processed {file}, ignoring", path);
|
||||
continue;
|
||||
}
|
||||
|
||||
processedFiles.Add(path, value: true);
|
||||
|
||||
long size = -1;
|
||||
long compressed = -1;
|
||||
if (splittedEntry.Length > 3)
|
||||
{
|
||||
if (long.TryParse(splittedEntry[3], CultureInfo.InvariantCulture, out long result))
|
||||
{
|
||||
size = result;
|
||||
}
|
||||
if (long.TryParse(splittedEntry[4], CultureInfo.InvariantCulture, out long resultCompressed))
|
||||
{
|
||||
compressed = resultCompressed;
|
||||
}
|
||||
}
|
||||
AddHashedFile(ReplacePathPrefixes(new FileCacheEntity(hash, path, time, size, compressed)));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to initialize entry {entry}, ignoring", entry);
|
||||
}
|
||||
}
|
||||
|
||||
if (processedFiles.Count != entries.Length)
|
||||
{
|
||||
WriteOutFullCsv();
|
||||
}
|
||||
}
|
||||
|
||||
_logger.LogInformation("Started FileCacheManager");
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task StopAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
WriteOutFullCsv();
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
250
MareSynchronos/FileCache/FileCompactor.cs
Normal file
250
MareSynchronos/FileCache/FileCompactor.cs
Normal file
@@ -0,0 +1,250 @@
|
||||
using MareSynchronos.MareConfiguration;
|
||||
using MareSynchronos.Services;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace MareSynchronos.FileCache;
|
||||
|
||||
public sealed class FileCompactor
|
||||
{
|
||||
public const uint FSCTL_DELETE_EXTERNAL_BACKING = 0x90314U;
|
||||
public const ulong WOF_PROVIDER_FILE = 2UL;
|
||||
|
||||
private readonly Dictionary<string, int> _clusterSizes;
|
||||
|
||||
private readonly WOF_FILE_COMPRESSION_INFO_V1 _efInfo;
|
||||
private readonly ILogger<FileCompactor> _logger;
|
||||
|
||||
private readonly MareConfigService _mareConfigService;
|
||||
private readonly DalamudUtilService _dalamudUtilService;
|
||||
|
||||
public FileCompactor(ILogger<FileCompactor> logger, MareConfigService mareConfigService, DalamudUtilService dalamudUtilService)
|
||||
{
|
||||
_clusterSizes = new(StringComparer.Ordinal);
|
||||
_logger = logger;
|
||||
_mareConfigService = mareConfigService;
|
||||
_dalamudUtilService = dalamudUtilService;
|
||||
_efInfo = new WOF_FILE_COMPRESSION_INFO_V1
|
||||
{
|
||||
Algorithm = CompressionAlgorithm.XPRESS8K,
|
||||
Flags = 0
|
||||
};
|
||||
}
|
||||
|
||||
private enum CompressionAlgorithm
|
||||
{
|
||||
NO_COMPRESSION = -2,
|
||||
LZNT1 = -1,
|
||||
XPRESS4K = 0,
|
||||
LZX = 1,
|
||||
XPRESS8K = 2,
|
||||
XPRESS16K = 3
|
||||
}
|
||||
|
||||
public bool MassCompactRunning { get; private set; } = false;
|
||||
|
||||
public string Progress { get; private set; } = string.Empty;
|
||||
|
||||
public void CompactStorage(bool compress)
|
||||
{
|
||||
MassCompactRunning = true;
|
||||
|
||||
int currentFile = 1;
|
||||
var allFiles = Directory.EnumerateFiles(_mareConfigService.Current.CacheFolder).ToList();
|
||||
int allFilesCount = allFiles.Count;
|
||||
foreach (var file in allFiles)
|
||||
{
|
||||
Progress = $"{currentFile}/{allFilesCount}";
|
||||
if (compress)
|
||||
CompactFile(file);
|
||||
else
|
||||
DecompressFile(file);
|
||||
currentFile++;
|
||||
}
|
||||
|
||||
MassCompactRunning = false;
|
||||
}
|
||||
|
||||
public long GetFileSizeOnDisk(FileInfo fileInfo, bool? isNTFS = null)
|
||||
{
|
||||
bool ntfs = isNTFS ?? string.Equals(new DriveInfo(fileInfo.Directory!.Root.FullName).DriveFormat, "NTFS", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
if (_dalamudUtilService.IsWine || !ntfs) return fileInfo.Length;
|
||||
|
||||
var clusterSize = GetClusterSize(fileInfo);
|
||||
if (clusterSize == -1) return fileInfo.Length;
|
||||
var losize = GetCompressedFileSizeW(fileInfo.FullName, out uint hosize);
|
||||
var size = (long)hosize << 32 | losize;
|
||||
return ((size + clusterSize - 1) / clusterSize) * clusterSize;
|
||||
}
|
||||
|
||||
public async Task WriteAllBytesAsync(string filePath, byte[] decompressedFile, CancellationToken token)
|
||||
{
|
||||
await File.WriteAllBytesAsync(filePath, decompressedFile, token).ConfigureAwait(false);
|
||||
|
||||
if (_dalamudUtilService.IsWine || !_mareConfigService.Current.UseCompactor)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
CompactFile(filePath);
|
||||
}
|
||||
|
||||
public void RenameAndCompact(string filePath, string originalFilePath)
|
||||
{
|
||||
try
|
||||
{
|
||||
File.Move(originalFilePath, filePath);
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
// File already exists
|
||||
return;
|
||||
}
|
||||
|
||||
if (_dalamudUtilService.IsWine || !_mareConfigService.Current.UseCompactor)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
CompactFile(filePath);
|
||||
}
|
||||
|
||||
[DllImport("kernel32.dll")]
|
||||
private static extern int DeviceIoControl(IntPtr hDevice, uint dwIoControlCode, IntPtr lpInBuffer, uint nInBufferSize, IntPtr lpOutBuffer, uint nOutBufferSize, out IntPtr lpBytesReturned, out IntPtr lpOverlapped);
|
||||
|
||||
[DllImport("kernel32.dll")]
|
||||
private static extern uint GetCompressedFileSizeW([In, MarshalAs(UnmanagedType.LPWStr)] string lpFileName,
|
||||
[Out, MarshalAs(UnmanagedType.U4)] out uint lpFileSizeHigh);
|
||||
|
||||
[DllImport("kernel32.dll", SetLastError = true, PreserveSig = true)]
|
||||
private static extern int GetDiskFreeSpaceW([In, MarshalAs(UnmanagedType.LPWStr)] string lpRootPathName,
|
||||
out uint lpSectorsPerCluster, out uint lpBytesPerSector, out uint lpNumberOfFreeClusters,
|
||||
out uint lpTotalNumberOfClusters);
|
||||
|
||||
[DllImport("WoFUtil.dll")]
|
||||
private static extern int WofIsExternalFile([MarshalAs(UnmanagedType.LPWStr)] string Filepath, out int IsExternalFile, out uint Provider, out WOF_FILE_COMPRESSION_INFO_V1 Info, ref uint BufferLength);
|
||||
|
||||
[DllImport("WofUtil.dll")]
|
||||
private static extern int WofSetFileDataLocation(IntPtr FileHandle, ulong Provider, IntPtr ExternalFileInfo, ulong Length);
|
||||
|
||||
private void CompactFile(string filePath)
|
||||
{
|
||||
var fs = new DriveInfo(new FileInfo(filePath).Directory!.Root.FullName);
|
||||
bool isNTFS = string.Equals(fs.DriveFormat, "NTFS", StringComparison.OrdinalIgnoreCase);
|
||||
if (!isNTFS)
|
||||
{
|
||||
_logger.LogWarning("Drive for file {file} is not NTFS", filePath);
|
||||
return;
|
||||
}
|
||||
|
||||
var fi = new FileInfo(filePath);
|
||||
var oldSize = fi.Length;
|
||||
var clusterSize = GetClusterSize(fi);
|
||||
|
||||
if (oldSize < Math.Max(clusterSize, 8 * 1024))
|
||||
{
|
||||
_logger.LogDebug("File {file} is smaller than cluster size ({size}), ignoring", filePath, clusterSize);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!IsCompactedFile(filePath))
|
||||
{
|
||||
_logger.LogDebug("Compacting file to XPRESS8K: {file}", filePath);
|
||||
|
||||
WOFCompressFile(filePath);
|
||||
|
||||
var newSize = GetFileSizeOnDisk(fi);
|
||||
|
||||
_logger.LogDebug("Compressed {file} from {orig}b to {comp}b", filePath, oldSize, newSize);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogDebug("File {file} already compressed", filePath);
|
||||
}
|
||||
}
|
||||
|
||||
private void DecompressFile(string path)
|
||||
{
|
||||
_logger.LogDebug("Removing compression from {file}", path);
|
||||
try
|
||||
{
|
||||
using (var fs = new FileStream(path, FileMode.Open))
|
||||
{
|
||||
#pragma warning disable S3869 // "SafeHandle.DangerousGetHandle" should not be called
|
||||
var hDevice = fs.SafeFileHandle.DangerousGetHandle();
|
||||
#pragma warning restore S3869 // "SafeHandle.DangerousGetHandle" should not be called
|
||||
_ = DeviceIoControl(hDevice, FSCTL_DELETE_EXTERNAL_BACKING, nint.Zero, 0, nint.Zero, 0, out _, out _);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Error decompressing file {path}", path);
|
||||
}
|
||||
}
|
||||
|
||||
private int GetClusterSize(FileInfo fi)
|
||||
{
|
||||
if (!fi.Exists) return -1;
|
||||
var root = fi.Directory?.Root.FullName.ToLower() ?? string.Empty;
|
||||
if (string.IsNullOrEmpty(root)) return -1;
|
||||
if (_clusterSizes.TryGetValue(root, out int value)) return value;
|
||||
_logger.LogDebug("Getting Cluster Size for {path}, root {root}", fi.FullName, root);
|
||||
int result = GetDiskFreeSpaceW(root, out uint sectorsPerCluster, out uint bytesPerSector, out _, out _);
|
||||
if (result == 0) return -1;
|
||||
_clusterSizes[root] = (int)(sectorsPerCluster * bytesPerSector);
|
||||
_logger.LogDebug("Determined Cluster Size for root {root}: {cluster}", root, _clusterSizes[root]);
|
||||
return _clusterSizes[root];
|
||||
}
|
||||
|
||||
private static bool IsCompactedFile(string filePath)
|
||||
{
|
||||
uint buf = 8;
|
||||
_ = WofIsExternalFile(filePath, out int isExtFile, out uint _, out var info, ref buf);
|
||||
if (isExtFile == 0) return false;
|
||||
return info.Algorithm == CompressionAlgorithm.XPRESS8K;
|
||||
}
|
||||
|
||||
private void WOFCompressFile(string path)
|
||||
{
|
||||
var efInfoPtr = Marshal.AllocHGlobal(Marshal.SizeOf(_efInfo));
|
||||
Marshal.StructureToPtr(_efInfo, efInfoPtr, fDeleteOld: true);
|
||||
ulong length = (ulong)Marshal.SizeOf(_efInfo);
|
||||
try
|
||||
{
|
||||
using (var fs = new FileStream(path, FileMode.Open))
|
||||
{
|
||||
#pragma warning disable S3869 // "SafeHandle.DangerousGetHandle" should not be called
|
||||
var hFile = fs.SafeFileHandle.DangerousGetHandle();
|
||||
#pragma warning restore S3869 // "SafeHandle.DangerousGetHandle" should not be called
|
||||
if (fs.SafeFileHandle.IsInvalid)
|
||||
{
|
||||
_logger.LogWarning("Invalid file handle to {file}", path);
|
||||
}
|
||||
else
|
||||
{
|
||||
var ret = WofSetFileDataLocation(hFile, WOF_PROVIDER_FILE, efInfoPtr, length);
|
||||
if (!(ret == 0 || ret == unchecked((int)0x80070158)))
|
||||
{
|
||||
_logger.LogWarning("Failed to compact {file}: {ret}", path, ret.ToString("X"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Error compacting file {path}", path);
|
||||
}
|
||||
finally
|
||||
{
|
||||
Marshal.FreeHGlobal(efInfoPtr);
|
||||
}
|
||||
}
|
||||
|
||||
[StructLayout(LayoutKind.Sequential)]
|
||||
private struct WOF_FILE_COMPRESSION_INFO_V1
|
||||
{
|
||||
public CompressionAlgorithm Algorithm;
|
||||
public ulong Flags;
|
||||
}
|
||||
}
|
||||
8
MareSynchronos/FileCache/FileState.cs
Normal file
8
MareSynchronos/FileCache/FileState.cs
Normal file
@@ -0,0 +1,8 @@
|
||||
namespace MareSynchronos.FileCache;
|
||||
|
||||
public enum FileState
|
||||
{
|
||||
Valid,
|
||||
RequireUpdate,
|
||||
RequireDeletion,
|
||||
}
|
||||
313
MareSynchronos/FileCache/TransientResourceManager.cs
Normal file
313
MareSynchronos/FileCache/TransientResourceManager.cs
Normal file
@@ -0,0 +1,313 @@
|
||||
using MareSynchronos.API.Data.Enum;
|
||||
using MareSynchronos.MareConfiguration;
|
||||
using MareSynchronos.PlayerData.Data;
|
||||
using MareSynchronos.PlayerData.Handlers;
|
||||
using MareSynchronos.Services;
|
||||
using MareSynchronos.Services.Mediator;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using System.Collections.Concurrent;
|
||||
|
||||
namespace MareSynchronos.FileCache;
|
||||
|
||||
public sealed class TransientResourceManager : DisposableMediatorSubscriberBase
|
||||
{
|
||||
private readonly Lock _cacheAdditionLock = new();
|
||||
private readonly HashSet<string> _cachedHandledPaths = new(StringComparer.Ordinal);
|
||||
private readonly TransientConfigService _configurationService;
|
||||
private readonly DalamudUtilService _dalamudUtil;
|
||||
private readonly string[] _fileTypesToHandle = ["tmb", "pap", "avfx", "atex", "sklb", "eid", "phyb", "scd", "skp", "shpk"];
|
||||
private readonly HashSet<GameObjectHandler> _playerRelatedPointers = [];
|
||||
private ConcurrentDictionary<IntPtr, ObjectKind> _cachedFrameAddresses = [];
|
||||
|
||||
public TransientResourceManager(ILogger<TransientResourceManager> logger, TransientConfigService configurationService,
|
||||
DalamudUtilService dalamudUtil, MareMediator mediator) : base(logger, mediator)
|
||||
{
|
||||
_configurationService = configurationService;
|
||||
_dalamudUtil = dalamudUtil;
|
||||
|
||||
Mediator.Subscribe<PenumbraResourceLoadMessage>(this, Manager_PenumbraResourceLoadEvent);
|
||||
Mediator.Subscribe<PenumbraModSettingChangedMessage>(this, (_) => Manager_PenumbraModSettingChanged());
|
||||
Mediator.Subscribe<PriorityFrameworkUpdateMessage>(this, (_) => DalamudUtil_FrameworkUpdate());
|
||||
Mediator.Subscribe<ClassJobChangedMessage>(this, (msg) =>
|
||||
{
|
||||
if (_playerRelatedPointers.Contains(msg.GameObjectHandler))
|
||||
{
|
||||
DalamudUtil_ClassJobChanged();
|
||||
}
|
||||
});
|
||||
Mediator.Subscribe<GameObjectHandlerCreatedMessage>(this, (msg) =>
|
||||
{
|
||||
if (!msg.OwnedObject) return;
|
||||
_playerRelatedPointers.Add(msg.GameObjectHandler);
|
||||
});
|
||||
Mediator.Subscribe<GameObjectHandlerDestroyedMessage>(this, (msg) =>
|
||||
{
|
||||
if (!msg.OwnedObject) return;
|
||||
_playerRelatedPointers.Remove(msg.GameObjectHandler);
|
||||
});
|
||||
}
|
||||
|
||||
private string PlayerPersistentDataKey => _dalamudUtil.GetPlayerNameAsync().GetAwaiter().GetResult() + "_" + _dalamudUtil.GetHomeWorldIdAsync().GetAwaiter().GetResult();
|
||||
private ConcurrentDictionary<ObjectKind, HashSet<string>>? _semiTransientResources = null;
|
||||
private ConcurrentDictionary<ObjectKind, HashSet<string>> SemiTransientResources
|
||||
{
|
||||
get
|
||||
{
|
||||
if (_semiTransientResources == null)
|
||||
{
|
||||
_semiTransientResources = new();
|
||||
_semiTransientResources.TryAdd(ObjectKind.Player, new HashSet<string>(StringComparer.Ordinal));
|
||||
if (_configurationService.Current.PlayerPersistentTransientCache.TryGetValue(PlayerPersistentDataKey, out var gamePaths))
|
||||
{
|
||||
int restored = 0;
|
||||
foreach (var gamePath in gamePaths)
|
||||
{
|
||||
if (string.IsNullOrEmpty(gamePath)) continue;
|
||||
|
||||
try
|
||||
{
|
||||
Logger.LogDebug("Loaded persistent transient resource {path}", gamePath);
|
||||
SemiTransientResources[ObjectKind.Player].Add(gamePath);
|
||||
restored++;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.LogWarning(ex, "Error during loading persistent transient resource {path}", gamePath);
|
||||
}
|
||||
}
|
||||
Logger.LogDebug("Restored {restored}/{total} semi persistent resources", restored, gamePaths.Count);
|
||||
}
|
||||
}
|
||||
|
||||
return _semiTransientResources;
|
||||
}
|
||||
}
|
||||
private ConcurrentDictionary<IntPtr, HashSet<string>> TransientResources { get; } = new();
|
||||
|
||||
public void CleanUpSemiTransientResources(ObjectKind objectKind, List<FileReplacement>? fileReplacement = null)
|
||||
{
|
||||
if (SemiTransientResources.TryGetValue(objectKind, out HashSet<string>? value))
|
||||
{
|
||||
if (fileReplacement == null)
|
||||
{
|
||||
value.Clear();
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (var replacement in fileReplacement.Where(p => !p.HasFileReplacement).SelectMany(p => p.GamePaths).ToList())
|
||||
{
|
||||
value.RemoveWhere(p => string.Equals(p, replacement, StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public HashSet<string> GetSemiTransientResources(ObjectKind objectKind)
|
||||
{
|
||||
if (SemiTransientResources.TryGetValue(objectKind, out var result))
|
||||
{
|
||||
return result ?? new HashSet<string>(StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
return new HashSet<string>(StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
public List<string> GetTransientResources(IntPtr gameObject)
|
||||
{
|
||||
if (TransientResources.TryGetValue(gameObject, out var result))
|
||||
{
|
||||
return [.. result];
|
||||
}
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
public void PersistTransientResources(IntPtr gameObject, ObjectKind objectKind)
|
||||
{
|
||||
if (!SemiTransientResources.TryGetValue(objectKind, out HashSet<string>? value))
|
||||
{
|
||||
value = new HashSet<string>(StringComparer.Ordinal);
|
||||
SemiTransientResources[objectKind] = value;
|
||||
}
|
||||
|
||||
if (!TransientResources.TryGetValue(gameObject, out var resources))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var transientResources = resources.ToList();
|
||||
Logger.LogDebug("Persisting {count} transient resources", transientResources.Count);
|
||||
foreach (var gamePath in transientResources)
|
||||
{
|
||||
value.Add(gamePath);
|
||||
}
|
||||
|
||||
if (objectKind == ObjectKind.Player && SemiTransientResources.TryGetValue(ObjectKind.Player, out var fileReplacements))
|
||||
{
|
||||
_configurationService.Current.PlayerPersistentTransientCache[PlayerPersistentDataKey] = fileReplacements.Where(f => !string.IsNullOrEmpty(f)).ToHashSet(StringComparer.Ordinal);
|
||||
_configurationService.Save();
|
||||
}
|
||||
TransientResources[gameObject].Clear();
|
||||
}
|
||||
|
||||
internal void AddSemiTransientResource(ObjectKind objectKind, string item)
|
||||
{
|
||||
if (!SemiTransientResources.TryGetValue(objectKind, out HashSet<string>? value))
|
||||
{
|
||||
value = new HashSet<string>(StringComparer.Ordinal);
|
||||
SemiTransientResources[objectKind] = value;
|
||||
}
|
||||
|
||||
value.Add(item.ToLowerInvariant());
|
||||
}
|
||||
|
||||
internal void ClearTransientPaths(IntPtr ptr, List<string> list)
|
||||
{
|
||||
if (TransientResources.TryGetValue(ptr, out var set))
|
||||
{
|
||||
foreach (var file in set.Where(p => list.Contains(p, StringComparer.OrdinalIgnoreCase)))
|
||||
{
|
||||
Logger.LogTrace("Removing From Transient: {file}", file);
|
||||
}
|
||||
|
||||
int removed = set.RemoveWhere(p => list.Contains(p, StringComparer.OrdinalIgnoreCase));
|
||||
Logger.LogInformation("Removed {removed} previously existing transient paths", removed);
|
||||
}
|
||||
}
|
||||
|
||||
protected override void Dispose(bool disposing)
|
||||
{
|
||||
base.Dispose(disposing);
|
||||
|
||||
try
|
||||
{
|
||||
TransientResources.Clear();
|
||||
SemiTransientResources.Clear();
|
||||
if (SemiTransientResources.TryGetValue(ObjectKind.Player, out HashSet<string>? value))
|
||||
{
|
||||
_configurationService.Current.PlayerPersistentTransientCache[PlayerPersistentDataKey] = value;
|
||||
_configurationService.Save();
|
||||
}
|
||||
}
|
||||
catch { }
|
||||
}
|
||||
|
||||
private void DalamudUtil_ClassJobChanged()
|
||||
{
|
||||
if (SemiTransientResources.TryGetValue(ObjectKind.Pet, out HashSet<string>? value))
|
||||
{
|
||||
value?.Clear();
|
||||
}
|
||||
}
|
||||
|
||||
private void DalamudUtil_FrameworkUpdate()
|
||||
{
|
||||
_cachedFrameAddresses = _cachedFrameAddresses = new ConcurrentDictionary<nint, ObjectKind>(_playerRelatedPointers.Where(k => k.Address != nint.Zero).ToDictionary(c => c.CurrentAddress(), c => c.ObjectKind));
|
||||
lock (_cacheAdditionLock)
|
||||
{
|
||||
_cachedHandledPaths.Clear();
|
||||
}
|
||||
foreach (var item in TransientResources.Where(item => !_dalamudUtil.IsGameObjectPresent(item.Key)).Select(i => i.Key).ToList())
|
||||
{
|
||||
Logger.LogDebug("Object not present anymore: {addr}", item.ToString("X"));
|
||||
TransientResources.TryRemove(item, out _);
|
||||
}
|
||||
}
|
||||
|
||||
private void Manager_PenumbraModSettingChanged()
|
||||
{
|
||||
_ = Task.Run(() =>
|
||||
{
|
||||
Logger.LogDebug("Penumbra Mod Settings changed, verifying SemiTransientResources");
|
||||
foreach (var item in _playerRelatedPointers)
|
||||
{
|
||||
Mediator.Publish(new TransientResourceChangedMessage(item.Address));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private void Manager_PenumbraResourceLoadEvent(PenumbraResourceLoadMessage msg)
|
||||
{
|
||||
var gamePath = msg.GamePath.ToLowerInvariant();
|
||||
var gameObject = msg.GameObject;
|
||||
var filePath = msg.FilePath;
|
||||
|
||||
// ignore files already processed this frame
|
||||
if (_cachedHandledPaths.Contains(gamePath)) return;
|
||||
|
||||
lock (_cacheAdditionLock)
|
||||
{
|
||||
_cachedHandledPaths.Add(gamePath);
|
||||
}
|
||||
|
||||
// replace individual mtrl stuff
|
||||
if (filePath.StartsWith("|", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
filePath = filePath.Split("|")[2];
|
||||
}
|
||||
// replace filepath
|
||||
filePath = filePath.ToLowerInvariant().Replace("\\", "/", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
// ignore files that are the same
|
||||
var replacedGamePath = gamePath.ToLowerInvariant().Replace("\\", "/", StringComparison.OrdinalIgnoreCase);
|
||||
if (string.Equals(filePath, replacedGamePath, StringComparison.OrdinalIgnoreCase)) return;
|
||||
|
||||
// ignore files to not handle
|
||||
if (!_fileTypesToHandle.Any(type => gamePath.EndsWith(type, StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
lock (_cacheAdditionLock)
|
||||
{
|
||||
_cachedHandledPaths.Add(gamePath);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// ignore files not belonging to anything player related
|
||||
if (!_cachedFrameAddresses.TryGetValue(gameObject, out var objectKind))
|
||||
{
|
||||
lock (_cacheAdditionLock)
|
||||
{
|
||||
_cachedHandledPaths.Add(gamePath);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (!TransientResources.TryGetValue(gameObject, out HashSet<string>? value))
|
||||
{
|
||||
value = new(StringComparer.OrdinalIgnoreCase);
|
||||
TransientResources[gameObject] = value;
|
||||
}
|
||||
|
||||
if (value.Contains(replacedGamePath) ||
|
||||
SemiTransientResources.SelectMany(k => k.Value).Any(f => string.Equals(f, gamePath, StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
Logger.LogTrace("Not adding {replacedPath} : {filePath}", replacedGamePath, filePath);
|
||||
}
|
||||
else
|
||||
{
|
||||
var thing = _playerRelatedPointers.FirstOrDefault(f => f.Address == gameObject);
|
||||
value.Add(replacedGamePath);
|
||||
Logger.LogDebug("Adding {replacedGamePath} for {gameObject} ({filePath})", replacedGamePath, thing?.ToString() ?? gameObject.ToString("X"), filePath);
|
||||
_ = Task.Run(async () =>
|
||||
{
|
||||
_sendTransientCts?.Cancel();
|
||||
_sendTransientCts?.Dispose();
|
||||
_sendTransientCts = new();
|
||||
var token = _sendTransientCts.Token;
|
||||
await Task.Delay(TimeSpan.FromSeconds(2), token).ConfigureAwait(false);
|
||||
Mediator.Publish(new TransientResourceChangedMessage(gameObject));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
internal void RemoveTransientResource(ObjectKind objectKind, string path)
|
||||
{
|
||||
if (SemiTransientResources.TryGetValue(objectKind, out var resources))
|
||||
{
|
||||
resources.RemoveWhere(f => string.Equals(path, f, StringComparison.OrdinalIgnoreCase));
|
||||
_configurationService.Current.PlayerPersistentTransientCache[PlayerPersistentDataKey] = resources;
|
||||
_configurationService.Save();
|
||||
}
|
||||
}
|
||||
|
||||
private CancellationTokenSource _sendTransientCts = new();
|
||||
}
|
||||
Reference in New Issue
Block a user