Initial

MareSynchronosServer/MareSynchronosStaticFilesServer/.config/dotnet-tools.json

@@ -0,0 +1,12 @@
+{
+  "version": 1,
+  "isRoot": true,
+  "tools": {
+    "dotnet-ef": {
+      "version": "6.0.8",
+      "commands": [
+        "dotnet-ef"
+      ]
+    }
+  }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/CacheController.cs

@@ -0,0 +1,51 @@
+using MareSynchronos.API.Routes;
+using MareSynchronosStaticFilesServer.Services;
+using MareSynchronosStaticFilesServer.Utils;
+using Microsoft.AspNetCore.Mvc;
+using System.Globalization;
+using System.Text;
+
+namespace MareSynchronosStaticFilesServer.Controllers;
+
+[Route(MareFiles.Cache)]
+public class CacheController : ControllerBase
+{
+    private readonly RequestBlockFileListResultFactory _requestBlockFileListResultFactory;
+    private readonly CachedFileProvider _cachedFileProvider;
+    private readonly RequestQueueService _requestQueue;
+    private readonly FileStatisticsService _fileStatisticsService;
+
+    public CacheController(ILogger<CacheController> logger, RequestBlockFileListResultFactory requestBlockFileListResultFactory,
+        CachedFileProvider cachedFileProvider, RequestQueueService requestQueue, FileStatisticsService fileStatisticsService) : base(logger)
+    {
+        _requestBlockFileListResultFactory = requestBlockFileListResultFactory;
+        _cachedFileProvider = cachedFileProvider;
+        _requestQueue = requestQueue;
+        _fileStatisticsService = fileStatisticsService;
+    }
+
+    [HttpGet(MareFiles.Cache_Get)]
+    public async Task<IActionResult> GetFiles(Guid requestId)
+    {
+        _logger.LogDebug($"GetFile:{MareUser}:{requestId}");
+
+        if (!_requestQueue.IsActiveProcessing(requestId, MareUser, out var request)) return BadRequest();
+
+        _requestQueue.ActivateRequest(requestId);
+
+        long requestSize = 0;
+        var fileList = new List<FileInfo>(request.FileIds.Count);
+
+        foreach (var file in request.FileIds)
+        {
+            var fi = await _cachedFileProvider.GetAndDownloadFile(file);
+            if (fi == null) continue;
+            requestSize += fi.Length;
+            fileList.Add(fi);
+        }
+
+        _fileStatisticsService.LogRequest(requestSize);
+
+        return _requestBlockFileListResultFactory.Create(requestId, fileList);
+    }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ControllerBase.cs

@@ -0,0 +1,18 @@
+using MareSynchronosShared.Utils;
+using Microsoft.AspNetCore.Mvc;
+
+namespace MareSynchronosStaticFilesServer.Controllers;
+
+public class ControllerBase : Controller
+{
+    protected ILogger _logger;
+
+    public ControllerBase(ILogger logger)
+    {
+        _logger = logger;
+    }
+
+    protected string MareUser => HttpContext.User.Claims.First(f => string.Equals(f.Type, MareClaimTypes.Uid, StringComparison.Ordinal)).Value;
+    protected string Continent => HttpContext.User.Claims.FirstOrDefault(f => string.Equals(f.Type, MareClaimTypes.Continent, StringComparison.Ordinal))?.Value ?? "*";
+    protected bool IsPriority => false;
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/DistributionController.cs

@@ -0,0 +1,60 @@
+using MareSynchronos.API.Routes;
+using MareSynchronosShared.Services;
+using MareSynchronosShared.Utils.Configuration;
+using MareSynchronosStaticFilesServer.Services;
+using Microsoft.AspNetCore.Authorization;
+using Microsoft.AspNetCore.Mvc;
+
+namespace MareSynchronosStaticFilesServer.Controllers;
+
+[Route(MareFiles.Distribution)]
+public class DistributionController : ControllerBase
+{
+    private readonly CachedFileProvider _cachedFileProvider;
+    private readonly IConfigurationService<StaticFilesServerConfiguration> _configuration;
+
+    public DistributionController(ILogger<DistributionController> logger, CachedFileProvider cachedFileProvider,
+        IConfigurationService<StaticFilesServerConfiguration> configuration) : base(logger)
+    {
+        _cachedFileProvider = cachedFileProvider;
+        _configuration = configuration;
+    }
+
+    [HttpGet(MareFiles.Distribution_Get)]
+    [Authorize(Policy = "Internal")]
+    public async Task<IActionResult> GetFile(string file)
+    {
+        _logger.LogInformation($"GetFile:{MareUser}:{file}");
+
+        var fi = await _cachedFileProvider.GetAndDownloadFile(file);
+        if (fi == null) return NotFound();
+
+        if (_configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseXAccelRedirect), false))
+        {
+            var prefix = _configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.XAccelRedirectPrefix));
+            Response.Headers.Append("X-Accel-Redirect", Path.Combine(prefix, file));
+            return Ok();
+        }
+        else
+        {
+            return PhysicalFile(fi.FullName, "application/octet-stream");
+        }
+    }
+
+    [HttpPost("touch")]
+    [Authorize(Policy = "Internal")]
+    public IActionResult TouchFiles([FromBody] string[] files)
+    {
+        _logger.LogInformation($"TouchFiles:{MareUser}:{files.Length}");
+
+        if (files.Length == 0)
+            return Ok();
+
+        Task.Run(() => {
+            foreach (var file in files)
+                _cachedFileProvider.TouchColdHash(file);
+        }).ConfigureAwait(false);
+
+        return Ok();
+    }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/MainController.cs

@@ -0,0 +1,25 @@
+using MareSynchronos.API.Routes;
+using MareSynchronosStaticFilesServer.Services;
+using Microsoft.AspNetCore.Authorization;
+using Microsoft.AspNetCore.Mvc;
+
+namespace MareSynchronosStaticFilesServer.Controllers;
+
+[Route(MareFiles.Main)]
+public class MainController : ControllerBase
+{
+    private readonly IClientReadyMessageService _messageService;
+
+    public MainController(ILogger<MainController> logger, IClientReadyMessageService mareHub) : base(logger)
+    {
+        _messageService = mareHub;
+    }
+
+    [HttpGet(MareFiles.Main_SendReady)]
+    [Authorize(Policy = "Internal")]
+    public IActionResult SendReadyToClients(string uid, Guid requestId)
+    {
+        _messageService.SendDownloadReady(uid, requestId);
+        return Ok();
+    }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/RequestController.cs

@@ -0,0 +1,72 @@
+using MareSynchronos.API.Routes;
+using MareSynchronosStaticFilesServer.Services;
+using Microsoft.AspNetCore.Mvc;
+
+namespace MareSynchronosStaticFilesServer.Controllers;
+
+[Route(MareFiles.Request)]
+public class RequestController : ControllerBase
+{
+    private readonly CachedFileProvider _cachedFileProvider;
+    private readonly RequestQueueService _requestQueue;
+    private readonly FilePreFetchService _preFetchService;
+
+    public RequestController(ILogger<RequestController> logger, CachedFileProvider cachedFileProvider, RequestQueueService requestQueue, FilePreFetchService preFetchService) : base(logger)
+    {
+        _cachedFileProvider = cachedFileProvider;
+        _requestQueue = requestQueue;
+        _preFetchService = preFetchService;
+    }
+
+    [HttpGet]
+    [Route(MareFiles.Request_Cancel)]
+    public async Task<IActionResult> CancelQueueRequest(Guid requestId)
+    {
+        try
+        {
+            _requestQueue.RemoveFromQueue(requestId, MareUser, IsPriority);
+            return Ok();
+        }
+        catch (OperationCanceledException) { return BadRequest(); }
+    }
+
+    [HttpPost]
+    [Route(MareFiles.Request_Enqueue)]
+    public async Task<IActionResult> PreRequestFilesAsync([FromBody] IEnumerable<string> files)
+    {
+        try
+        {
+            var hashList = files.ToList();
+            var fileList = new List<FileInfo>();
+
+            foreach (var file in hashList)
+            {
+                _logger.LogDebug("Prerequested file: " + file);
+                var fileInfo = await _cachedFileProvider.DownloadFileWhenRequired(file).ConfigureAwait(false);
+                if (fileInfo != null)
+                    fileList.Add(fileInfo);
+            }
+
+            _preFetchService.PrefetchFiles(fileList);
+
+            Guid g = Guid.NewGuid();
+            await _requestQueue.EnqueueUser(new(g, MareUser, hashList), IsPriority, HttpContext.RequestAborted);
+
+            return Ok(g);
+        }
+        catch (OperationCanceledException) { return BadRequest(); }
+    }
+
+    [HttpGet]
+    [Route(MareFiles.Request_Check)]
+    public async Task<IActionResult> CheckQueueAsync(Guid requestId, [FromBody] IEnumerable<string> files)
+    {
+        try
+        {
+            if (!_requestQueue.StillEnqueued(requestId, MareUser, IsPriority))
+                await _requestQueue.EnqueueUser(new(requestId, MareUser, files.ToList()), IsPriority, HttpContext.RequestAborted);
+            return Ok();
+        }
+        catch (OperationCanceledException) { return BadRequest(); }
+    }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs

@@ -0,0 +1,280 @@
+using K4os.Compression.LZ4.Streams;
+using MareSynchronos.API.Dto.Files;
+using MareSynchronos.API.Routes;
+using MareSynchronos.API.SignalR;
+using MareSynchronosServer.Hubs;
+using MareSynchronosShared.Data;
+using MareSynchronosShared.Metrics;
+using MareSynchronosShared.Models;
+using MareSynchronosShared.Services;
+using MareSynchronosShared.Utils.Configuration;
+using MareSynchronosStaticFilesServer.Services;
+using MareSynchronosStaticFilesServer.Utils;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.AspNetCore.SignalR;
+using Microsoft.EntityFrameworkCore;
+using System.Collections.Concurrent;
+using System.Security.Cryptography;
+using System.Security.Policy;
+using System.Text.Json;
+using System.Text.RegularExpressions;
+
+namespace MareSynchronosStaticFilesServer.Controllers;
+
+[Route(MareFiles.ServerFiles)]
+public class ServerFilesController : ControllerBase
+{
+    private static readonly SemaphoreSlim _fileLockDictLock = new(1);
+    private static readonly ConcurrentDictionary<string, SemaphoreSlim> _fileUploadLocks = new(StringComparer.Ordinal);
+    private readonly string _basePath;
+    private readonly string _coldBasePath;
+    private readonly CachedFileProvider _cachedFileProvider;
+    private readonly IConfigurationService<StaticFilesServerConfiguration> _configuration;
+    private readonly IHubContext<MareHub> _hubContext;
+    private readonly MareDbContext _mareDbContext;
+    private readonly MareMetrics _metricsClient;
+
+    public ServerFilesController(ILogger<ServerFilesController> logger, CachedFileProvider cachedFileProvider,
+        IConfigurationService<StaticFilesServerConfiguration> configuration,
+        IHubContext<MareHub> hubContext,
+        MareDbContext mareDbContext, MareMetrics metricsClient) : base(logger)
+    {
+        _configuration = configuration;
+        _basePath = configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
+        if (_configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false))
+            _basePath = configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.ColdStorageDirectory));
+        _cachedFileProvider = cachedFileProvider;
+        _hubContext = hubContext;
+        _mareDbContext = mareDbContext;
+        _metricsClient = metricsClient;
+    }
+
+    [HttpPost(MareFiles.ServerFiles_DeleteAll)]
+    public async Task<IActionResult> FilesDeleteAll()
+    {
+        return Ok();
+    }
+
+    [HttpGet(MareFiles.ServerFiles_GetSizes)]
+    public async Task<IActionResult> FilesGetSizes([FromBody] List<string> hashes)
+    {
+        var forbiddenFiles = await _mareDbContext.ForbiddenUploadEntries.
+            Where(f => hashes.Contains(f.Hash)).ToListAsync().ConfigureAwait(false);
+        List<DownloadFileDto> response = new();
+
+        var cacheFile = await _mareDbContext.Files.AsNoTracking().Where(f => hashes.Contains(f.Hash)).AsNoTracking().Select(k => new { k.Hash, k.Size }).AsNoTracking().ToListAsync().ConfigureAwait(false);
+
+        var allFileShards = new List<CdnShardConfiguration>(_configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CdnShardConfiguration), new List<CdnShardConfiguration>()));
+
+        foreach (var file in cacheFile)
+        {
+            var forbiddenFile = forbiddenFiles.SingleOrDefault(f => string.Equals(f.Hash, file.Hash, StringComparison.OrdinalIgnoreCase));
+            Uri? baseUrl = null;
+
+            if (forbiddenFile == null)
+            {
+                List<CdnShardConfiguration> selectedShards = new();
+                var matchingShards = allFileShards.Where(f => new Regex(f.FileMatch).IsMatch(file.Hash)).ToList();
+
+                if (string.Equals(Continent, "*", StringComparison.Ordinal))
+                {
+                    selectedShards = matchingShards;
+                }
+                else
+                {
+                    selectedShards = matchingShards.Where(c => c.Continents.Contains(Continent, StringComparer.OrdinalIgnoreCase)).ToList();
+                    if (!selectedShards.Any()) selectedShards = matchingShards;
+                }
+
+                var shard = selectedShards
+                    .OrderBy(s => !s.Continents.Any() ? 0 : 1)
+                    .ThenBy(s => s.Continents.Contains("*", StringComparer.Ordinal) ? 0 : 1)
+                    .ThenBy(g => Guid.NewGuid()).FirstOrDefault();
+
+                baseUrl = shard?.CdnFullUrl ?? _configuration.GetValue<Uri>(nameof(StaticFilesServerConfiguration.CdnFullUrl));
+            }
+
+            response.Add(new DownloadFileDto
+            {
+                FileExists = file.Size > 0,
+                ForbiddenBy = forbiddenFile?.ForbiddenBy ?? string.Empty,
+                IsForbidden = forbiddenFile != null,
+                Hash = file.Hash,
+                Size = file.Size,
+                Url = baseUrl?.ToString() ?? string.Empty,
+            });
+        }
+
+        return Ok(JsonSerializer.Serialize(response));
+    }
+
+    [HttpPost(MareFiles.ServerFiles_FilesSend)]
+    public async Task<IActionResult> FilesSend([FromBody] FilesSendDto filesSendDto)
+    {
+        var userSentHashes = new HashSet<string>(filesSendDto.FileHashes.Distinct(StringComparer.Ordinal).Select(s => string.Concat(s.Where(c => char.IsLetterOrDigit(c)))), StringComparer.Ordinal);
+        var notCoveredFiles = new Dictionary<string, UploadFileDto>(StringComparer.Ordinal);
+        var forbiddenFiles = await _mareDbContext.ForbiddenUploadEntries.AsNoTracking().Where(f => userSentHashes.Contains(f.Hash)).AsNoTracking().ToDictionaryAsync(f => f.Hash, f => f).ConfigureAwait(false);
+        var existingFiles = await _mareDbContext.Files.AsNoTracking().Where(f => userSentHashes.Contains(f.Hash)).AsNoTracking().ToDictionaryAsync(f => f.Hash, f => f).ConfigureAwait(false);
+
+        List<FileCache> fileCachesToUpload = new();
+        foreach (var hash in userSentHashes)
+        {
+            // Skip empty file hashes, duplicate file hashes, forbidden file hashes and existing file hashes
+            if (string.IsNullOrEmpty(hash)) { continue; }
+            if (notCoveredFiles.ContainsKey(hash)) { continue; }
+            if (forbiddenFiles.ContainsKey(hash))
+            {
+                notCoveredFiles[hash] = new UploadFileDto()
+                {
+                    ForbiddenBy = forbiddenFiles[hash].ForbiddenBy,
+                    Hash = hash,
+                    IsForbidden = true,
+                };
+
+                continue;
+            }
+            if (existingFiles.TryGetValue(hash, out var file) && file.Uploaded) { continue; }
+
+            notCoveredFiles[hash] = new UploadFileDto()
+            {
+                Hash = hash,
+            };
+        }
+
+        if (notCoveredFiles.Any(p => !p.Value.IsForbidden))
+        {
+            await _hubContext.Clients.Users(filesSendDto.UIDs).SendAsync(nameof(IMareHub.Client_UserReceiveUploadStatus), new MareSynchronos.API.Dto.User.UserDto(new(MareUser)))
+                .ConfigureAwait(false);
+        }
+
+        return Ok(JsonSerializer.Serialize(notCoveredFiles.Values.ToList()));
+    }
+
+    [HttpPost(MareFiles.ServerFiles_Upload + "/{hash}")]
+    [RequestSizeLimit(200 * 1024 * 1024)]
+    public async Task<IActionResult> UploadFile(string hash, CancellationToken requestAborted)
+    {
+        _logger.LogInformation("{user} uploading file {file}", MareUser, hash);
+        hash = hash.ToUpperInvariant();
+        var existingFile = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash);
+        if (existingFile != null) return Ok();
+
+        SemaphoreSlim? fileLock = null;
+        bool successfullyWaited = false;
+        while (!successfullyWaited && !requestAborted.IsCancellationRequested)
+        {
+            lock (_fileUploadLocks)
+            {
+                if (!_fileUploadLocks.TryGetValue(hash, out fileLock))
+                    _fileUploadLocks[hash] = fileLock = new SemaphoreSlim(1);
+            }
+
+            try
+            {
+                await fileLock.WaitAsync(requestAborted).ConfigureAwait(false);
+                successfullyWaited = true;
+            }
+            catch (ObjectDisposedException)
+            {
+                _logger.LogWarning("Semaphore disposed for {hash}, recreating", hash);
+            }
+        }
+
+        try
+        {
+            var existingFileCheck2 = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash);
+            if (existingFileCheck2 != null)
+            {
+                return Ok();
+            }
+
+            var path = FilePathUtil.GetFilePath(_basePath, hash);
+            var tmpPath = path + ".tmp";
+            long compressedSize = -1;
+
+            try
+            {
+                // Write incoming file to a temporary file while also hashing the decompressed content
+
+                // Stream flow diagram:
+                // Request.Body ==> (Tee) ==> FileStream
+                //                        ==> CountedStream ==> LZ4DecoderStream ==> HashingStream ==> Stream.Null
+
+                // Reading via TeeStream causes the request body to be copied to tmpPath
+                using var tmpFileStream = new FileStream(tmpPath, FileMode.Create);
+                using var teeStream = new TeeStream(Request.Body, tmpFileStream);
+                teeStream.DisposeUnderlying = false;
+                // Read via CountedStream to count the number of compressed bytes
+                using var countStream = new CountedStream(teeStream);
+                countStream.DisposeUnderlying = false;
+
+                // The decompressed file content is read through LZ4DecoderStream, and written out to HashingStream
+                using var decStream = LZ4Stream.Decode(countStream, extraMemory: 0, leaveOpen: true);
+                // HashingStream simply hashes the decompressed bytes without writing them anywhere
+                using var hashStream = new HashingStream(Stream.Null, SHA1.Create());
+                hashStream.DisposeUnderlying = false;
+
+                await decStream.CopyToAsync(hashStream, requestAborted).ConfigureAwait(false);
+                decStream.Close();
+
+                var hashString = BitConverter.ToString(hashStream.Finish())
+                    .Replace("-", "", StringComparison.Ordinal).ToUpperInvariant();
+                if (!string.Equals(hashString, hash, StringComparison.Ordinal))
+                    throw new InvalidOperationException($"Hash does not match file, computed: {hashString}, expected: {hash}");
+
+                compressedSize = countStream.BytesRead;
+
+                // File content is verified -- move it to its final location
+                System.IO.File.Move(tmpPath, path, true);
+            }
+            catch
+            {
+                try
+                {
+                    System.IO.File.Delete(tmpPath);
+                }
+                catch { }
+                throw;
+            }
+
+            // update on db
+            await _mareDbContext.Files.AddAsync(new FileCache()
+            {
+                Hash = hash,
+                UploadDate = DateTime.UtcNow,
+                UploaderUID = MareUser,
+                Size = compressedSize,
+                Uploaded = true
+            }).ConfigureAwait(false);
+            await _mareDbContext.SaveChangesAsync().ConfigureAwait(false);
+
+            _metricsClient.IncGauge(MetricsAPI.GaugeFilesTotal, 1);
+            _metricsClient.IncGauge(MetricsAPI.GaugeFilesTotalSize, compressedSize);
+
+            _fileUploadLocks.TryRemove(hash, out _);
+
+            return Ok();
+        }
+        catch (Exception e)
+        {
+            _logger.LogError(e, "Error during file upload");
+            return BadRequest();
+        }
+        finally
+        {
+            try
+            {
+                fileLock?.Release();
+                fileLock?.Dispose();
+            }
+            catch (ObjectDisposedException)
+            {
+                // it's disposed whatever
+            }
+            finally
+            {
+                _fileUploadLocks.TryRemove(hash, out _);
+            }
+        }
+    }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/DummyHub.cs

@@ -0,0 +1,25 @@
+using Microsoft.AspNetCore.SignalR;
+
+// this is a very hacky way to attach this file server to the main mare hub signalr instance via redis
+// signalr publishes the namespace and hubname into the redis backend so this needs to be equal to the original
+// but I don't need to reimplement the hub completely as I only exclusively use it for internal connection calling
+// from the queue service so I keep the namespace and name of the class the same so it can connect to the same channel
+// if anyone finds a better way to do this let me know
+
+#pragma warning disable IDE0130 // Namespace does not match folder structure
+#pragma warning disable MA0048 // File name must match type name
+namespace MareSynchronosServer.Hubs;
+public class MareHub : Hub
+{
+    public override Task OnConnectedAsync()
+    {
+        throw new NotSupportedException();
+    }
+
+    public override Task OnDisconnectedAsync(Exception exception)
+    {
+        throw new NotSupportedException();
+    }
+}
+#pragma warning restore IDE0130 // Namespace does not match folder structure
+#pragma warning restore MA0048 // File name must match type name

MareSynchronosServer/MareSynchronosStaticFilesServer/MareSynchronosStaticFilesServer.csproj

@@ -0,0 +1,38 @@
+<Project Sdk="Microsoft.NET.Sdk.Web">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <Content Remove="appsettings.Development.json" />
+    <Content Remove="appsettings.json" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <None Include="appsettings.Development.json" />
+    <None Include="appsettings.json">
+      <CopyToOutputDirectory>Never</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="IDisposableAnalyzers" Version="4.0.7">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
+    <PackageReference Include="K4os.Compression.LZ4.Streams" Version="1.3.6" />
+    <PackageReference Include="Meziantou.Analyzer" Version="2.0.150">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
+    <PackageReference Include="Microsoft.Extensions.Hosting.Systemd" Version="8.0.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\MareAPI\MareSynchronosAPI\MareSynchronos.API.csproj" />
+    <ProjectReference Include="..\MareSynchronosShared\MareSynchronosShared.csproj" />
+  </ItemGroup>
+
+</Project>

MareSynchronosServer/MareSynchronosStaticFilesServer/Program.cs

@@ -0,0 +1,43 @@
+using MareSynchronosShared.Services;
+using MareSynchronosShared.Utils.Configuration;
+
+namespace MareSynchronosStaticFilesServer;
+
+public class Program
+{
+    public static void Main(string[] args)
+    {
+        var hostBuilder = CreateHostBuilder(args);
+        var host = hostBuilder.Build();
+
+        using (var scope = host.Services.CreateScope())
+        {
+            var options = host.Services.GetService<IConfigurationService<StaticFilesServerConfiguration>>();
+            var optionsServer = host.Services.GetService<IConfigurationService<MareConfigurationBase>>();
+            var logger = host.Services.GetService<ILogger<Program>>();
+            logger.LogInformation("Loaded MareSynchronos Static Files Server Configuration (IsMain: {isMain})", options.IsMain);
+            logger.LogInformation(options.ToString());
+            logger.LogInformation("Loaded MareSynchronos Server Auth Configuration (IsMain: {isMain})", optionsServer.IsMain);
+            logger.LogInformation(optionsServer.ToString());
+        }
+
+        host.Run();
+    }
+
+    public static IHostBuilder CreateHostBuilder(string[] args)
+    {
+        var loggerFactory = LoggerFactory.Create(builder =>
+        {
+            builder.ClearProviders();
+            builder.AddConsole();
+        });
+        var logger = loggerFactory.CreateLogger<Startup>();
+        return Host.CreateDefaultBuilder(args)
+            .UseSystemd()
+            .ConfigureWebHostDefaults(webBuilder =>
+            {
+                webBuilder.UseContentRoot(AppContext.BaseDirectory);
+                webBuilder.UseStartup(ctx => new Startup(ctx.Configuration, logger));
+            });
+    }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/Properties/launchSettings.json

@@ -0,0 +1,28 @@
+{
+  "iisSettings": {
+    "windowsAuthentication": false,
+    "anonymousAuthentication": true,
+    "iisExpress": {
+      "applicationUrl": "http://localhost:21378",
+      "sslPort": 44331
+    }
+  },
+  "profiles": {
+    "MareSynchronosStaticFilesServer": {
+      "commandName": "Project",
+      "dotnetRunMessages": true,
+      "launchBrowser": true,
+      "applicationUrl": "https://localhost:7094;http://localhost:5094",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development"
+      }
+    },
+    "IIS Express": {
+      "commandName": "IISExpress",
+      "launchBrowser": true,
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development"
+      }
+    }
+  }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs

@@ -0,0 +1,243 @@
+using MareSynchronosShared.Metrics;
+using MareSynchronosShared.Services;
+using MareSynchronosStaticFilesServer.Utils;
+using System.Collections.Concurrent;
+using System.Net.Http.Headers;
+using MareSynchronosShared.Utils;
+using MareSynchronos.API.Routes;
+using MareSynchronosShared.Utils.Configuration;
+
+namespace MareSynchronosStaticFilesServer.Services;
+
+public sealed class CachedFileProvider : IDisposable
+{
+    private readonly IConfigurationService<StaticFilesServerConfiguration> _configuration;
+    private readonly ILogger<CachedFileProvider> _logger;
+    private readonly FileStatisticsService _fileStatisticsService;
+    private readonly MareMetrics _metrics;
+    private readonly ServerTokenGenerator _generator;
+    private readonly ITouchHashService _touchService;
+    private readonly Uri _remoteCacheSourceUri;
+    private readonly bool _useColdStorage;
+    private readonly string _hotStoragePath;
+    private readonly string _coldStoragePath;
+    private readonly ConcurrentDictionary<string, Task> _currentTransfers = new(StringComparer.Ordinal);
+    private readonly HttpClient _httpClient;
+    private readonly SemaphoreSlim _downloadSemaphore = new(1, 1);
+    private bool _disposed;
+
+    private bool IsMainServer => _remoteCacheSourceUri == null && _isDistributionServer;
+    private bool _isDistributionServer;
+
+    public CachedFileProvider(IConfigurationService<StaticFilesServerConfiguration> configuration, ILogger<CachedFileProvider> logger,
+        FileStatisticsService fileStatisticsService, MareMetrics metrics, ServerTokenGenerator generator, ITouchHashService touchService)
+    {
+        AppContext.SetSwitch("System.Net.Http.SocketsHttpHandler.Http2UnencryptedSupport", true);
+        _configuration = configuration;
+        _logger = logger;
+        _fileStatisticsService = fileStatisticsService;
+        _metrics = metrics;
+        _generator = generator;
+        _touchService = touchService;
+        _remoteCacheSourceUri = configuration.GetValueOrDefault<Uri>(nameof(StaticFilesServerConfiguration.DistributionFileServerAddress), null);
+        _isDistributionServer = configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.IsDistributionNode), false);
+        _useColdStorage = configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false);
+        _hotStoragePath = configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
+        _coldStoragePath = configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.ColdStorageDirectory));
+        _httpClient = new();
+        _httpClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("MareSynchronosServer", "1.0.0.0"));
+    }
+
+    public void Dispose()
+    {
+        if (_disposed)
+        {
+            return;
+        }
+
+        _disposed = true;
+        _httpClient?.Dispose();
+    }
+
+    private async Task DownloadTask(string hash)
+    {
+        var destinationFilePath = FilePathUtil.GetFilePath(_useColdStorage ? _coldStoragePath : _hotStoragePath, hash);
+
+        // if cold storage is not configured or file not found or error is present try to download file from remote
+        var downloadUrl = MareFiles.DistributionGetFullPath(_remoteCacheSourceUri, hash);
+        _logger.LogInformation("Did not find {hash}, downloading from {server}", hash, downloadUrl);
+
+        using var requestMessage = new HttpRequestMessage(HttpMethod.Get, downloadUrl);
+        requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _generator.Token);
+        if (_configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.DistributionFileServerForceHTTP2), false))
+        {
+            requestMessage.Version = new Version(2, 0);
+            requestMessage.VersionPolicy = HttpVersionPolicy.RequestVersionExact;
+        }
+        HttpResponseMessage? response = null;
+
+        try
+        {
+            response = await _httpClient.SendAsync(requestMessage).ConfigureAwait(false);
+            response.EnsureSuccessStatusCode();
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "Failed to download {url}", downloadUrl);
+            response?.Dispose();
+            return;
+        }
+
+        var tempFileName = destinationFilePath + ".dl";
+        var fileStream = new FileStream(tempFileName, FileMode.Create, FileAccess.ReadWrite);
+        var bufferSize = 4096;
+        var buffer = new byte[bufferSize];
+
+        var bytesRead = 0;
+        using var content = await response.Content.ReadAsStreamAsync().ConfigureAwait(false);
+        while ((bytesRead = await content.ReadAsync(buffer).ConfigureAwait(false)) > 0)
+        {
+            await fileStream.WriteAsync(buffer.AsMemory(0, bytesRead)).ConfigureAwait(false);
+        }
+        await fileStream.FlushAsync().ConfigureAwait(false);
+        await fileStream.DisposeAsync().ConfigureAwait(false);
+        File.Move(tempFileName, destinationFilePath, true);
+
+        _metrics.IncGauge(_useColdStorage ? MetricsAPI.GaugeFilesTotalColdStorage : MetricsAPI.GaugeFilesTotal);
+        _metrics.IncGauge(_useColdStorage ? MetricsAPI.GaugeFilesTotalSizeColdStorage : MetricsAPI.GaugeFilesTotalSize, new FileInfo(destinationFilePath).Length);
+        response.Dispose();
+    }
+
+    private bool TryCopyFromColdStorage(string hash, string destinationFilePath)
+    {
+        if (!_useColdStorage) return false;
+
+        if (string.IsNullOrEmpty(_coldStoragePath)) return false;
+
+        var coldStorageFilePath = FilePathUtil.GetFilePath(_coldStoragePath, hash);
+        if (!File.Exists(coldStorageFilePath)) return false;
+
+        try
+        {
+            _logger.LogDebug("Copying {hash} from cold storage: {path}", hash, coldStorageFilePath);
+            var tempFileName = destinationFilePath + ".dl";
+            File.Copy(coldStorageFilePath, tempFileName, true);
+            File.Move(tempFileName, destinationFilePath, true);
+            var destinationFile = new FileInfo(destinationFilePath);
+            _metrics.IncGauge(MetricsAPI.GaugeFilesTotal);
+            _metrics.IncGauge(MetricsAPI.GaugeFilesTotalSize, new FileInfo(destinationFilePath).Length);
+            return true;
+        }
+        catch (Exception ex)
+        {
+            // Recover from a fairly common race condition -- max wait time is 75ms
+            // Having TryCopyFromColdStorage protected by the downloadtask mutex doesn't work for some reason?
+            for (int retry = 0; retry < 5; ++retry)
+            {
+                Thread.Sleep(5 + retry * 5);
+                if (File.Exists(destinationFilePath))
+                    return true;
+            }
+            _logger.LogWarning(ex, "Could not copy {coldStoragePath} from cold storage", coldStorageFilePath);
+        }
+
+        return false;
+    }
+
+    // Returns FileInfo ONLY if the hot file was immediately available without downloading
+    // Since the intended use is for pre-fetching files from hot storage, this is exactly what we need anyway
+    public async Task<FileInfo?> DownloadFileWhenRequired(string hash)
+    {
+        var fi = FilePathUtil.GetFileInfoForHash(_hotStoragePath, hash);
+
+        if (fi != null && fi.Length != 0)
+            return fi;
+
+        // first check cold storage
+        if (TryCopyFromColdStorage(hash, FilePathUtil.GetFilePath(_hotStoragePath, hash)))
+            return null;
+
+        // no distribution server configured to download from
+        if (_remoteCacheSourceUri == null)
+            return null;
+
+        await _downloadSemaphore.WaitAsync().ConfigureAwait(false);
+        if (!_currentTransfers.TryGetValue(hash, out var downloadTask) || (downloadTask?.IsCompleted ?? true))
+        {
+            _currentTransfers[hash] = Task.Run(async () =>
+            {
+                try
+                {
+                    _metrics.IncGauge(MetricsAPI.GaugeFilesDownloadingFromCache);
+                    await DownloadTask(hash).ConfigureAwait(false);
+                    TryCopyFromColdStorage(hash, FilePathUtil.GetFilePath(_hotStoragePath, hash));
+                }
+                catch (Exception ex)
+                {
+                    _logger.LogWarning(ex, "Error during Download Task for {hash}", hash);
+                }
+                finally
+                {
+                    _metrics.DecGauge(MetricsAPI.GaugeFilesDownloadingFromCache);
+                    _currentTransfers.Remove(hash, out _);
+                }
+            });
+        }
+        _downloadSemaphore.Release();
+
+        return null;
+    }
+
+    public async Task<FileInfo?> GetAndDownloadFile(string hash)
+    {
+        var fi = await DownloadFileWhenRequired(hash).ConfigureAwait(false);
+
+        if (fi == null && _currentTransfers.TryGetValue(hash, out var downloadTask))
+        {
+            try
+            {
+                using CancellationTokenSource cts = new();
+                cts.CancelAfter(TimeSpan.FromSeconds(120));
+                _metrics.IncGauge(MetricsAPI.GaugeFilesTasksWaitingForDownloadFromCache);
+                await downloadTask.WaitAsync(cts.Token).ConfigureAwait(false);
+            }
+            catch (Exception e)
+            {
+                _logger.LogWarning(e, "Failed while waiting for download task for {hash}", hash);
+                return null;
+            }
+            finally
+            {
+                _metrics.DecGauge(MetricsAPI.GaugeFilesTasksWaitingForDownloadFromCache);
+            }
+        }
+
+        fi ??= FilePathUtil.GetFileInfoForHash(_hotStoragePath, hash);
+
+        if (fi == null)
+            return null;
+
+        fi.LastAccessTimeUtc = DateTime.UtcNow;
+        _touchService.TouchColdHash(hash);
+
+        _fileStatisticsService.LogFile(hash, fi.Length);
+
+        return fi;
+    }
+
+    public async Task<FileStream?> GetAndDownloadFileStream(string hash)
+    {
+        var fi = await GetAndDownloadFile(hash).ConfigureAwait(false);
+        return new FileStream(fi.FullName, FileMode.Open, FileAccess.Read, FileShare.Inheritable | FileShare.Read);
+    }
+
+    public void TouchColdHash(string hash)
+    {
+        _touchService.TouchColdHash(hash);
+    }
+
+    public bool AnyFilesDownloading(List<string> hashes)
+    {
+        return hashes.Exists(_currentTransfers.Keys.Contains);
+    }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/Services/ColdTouchHashService.cs

@@ -0,0 +1,82 @@
+using MareSynchronosShared.Services;
+using MareSynchronosShared.Utils.Configuration;
+using MareSynchronosStaticFilesServer.Utils;
+using System.Collections.Concurrent;
+
+namespace MareSynchronosStaticFilesServer.Services;
+
+// Perform access time updates for cold cache files accessed via hot cache or shard servers
+public class ColdTouchHashService : ITouchHashService
+{
+    private readonly ILogger<ColdTouchHashService> _logger;
+    private readonly IConfigurationService<StaticFilesServerConfiguration> _configuration;
+
+    private readonly bool _useColdStorage;
+    private readonly string _coldStoragePath;
+
+    // Debounce multiple updates towards the same file
+    private readonly ConcurrentDictionary<string, DateTime> _lastUpdateTimesUtc = new(StringComparer.Ordinal);
+    private int _cleanupCounter = 0;
+    private object _cleanupLockObj = new();
+    private const double _debounceTimeSecs = 900.0;
+
+    public ColdTouchHashService(ILogger<ColdTouchHashService> logger, IConfigurationService<StaticFilesServerConfiguration> configuration)
+    {
+        _logger = logger;
+        _configuration = configuration;
+        _useColdStorage = configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false);
+        _coldStoragePath = configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.ColdStorageDirectory));
+    }
+
+    public Task StartAsync(CancellationToken cancellationToken)
+    {
+        return Task.CompletedTask;
+    }
+
+    public Task StopAsync(CancellationToken cancellationToken)
+    {
+        return Task.CompletedTask;
+    }
+
+    public void TouchColdHash(string hash)
+    {
+        if (!_useColdStorage)
+            return;
+
+        var nowUtc = DateTime.UtcNow;
+
+        // Clean up debounce dictionary regularly
+        if (_cleanupCounter++ >= 1000)
+        {
+            _cleanupCounter = 0;
+            if (Monitor.TryEnter(_cleanupLockObj))
+            {
+                try
+                {
+                    foreach (var entry in _lastUpdateTimesUtc.Where(entry => (nowUtc - entry.Value).TotalSeconds >= _debounceTimeSecs).ToList())
+                        _lastUpdateTimesUtc.TryRemove(entry.Key, out _);
+                }
+                finally
+                {
+                    Monitor.Exit(_cleanupLockObj);
+                }
+            }
+        }
+
+        // Ignore multiple updates within a time window of the first
+        if (_lastUpdateTimesUtc.TryGetValue(hash, out var lastUpdateTimeUtc) && (nowUtc - lastUpdateTimeUtc).TotalSeconds < _debounceTimeSecs)
+            return;
+
+        var fileInfo = FilePathUtil.GetFileInfoForHash(_coldStoragePath, hash);
+        if (fileInfo != null)
+        {
+            _logger.LogTrace("Touching {fileName}", fileInfo.Name);
+            try
+            {
+                fileInfo.LastAccessTimeUtc = nowUtc;
+            }
+            catch (IOException) { return; }
+            _lastUpdateTimesUtc.TryAdd(hash, nowUtc);
+        }
+    }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/Services/FileCleanupService.cs

@@ -0,0 +1,359 @@
+using ByteSizeLib;
+using MareSynchronos.API.Dto.Files;
+using MareSynchronosShared.Data;
+using MareSynchronosShared.Metrics;
+using MareSynchronosShared.Models;
+using MareSynchronosShared.Services;
+using MareSynchronosShared.Utils.Configuration;
+using MareSynchronosStaticFilesServer.Utils;
+using MessagePack.Formatters;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Hosting.Systemd;
+
+namespace MareSynchronosStaticFilesServer.Services;
+
+public class FileCleanupService : IHostedService
+{
+    private readonly IConfigurationService<StaticFilesServerConfiguration> _configuration;
+    private readonly ILogger<FileCleanupService> _logger;
+    private readonly MareMetrics _metrics;
+    private readonly IServiceProvider _services;
+
+    private readonly string _hotStoragePath;
+    private readonly string _coldStoragePath;
+    private readonly bool _isMain = false;
+    private readonly bool _isDistributionNode = false;
+    private readonly bool _useColdStorage = false;
+    private HashSet<string> _orphanedFiles = new(StringComparer.Ordinal);
+
+    private CancellationTokenSource _cleanupCts;
+
+    private int HotStorageMinimumRetention => _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.MinimumFileRetentionPeriodInDays), 7);
+    private int HotStorageRetention => _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UnusedFileRetentionPeriodInDays), 14);
+    private double HotStorageSize => _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CacheSizeHardLimitInGiB), -1.0);
+
+    private int ColdStorageMinimumRetention => _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ColdStorageMinimumFileRetentionPeriodInDays), 60);
+    private int ColdStorageRetention => _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ColdStorageUnusedFileRetentionPeriodInDays), 60);
+    private double ColdStorageSize => _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ColdStorageSizeHardLimitInGiB), -1.0);
+
+    private double SmallSizeKiB => _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CacheSmallSizeThresholdKiB), 64.0);
+    private double LargeSizeKiB => _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CacheLargeSizeThresholdKiB), 1024.0);
+
+    private int ForcedDeletionAfterHours => _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ForcedDeletionOfFilesAfterHours), -1);
+    private int CleanupCheckMinutes => _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CleanupCheckInMinutes), 15);
+
+    private List<FileInfo> GetAllHotFiles() => new DirectoryInfo(_hotStoragePath).GetFiles("*", SearchOption.AllDirectories)
+                .Where(f => f != null && f.Name.Length == 40)
+                .OrderBy(f => f.LastAccessTimeUtc).ToList();
+
+    private List<FileInfo> GetAllColdFiles() => new DirectoryInfo(_coldStoragePath).GetFiles("*", SearchOption.AllDirectories)
+                .Where(f => f != null && f.Name.Length == 40)
+                .OrderBy(f => f.LastAccessTimeUtc).ToList();
+
+    private List<FileInfo> GetTempFiles() => new DirectoryInfo(_useColdStorage ? _coldStoragePath : _hotStoragePath).GetFiles("*", SearchOption.AllDirectories)
+                .Where(f => f != null && (f.Name.EndsWith(".dl", StringComparison.InvariantCultureIgnoreCase) || f.Name.EndsWith(".tmp", StringComparison.InvariantCultureIgnoreCase))).ToList();
+
+    public FileCleanupService(MareMetrics metrics, ILogger<FileCleanupService> logger,
+        IServiceProvider services, IConfigurationService<StaticFilesServerConfiguration> configuration)
+    {
+        _metrics = metrics;
+        _logger = logger;
+        _services = services;
+        _configuration = configuration;
+        _useColdStorage = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false);
+        _hotStoragePath = configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
+        _coldStoragePath = configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.ColdStorageDirectory));
+        _isDistributionNode = configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.IsDistributionNode), false);
+        _isMain = configuration.GetValue<Uri>(nameof(StaticFilesServerConfiguration.MainFileServerAddress)) == null && _isDistributionNode;
+    }
+
+    public Task StartAsync(CancellationToken cancellationToken)
+    {
+        _logger.LogInformation("Cleanup Service started");
+
+        InitializeGauges();
+
+        _cleanupCts = new();
+
+        _ = CleanUpTask(_cleanupCts.Token);
+
+        return Task.CompletedTask;
+    }
+
+    public Task StopAsync(CancellationToken cancellationToken)
+    {
+        _cleanupCts.Cancel();
+
+        return Task.CompletedTask;
+    }
+
+    private List<string> CleanUpFilesBeyondSizeLimit(List<FileInfo> files, double sizeLimit, double minTTL, double maxTTL, CancellationToken ct)
+    {
+        var removedFiles = new List<string>();
+        if (sizeLimit <= 0)
+        {
+            return removedFiles;
+        }
+
+        var smallSize = SmallSizeKiB * 1024.0;
+        var largeSize = LargeSizeKiB * 1024.0;
+        var now = DateTime.Now;
+
+        // Avoid nonsense in future calculations
+        if (smallSize < 0.0)
+            smallSize = 0.0;
+
+        if (largeSize < smallSize)
+            largeSize = smallSize;
+
+        if (minTTL < 0.0)
+            minTTL = 0.0;
+
+        if (maxTTL < minTTL)
+            maxTTL = minTTL;
+
+        // Calculates a deletion priority to prioritize deletion of larger files over a configured TTL range based on a file's size.
+        // This is intended to be applied to the hot cache, as the cost of recovering many small files is greater than a single large file.
+        // Example (minTTL=7, maxTTL=30):
+        //  - A 10MB file was last accessed 5 days ago. Its calculated optimum TTL is 7 days. result = 0.7143
+        //  - A 50kB file was last accessed 10 days ago. Its calculated optimum TTL is 30 days. result = 0.3333
+        //  The larger file will be deleted with a higher priority than the smaller file.
+        double CalculateTTLProgression(FileInfo file)
+        {
+            var fileLength = (double)file.Length;
+            var fileAgeDays = (now - file.LastAccessTime).TotalDays;
+            var sizeNorm = Math.Clamp((fileLength - smallSize) / (largeSize - smallSize), 0.0, 1.0);
+            // Using Math.Sqrt(sizeNorm) would create a more logical scaling curve, but it barely matters
+            var ttlDayRange = (maxTTL - minTTL) * (1.0 - sizeNorm);
+            var daysPastMinTTL = Math.Max(fileAgeDays - minTTL, 0.0);
+            // There is some creativity in choosing an upper bound here:
+            //  - With no upper bound, any file larger than `largeSize` is always the highest priority for deletion once it passes its calculated TTL
+            //  - With 1.0 as an upper bound, all files older than `maxTTL` will have the same priority regardless of size
+            //  - Using maxTTL/minTTL chooses a logical cut-off point where any files old enough to be affected would have been cleaned up already
+            var ttlProg = Math.Clamp(daysPastMinTTL / ttlDayRange, 0.0, maxTTL / minTTL);
+            return ttlProg;
+        }
+
+        try
+        {
+            // Since we already have the file list sorted by access time, the list index is incorporated in to
+            // the dictionary key to preserve it as a secondary ordering
+            var sortedFiles = new PriorityQueue<FileInfo, (double, int)>();
+
+            foreach (var (file, i) in files.Select((file, i) => ( file, i )))
+            {
+                double ttlProg = CalculateTTLProgression(file);
+                sortedFiles.Enqueue(file, (-ttlProg, i));
+            }
+
+            _logger.LogInformation("Cleaning up files beyond the cache size limit of {cacheSizeLimit} GiB", sizeLimit);
+            var totalCacheSizeInBytes = files.Sum(s => s.Length);
+            long cacheSizeLimitInBytes = (long)ByteSize.FromGibiBytes(sizeLimit).Bytes;
+            while (totalCacheSizeInBytes > cacheSizeLimitInBytes && sortedFiles.Count != 0 && !ct.IsCancellationRequested)
+            {
+                var file = sortedFiles.Dequeue();
+                totalCacheSizeInBytes -= file.Length;
+                _logger.LogInformation("Deleting {file} with size {size:N2}MiB", file.FullName, ByteSize.FromBytes(file.Length).MebiBytes);
+                file.Delete();
+                removedFiles.Add(file.Name);
+            }
+            files.RemoveAll(f => removedFiles.Contains(f.Name, StringComparer.InvariantCultureIgnoreCase));
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "Error during cache size limit cleanup");
+        }
+
+        return removedFiles;
+    }
+
+    private void CleanUpOrphanedFiles(HashSet<string> allDbFileHashes, List<FileInfo> allPhysicalFiles, CancellationToken ct)
+    {
+        // To avoid race conditions with file uploads, only delete files on a second pass
+        var newOrphanedFiles = new HashSet<string>(StringComparer.Ordinal);
+
+        foreach (var file in allPhysicalFiles.ToList())
+        {
+            if (!allDbFileHashes.Contains(file.Name.ToUpperInvariant()))
+            {
+                _logger.LogInformation("File not in DB, marking: {fileName}", file.Name);
+                newOrphanedFiles.Add(file.FullName);
+            }
+
+            ct.ThrowIfCancellationRequested();
+        }
+
+        foreach (var fullName in _orphanedFiles.Where(f => newOrphanedFiles.Contains(f)))
+        {
+            var name = Path.GetFileName(fullName);
+            File.Delete(fullName);
+            _logger.LogInformation("File still not in DB, deleting: {fileName}", name);
+            allPhysicalFiles.RemoveAll(f => f.FullName.Equals(fullName, StringComparison.InvariantCultureIgnoreCase));
+        }
+
+        _orphanedFiles = newOrphanedFiles;
+    }
+
+    private List<string> CleanUpOutdatedFiles(List<FileInfo> files, int unusedRetention, int forcedDeletionAfterHours, CancellationToken ct)
+    {
+        var removedFiles = new List<string>();
+        try
+        {
+            _logger.LogInformation("Cleaning up files older than {filesOlderThanDays} days", unusedRetention);
+            if (forcedDeletionAfterHours > 0)
+            {
+                _logger.LogInformation("Cleaning up files written to longer than {hours}h ago", forcedDeletionAfterHours);
+            }
+
+            var lastAccessCutoffTime = DateTime.Now.Subtract(TimeSpan.FromDays(unusedRetention));
+            var forcedDeletionCutoffTime = DateTime.Now.Subtract(TimeSpan.FromHours(forcedDeletionAfterHours));
+
+            foreach (var file in files)
+            {
+                if (file.LastAccessTime < lastAccessCutoffTime)
+                {
+                    _logger.LogInformation("File outdated: {fileName}, {fileSize:N2}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes);
+                    file.Delete();
+                    removedFiles.Add(file.Name);
+                }
+                else if (forcedDeletionAfterHours > 0 && file.LastWriteTime < forcedDeletionCutoffTime)
+                {
+                    _logger.LogInformation("File forcefully deleted: {fileName}, {fileSize:N2}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes);
+                    file.Delete();
+                    removedFiles.Add(file.Name);
+                }
+
+                ct.ThrowIfCancellationRequested();
+            }
+            files.RemoveAll(f => removedFiles.Contains(f.Name, StringComparer.InvariantCultureIgnoreCase));
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "Error during file cleanup of old files");
+        }
+
+        return removedFiles;
+    }
+
+    private void CleanUpTempFiles()
+    {
+        var pastTime = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(20));
+        var tempFiles = GetTempFiles();
+        foreach (var tempFile in tempFiles.Where(f => f.LastWriteTimeUtc < pastTime))
+            tempFile.Delete();
+    }
+
+    private async Task CleanUpTask(CancellationToken ct)
+    {
+        while (!ct.IsCancellationRequested)
+        {
+            try
+            {
+                using var scope = _services.CreateScope();
+                using var dbContext = _isMain ? scope.ServiceProvider.GetService<MareDbContext>()! : null;
+
+                HashSet<string> allDbFileHashes = null;
+
+                // Database operations only performed on main server
+                if (_isMain)
+                {
+                    var allDbFiles = await dbContext.Files.ToListAsync(ct).ConfigureAwait(false);
+                    allDbFileHashes = new HashSet<string>(allDbFiles.Select(a => a.Hash.ToUpperInvariant()), StringComparer.Ordinal);
+                }
+
+                if (_useColdStorage)
+                {
+                    var coldFiles = GetAllColdFiles();
+                    var removedColdFiles = new List<string>();
+
+                    removedColdFiles.AddRange(
+                        CleanUpOutdatedFiles(coldFiles, ColdStorageRetention, ForcedDeletionAfterHours, ct)
+                    );
+                    removedColdFiles.AddRange(
+                        CleanUpFilesBeyondSizeLimit(coldFiles, ColdStorageSize, ColdStorageMinimumRetention, ColdStorageRetention, ct)
+                    );
+
+                    // Remove cold storage files are deleted from the database, if we are the main file server
+                    if (_isMain)
+                    {
+                        dbContext.Files.RemoveRange(
+                            dbContext.Files.Where(f => removedColdFiles.Contains(f.Hash))
+                        );
+                        allDbFileHashes.ExceptWith(removedColdFiles);
+                        CleanUpOrphanedFiles(allDbFileHashes, coldFiles, ct);
+                    }
+
+                    // Remove hot copies of files now that the authoritative copy is gone
+                    foreach (var removedFile in removedColdFiles)
+                    {
+                        var hotFile = FilePathUtil.GetFileInfoForHash(_hotStoragePath, removedFile);
+                        hotFile?.Delete();
+                    }
+
+                    _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSizeColdStorage, coldFiles.Sum(f => { try { return f.Length; } catch { return 0; } }));
+                    _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalColdStorage, coldFiles.Count);
+                }
+
+                var hotFiles = GetAllHotFiles();
+                var removedHotFiles = new List<string>();
+
+                removedHotFiles.AddRange(
+                    CleanUpOutdatedFiles(hotFiles, HotStorageRetention, forcedDeletionAfterHours: _useColdStorage ? ForcedDeletionAfterHours : -1, ct)
+                );
+                removedHotFiles.AddRange(
+                    CleanUpFilesBeyondSizeLimit(hotFiles, HotStorageSize, HotStorageMinimumRetention, HotStorageRetention, ct)
+                );
+
+                if (_isMain)
+                {
+                    // If cold storage is not active, then "hot" files are deleted from the database instead
+                    if (!_useColdStorage)
+                    {
+                        dbContext.Files.RemoveRange(
+                            dbContext.Files.Where(f => removedHotFiles.Contains(f.Hash))
+                        );
+                        allDbFileHashes.ExceptWith(removedHotFiles);
+                    }
+
+                    CleanUpOrphanedFiles(allDbFileHashes, hotFiles, ct);
+
+                    await dbContext.SaveChangesAsync(ct).ConfigureAwait(false);
+                }
+
+                _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSize, hotFiles.Sum(f => { try { return f.Length; } catch { return 0; } }));
+                _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotal, hotFiles.Count);
+
+                CleanUpTempFiles();
+            }
+            catch (Exception e)
+            {
+                _logger.LogError(e, "Error during cleanup task");
+            }
+
+            var cleanupCheckMinutes = CleanupCheckMinutes;
+            var now = DateTime.Now;
+            TimeOnly currentTime = new(now.Hour, now.Minute, now.Second);
+            TimeOnly futureTime = new(now.Hour, now.Minute - now.Minute % cleanupCheckMinutes, 0);
+            var span = futureTime.AddMinutes(cleanupCheckMinutes) - currentTime;
+
+            _logger.LogInformation("File Cleanup Complete, next run at {date}", now.Add(span));
+            await Task.Delay(span, ct).ConfigureAwait(false);
+        }
+    }
+
+    private void InitializeGauges()
+    {
+        if (_useColdStorage)
+        {
+            var allFilesInColdStorageDir = GetAllColdFiles();
+
+            _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSizeColdStorage, allFilesInColdStorageDir.Sum(f => f.Length));
+            _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalColdStorage, allFilesInColdStorageDir.Count);
+        }
+
+        var allFilesInHotStorage = GetAllHotFiles();
+
+        _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSize, allFilesInHotStorage.Sum(f => { try { return f.Length; } catch { return 0; } }));
+        _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotal, allFilesInHotStorage.Count);
+    }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/Services/FilePreFetchService.cs

@@ -0,0 +1,125 @@
+using Microsoft.Win32.SafeHandles;
+using System.Runtime.InteropServices;
+using System.Threading.Channels;
+
+namespace MareSynchronosStaticFilesServer.Services;
+
+// Pre-fetch files from cache storage in to memory
+public class FilePreFetchService : IHostedService
+{
+    private struct PreFetchRequest
+    {
+        public FileInfo FileInfo;
+        public DateTime ExpiryUtc;
+    }
+
+    private readonly ILogger<FilePreFetchService> _logger;
+
+    private CancellationTokenSource _prefetchCts;
+    private readonly Channel<PreFetchRequest> _prefetchChannel;
+
+    private const int _readAheadBytes = 8 * 1024 * 1024; // Maximum number of of bytes to prefetch per file (8MB)
+    private const int _preFetchTasks = 4; // Maximum number of tasks to process prefetches concurrently
+
+    // Use readahead() on linux if its available
+    [DllImport("libc", EntryPoint = "readahead")]
+    static extern int LinuxReadAheadExternal(SafeFileHandle fd, Int64 offset, int count);
+
+    private bool _hasLinuxReadAhead = true;
+
+    public FilePreFetchService(ILogger<FilePreFetchService> logger)
+    {
+        _logger = logger;
+        _prefetchChannel = Channel.CreateUnbounded<PreFetchRequest>();
+    }
+
+    public Task StartAsync(CancellationToken cancellationToken)
+    {
+        _logger.LogInformation("File PreFetch Service started");
+        _prefetchCts = new();
+        for (int i = 0; i < _preFetchTasks; ++i)
+            _ = PrefetchTask(_prefetchCts.Token);
+        return Task.CompletedTask;
+    }
+
+    public Task StopAsync(CancellationToken cancellationToken)
+    {
+        _prefetchCts.Cancel();
+        return Task.CompletedTask;
+    }
+
+    // Queue a list of hashes to be prefetched in a background task
+    public void PrefetchFiles(ICollection<FileInfo> fileList)
+    {
+        if (!_hasLinuxReadAhead)
+        {
+            if (!_prefetchCts.IsCancellationRequested)
+            {
+                _logger.LogError("readahead() is not available - aborting File PreFetch Service");
+                _prefetchCts.Cancel();
+            }
+            return;
+        }
+
+        var nowUtc = DateTime.UtcNow;
+
+        // Expire prefetch requests that aren't picked up within 500ms
+        // By this point the request is probably already being served, or things are moving too slow to matter anyway
+        var expiry = nowUtc + TimeSpan.FromMilliseconds(500);
+
+        foreach (var fileInfo in fileList)
+        {
+            _ = _prefetchChannel.Writer.TryWrite(new PreFetchRequest(){
+                FileInfo = fileInfo,
+                ExpiryUtc = expiry,
+            });
+        }
+    }
+
+    private async Task PrefetchTask(CancellationToken ct)
+    {
+        var reader = _prefetchChannel.Reader;
+
+        while (!ct.IsCancellationRequested)
+        {
+            try
+            {
+                var req = await reader.ReadAsync(ct).ConfigureAwait(false);
+                var nowUtc = DateTime.UtcNow;
+
+                if (nowUtc >= req.ExpiryUtc)
+                {
+                    _logger.LogDebug("Skipped expired prefetch for {hash}", req.FileInfo.Name);
+                    continue;
+                }
+
+                try
+                {
+                    var fs = new FileStream(req.FileInfo.FullName, FileMode.Open, FileAccess.Read, FileShare.Inheritable | FileShare.Read);
+
+                    await using (fs.ConfigureAwait(false))
+                    {
+                        try
+                        {
+                            _ = LinuxReadAheadExternal(fs.SafeFileHandle, 0, _readAheadBytes);
+                            _logger.LogTrace("Prefetched {hash}", req.FileInfo.Name);
+                        }
+                        catch (EntryPointNotFoundException)
+                        {
+                            _hasLinuxReadAhead = false;
+                        }
+                    }
+                }
+                catch (IOException) { }
+            }
+            catch (OperationCanceledException)
+            {
+                continue;
+            }
+            catch (Exception e)
+            {
+                _logger.LogError(e, "Error during prefetch task");
+            }
+        }
+    }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/Services/FileStatisticsService.cs

@@ -0,0 +1,94 @@
+using MareSynchronosShared.Metrics;
+using System.Collections.Concurrent;
+
+namespace MareSynchronosStaticFilesServer.Services;
+
+public class FileStatisticsService : IHostedService
+{
+    private readonly MareMetrics _metrics;
+    private readonly ILogger<FileStatisticsService> _logger;
+    private CancellationTokenSource _resetCancellationTokenSource;
+    private ConcurrentDictionary<string, long> _pastHourFiles = new(StringComparer.Ordinal);
+    private ConcurrentDictionary<string, long> _pastDayFiles = new(StringComparer.Ordinal);
+
+    public FileStatisticsService(MareMetrics metrics, ILogger<FileStatisticsService> logger)
+    {
+        _metrics = metrics;
+        _logger = logger;
+    }
+
+    public void LogFile(string fileHash, long length)
+    {
+        if (!_pastHourFiles.ContainsKey(fileHash))
+        {
+            _pastHourFiles[fileHash] = length;
+            _metrics.IncGauge(MetricsAPI.GaugeFilesUniquePastHour);
+            _metrics.IncGauge(MetricsAPI.GaugeFilesUniquePastHourSize, length);
+        }
+        if (!_pastDayFiles.ContainsKey(fileHash))
+        {
+            _pastDayFiles[fileHash] = length;
+            _metrics.IncGauge(MetricsAPI.GaugeFilesUniquePastDay);
+            _metrics.IncGauge(MetricsAPI.GaugeFilesUniquePastDaySize, length);
+        }
+    }
+
+    public void LogRequest(long requestSize)
+    {
+        _metrics.IncCounter(MetricsAPI.CounterFileRequests, 1);
+        _metrics.IncCounter(MetricsAPI.CounterFileRequestSize, requestSize);
+    }
+
+    public Task StartAsync(CancellationToken cancellationToken)
+    {
+        _logger.LogInformation("Starting FileStatisticsService");
+        _resetCancellationTokenSource = new();
+        _ = ResetHourlyFileData();
+        _ = ResetDailyFileData();
+        return Task.CompletedTask;
+    }
+
+    public async Task ResetHourlyFileData()
+    {
+        while (!_resetCancellationTokenSource.Token.IsCancellationRequested)
+        {
+            _logger.LogInformation("Resetting 1h Data");
+
+            _pastHourFiles = new(StringComparer.Ordinal);
+            _metrics.SetGaugeTo(MetricsAPI.GaugeFilesUniquePastHour, 0);
+            _metrics.SetGaugeTo(MetricsAPI.GaugeFilesUniquePastHourSize, 0);
+
+            var now = DateTime.UtcNow;
+            TimeOnly currentTime = new(now.Hour, now.Minute, now.Second);
+            TimeOnly futureTime = new(now.Hour, 0, 0);
+            var span = futureTime.AddHours(1) - currentTime;
+
+            await Task.Delay(span, _resetCancellationTokenSource.Token).ConfigureAwait(false);
+        }
+    }
+
+    public async Task ResetDailyFileData()
+    {
+        while (!_resetCancellationTokenSource.Token.IsCancellationRequested)
+        {
+            _logger.LogInformation("Resetting 24h Data");
+
+            _pastDayFiles = new(StringComparer.Ordinal);
+            _metrics.SetGaugeTo(MetricsAPI.GaugeFilesUniquePastDay, 0);
+            _metrics.SetGaugeTo(MetricsAPI.GaugeFilesUniquePastDaySize, 0);
+
+            var now = DateTime.UtcNow;
+            DateTime midnight = new(new DateOnly(now.Date.Year, now.Date.Month, now.Date.Day), new(0, 0, 0));
+            var span = midnight.AddDays(1) - now;
+
+            await Task.Delay(span, _resetCancellationTokenSource.Token).ConfigureAwait(false);
+        }
+    }
+
+    public Task StopAsync(CancellationToken cancellationToken)
+    {
+        _resetCancellationTokenSource.Cancel();
+        _logger.LogInformation("Stopping FileStatisticsService");
+        return Task.CompletedTask;
+    }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/Services/IClientReadyMessageService.cs

@@ -0,0 +1,6 @@
+namespace MareSynchronosStaticFilesServer.Services;
+
+public interface IClientReadyMessageService
+{
+    void SendDownloadReady(string uid, Guid requestId);
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/Services/ITouchHashService.cs

@@ -0,0 +1,6 @@
+namespace MareSynchronosStaticFilesServer.Services;
+
+public interface ITouchHashService : IHostedService
+{
+    void TouchColdHash(string hash);
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/Services/MainClientReadyMessageService.cs

@@ -0,0 +1,26 @@
+using Microsoft.AspNetCore.SignalR;
+using MareSynchronos.API.SignalR;
+using MareSynchronosServer.Hubs;
+
+namespace MareSynchronosStaticFilesServer.Services;
+
+public class MainClientReadyMessageService : IClientReadyMessageService
+{
+    private readonly ILogger<MainClientReadyMessageService> _logger;
+    private readonly IHubContext<MareHub> _mareHub;
+
+    public MainClientReadyMessageService(ILogger<MainClientReadyMessageService> logger, IHubContext<MareHub> mareHub)
+    {
+        _logger = logger;
+        _mareHub = mareHub;
+    }
+
+    public void SendDownloadReady(string uid, Guid requestId)
+    {
+        _ = Task.Run(async () =>
+        {
+            _logger.LogDebug("Sending Client Ready for {uid}:{requestId} to SignalR", uid, requestId);
+            await _mareHub.Clients.User(uid).SendAsync(nameof(IMareHub.Client_DownloadReady), requestId).ConfigureAwait(false);
+        });
+    }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/Services/RequestQueueService.cs

@@ -0,0 +1,222 @@
+using MareSynchronosShared.Metrics;
+using MareSynchronosShared.Services;
+using MareSynchronosShared.Utils.Configuration;
+using MareSynchronosStaticFilesServer.Utils;
+using System.Collections.Concurrent;
+using System.Timers;
+
+namespace MareSynchronosStaticFilesServer.Services;
+
+public class RequestQueueService : IHostedService
+{
+    private readonly IClientReadyMessageService _clientReadyMessageService;
+    private readonly CachedFileProvider _cachedFileProvider;
+    private readonly ILogger<RequestQueueService> _logger;
+    private readonly MareMetrics _metrics;
+    private readonly ConcurrentQueue<UserRequest> _queue = new();
+    private readonly ConcurrentQueue<UserRequest> _priorityQueue = new();
+    private readonly int _queueExpirationSeconds;
+    private readonly SemaphoreSlim _queueProcessingSemaphore = new(1);
+    private readonly UserQueueEntry[] _userQueueRequests;
+    private int _queueLimitForReset;
+    private readonly int _queueReleaseSeconds;
+    private System.Timers.Timer _queueTimer;
+
+    public RequestQueueService(MareMetrics metrics, IConfigurationService<StaticFilesServerConfiguration> configurationService,
+        ILogger<RequestQueueService> logger, IClientReadyMessageService hubContext, CachedFileProvider cachedFileProvider)
+    {
+        _userQueueRequests = new UserQueueEntry[configurationService.GetValueOrDefault(nameof(StaticFilesServerConfiguration.DownloadQueueSize), 50)];
+        _queueExpirationSeconds = configurationService.GetValueOrDefault(nameof(StaticFilesServerConfiguration.DownloadTimeoutSeconds), 5);
+        _queueLimitForReset = configurationService.GetValueOrDefault(nameof(StaticFilesServerConfiguration.DownloadQueueClearLimit), 15000);
+        _queueReleaseSeconds = configurationService.GetValueOrDefault(nameof(StaticFilesServerConfiguration.DownloadQueueReleaseSeconds), 15);
+        _metrics = metrics;
+        _logger = logger;
+        _clientReadyMessageService = hubContext;
+        _cachedFileProvider = cachedFileProvider;
+    }
+
+    public void ActivateRequest(Guid request)
+    {
+        _logger.LogDebug("Activating request {guid}", request);
+        var req = _userQueueRequests.First(f => f != null && f.UserRequest.RequestId == request);
+        req.MarkActive();
+    }
+
+    public async Task EnqueueUser(UserRequest request, bool isPriority, CancellationToken token)
+    {
+        while (_queueProcessingSemaphore.CurrentCount == 0)
+        {
+            await Task.Delay(50, token).ConfigureAwait(false);
+        }
+
+        _logger.LogDebug("Enqueueing req {guid} from {user} for {file}", request.RequestId, request.User, string.Join(", ", request.FileIds));
+
+        GetQueue(isPriority).Enqueue(request);
+    }
+
+    public void FinishRequest(Guid request)
+    {
+        var req = _userQueueRequests.FirstOrDefault(f => f != null && f.UserRequest.RequestId == request);
+        if (req != null)
+        {
+            var idx = Array.IndexOf(_userQueueRequests, req);
+            _logger.LogDebug("Finishing Request {guid}, clearing slot {idx}", request, idx);
+            _userQueueRequests[idx] = null;
+        }
+        else
+        {
+            _logger.LogDebug("Request {guid} already cleared", request);
+        }
+    }
+
+    public bool IsActiveProcessing(Guid request, string user, out UserRequest userRequest)
+    {
+        var userQueueRequest = _userQueueRequests.FirstOrDefault(u => u != null && u.UserRequest.RequestId == request && string.Equals(u.UserRequest.User, user, StringComparison.Ordinal));
+        userRequest = userQueueRequest?.UserRequest ?? null;
+        return userQueueRequest != null && userRequest != null && userQueueRequest.ExpirationDate > DateTime.UtcNow;
+    }
+
+    public void RemoveFromQueue(Guid requestId, string user, bool isPriority)
+    {
+        var existingRequest = GetQueue(isPriority).FirstOrDefault(f => f.RequestId == requestId && string.Equals(f.User, user, StringComparison.Ordinal));
+        if (existingRequest == null)
+        {
+            var activeSlot = _userQueueRequests.FirstOrDefault(r => r != null && string.Equals(r.UserRequest.User, user, StringComparison.Ordinal) && r.UserRequest.RequestId == requestId);
+            if (activeSlot != null)
+            {
+                var idx = Array.IndexOf(_userQueueRequests, activeSlot);
+                if (idx >= 0)
+                {
+                    _userQueueRequests[idx] = null;
+                }
+            }
+        }
+        else
+        {
+            existingRequest.IsCancelled = true;
+        }
+    }
+
+    public Task StartAsync(CancellationToken cancellationToken)
+    {
+        _queueTimer = new System.Timers.Timer(500);
+        _queueTimer.Elapsed += ProcessQueue;
+        _queueTimer.AutoReset = true;
+        _queueTimer.Start();
+        return Task.CompletedTask;
+    }
+
+    private ConcurrentQueue<UserRequest> GetQueue(bool isPriority) => isPriority ? _priorityQueue : _queue;
+
+    public bool StillEnqueued(Guid request, string user, bool isPriority)
+    {
+        return GetQueue(isPriority).Any(c => c.RequestId == request && string.Equals(c.User, user, StringComparison.Ordinal));
+    }
+
+    public Task StopAsync(CancellationToken cancellationToken)
+    {
+        _queueTimer.Stop();
+        return Task.CompletedTask;
+    }
+
+    private void DequeueIntoSlot(UserRequest userRequest, int slot)
+    {
+        _logger.LogDebug("Dequeueing {req} into {i}: {user} with {file}", userRequest.RequestId, slot, userRequest.User, string.Join(", ", userRequest.FileIds));
+        _userQueueRequests[slot] = new(userRequest, DateTime.UtcNow.AddSeconds(_queueExpirationSeconds));
+        _clientReadyMessageService.SendDownloadReady(userRequest.User, userRequest.RequestId);
+    }
+
+    private void ProcessQueue(object src, ElapsedEventArgs e)
+    {
+        if (_queueProcessingSemaphore.CurrentCount == 0) return;
+
+        _queueProcessingSemaphore.Wait();
+
+        try
+        {
+            if (_queue.Count(c => !c.IsCancelled) > _queueLimitForReset)
+            {
+                _queue.Clear();
+                return;
+            }
+
+            for (int i = 0; i < _userQueueRequests.Length; i++)
+            {
+                try
+                {
+                    if (_userQueueRequests[i] != null
+                        && (((!_userQueueRequests[i].IsActive && _userQueueRequests[i].ExpirationDate < DateTime.UtcNow))
+                            || (_userQueueRequests[i].IsActive && _userQueueRequests[i].ActivationDate < DateTime.UtcNow.Subtract(TimeSpan.FromSeconds(_queueReleaseSeconds))))
+                            )
+                    {
+                        _logger.LogDebug("Expiring request {guid} slot {slot}", _userQueueRequests[i].UserRequest.RequestId, i);
+                        _userQueueRequests[i] = null;
+                    }
+
+                    if (_userQueueRequests[i] != null) continue;
+
+                    while (true)
+                    {
+                        if (!_priorityQueue.All(u => _cachedFileProvider.AnyFilesDownloading(u.FileIds))
+                            && _priorityQueue.TryDequeue(out var prioRequest))
+                        {
+                            if (prioRequest.IsCancelled)
+                            {
+                                continue;
+                            }
+
+                            if (_cachedFileProvider.AnyFilesDownloading(prioRequest.FileIds))
+                            {
+                                _priorityQueue.Enqueue(prioRequest);
+                                continue;
+                            }
+
+                            DequeueIntoSlot(prioRequest, i);
+                            break;
+                        }
+
+                        if (!_queue.All(u => _cachedFileProvider.AnyFilesDownloading(u.FileIds))
+                            && _queue.TryDequeue(out var request))
+                        {
+                            if (request.IsCancelled)
+                            {
+                                continue;
+                            }
+
+                            if (_cachedFileProvider.AnyFilesDownloading(request.FileIds))
+                            {
+                                _queue.Enqueue(request);
+                                continue;
+                            }
+
+                            DequeueIntoSlot(request, i);
+                            break;
+                        }
+
+                        break;
+                    }
+                }
+                catch (Exception ex)
+                {
+                    _logger.LogWarning(ex, "Error during inside queue processing");
+                }
+            }
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "Error during Queue processing");
+        }
+        finally
+        {
+            _queueProcessingSemaphore.Release();
+        }
+
+        _metrics.SetGaugeTo(MetricsAPI.GaugeQueueFree, _userQueueRequests.Count(c => c == null));
+        _metrics.SetGaugeTo(MetricsAPI.GaugeQueueActive, _userQueueRequests.Count(c => c != null && c.IsActive));
+        _metrics.SetGaugeTo(MetricsAPI.GaugeQueueInactive, _userQueueRequests.Count(c => c != null && !c.IsActive));
+        _metrics.SetGaugeTo(MetricsAPI.GaugeDownloadQueue, _queue.Count(q => !q.IsCancelled));
+        _metrics.SetGaugeTo(MetricsAPI.GaugeDownloadQueueCancelled, _queue.Count(q => q.IsCancelled));
+        _metrics.SetGaugeTo(MetricsAPI.GaugeDownloadPriorityQueue, _priorityQueue.Count(q => !q.IsCancelled));
+        _metrics.SetGaugeTo(MetricsAPI.GaugeDownloadPriorityQueueCancelled, _priorityQueue.Count(q => q.IsCancelled));
+    }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/Services/ShardClientReadyMessageService.cs

@@ -0,0 +1,51 @@
+using MareSynchronos.API.Routes;
+using MareSynchronosShared.Services;
+using MareSynchronosShared.Utils;
+using MareSynchronosShared.Utils.Configuration;
+using System.Net.Http.Headers;
+
+namespace MareSynchronosStaticFilesServer.Services;
+
+public class ShardClientReadyMessageService : IClientReadyMessageService
+{
+    private readonly ILogger<ShardClientReadyMessageService> _logger;
+    private readonly ServerTokenGenerator _tokenGenerator;
+    private readonly IConfigurationService<StaticFilesServerConfiguration> _configurationService;
+    private readonly HttpClient _httpClient;
+
+    public ShardClientReadyMessageService(ILogger<ShardClientReadyMessageService> logger, ServerTokenGenerator tokenGenerator, IConfigurationService<StaticFilesServerConfiguration> configurationService)
+    {
+        _logger = logger;
+        _tokenGenerator = tokenGenerator;
+        _configurationService = configurationService;
+        _httpClient = new();
+        _httpClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("MareSynchronosServer", "1.0.0.0"));
+    }
+
+    public void SendDownloadReady(string uid, Guid requestId)
+    {
+        _ = Task.Run(async () =>
+        {
+            var mainUrlConfigKey = _configurationService.GetValue<bool>(nameof(StaticFilesServerConfiguration.NotifyMainServerDirectly))
+                ? nameof(StaticFilesServerConfiguration.MainServerAddress)
+                : nameof(StaticFilesServerConfiguration.MainFileServerAddress);
+            var mainUrl = _configurationService.GetValue<Uri>(mainUrlConfigKey);
+            var path = MareFiles.MainSendReadyFullPath(mainUrl, uid, requestId);
+            using HttpRequestMessage msg = new()
+            {
+                RequestUri = path
+            };
+            msg.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _tokenGenerator.Token);
+
+            _logger.LogDebug("Sending Client Ready for {uid}:{requestId} to {path}", uid, requestId, path);
+            try
+            {
+                using var result = await _httpClient.SendAsync(msg).ConfigureAwait(false);
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "Failure to send for {uid}:{requestId}", uid, requestId);
+            }
+        });
+    }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/Services/ShardTouchMessageService.cs

@@ -0,0 +1,132 @@
+using MareSynchronos.API.Routes;
+using MareSynchronosShared.Services;
+using MareSynchronosShared.Utils;
+using MareSynchronosShared.Utils.Configuration;
+using System.Net.Http.Headers;
+
+namespace MareSynchronosStaticFilesServer.Services;
+
+// Notify distribution server of file hashes downloaded via shards, so they are not prematurely purged from its cold cache
+public class ShardTouchMessageService : ITouchHashService
+{
+    private readonly ILogger<ShardTouchMessageService> _logger;
+    private readonly ServerTokenGenerator _tokenGenerator;
+    private readonly IConfigurationService<StaticFilesServerConfiguration> _configuration;
+    private readonly HttpClient _httpClient;
+    private readonly Uri _remoteCacheSourceUri;
+    private readonly HashSet<string> _touchHashSet = new();
+    private readonly ColdTouchHashService _nestedService = null;
+
+    private CancellationTokenSource _touchmsgCts;
+
+    public ShardTouchMessageService(ILogger<ShardTouchMessageService> logger, ILogger<ColdTouchHashService> nestedLogger,
+        ServerTokenGenerator tokenGenerator, IConfigurationService<StaticFilesServerConfiguration> configuration)
+    {
+        _logger = logger;
+        _tokenGenerator = tokenGenerator;
+        _configuration = configuration;
+        _remoteCacheSourceUri = _configuration.GetValueOrDefault<Uri>(nameof(StaticFilesServerConfiguration.DistributionFileServerAddress), null);
+        _httpClient = new();
+        _httpClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("MareSynchronosServer", "1.0.0.0"));
+
+        if (configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false))
+        {
+            _nestedService = new ColdTouchHashService(nestedLogger, configuration);
+        }
+    }
+
+    public Task StartAsync(CancellationToken cancellationToken)
+    {
+        if (_remoteCacheSourceUri == null)
+            return Task.CompletedTask;
+
+        _logger.LogInformation("Touch Message Service started");
+
+        _touchmsgCts = new();
+
+        _ = TouchMessageTask(_touchmsgCts.Token);
+
+        return Task.CompletedTask;
+    }
+
+    public Task StopAsync(CancellationToken cancellationToken)
+    {
+        if (_remoteCacheSourceUri == null)
+            return Task.CompletedTask;
+
+        _touchmsgCts.Cancel();
+
+        return Task.CompletedTask;
+    }
+
+    private async Task SendTouches(IEnumerable<string> hashes)
+    {
+        var mainUrl = _remoteCacheSourceUri;
+        var path = new Uri(mainUrl, MareFiles.Distribution + "/touch");
+        using HttpRequestMessage msg = new()
+        {
+            RequestUri = path
+        };
+        msg.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _tokenGenerator.Token);
+        msg.Method = HttpMethod.Post;
+        msg.Content = JsonContent.Create(hashes);
+        if (_configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.DistributionFileServerForceHTTP2), false))
+        {
+            msg.Version = new Version(2, 0);
+            msg.VersionPolicy = HttpVersionPolicy.RequestVersionExact;
+        }
+
+        _logger.LogDebug("Sending remote touch to {path}", path);
+        try
+        {
+            using var result = await _httpClient.SendAsync(msg).ConfigureAwait(false);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "Failure to send touches for {hashChunk}", hashes);
+        }
+    }
+
+    private async Task TouchMessageTask(CancellationToken ct)
+    {
+        List<string> hashes;
+
+        while (!ct.IsCancellationRequested)
+        {
+            try
+            {
+                lock (_touchHashSet)
+                {
+                    hashes = _touchHashSet.ToList();
+                    _touchHashSet.Clear();
+                }
+                if (hashes.Count > 0)
+                    await SendTouches(hashes).ConfigureAwait(false);
+                await Task.Delay(TimeSpan.FromSeconds(60), ct).ConfigureAwait(false);
+            }
+            catch (Exception e)
+            {
+                _logger.LogError(e, "Error during touch message task");
+            }
+        }
+
+        lock (_touchHashSet)
+        {
+            hashes = _touchHashSet.ToList();
+            _touchHashSet.Clear();
+        }
+        if (hashes.Count > 0)
+            await SendTouches(hashes).ConfigureAwait(false);
+    }
+
+    public void TouchColdHash(string hash)
+    {
+        if (_nestedService != null)
+            _nestedService.TouchColdHash(hash);
+
+        lock (_touchHashSet)
+        {
+            _touchHashSet.Add(hash);
+        }
+    }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs

@@ -0,0 +1,271 @@
+using MareSynchronosShared.Data;
+using MareSynchronosShared.Metrics;
+using MareSynchronosShared.Services;
+using MareSynchronosShared.Utils;
+using MareSynchronosStaticFilesServer.Controllers;
+using MareSynchronosStaticFilesServer.Services;
+using MareSynchronosStaticFilesServer.Utils;
+using MessagePack;
+using MessagePack.Resolvers;
+using Microsoft.AspNetCore.Authentication.JwtBearer;
+using Microsoft.AspNetCore.Authorization;
+using Microsoft.AspNetCore.Mvc.Controllers;
+using Microsoft.AspNetCore.Server.Kestrel.Core;
+using Microsoft.AspNetCore.SignalR;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.IdentityModel.Tokens;
+using Prometheus;
+using StackExchange.Redis.Extensions.Core.Configuration;
+using StackExchange.Redis.Extensions.System.Text.Json;
+using StackExchange.Redis;
+using System.Net;
+using System.Text;
+using MareSynchronosShared.Utils.Configuration;
+
+namespace MareSynchronosStaticFilesServer;
+
+public class Startup
+{
+    private bool _isMain;
+    private bool _isDistributionNode;
+    private bool _hasDistributionUpstream;
+    private readonly ILogger<Startup> _logger;
+
+    public Startup(IConfiguration configuration, ILogger<Startup> logger)
+    {
+        Configuration = configuration;
+        _logger = logger;
+        var mareSettings = Configuration.GetRequiredSection("MareSynchronos");
+        _isDistributionNode = mareSettings.GetValue(nameof(StaticFilesServerConfiguration.IsDistributionNode), false);
+        _hasDistributionUpstream = !string.IsNullOrEmpty(mareSettings.GetValue(nameof(StaticFilesServerConfiguration.DistributionFileServerAddress), string.Empty));
+        _isMain = string.IsNullOrEmpty(mareSettings.GetValue(nameof(StaticFilesServerConfiguration.MainFileServerAddress), string.Empty)) && _isDistributionNode;
+    }
+
+    public IConfiguration Configuration { get; }
+
+    public void ConfigureServices(IServiceCollection services)
+    {
+        services.AddHttpContextAccessor();
+
+        services.AddLogging();
+
+        services.Configure<StaticFilesServerConfiguration>(Configuration.GetRequiredSection("MareSynchronos"));
+        services.Configure<MareConfigurationBase>(Configuration.GetRequiredSection("MareSynchronos"));
+        services.Configure<KestrelServerOptions>(Configuration.GetSection("Kestrel"));
+        services.AddSingleton(Configuration);
+
+        var mareConfig = Configuration.GetRequiredSection("MareSynchronos");
+
+        // metrics configuration
+        services.AddSingleton(m => new MareMetrics(m.GetService<ILogger<MareMetrics>>(), new List<string>
+        {
+            MetricsAPI.CounterFileRequests,
+            MetricsAPI.CounterFileRequestSize
+        }, new List<string>
+        {
+            MetricsAPI.GaugeFilesTotalColdStorage,
+            MetricsAPI.GaugeFilesTotalSizeColdStorage,
+            MetricsAPI.GaugeFilesTotalSize,
+            MetricsAPI.GaugeFilesTotal,
+            MetricsAPI.GaugeFilesUniquePastDay,
+            MetricsAPI.GaugeFilesUniquePastDaySize,
+            MetricsAPI.GaugeFilesUniquePastHour,
+            MetricsAPI.GaugeFilesUniquePastHourSize,
+            MetricsAPI.GaugeCurrentDownloads,
+            MetricsAPI.GaugeDownloadQueue,
+            MetricsAPI.GaugeDownloadQueueCancelled,
+            MetricsAPI.GaugeDownloadPriorityQueue,
+            MetricsAPI.GaugeDownloadPriorityQueueCancelled,
+            MetricsAPI.GaugeQueueFree,
+            MetricsAPI.GaugeQueueInactive,
+            MetricsAPI.GaugeQueueActive,
+            MetricsAPI.GaugeFilesDownloadingFromCache,
+            MetricsAPI.GaugeFilesTasksWaitingForDownloadFromCache
+        }));
+
+        // generic services
+        services.AddSingleton<CachedFileProvider>();
+        services.AddHostedService<FileCleanupService>();
+        services.AddSingleton<FileStatisticsService>();
+        services.AddSingleton<RequestBlockFileListResultFactory>();
+        services.AddSingleton<ServerTokenGenerator>();
+        services.AddSingleton<RequestQueueService>();
+        services.AddHostedService(p => p.GetService<RequestQueueService>());
+        services.AddSingleton<FilePreFetchService>();
+        services.AddHostedService(p => p.GetService<FilePreFetchService>());
+        services.AddHostedService(m => m.GetService<FileStatisticsService>());
+        services.AddSingleton<IConfigurationService<MareConfigurationBase>, MareConfigurationServiceClient<MareConfigurationBase>>();
+        services.AddHostedService(p => (MareConfigurationServiceClient<MareConfigurationBase>)p.GetService<IConfigurationService<MareConfigurationBase>>());
+
+        // specific services
+        if (_isMain)
+        {
+            services.AddSingleton<IClientReadyMessageService, MainClientReadyMessageService>();
+            services.AddSingleton<IConfigurationService<StaticFilesServerConfiguration>, MareConfigurationServiceServer<StaticFilesServerConfiguration>>();
+            services.AddDbContextPool<MareDbContext>(options =>
+            {
+                options.UseNpgsql(Configuration.GetConnectionString("DefaultConnection"), builder =>
+                {
+                    builder.MigrationsHistoryTable("_efmigrationshistory", "public");
+                }).UseSnakeCaseNamingConvention();
+                options.EnableThreadSafetyChecks(false);
+            }, mareConfig.GetValue(nameof(MareConfigurationBase.DbContextPoolSize), 1024));
+
+            var signalRServiceBuilder = services.AddSignalR(hubOptions =>
+            {
+                hubOptions.MaximumReceiveMessageSize = long.MaxValue;
+                hubOptions.EnableDetailedErrors = true;
+                hubOptions.MaximumParallelInvocationsPerClient = 10;
+                hubOptions.StreamBufferCapacity = 200;
+            }).AddMessagePackProtocol(opt =>
+            {
+                var resolver = CompositeResolver.Create(StandardResolverAllowPrivate.Instance,
+                    BuiltinResolver.Instance,
+                    AttributeFormatterResolver.Instance,
+                    // replace enum resolver
+                    DynamicEnumAsStringResolver.Instance,
+                    DynamicGenericResolver.Instance,
+                    DynamicUnionResolver.Instance,
+                    DynamicObjectResolver.Instance,
+                    PrimitiveObjectResolver.Instance,
+                    // final fallback(last priority)
+                    StandardResolver.Instance);
+
+                opt.SerializerOptions = MessagePackSerializerOptions.Standard
+                    .WithCompression(MessagePackCompression.Lz4Block)
+                    .WithResolver(resolver);
+            });
+
+            // configure redis for SignalR
+            var redisConnection = mareConfig.GetValue(nameof(ServerConfiguration.RedisConnectionString), string.Empty);
+            signalRServiceBuilder.AddStackExchangeRedis(redisConnection, options => { });
+
+            var options = ConfigurationOptions.Parse(redisConnection);
+
+            var endpoint = options.EndPoints[0];
+            string address = "";
+            int port = 0;
+            if (endpoint is DnsEndPoint dnsEndPoint) { address = dnsEndPoint.Host; port = dnsEndPoint.Port; }
+            if (endpoint is IPEndPoint ipEndPoint) { address = ipEndPoint.Address.ToString(); port = ipEndPoint.Port; }
+            var redisConfiguration = new RedisConfiguration()
+            {
+                AbortOnConnectFail = true,
+                KeyPrefix = "",
+                Hosts = new RedisHost[]
+                {
+                new RedisHost(){ Host = address, Port = port },
+                },
+                AllowAdmin = true,
+                ConnectTimeout = options.ConnectTimeout,
+                Database = 0,
+                Ssl = false,
+                Password = options.Password,
+                ServerEnumerationStrategy = new ServerEnumerationStrategy()
+                {
+                    Mode = ServerEnumerationStrategy.ModeOptions.All,
+                    TargetRole = ServerEnumerationStrategy.TargetRoleOptions.Any,
+                    UnreachableServerAction = ServerEnumerationStrategy.UnreachableServerActionOptions.Throw,
+                },
+                MaxValueLength = 1024,
+                PoolSize = mareConfig.GetValue(nameof(ServerConfiguration.RedisPool), 50),
+                SyncTimeout = options.SyncTimeout,
+            };
+
+            services.AddStackExchangeRedisExtensions<SystemTextJsonSerializer>(redisConfiguration);
+        }
+        else
+        {
+            services.AddSingleton<IClientReadyMessageService, ShardClientReadyMessageService>();
+            services.AddSingleton<IConfigurationService<StaticFilesServerConfiguration>, MareConfigurationServiceClient<StaticFilesServerConfiguration>>();
+            services.AddHostedService(p => (MareConfigurationServiceClient<StaticFilesServerConfiguration>)p.GetService<IConfigurationService<StaticFilesServerConfiguration>>());
+        }
+
+        if (!_hasDistributionUpstream)
+        {
+            services.AddSingleton<ITouchHashService, ColdTouchHashService>();
+            services.AddHostedService(p => p.GetService<ITouchHashService>());
+        }
+        else
+        {
+            services.AddSingleton<ITouchHashService, ShardTouchMessageService>();
+            services.AddHostedService(p => p.GetService<ITouchHashService>());
+        }
+
+        // controller setup
+        services.AddControllers().ConfigureApplicationPartManager(a =>
+        {
+            a.FeatureProviders.Remove(a.FeatureProviders.OfType<ControllerFeatureProvider>().First());
+            if (_isMain)
+            {
+                a.FeatureProviders.Add(new AllowedControllersFeatureProvider(typeof(MareStaticFilesServerConfigurationController),
+                    typeof(CacheController), typeof(RequestController), typeof(ServerFilesController),
+                    typeof(DistributionController), typeof(MainController)));
+            }
+            else if (_isDistributionNode)
+            {
+                a.FeatureProviders.Add(new AllowedControllersFeatureProvider(typeof(CacheController), typeof(RequestController), typeof(DistributionController)));
+            }
+            else
+            {
+                a.FeatureProviders.Add(new AllowedControllersFeatureProvider(typeof(CacheController), typeof(RequestController)));
+            }
+        });
+
+        // authentication and authorization
+        services.AddOptions<JwtBearerOptions>(JwtBearerDefaults.AuthenticationScheme)
+            .Configure<IConfigurationService<MareConfigurationBase>>((o, s) =>
+            {
+                o.TokenValidationParameters = new()
+                {
+                    ValidateIssuer = false,
+                    ValidateLifetime = false,
+                    ValidateAudience = false,
+                    ValidateIssuerSigningKey = true,
+                    IssuerSigningKey = new SymmetricSecurityKey(Encoding.ASCII.GetBytes(s.GetValue<string>(nameof(MareConfigurationBase.Jwt))))
+                };
+            });
+        services.AddAuthentication(o =>
+        {
+            o.DefaultAuthenticateScheme = JwtBearerDefaults.AuthenticationScheme;
+            o.DefaultChallengeScheme = JwtBearerDefaults.AuthenticationScheme;
+            o.DefaultScheme = JwtBearerDefaults.AuthenticationScheme;
+        }).AddJwtBearer();
+        services.AddAuthorization(options =>
+        {
+            options.FallbackPolicy = new AuthorizationPolicyBuilder().RequireAuthenticatedUser().Build();
+            options.AddPolicy("Internal", new AuthorizationPolicyBuilder().RequireClaim(MareClaimTypes.Internal, "true").Build());
+        });
+        services.AddSingleton<IUserIdProvider, IdBasedUserIdProvider>();
+
+        services.AddHealthChecks();
+        services.AddHttpLogging(e => e = new Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions());
+    }
+
+    public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
+    {
+        app.UseHttpLogging();
+
+        app.UseRouting();
+
+        var config = app.ApplicationServices.GetRequiredService<IConfigurationService<MareConfigurationBase>>();
+
+        var metricServer = new KestrelMetricServer(config.GetValueOrDefault<int>(nameof(MareConfigurationBase.MetricsPort), 4981));
+        metricServer.Start();
+
+        app.UseHttpMetrics();
+
+        app.UseAuthentication();
+        app.UseAuthorization();
+
+        app.UseEndpoints(e =>
+        {
+            if (_isMain)
+            {
+                e.MapHub<MareSynchronosServer.Hubs.MareHub>("/dummyhub");
+            }
+
+            e.MapControllers();
+            e.MapHealthChecks("/health").WithMetadata(new AllowAnonymousAttribute());
+        });
+    }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/CountedStream.cs

@@ -0,0 +1,73 @@
+
+namespace MareSynchronosStaticFilesServer.Utils;
+
+// Counts the number of bytes read/written to an underlying stream
+public class CountedStream : Stream
+{
+    private readonly Stream _stream;
+    public long BytesRead { get; private set; }
+    public long BytesWritten { get; private set; }
+    public bool DisposeUnderlying = true;
+
+    public Stream UnderlyingStream { get => _stream; }
+
+    public CountedStream(Stream underlyingStream)
+    {
+        _stream = underlyingStream;
+    }
+
+    protected override void Dispose(bool disposing)
+    {
+        if (!DisposeUnderlying)
+            return;
+        _stream.Dispose();
+    }
+
+    public override bool CanRead => _stream.CanRead;
+    public override bool CanSeek => _stream.CanSeek;
+    public override bool CanWrite => _stream.CanWrite;
+    public override long Length => _stream.Length;
+
+    public override long Position { get => _stream.Position; set => _stream.Position = value; }
+
+    public override void Flush()
+    {
+        _stream.Flush();
+    }
+
+    public override int Read(byte[] buffer, int offset, int count)
+    {
+        int n = _stream.Read(buffer, offset, count);
+        BytesRead += n;
+        return n;
+    }
+
+    public async override Task<int> ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
+    {
+        int n = await _stream.ReadAsync(buffer, offset, count, cancellationToken);
+        BytesRead += n;
+        return n;
+    }
+
+    public override long Seek(long offset, SeekOrigin origin)
+    {
+        return _stream.Seek(offset, origin);
+    }
+
+    public override void SetLength(long value)
+    {
+        _stream.SetLength(value);
+    }
+
+    public override void Write(byte[] buffer, int offset, int count)
+    {
+        _stream.Write(buffer, offset, count);
+        BytesWritten += count;
+    }
+
+    public async override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
+    {
+        await _stream.WriteAsync(buffer, offset, count, cancellationToken);
+        BytesWritten += count;
+    }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/FilePathUtil.cs

@@ -0,0 +1,31 @@
+namespace MareSynchronosStaticFilesServer.Utils;
+
+public static partial class FilePathUtil
+{
+    public static FileInfo GetFileInfoForHash(string basePath, string hash)
+    {
+        if (hash.Length != 40 || !hash.All(char.IsAsciiLetterOrDigit)) throw new InvalidOperationException();
+
+        FileInfo fi = new(Path.Join(basePath, hash[0].ToString(), hash));
+        if (!fi.Exists)
+        {
+            fi = new FileInfo(Path.Join(basePath, hash));
+            if (!fi.Exists)
+            {
+                return null;
+            }
+        }
+
+        return fi;
+    }
+
+    public static string GetFilePath(string basePath, string hash)
+    {
+        if (hash.Length != 40 || !hash.All(char.IsAsciiLetterOrDigit)) throw new InvalidOperationException();
+
+        var dirPath = Path.Join(basePath, hash[0].ToString());
+        var path = Path.Join(dirPath, hash);
+        if (!Directory.Exists(dirPath)) Directory.CreateDirectory(dirPath);
+        return path;
+    }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/HashingStream.cs

@@ -0,0 +1,82 @@
+using System.Security.Cryptography;
+
+namespace MareSynchronosStaticFilesServer.Utils;
+
+// Calculates the hash of content read or written to a stream
+public class HashingStream : Stream
+{
+    private readonly Stream _stream;
+    private readonly HashAlgorithm _hashAlgo;
+    private bool _finished = false;
+    public bool DisposeUnderlying = true;
+
+    public Stream UnderlyingStream { get => _stream; }
+
+    public HashingStream(Stream underlyingStream, HashAlgorithm hashAlgo)
+    {
+        _stream = underlyingStream;
+        _hashAlgo = hashAlgo;
+    }
+
+    protected override void Dispose(bool disposing)
+    {
+        if (!DisposeUnderlying)
+            return;
+        if (!_finished)
+            _stream.Dispose();
+        _hashAlgo.Dispose();
+    }
+
+    public override bool CanRead => _stream.CanRead;
+    public override bool CanSeek => false;
+    public override bool CanWrite => _stream.CanWrite;
+    public override long Length => _stream.Length;
+
+    public override long Position { get => _stream.Position; set => throw new NotSupportedException(); }
+
+    public override void Flush()
+    {
+        _stream.Flush();
+    }
+
+    public override int Read(byte[] buffer, int offset, int count)
+    {
+        if (_finished)
+            throw new ObjectDisposedException("HashingStream");
+        int n = _stream.Read(buffer, offset, count);
+        if (n > 0)
+            _hashAlgo.TransformBlock(buffer, offset, n, buffer, offset);
+        return n;
+    }
+
+    public override long Seek(long offset, SeekOrigin origin)
+    {
+        throw new NotSupportedException();
+    }
+
+    public override void SetLength(long value)
+    {
+        if (_finished)
+            throw new ObjectDisposedException("HashingStream");
+        _stream.SetLength(value);
+    }
+
+    public override void Write(byte[] buffer, int offset, int count)
+    {
+        if (_finished)
+            throw new ObjectDisposedException("HashingStream");
+        _stream.Write(buffer, offset, count);
+        string x = new(System.Text.Encoding.ASCII.GetChars(buffer.AsSpan().Slice(offset, count).ToArray()));
+        _hashAlgo.TransformBlock(buffer, offset, count, buffer, offset);
+    }
+
+    public byte[] Finish()
+    {
+        if (_finished)
+            return _hashAlgo.Hash;
+        _hashAlgo.TransformFinalBlock(Array.Empty<byte>(), 0, 0);
+        if (DisposeUnderlying)
+            _stream.Dispose();
+        return _hashAlgo.Hash;
+    }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/RequestBlockFileListResult.cs

@@ -0,0 +1,76 @@
+using MareSynchronosShared.Metrics;
+using MareSynchronosShared.Services;
+using MareSynchronosShared.Utils.Configuration;
+using MareSynchronosStaticFilesServer.Services;
+using Microsoft.AspNetCore.Mvc;
+using System.Globalization;
+using System.Text;
+
+namespace MareSynchronosStaticFilesServer.Utils;
+
+public class RequestBlockFileListResult : IActionResult
+{
+    private readonly Guid _requestId;
+    private readonly RequestQueueService _requestQueueService;
+    private readonly MareMetrics _mareMetrics;
+    private readonly IEnumerable<FileInfo> _fileList;
+    private readonly IConfigurationService<StaticFilesServerConfiguration> _configurationService;
+
+    public RequestBlockFileListResult(Guid requestId, RequestQueueService requestQueueService, MareMetrics mareMetrics, IEnumerable<FileInfo> fileList,
+        IConfigurationService<StaticFilesServerConfiguration> configurationService)
+    {
+        _requestId = requestId;
+        _requestQueueService = requestQueueService;
+        _mareMetrics = mareMetrics;
+        _mareMetrics.IncGauge(MetricsAPI.GaugeCurrentDownloads);
+        _fileList = fileList;
+        _configurationService = configurationService;
+    }
+
+    public async Task ExecuteResultAsync(ActionContext context)
+    {
+        try
+        {
+            ArgumentNullException.ThrowIfNull(context);
+
+            var useSSI = _configurationService.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseSSI), false);
+
+            context.HttpContext.Response.StatusCode = 200;
+
+            if (useSSI)
+                context.HttpContext.Response.ContentType = _configurationService.GetValue<string>(nameof(StaticFilesServerConfiguration.SSIContentType));
+            else
+                context.HttpContext.Response.ContentType = "application/octet-stream";
+
+            string ssiFilePrefix = null;
+
+            if (useSSI)
+                ssiFilePrefix = _configurationService.GetValue<string>(nameof(StaticFilesServerConfiguration.XAccelRedirectPrefix));
+
+            foreach (var file in _fileList)
+            {
+                if (useSSI)
+                {
+                    var internalName = Path.Combine(ssiFilePrefix, file.Name);
+                    await context.HttpContext.Response.WriteAsync(
+                        "#" + file.Name + ":" + file.Length.ToString(CultureInfo.InvariantCulture) + "#"
+                        + "<!--#include file=\"" + internalName + "\" -->", Encoding.ASCII);
+                }
+                else
+                {
+                    await context.HttpContext.Response.WriteAsync("#" + file.Name + ":" + file.Length.ToString(CultureInfo.InvariantCulture) + "#", Encoding.ASCII);
+                    await context.HttpContext.Response.SendFileAsync(file.FullName);
+                }
+            }
+        }
+        catch
+        {
+            throw;
+        }
+        finally
+        {
+            _requestQueueService.FinishRequest(_requestId);
+            _mareMetrics.DecGauge(MetricsAPI.GaugeCurrentDownloads);
+        }
+    }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/RequestBlockFileListResultFactory.cs

@@ -0,0 +1,25 @@
+using MareSynchronosShared.Metrics;
+using MareSynchronosShared.Services;
+using MareSynchronosShared.Utils.Configuration;
+using MareSynchronosStaticFilesServer.Services;
+
+namespace MareSynchronosStaticFilesServer.Utils;
+
+public class RequestBlockFileListResultFactory
+{
+    private readonly MareMetrics _metrics;
+    private readonly RequestQueueService _requestQueueService;
+    private readonly IConfigurationService<StaticFilesServerConfiguration> _configurationService;
+
+    public RequestBlockFileListResultFactory(MareMetrics metrics, RequestQueueService requestQueueService, IConfigurationService<StaticFilesServerConfiguration> configurationService)
+    {
+        _metrics = metrics;
+        _requestQueueService = requestQueueService;
+        _configurationService = configurationService;
+    }
+
+    public RequestBlockFileListResult Create(Guid requestId, IEnumerable<FileInfo> fileList)
+    {
+        return new RequestBlockFileListResult(requestId, _requestQueueService, _metrics, fileList, _configurationService);
+    }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/TeeStream.cs

@@ -0,0 +1,74 @@
+namespace MareSynchronosStaticFilesServer.Utils;
+
+// Writes data read from one stream out to a second stream
+public class TeeStream : Stream
+{
+    private readonly Stream _inStream;
+    private readonly Stream _outStream;
+    public bool DisposeUnderlying = true;
+
+    public Stream InStream { get => _inStream; }
+    public Stream OutStream { get => _outStream; }
+
+    public TeeStream(Stream inStream, Stream outStream)
+    {
+        _inStream = inStream;
+        _outStream = outStream;
+    }
+
+    protected override void Dispose(bool disposing)
+    {
+        if (!DisposeUnderlying)
+            return;
+        _inStream.Dispose();
+        _outStream.Dispose();
+    }
+
+    public override bool CanRead => _inStream.CanRead;
+    public override bool CanSeek => _inStream.CanSeek;
+    public override bool CanWrite => false;
+    public override long Length => _inStream.Length;
+
+    public override long Position
+    {
+        get => _inStream.Position;
+        set => _inStream.Position = value;
+    }
+
+    public override void Flush()
+    {
+        _inStream.Flush();
+        _outStream.Flush();
+    }
+
+    public override int Read(byte[] buffer, int offset, int count)
+    {
+        int n = _inStream.Read(buffer, offset, count);
+        if (n > 0)
+            _outStream.Write(buffer, offset, n);
+        return n;
+    }
+
+    public async override Task<int> ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
+    {
+        int n = await _inStream.ReadAsync(buffer, offset, count, cancellationToken);
+        if (n > 0)
+            await _outStream.WriteAsync(buffer, offset, n);
+        return n;
+    }
+
+    public override long Seek(long offset, SeekOrigin origin)
+    {
+        return _inStream.Seek(offset, origin);
+    }
+
+    public override void SetLength(long value)
+    {
+        _inStream.SetLength(value);
+    }
+
+    public override void Write(byte[] buffer, int offset, int count)
+    {
+        throw new NotSupportedException();
+    }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/UserQueueEntry.cs

@@ -0,0 +1,21 @@
+namespace MareSynchronosStaticFilesServer.Utils;
+
+public class UserQueueEntry
+{
+    public UserQueueEntry(UserRequest userRequest, DateTime expirationDate)
+    {
+        UserRequest = userRequest;
+        ExpirationDate = expirationDate;
+    }
+
+    public void MarkActive()
+    {
+        IsActive = true;
+        ActivationDate = DateTime.UtcNow;
+    }
+
+    public UserRequest UserRequest { get; }
+    public DateTime ExpirationDate { get; }
+    public bool IsActive { get; private set; } = false;
+    public DateTime ActivationDate { get; private set; }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/UserRequest.cs

@@ -0,0 +1,6 @@
+namespace MareSynchronosStaticFilesServer.Utils;
+
+public record UserRequest(Guid RequestId, string User, List<string> FileIds)
+{
+    public bool IsCancelled { get; set; } = false;
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/appsettings.Development.json

@@ -0,0 +1,9 @@
+{
+  "DetailedErrors": true,
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/appsettings.json

@@ -0,0 +1,31 @@
+{
+  "ConnectionStrings": {
+    "DefaultConnection": "Host=localhost;Port=5432;Database=mare;Username=postgres"
+  },
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  },
+  "Kestrel": {
+    "Endpoints": {
+      "Http": {
+        "Url": "http://+:5001"
+      },
+      "Gprc": {
+        "Protocols": "Http2",
+        "Url": "http://+:5003"
+      }
+    }
+  },
+  "MareSynchronos": {
+    "ForcedDeletionOfFilesAfterHours": -1,
+    "CacheSizeHardLimitInGiB": -1,
+    "UnusedFileRetentionPeriodInDays": 7,
+    "CacheDirectory": "G:\\ServerTest",
+    "ServiceAddress": "http://localhost:5002",
+    "RemoteCacheSourceUri": ""
+  },
+  "AllowedHosts": "*"
+}