using MareSynchronosShared.Data; using MareSynchronosShared.Metrics; using MareSynchronosShared.Services; using MareSynchronosShared.Utils; using MareSynchronosStaticFilesServer.Controllers; using MareSynchronosStaticFilesServer.Services; using MareSynchronosStaticFilesServer.Utils; using MessagePack; using MessagePack.Resolvers; using Microsoft.AspNetCore.Authentication.JwtBearer; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc.Controllers; using Microsoft.AspNetCore.Server.Kestrel.Core; using Microsoft.AspNetCore.SignalR; using Microsoft.EntityFrameworkCore; using Microsoft.IdentityModel.Tokens; using Prometheus; using StackExchange.Redis.Extensions.Core.Configuration; using StackExchange.Redis.Extensions.System.Text.Json; using StackExchange.Redis; using System.Net; using System.Text; using MareSynchronosShared.Utils.Configuration; namespace MareSynchronosStaticFilesServer; public class Startup { private bool _isMain; private bool _isDistributionNode; private bool _hasDistributionUpstream; private readonly ILogger _logger; public Startup(IConfiguration configuration, ILogger logger) { Configuration = configuration; _logger = logger; var mareSettings = Configuration.GetRequiredSection("MareSynchronos"); _isDistributionNode = mareSettings.GetValue(nameof(StaticFilesServerConfiguration.IsDistributionNode), false); _hasDistributionUpstream = !string.IsNullOrEmpty(mareSettings.GetValue(nameof(StaticFilesServerConfiguration.DistributionFileServerAddress), string.Empty)); _isMain = string.IsNullOrEmpty(mareSettings.GetValue(nameof(StaticFilesServerConfiguration.MainFileServerAddress), string.Empty)) && _isDistributionNode; } public IConfiguration Configuration { get; } public void ConfigureServices(IServiceCollection services) { services.AddHttpContextAccessor(); services.AddLogging(); services.Configure(Configuration.GetRequiredSection("MareSynchronos")); services.Configure(Configuration.GetRequiredSection("MareSynchronos")); services.Configure(Configuration.GetSection("Kestrel")); services.AddSingleton(Configuration); var mareConfig = Configuration.GetRequiredSection("MareSynchronos"); // metrics configuration services.AddSingleton(m => new MareMetrics(m.GetService>(), new List { MetricsAPI.CounterFileRequests, MetricsAPI.CounterFileRequestSize }, new List { MetricsAPI.GaugeFilesTotalColdStorage, MetricsAPI.GaugeFilesTotalSizeColdStorage, MetricsAPI.GaugeFilesTotalSize, MetricsAPI.GaugeFilesTotal, MetricsAPI.GaugeFilesUniquePastDay, MetricsAPI.GaugeFilesUniquePastDaySize, MetricsAPI.GaugeFilesUniquePastHour, MetricsAPI.GaugeFilesUniquePastHourSize, MetricsAPI.GaugeCurrentDownloads, MetricsAPI.GaugeDownloadQueue, MetricsAPI.GaugeDownloadQueueCancelled, MetricsAPI.GaugeDownloadPriorityQueue, MetricsAPI.GaugeDownloadPriorityQueueCancelled, MetricsAPI.GaugeQueueFree, MetricsAPI.GaugeQueueInactive, MetricsAPI.GaugeQueueActive, MetricsAPI.GaugeFilesDownloadingFromCache, MetricsAPI.GaugeFilesTasksWaitingForDownloadFromCache })); // generic services services.AddSingleton(); services.AddHostedService(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); services.AddHostedService(p => p.GetService()); services.AddSingleton(); services.AddHostedService(p => p.GetService()); services.AddHostedService(m => m.GetService()); services.AddSingleton, MareConfigurationServiceClient>(); services.AddHostedService(p => (MareConfigurationServiceClient)p.GetService>()); // specific services if (_isMain) { services.AddSingleton(); services.AddSingleton, MareConfigurationServiceServer>(); services.AddDbContextPool(options => { options.UseNpgsql(Configuration.GetConnectionString("DefaultConnection"), builder => { builder.MigrationsHistoryTable("_efmigrationshistory", "public"); }).UseSnakeCaseNamingConvention(); options.EnableThreadSafetyChecks(false); }, mareConfig.GetValue(nameof(MareConfigurationBase.DbContextPoolSize), 1024)); var signalRServiceBuilder = services.AddSignalR(hubOptions => { hubOptions.MaximumReceiveMessageSize = long.MaxValue; hubOptions.EnableDetailedErrors = true; hubOptions.MaximumParallelInvocationsPerClient = 10; hubOptions.StreamBufferCapacity = 200; }).AddMessagePackProtocol(opt => { var resolver = CompositeResolver.Create(StandardResolverAllowPrivate.Instance, BuiltinResolver.Instance, AttributeFormatterResolver.Instance, // replace enum resolver DynamicEnumAsStringResolver.Instance, DynamicGenericResolver.Instance, DynamicUnionResolver.Instance, DynamicObjectResolver.Instance, PrimitiveObjectResolver.Instance, // final fallback(last priority) StandardResolver.Instance); opt.SerializerOptions = MessagePackSerializerOptions.Standard .WithCompression(MessagePackCompression.Lz4Block) .WithResolver(resolver); }); // configure redis for SignalR var redisConnection = mareConfig.GetValue(nameof(ServerConfiguration.RedisConnectionString), string.Empty); signalRServiceBuilder.AddStackExchangeRedis(redisConnection, options => { }); var options = ConfigurationOptions.Parse(redisConnection); var endpoint = options.EndPoints[0]; string address = ""; int port = 0; if (endpoint is DnsEndPoint dnsEndPoint) { address = dnsEndPoint.Host; port = dnsEndPoint.Port; } if (endpoint is IPEndPoint ipEndPoint) { address = ipEndPoint.Address.ToString(); port = ipEndPoint.Port; } var redisConfiguration = new RedisConfiguration() { AbortOnConnectFail = true, KeyPrefix = "", Hosts = new RedisHost[] { new RedisHost(){ Host = address, Port = port }, }, AllowAdmin = true, ConnectTimeout = options.ConnectTimeout, Database = 0, Ssl = false, Password = options.Password, ServerEnumerationStrategy = new ServerEnumerationStrategy() { Mode = ServerEnumerationStrategy.ModeOptions.All, TargetRole = ServerEnumerationStrategy.TargetRoleOptions.Any, UnreachableServerAction = ServerEnumerationStrategy.UnreachableServerActionOptions.Throw, }, MaxValueLength = 1024, PoolSize = mareConfig.GetValue(nameof(ServerConfiguration.RedisPool), 50), SyncTimeout = options.SyncTimeout, }; services.AddStackExchangeRedisExtensions(redisConfiguration); } else { services.AddSingleton(); services.AddSingleton, MareConfigurationServiceClient>(); services.AddHostedService(p => (MareConfigurationServiceClient)p.GetService>()); } if (!_hasDistributionUpstream) { services.AddSingleton(); services.AddHostedService(p => p.GetService()); } else { services.AddSingleton(); services.AddHostedService(p => p.GetService()); } // controller setup services.AddControllers().ConfigureApplicationPartManager(a => { a.FeatureProviders.Remove(a.FeatureProviders.OfType().First()); if (_isMain) { a.FeatureProviders.Add(new AllowedControllersFeatureProvider(typeof(MareStaticFilesServerConfigurationController), typeof(CacheController), typeof(RequestController), typeof(ServerFilesController), typeof(DistributionController), typeof(MainController))); } else if (_isDistributionNode) { a.FeatureProviders.Add(new AllowedControllersFeatureProvider(typeof(CacheController), typeof(RequestController), typeof(DistributionController))); } else { a.FeatureProviders.Add(new AllowedControllersFeatureProvider(typeof(CacheController), typeof(RequestController))); } }); // authentication and authorization services.AddOptions(JwtBearerDefaults.AuthenticationScheme) .Configure>((o, s) => { o.TokenValidationParameters = new() { ValidateIssuer = false, ValidateLifetime = false, ValidateAudience = false, ValidateIssuerSigningKey = true, IssuerSigningKey = new SymmetricSecurityKey(Encoding.ASCII.GetBytes(s.GetValue(nameof(MareConfigurationBase.Jwt)))) }; }); services.AddAuthentication(o => { o.DefaultAuthenticateScheme = JwtBearerDefaults.AuthenticationScheme; o.DefaultChallengeScheme = JwtBearerDefaults.AuthenticationScheme; o.DefaultScheme = JwtBearerDefaults.AuthenticationScheme; }).AddJwtBearer(); services.AddAuthorization(options => { options.FallbackPolicy = new AuthorizationPolicyBuilder().RequireAuthenticatedUser().Build(); options.AddPolicy("Internal", new AuthorizationPolicyBuilder().RequireClaim(MareClaimTypes.Internal, "true").Build()); }); services.AddSingleton(); services.AddHealthChecks(); services.AddHttpLogging(e => e = new Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions()); } public void Configure(IApplicationBuilder app, IWebHostEnvironment env) { app.UseHttpLogging(); app.UseRouting(); var config = app.ApplicationServices.GetRequiredService>(); var metricServer = new KestrelMetricServer(config.GetValueOrDefault(nameof(MareConfigurationBase.MetricsPort), 4981)); metricServer.Start(); app.UseHttpMetrics(); app.UseAuthentication(); app.UseAuthorization(); app.UseEndpoints(e => { if (_isMain) { e.MapHub("/dummyhub"); } e.MapControllers(); e.MapHealthChecks("/health").WithMetadata(new AllowAnonymousAttribute()); }); } }