diff --git a/PCL.Core/IO/Net/Http/Cache/HttpCacheHandler.cs b/PCL.Core/IO/Net/Http/Cache/HttpCacheHandler.cs new file mode 100644 index 000000000..af7bd8c6c --- /dev/null +++ b/PCL.Core/IO/Net/Http/Cache/HttpCacheHandler.cs @@ -0,0 +1,52 @@ +using System; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Threading; +using System.Threading.Tasks; +using PCL.Core.IO.Net.Http.Cache.Models; + +namespace PCL.Core.IO.Net.Http.Cache; + +/// +/// HTTP 缓存处理器 +/// +public class HttpCacheHandler:DelegatingHandler +{ + private HttpCacheRepository _repository; + public HttpCacheHandler(HttpMessageHandler invoker, HttpCacheRepository repo) + { + InnerHandler = invoker; + _repository = repo; + } + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + if(!_repository.TryGetCacheData(request.RequestUri!.ToString(),out var details)) + return await base.SendAsync(request, cancellationToken); + if (details.ExpiredAt is not null && + details.LastUpdate.AddSeconds((double)details.ExpiredAt) < DateTimeOffset.Now + && _repository.TryGetCacheResponse(request,out var cacheResponse) && !details.EnsureValidate + ) + return cacheResponse; + + if(details.Tag is not null) request.Headers.IfNoneMatch.Add(new EntityTagHeaderValue(details.Tag)); + if(details.LastModify is not null) request.Headers.IfModifiedSince = DateTimeOffset.Parse(details.LastModify); + var response = await base.SendAsync(request, cancellationToken); + if (response.Headers.CacheControl?.NoStore ?? false) return response; + if(response.StatusCode == HttpStatusCode.NotModified && _repository.TryGetCacheResponse(request,out cacheResponse)) + return cacheResponse; + var handle = await _repository.TryBeginUpdateAsync(request.RequestUri.ToString()); + var newDetails = handle?.Details; + newDetails?.RequestUri = request.RequestUri.ToString(); + newDetails?.LastUpdate = DateTimeOffset.Now; + newDetails?.EnsureValidate = response.Headers.CacheControl?.NoCache ?? false; + newDetails?.LastModify = response.Content.Headers.LastModified.ToString(); + newDetails?.Tag = response.Headers.ETag?.Tag; + if (handle is not null) + response.Content = new StreamContent(new CacheStream(handle, + await response.Content.ReadAsStreamAsync(cancellationToken))); + return response; + } +} diff --git a/PCL.Core/IO/Net/Http/Cache/HttpCacheRepository.cs b/PCL.Core/IO/Net/Http/Cache/HttpCacheRepository.cs new file mode 100644 index 000000000..edc930acf --- /dev/null +++ b/PCL.Core/IO/Net/Http/Cache/HttpCacheRepository.cs @@ -0,0 +1,374 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Data.Sqlite; +using PCL.Core.IO.Net.Http.Cache.Models; +using PCL.Core.IO.Storage; +using PCL.Core.Logging; +using PCL.Core.Utils.Hash; + +namespace PCL.Core.IO.Net.Http.Cache; + +/// +/// HTTP 缓存储存库,支持 LazyGC +/// +/// SQLite 数据库路径 +/// 存储位置 +public class HttpCacheRepository(string dbPath,string destLocation) +{ + + #region "预设 SQL 命令" + + private const string FindTable = "SELECT * FROM HttpCache WHERE RequestUri = @Uri"; + + private const string CreateTable = """ + + CREATE TABLE IF NOT EXISTS HttpCache ( + RequestUri TEXT NOT NULL PRIMARY KEY, + Tag TEXT NULL, + LastModify TEXT NULL, + ExpiredAt INTEGER NOT NULL, + EnsureValidate INTEGER NOT NULL DEFAULT 0, + Status INTEGER NOT NULL DEFAULT 0, + LastUpdate TEXT NOT NULL, + Hash TEXT NULL + ) + """; + + private const string InsertTable = """ + INSERT OR REPLACE INTO HttpCache ( + RequestUri, Tag, LastModify, ExpiredAt, EnsureValidate, Status, LastUpdate, Hash + ) VALUES ( + @Uri, @Tag, @LastModify, @ExpiredAt, @EnsureValidate, @Status, @LastUpdate, @Hash + ) + """; + + + private const string DeleteTable = "DELETE FROM HttpCache WHERE RequestUri = @Uri"; + + + #endregion + + #region "配置" + + + private readonly Func _connectionFactory = () => + { + var c = new SqliteConnection($"Data Source={dbPath}"); + c.Open(); + return c; + }; + + private readonly HashStorage _store = new(destLocation, SHA256Provider.Instance, true); + + #endregion + + #region "HTTP 缓存处理" + + /// + /// 初始化数据库 + /// + public void Initialize() + { + try + { + if (!Directory.Exists(destLocation)) Directory.CreateDirectory(destLocation); + } + catch (IOException) + { + File.Delete(destLocation); + Directory.CreateDirectory(destLocation); + } + + using var connection = _connectionFactory.Invoke(); + var cmd = connection.CreateCommand(); + cmd.CommandText = CreateTable; + cmd.ExecuteNonQuery(); + } + + /// + /// 获取缓存数据 + /// + /// + /// + /// + public bool TryGetCacheData(string uri,[NotNullWhen(true)] out HttpCacheDetails? details) + { + details = null; + using var conn = _connectionFactory.Invoke(); + using var cmd = _FindTableWithUri(uri, conn); + using var result = cmd.ExecuteReader(); + if (!result.Read()) return false; + if ((HttpCacheStatus)result.GetInt16(6) is HttpCacheStatus.Invalid or HttpCacheStatus.Expired) + { + _DeleteTable(result.GetString(0), conn); + return false; + } + details = new HttpCacheDetails(this) + { + RequestUri = result.GetString(0), + Tag = result.GetString(1), + LastModify = result.GetString(2), + ExpiredAt = result.GetInt32(3), + EnsureValidate = result.GetBoolean(4), + Status = (HttpCacheStatus)result.GetInt16(5), + LastUpdate = DateTimeOffset.Parse(result.GetString(6)) + }; + return true; + } + + /// + /// 获取已缓存的响应 + /// + /// 发出的 HTTP 请求 + /// 缓存响应 + /// 如果缓存存在,返回 true + public bool TryGetCacheResponse(HttpRequestMessage request,[NotNullWhen(true)] out HttpResponseMessage? response) + { + response = null; + if (request.RequestUri is null) return false; + if (!TryGetCacheData(request.RequestUri.ToString(), out var details)) return false; + if (details.Status is HttpCacheStatus.Updating) + return false; + response = new HttpResponseMessage + { + StatusCode = HttpStatusCode.OK, + Content = new StreamContent(_store.Get(details.Hash!) ?? throw new NullReferenceException("Hash Storage return null.")), + RequestMessage = request + }; + response.Headers.TryAddWithoutValidation("X-Cache-Repository-Status", "Hit"); + + return true; + } + + /// + /// 获取缓存更新句柄 + /// + /// URL + /// + public async ValueTask TryBeginUpdateAsync(string uri) + { + await using var conn = _connectionFactory.Invoke(); + if (!TryGetCacheData(uri, out var details)) + { + Span buffer = stackalloc byte[16]; + Random.Shared.NextBytes(buffer); + details = new HttpCacheDetails(this) + { + LastUpdate = DateTimeOffset.Now, + RequestUri = uri, + EnsureValidate = false, + ExpiredAt = null, + Tag = null, + Status = HttpCacheStatus.Updating, + Hash = null + }; + await using var cmd = _InsertDatabase(details, conn); + cmd.ExecuteNonQuery(); + // 互斥锁,避免线程冲突 + }else if (details.Status == HttpCacheStatus.Updating) return null; + + var handle = details.GetUpdateHandle(); + await _store.PutAsync(handle.GetOutputStream()); + return handle; + } + /// + /// 异步结束更新并设置缓存状态 + /// + /// + /// + public async ValueTask TryEndUpdateAsync(HttpCacheUpdateHandle handle) + { + await using var conn = _connectionFactory.Invoke(); + var details = handle.Details; + if (details is null) return false; + details.Status = HttpCacheStatus.Ok; + await using var cmd = _UpdateTable(details,conn); + if (cmd is null) return true; + await cmd.ExecuteNonQueryAsync(); + return true; + } + + + /// + /// 删除一个缓存 + /// + /// 请求 + /// + public bool TryRemove(HttpRequestMessage request) + { + using var conn = _connectionFactory.Invoke(); + try + { + if (!TryGetCacheData(request.RequestUri!.ToString(), out var details) && details?.Hash is null) return false; + if (request.RequestUri is null) return false; + using var cmd = _DeleteTable(request.RequestUri.ToString(), conn); + cmd.ExecuteNonQuery(); + _store.DeleteAsync(details.Hash!).GetAwaiter().GetResult(); + return true; + } + catch(Exception ex) + { + LogWrapper.Error(ex,"Http", "删除缓存文件失败"); + } + + return false; + } + + /// + /// 删除一个缓存 + /// + /// 请求 + /// + public async ValueTask TryRemoveAsync(HttpRequestMessage request) + { + await using var conn = _connectionFactory.Invoke(); + try + { + if (!TryGetCacheData(request.RequestUri!.ToString(), out var details) && details?.Hash is null) return false; + if (request.RequestUri is null) return false; + await using var cmd = _DeleteTable(request.RequestUri.ToString(), conn); + await cmd.ExecuteNonQueryAsync(); + await _store.DeleteAsync(details.Hash!).ConfigureAwait(false); + return true; + } + catch(Exception ex) + { + LogWrapper.Error(ex,"Http", "删除缓存文件失败"); + } + + return false; + } + + /// + /// 将全部对象标记为过期 + /// + public void MarkAllObjectAsExpired() + { + using var conn = _connectionFactory.Invoke(); + using var cmd = conn.CreateCommand(); + cmd.CommandText = "UPDATE HttpCache SET Status = 2"; + cmd.ExecuteNonQuery(); + } + + + + #endregion + + #region "SQL 执行函数" + + + private static SqliteCommand _InsertDatabase(HttpCacheDetails details, SqliteConnection conn) + { + var cmd = conn.CreateCommand(); + cmd.CommandText = InsertTable; + cmd.Parameters.AddWithValue("@Uri", details.RequestUri); + cmd.Parameters.AddWithValue("@Tag", details.Tag); + cmd.Parameters.AddWithValue("@LastModify", details.LastModify); + cmd.Parameters.AddWithValue("@ExpiredAt", details.ExpiredAt); + cmd.Parameters.AddWithValue("@EnsureValidate", details.EnsureValidate); + cmd.Parameters.AddWithValue("@Status", (int)details.Status); + cmd.Parameters.AddWithValue("@Hash", details.Hash); + return cmd; + } + + private static SqliteCommand _DeleteTable(string uri, SqliteConnection conn) + { + var cmd = conn.CreateCommand(); + cmd.CommandText = DeleteTable; + cmd.Parameters.AddWithValue("@Uri", uri); + return cmd; + } + + private static SqliteCommand _FindTableWithUri(string uri, SqliteConnection conn) + { + var queryCmd = conn.CreateCommand(); + queryCmd.CommandText = FindTable; + queryCmd.Parameters.AddWithValue("@Uri", uri); + return queryCmd; + } + + private SqliteCommand? _UpdateTable(HttpCacheDetails details, SqliteConnection conn) + { + using var queryCmd = _FindTableWithUri(details.RequestUri, conn); + // 获取用于比较的原始内容 + using var reader = queryCmd.ExecuteReader(); + if (!reader.Read()) + { + // 可能已经被删掉了,添加就好 + return _InsertDatabase(details,conn); + + } + var sb = new StringBuilder(); + sb.Append("UPDATE HttpCache "); + var writeCmd = conn.CreateCommand(); + writeCmd.Disposed += (_, _) => conn.Dispose(); + var setCount = 0; + // 按需更新以减少开销 + if (reader.GetString(0) != details.RequestUri) + { + setCount++; + sb.Append("SET RequestUri = @Uri, "); + writeCmd.Parameters.AddWithValue("@Uri", details.RequestUri); + } + + if (reader.GetString(1) != details.Tag) + { + setCount++; + sb.Append("SET Tag = @Tag, "); + writeCmd.Parameters.AddWithValue("@Tag", details.Tag); + } + if (reader.GetString(2) != details.LastModify) + { + setCount++; + sb.Append("SET LastModify = @LastModify, "); + writeCmd.Parameters.AddWithValue("@LastModify", details.LastModify); + } + if (reader.GetInt32(3) != details.ExpiredAt) + { + setCount++; + sb.Append("SET ExpiredAt = @ExpiredAt, "); + writeCmd.Parameters.AddWithValue("@ExpiredAt", details.ExpiredAt); + } + + if (reader.GetBoolean(4) != details.EnsureValidate) + { + setCount++; + sb.Append("SET EnsureValidate = @EnsureValidate,"); + writeCmd.Parameters.AddWithValue("@EnsureValidate", details.EnsureValidate); + } + if ((HttpCacheStatus)reader.GetInt16(5) != details.Status) + { + setCount++; + sb.Append("SET Status = @Status,"); + writeCmd.Parameters.AddWithValue("@Status", (int)details.Status); + } + + if (reader.GetString(6) != details.LastUpdate.ToString()) + { + setCount++; + sb.Append("SET LastUpdate = @LastUpdate,"); + writeCmd.Parameters.AddWithValue("@LastUpdate", details.LastUpdate.ToString()); + } + + if (reader.GetString(7) != details.Hash) + { + setCount++; + sb.Append("SET Hash = @Hash"); + writeCmd.Parameters.AddWithValue("@Hash", details.Hash); + + } + + if (setCount == 0) return null; + sb.Append("WHERE RequestUri = @Uri"); + writeCmd.CommandText = sb.ToString(); + return writeCmd; + } + + + #endregion +} \ No newline at end of file diff --git a/PCL.Core/IO/Net/Http/Cache/Models/BlockingStream.cs b/PCL.Core/IO/Net/Http/Cache/Models/BlockingStream.cs new file mode 100644 index 000000000..c75a1226b --- /dev/null +++ b/PCL.Core/IO/Net/Http/Cache/Models/BlockingStream.cs @@ -0,0 +1,39 @@ +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +namespace PCL.Core.IO.Net.Http.Cache.Models; + +public class BlockingStream:MemoryStream +{ + private SemaphoreSlim _lock = new(0); + + [Obsolete("请使用支持取消重载的 Read", error:true)] + public new int Read(byte[] buffer, int offset, int count) + { + return base.Read(buffer, offset, count); + } + + public int Read(Span buffer, CancellationToken token) + { + _lock.Wait(token); + return base.Read(buffer); + } + + public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = new CancellationToken()) + { + _lock.Wait(cancellationToken); + return base.ReadAsync(buffer, cancellationToken); + } + + internal void Readable() + { + _lock.Release(); + } + + protected override void Dispose(bool disposing) + { + _lock.Dispose(); + base.Dispose(disposing); + } +} \ No newline at end of file diff --git a/PCL.Core/IO/Net/Http/Cache/Models/CacheStream.cs b/PCL.Core/IO/Net/Http/Cache/Models/CacheStream.cs new file mode 100644 index 000000000..66dccc7b1 --- /dev/null +++ b/PCL.Core/IO/Net/Http/Cache/Models/CacheStream.cs @@ -0,0 +1,67 @@ +using System; +using System.IO; + +namespace PCL.Core.IO.Net.Http.Cache.Models; + +public class CacheStream: Stream +{ + private Stream _responseStream; + private BlockingStream? _destStream; + private HttpCacheUpdateHandle _handle; + + public CacheStream(HttpCacheUpdateHandle handle, byte[] data) + { + _responseStream = new MemoryStream(data); + _destStream = handle.GetOutputStream(); + _handle = handle; + } + + public CacheStream(HttpCacheUpdateHandle handle, Stream responseStream) + { + _responseStream = responseStream; + _destStream = handle.GetOutputStream(); + _handle = handle; + } + + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) + { + var read = _responseStream.Read(buffer, offset, count); + if (read == 0) return read; + _destStream?.Write(buffer,0, read); + _destStream?.Readable(); + return read; + } + + public override long Seek(long offset, SeekOrigin origin) + { + throw new InvalidOperationException("This stream is readonly."); + } + + public override void SetLength(long value) + { + throw new InvalidOperationException("This stream is readonly."); + } + + public override void Write(byte[] buffer, int offset, int count) + { + throw new InvalidOperationException("This stream is readonly."); + } + + public override bool CanRead => _responseStream.CanRead; + public override bool CanSeek => _responseStream.CanSeek; + public override bool CanWrite => false; + public override long Length => _responseStream.Length; + public override long Position { get => _responseStream.Length; + set => throw new InvalidOperationException("can not set position on readonly stream"); + } + + protected override void Dispose(bool disposing) + { + _destStream?.Dispose(); + _handle.Dispose(); + base.Dispose(disposing); + } +} \ No newline at end of file diff --git a/PCL.Core/IO/Net/Http/Cache/Models/HttpCacheDetails.cs b/PCL.Core/IO/Net/Http/Cache/Models/HttpCacheDetails.cs new file mode 100644 index 000000000..3cf8867d8 --- /dev/null +++ b/PCL.Core/IO/Net/Http/Cache/Models/HttpCacheDetails.cs @@ -0,0 +1,26 @@ +using System; + +namespace PCL.Core.IO.Net.Http.Cache.Models; + +/// +/// HTTP 缓存信息 +/// +public class HttpCacheDetails(HttpCacheRepository repo) +{ + public required DateTimeOffset LastUpdate { get; set; } + public required string RequestUri { get; set; } + public string? Tag { get; set; } + public string? LastModify { get; set; } + public int? ExpiredAt { get; set; } + public bool EnsureValidate { get; set; } + public string? Hash { get; set; } + public HttpCacheStatus Status = HttpCacheStatus.Invalid; + + public HttpCacheUpdateHandle GetUpdateHandle() + { + return new HttpCacheUpdateHandle(repo) + { + Details = this + }; + } +} \ No newline at end of file diff --git a/PCL.Core/IO/Net/Http/Cache/Models/HttpCacheStatus.cs b/PCL.Core/IO/Net/Http/Cache/Models/HttpCacheStatus.cs new file mode 100644 index 000000000..3b3fd6653 --- /dev/null +++ b/PCL.Core/IO/Net/Http/Cache/Models/HttpCacheStatus.cs @@ -0,0 +1,12 @@ +namespace PCL.Core.IO.Net.Http.Cache.Models; + +public enum HttpCacheStatus +{ + /// + /// 缓存无效(例如文件已经删除) + /// + Invalid, + Ok, + Expired, + Updating +} \ No newline at end of file diff --git a/PCL.Core/IO/Net/Http/Cache/Models/HttpCacheUpdateHandle.cs b/PCL.Core/IO/Net/Http/Cache/Models/HttpCacheUpdateHandle.cs new file mode 100644 index 000000000..d383460b1 --- /dev/null +++ b/PCL.Core/IO/Net/Http/Cache/Models/HttpCacheUpdateHandle.cs @@ -0,0 +1,51 @@ +using System; +using System.IO; +using System.Threading.Tasks; + +namespace PCL.Core.IO.Net.Http.Cache.Models; + +public class HttpCacheUpdateHandle(HttpCacheRepository repo) : IDisposable +{ + + private BlockingStream? _fileStream; + private bool _disposed; + /// + /// 该对象对应的缓存信息 + /// + public HttpCacheDetails? Details { get; set; } + /// + /// 获取当前文件的写入流 + /// + /// + /// + public BlockingStream GetOutputStream() + { + ObjectDisposedException.ThrowIf(_disposed, typeof(HttpCacheUpdateHandle)); + return _fileStream ??= new BlockingStream(); + } + + ~HttpCacheUpdateHandle() + { + _Dispose(false); + } + + public void Dispose() + { + _Dispose(true); + GC.SuppressFinalize(this); + } + + private void _Dispose(bool dispose) + { + _DisposeAsync(dispose).GetAwaiter().GetResult(); + } + + private async Task _DisposeAsync(bool dispose) + { + if (_disposed) return; + await repo.TryEndUpdateAsync(this); + Details?.Status = HttpCacheStatus.Ok; + _disposed = true; + if (dispose) _fileStream?.Dispose(); + } +} \ No newline at end of file diff --git a/PCL.Core/IO/Net/NetworkService.cs b/PCL.Core/IO/Net/NetworkService.cs index 75e79625e..b7a471240 100644 --- a/PCL.Core/IO/Net/NetworkService.cs +++ b/PCL.Core/IO/Net/NetworkService.cs @@ -1,9 +1,11 @@ using System; +using System.IO; using System.Net; using System.Net.Http; using Microsoft.Extensions.DependencyInjection; using PCL.Core.App; using PCL.Core.App.IoC; +using PCL.Core.IO.Net.Http.Cache; using PCL.Core.IO.Net.Http.Client; using PCL.Core.Logging; using Polly; @@ -12,14 +14,16 @@ namespace PCL.Core.IO.Net; [LifecycleService(LifecycleState.Loading)] [LifecycleScope("network", "网络服务")] -public partial class NetworkService { - +public partial class NetworkService +{ + private static HttpCacheRepository _repo = new(Path.Combine(Paths.Temp,"cache","cache.db"),Path.Combine(Paths.Temp,"cache")); + private static ServiceProvider? _provider; private static IHttpClientFactory? _factory; [LifecycleStart] private static void _Start() - { + { _repo.Initialize(); var services = new ServiceCollection(); services.AddHttpClient("default") .ConfigurePrimaryHttpMessageHandler(() => new SocketsHttpHandler @@ -35,6 +39,20 @@ private static void _Start() : null } ); + services.AddHttpClient("cache").ConfigurePrimaryHttpMessageHandler(() => new HttpCacheHandler( + new SocketsHttpHandler + { + UseProxy = true, + UseCookies = false, + AutomaticDecompression = DecompressionMethods.All, + Proxy = HttpProxyManager.Instance, + AllowAutoRedirect = true, + MaxAutomaticRedirections = 20, + ConnectCallback = Config.Network.EnableDoH + ? HostConnectionHandler.Instance.GetConnectionAsync + : null + },_repo)); + _provider = services.BuildServiceProvider(); _factory = _provider.GetRequiredService(); diff --git a/PCL.Core/PCL.Core.csproj b/PCL.Core/PCL.Core.csproj index e07c065d0..4b17bc1e3 100644 --- a/PCL.Core/PCL.Core.csproj +++ b/PCL.Core/PCL.Core.csproj @@ -56,6 +56,7 @@ +