diff --git a/rss-generator/Class/CmnUtil.cs b/rss-generator/Class/CmnUtil.cs index 4a51ce7..9ce6898 100644 --- a/rss-generator/Class/CmnUtil.cs +++ b/rss-generator/Class/CmnUtil.cs @@ -1,4 +1,4 @@ -using AngleSharp; +using AngleSharp; using AngleSharp.Dom; using AngleSharp.Html.Parser; using PuppeteerSharp; @@ -7,15 +7,18 @@ using System.Collections.Generic; using System.Diagnostics; using System.Net; using System.Net.Http.Headers; +using System.Net.Http.Json; using System.Security.Cryptography; using System.ServiceModel.Syndication; using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; using System.Text.RegularExpressions; using System.Xml; namespace rss_generator.Class; -internal class CmnUtil +public class CmnUtil { /// /// XMLファイルの出力先パスを取得 @@ -36,17 +39,72 @@ internal class CmnUtil return filePath; } - public static void AddLog(string? message) + /// + /// 例外発生箇所からポストバックのイベントハンドラまでの呼び出し履歴を出力します + /// + public static void PrintCallStackToEvent(Exception ex) { - var filrDir = Path.Combine(GetExportXmlPath(""), "Log"); - if (Directory.Exists(filrDir) == false) +#if DEBUG + // 1. 現在のスレッドの実行スタック全体を取得 (PDB情報を含む) + StackTrace st = new StackTrace(true); + StackFrame[] frames = st.GetFrames(); + + if (frames == null || frames.Length == 0) return; + + Debug.Print("================================================================================"); + Debug.Print(string.Format("[例外種別] {0}", ex != null ? ex.GetType().FullName : "Unknown Exception")); + Debug.Print(string.Format("[エラー内容] {0}", ex != null ? ex.Message.Trim() : "なし")); + Debug.Print("--------------------------------------------------------------------------------"); + Debug.Print("【呼び出し経路】"); + + List userFrames = new List(); + + foreach (StackFrame frame in frames) { - Directory.CreateDirectory(filrDir); + var method = frame.GetMethod(); + if (method == null) continue; + + var declaringType = method.DeclaringType; + var typeFullName = (declaringType != null ? declaringType.FullName : "") ?? ""; + + // ASP.NET 内部のパイプラインに到達したら、それより上位(IIS内部)は除外して打ち切り + if (typeFullName.StartsWith("System.Web.") || typeFullName.StartsWith("System.Web.UI.")) + { + break; + } + + userFrames.Add(frame); } - var fileName = $"{DateTime.Now:yyyy-MM-dd}.log"; - var filePath = Path.Combine(filrDir, fileName); - using var sw = File.AppendText(filePath); - sw.WriteLine(message); + + // 時系列順に並び替える + userFrames.Reverse(); + + for (int i = 0; i < userFrames.Count; i++) + { + StackFrame frame = userFrames[i]; + var method = frame.GetMethod(); + var declaringType = method != null ? method.DeclaringType : null; + string className = declaringType != null ? declaringType.Name : "Unknown"; + string methodName = method != null ? method.Name : ""; + if (methodName == "PrintCallStackToEvent") + { + continue; + } + + var fileName = frame.GetFileName(); + int lineNumber = frame.GetFileLineNumber(); + string fileInfo = ""; + + if (!string.IsNullOrEmpty(fileName) && lineNumber > 0) + { + fileInfo = string.Format(" ({0} : {1} 行目)", Path.GetFileName(fileName), lineNumber); + } + + Debug.Print($"-> {className}.{methodName}{fileInfo}"); + } + + Debug.Print("================================================================================"); +#endif } /// @@ -103,6 +161,7 @@ internal class CmnUtil { // 404の場合などのログをここに書くことも可能 //AddLog($"[HTTP Error] Status Code: {response.StatusCode} for {url}"); + LogManager.AddLog($"[HTTP Error] Status Code: {response.StatusCode} for {url}"); return null; } @@ -112,7 +171,7 @@ internal class CmnUtil catch (HttpRequestException ex) { // ネットワーク不通やタイムアウトなどの例外処理 - AddLog($"[Network Error] {ex.Message}"); + LogManager.AddLog($"[Network Error] {ex.Message}"); return null; } catch (Exception) @@ -122,25 +181,33 @@ internal class CmnUtil } /// - /// IDocumentの取得 + /// 直接HTTP通信により対象URLのHTMLを取得し、AngleSharpのIDocumentを生成して返します。 + /// ストリーム直接パース、並列アクセスセーフ、メモリリーク防止、高速化を施しています。 /// - /// - /// - /// - public static async Task GetDocument(HttpClient client, string pageURL) + /// HttpClient インスタンス + /// 取得対象のWebページURL + /// キャッシュ回避用パラメータ(_tutc)を付与するかどうか(デフォルト: true) + /// キャンセレーショントークン + /// AngleSharpのIDocument + public static async Task GetDocument( + HttpClient client, + string pageURL, + bool useCacheBuster = true, + CancellationToken cancellationToken = default) { Console.WriteLine($"Fetching: {pageURL}"); - // 1. User-Agent が未設定なら付与 - if (!client.DefaultRequestHeaders.UserAgent.Any()) + // 1. リクエストURLの構築(キャッシュ回避パラメータ付与) + string requestUrl = pageURL; + if (useCacheBuster) { - client.DefaultRequestHeaders.UserAgent.ParseAdd("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"); + string separator = pageURL.Contains('?') ? "&" : "?"; + requestUrl = $"{pageURL}{separator}_tutc={DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()}"; } - // 2. キャッシュ無効化ヘッダーを付与したリクエストを生成 - string separator = pageURL.Contains('?') ? "&" : "?"; - string cacheBusterUrl = $"{pageURL}{separator}_tutc={DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()}"; - using var request = new HttpRequestMessage(HttpMethod.Get, cacheBusterUrl); + using var request = new HttpRequestMessage(HttpMethod.Get, requestUrl); + + // 2. キャッシュ制御ヘッダー request.Headers.CacheControl = new CacheControlHeaderValue { NoCache = true, @@ -149,26 +216,256 @@ internal class CmnUtil }; request.Headers.Pragma.ParseAdd("no-cache"); - // 3. HTMLのダウンロード - using var response = await client.SendAsync(request); - response.EnsureSuccessStatusCode(); - string htmlContent = await response.Content.ReadAsStringAsync(); - - if (string.IsNullOrWhiteSpace(htmlContent)) + // 3. User-Agent(クライアント側に未設定の場合のみリクエスト単位で設定し、共有clientの競合を防止) + if (!client.DefaultRequestHeaders.UserAgent.Any() && !request.Headers.UserAgent.Any()) { - Console.WriteLine("Warning: 取得したHTMLが空です。"); + request.Headers.UserAgent.ParseAdd("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"); } - // 4. AngleSharpのコンテキスト生成(メモリ内パース) + // 4. Accept ヘッダー + if (!client.DefaultRequestHeaders.Accept.Any() && !request.Headers.Accept.Any()) + { + request.Headers.Accept.ParseAdd("text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"); + } + + // 5. ストリームによるレスポンス受信と直接パース(ResponseHeadersRead で中間バッファリングと巨大文字列生成を排除) + using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken); + response.EnsureSuccessStatusCode(); + + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken); + + // Content-Type (charset 等) を取得して AngleSharp に通知 + var contentType = response.Content.Headers.ContentType?.ToString(); + + // 6. AngleSharp の軽量コンテキスト生成とストリーム直接パース var config = Configuration.Default; var context = BrowsingContext.New(config); - // 5. アドレス(pageURL)を紐付けてパース + IDocument document = await context.OpenAsync(req => + { + req.Content(stream); + req.Address(pageURL); // 基準URLは元のpageURLに設定 + if (!string.IsNullOrEmpty(contentType)) + { + req.Header("Content-Type", contentType); + } + }, cancellationToken); + + return document; + } + + #region FlareSolverr 連携 + + /// + /// FlareSolverr 用の JSON シリアライズオプション(静的インスタンスとしてキャッシュしメモリ浪費を防止) + /// + private static readonly JsonSerializerOptions FlareSolverrJsonOptions = new() + { + PropertyNameCaseInsensitive = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull + }; + + /// + /// FlareSolverr 経由で対象 URL の HTML を取得し、AngleSharp の IDocument を生成して返します。 + /// メモリ割り当ての最小化およびストリーム直接デシリアライズにより、メモリリーク防止と高速化を行っています。 + /// + /// HttpClient インスタンス + /// 取得対象のWebページURL + /// FlareSolverrのエンドポイントURL(デフォルト: https://udukipc-flaresolverr.armonica-vps.work/v1) + /// FlareSolverr側の最大タイムアウトミリ秒(デフォルト: 60000ms) + /// セッションID(セッションを再利用して高速化する場合に指定) + /// AngleSharpのIDocument + public static async Task GetDocumentFlareSolverr(HttpClient client, string pageURL, string flareSolverrEndpoint = "https://udukipc-flaresolverr.armonica-vps.work/v1", int maxTimeout = 60000, string? session = null) + { + Console.WriteLine($"Fetching via FlareSolverr: {pageURL}"); + + var payload = new FlareSolverrRequest + { + Cmd = "request.get", + Url = pageURL, + MaxTimeout = maxTimeout, + Session = session + }; + + using var request = new HttpRequestMessage(HttpMethod.Post, flareSolverrEndpoint) + { + Content = JsonContent.Create(payload, options: FlareSolverrJsonOptions) + }; + + // レスポンスヘッダー受信時点で制御を戻し、ストリームから直接デシリアライズしてメモリ消費を抑制 + using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead); + try + { + response.EnsureSuccessStatusCode(); + } + catch (Exception ex) + { + LogManager.AddLog($"[エラー] FlareSolverr {pageURL} {ex.Message}"); + PrintCallStackToEvent(ex); + throw; + } + + using var responseStream = await response.Content.ReadAsStreamAsync(); + var flareResponse = await JsonSerializer.DeserializeAsync(responseStream, FlareSolverrJsonOptions); + + if (flareResponse == null) + { + throw new InvalidOperationException($"FlareSolverr からのレスポンスのデシリアライズに失敗しました: {pageURL}"); + } + + if (!string.Equals(flareResponse.Status, "ok", StringComparison.OrdinalIgnoreCase)) + { + string errorMsg = $"FlareSolverr エラー (Status: {flareResponse.Status}, Message: {flareResponse.Message}): {pageURL}"; + LogManager.AddLog($"[エラー] {errorMsg}"); + throw new InvalidOperationException(errorMsg); + } + + string htmlContent = flareResponse.Solution?.Response ?? string.Empty; + + if (string.IsNullOrWhiteSpace(htmlContent)) + { + Console.WriteLine($"Warning: FlareSolverr から取得した HTML が空です ({pageURL})"); + } + + // AngleSharp の軽量コンテキスト生成とメモリ内パース + var config = Configuration.Default; + var context = BrowsingContext.New(config); IDocument document = await context.OpenAsync(req => req.Content(htmlContent).Address(pageURL)); return document; } + /// + /// FlareSolverr の新規セッションを作成します。セッションを再利用することで2回目以降のリクエストが大幅に高速化します。 + /// + /// HttpClient インスタンス + /// セッションID(省略時はFlareSolverr側で自動生成) + /// FlareSolverrのエンドポイントURL + /// 作成されたセッションID + public static async Task CreateFlareSolverrSession(HttpClient client, string? session = null, string flareSolverrEndpoint = "https://udukipc-flaresolverr.armonica-vps.work/v1") + { + var payload = new FlareSolverrRequest + { + Cmd = "sessions.create", + Session = session + }; + + using var request = new HttpRequestMessage(HttpMethod.Post, flareSolverrEndpoint) + { + Content = JsonContent.Create(payload, options: FlareSolverrJsonOptions) + }; + + using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead); + response.EnsureSuccessStatusCode(); + + using var responseStream = await response.Content.ReadAsStreamAsync(); + var flareResponse = await JsonSerializer.DeserializeAsync(responseStream, FlareSolverrJsonOptions); + + return flareResponse?.Session; + } + + /// + /// FlareSolverr のセッションを破棄します。 + /// + /// HttpClient インスタンス + /// 破棄対象のセッションID + /// FlareSolverrのエンドポイントURL + /// 破棄が成功したかどうか + public static async Task DestroyFlareSolverrSession( + HttpClient client, + string session, + string flareSolverrEndpoint = "https://udukipc-flaresolverr.armonica-vps.work/v1") + { + if (string.IsNullOrWhiteSpace(session)) return false; + + var payload = new FlareSolverrRequest + { + Cmd = "sessions.destroy", + Session = session + }; + + using var request = new HttpRequestMessage(HttpMethod.Post, flareSolverrEndpoint) + { + Content = JsonContent.Create(payload, options: FlareSolverrJsonOptions) + }; + + using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead); + response.EnsureSuccessStatusCode(); + + using var responseStream = await response.Content.ReadAsStreamAsync(); + var flareResponse = await JsonSerializer.DeserializeAsync(responseStream, FlareSolverrJsonOptions); + + return string.Equals(flareResponse?.Status, "ok", StringComparison.OrdinalIgnoreCase); + } + + /// + /// FlareSolverr リクエスト用 DTO + /// + private sealed class FlareSolverrRequest + { + [JsonPropertyName("cmd")] + public string Cmd { get; set; } = "request.get"; + + [JsonPropertyName("url")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Url { get; set; } + + [JsonPropertyName("maxTimeout")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? MaxTimeout { get; set; } + + [JsonPropertyName("session")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Session { get; set; } + } + + /// + /// FlareSolverr レスポンス用 DTO + /// + private sealed class FlareSolverrResponse + { + [JsonPropertyName("status")] + public string Status { get; set; } = string.Empty; + + [JsonPropertyName("message")] + public string Message { get; set; } = string.Empty; + + [JsonPropertyName("session")] + public string? Session { get; set; } + + [JsonPropertyName("solution")] + public FlareSolverrSolution? Solution { get; set; } + + [JsonPropertyName("startTimestamp")] + public long? StartTimestamp { get; set; } + + [JsonPropertyName("endTimestamp")] + public long? EndTimestamp { get; set; } + + [JsonPropertyName("version")] + public string? Version { get; set; } + } + + /// + /// FlareSolverr 解決結果用 DTO + /// + private sealed class FlareSolverrSolution + { + [JsonPropertyName("url")] + public string? Url { get; set; } + + [JsonPropertyName("status")] + public int Status { get; set; } + + [JsonPropertyName("response")] + public string? Response { get; set; } + + [JsonPropertyName("userAgent")] + public string? UserAgent { get; set; } + } + + #endregion + public static async Task> GetFeedItems(HttpClient client, string url) { try @@ -181,7 +478,7 @@ internal class CmnUtil } catch (Exception ex) { - AddLog($"[エラー] {url} の取得に失敗: {ex.Message}"); + LogManager.AddLog($"[エラー] {url} の取得に失敗: {ex.Message}"); return Enumerable.Empty(); } } diff --git a/rss-generator/Class/RSS/Danbooru/rank.cs b/rss-generator/Class/RSS/Danbooru/rank.cs index 2b14be7..9f6f3e3 100644 --- a/rss-generator/Class/RSS/Danbooru/rank.cs +++ b/rss-generator/Class/RSS/Danbooru/rank.cs @@ -1,4 +1,4 @@ -using AngleSharp.Dom; +using AngleSharp.Dom; using AngleSharp.Html.Parser; using System; using System.Collections.Generic; @@ -147,9 +147,7 @@ internal class rank body.Append(originalSummary); if (link != null) { - var task = CmnUtil.GetDocument(client, link.ToString()); - task.Wait(); - var docContents = task.Result; + var docContents = await CmnUtil.GetDocument(client, link.ToString()); var imgElements = docContents.QuerySelectorAll("#post-hentai img"); foreach (var img in imgElements) { diff --git a/rss-generator/Class/RSS/aurahentai.cs b/rss-generator/Class/RSS/aurahentai.cs index 3f00474..e6aea58 100644 --- a/rss-generator/Class/RSS/aurahentai.cs +++ b/rss-generator/Class/RSS/aurahentai.cs @@ -1,10 +1,11 @@ -using AngleSharp.Dom; +using AngleSharp.Dom; using AngleSharp.Html.Dom; using System; using System.Collections.Generic; using System.Reflection.Metadata; using System.ServiceModel.Syndication; using System.Text; +using System.Text.Json; using System.Text.RegularExpressions; using System.Xml.Linq; using WebDriverBiDi.Script; @@ -22,17 +23,27 @@ internal class aurahentai var items = new List(); try { - // 複数ページをまとめる場合に先に取得する - var docList = new List(); - for (int pageId = 1; pageId <= 5; pageId++) + // セッションを作成して複数ページおよび詳細ページ取得を高速化 + string? sessionId = await CmnUtil.CreateFlareSolverrSession(client); + try { - query = $"{pageId}"; - var pageURL = $"{baseURL}?page={query}"; - var document = await CmnUtil.GetDocument(client, pageURL); - var newItems = await CreateFeedItem(client, document); - foreach (var newItem in newItems) + for (int pageId = 1; pageId <= 5; pageId++) { - items.Add(newItem); + query = $"{pageId}"; + var pageURL = $"{baseURL}?page={query}"; + var document = await CmnUtil.GetDocumentFlareSolverr(client, pageURL, session: sessionId); + var newItems = await CreateFeedItem(client, document, sessionId); + foreach (var newItem in newItems) + { + items.Add(newItem); + } + } + } + finally + { + if (sessionId != null) + { + await CmnUtil.DestroyFlareSolverrSession(client, sessionId); } } feed.Items = items; @@ -45,7 +56,7 @@ internal class aurahentai } } - private static async Task> CreateFeedItem(HttpClient client, IDocument docListPage) + private static async Task> CreateFeedItem(HttpClient client, IDocument docListPage, string? sessionId = null) { var items = new List(); var elements = docListPage.QuerySelectorAll("main article>a"); @@ -58,7 +69,7 @@ internal class aurahentai var contentsUri = CmnUtil.GetAbsoluteUri(docListPage.BaseUrl!, relLink)!; var linkHref = contentsUri.AbsoluteUri; - var docContents = await CmnUtil.GetDocument(client, linkHref); + var docContents = await CmnUtil.GetDocumentFlareSolverr(client, linkHref, session: sessionId); var divDetail = docContents.QuerySelector("main>div:nth-of-type(2)")!; var title = ""; @@ -82,6 +93,20 @@ internal class aurahentai var replaceTo = "https://img.aurahentai.com/g/"; imgUrl = $"{linkHref.Replace(replaceFr, replaceTo)}/001.webp"; } + var pubDate = new DateTime(); + { + var pubDateStr = GetDatePublished(docContents); + // 3. DateTimeOffset型としてパース(時差情報を維持するため) + if (DateTimeOffset.TryParse(pubDateStr, out DateTimeOffset parsedOffset)) + { + // 4. 日本時間(UTC+9時間)のオフセットに変換 + DateTimeOffset jstOffset = parsedOffset.ToOffset(TimeSpan.FromHours(9)); + + // 5. DateTime型(KindはUnspecifiedまたはLocal扱い)として戻す + pubDate = jstOffset.DateTime; + } + + } var body = new StringBuilder(); { @@ -93,10 +118,88 @@ internal class aurahentai item.Summary = new TextSyndicationContent(body.ToString(), TextSyndicationContentKind.Html); // ID item.Id = CmnUtil.CreateGuid($"{linkHref}"); + // Date + item.PublishDate = pubDate; + item.LastUpdatedTime = pubDate; items.Add(item); } return items; } + + /// + /// IDocument 内の複数ある JSON-LD から、@type が "ImageGallery" の datePublished を取得します。 + /// + /// AngleSharp の IDocument + /// 取得した datePublished 文字列(見つからない場合は null) + private static string? GetDatePublished(IDocument document) + { + // type="application/ld+json" を持つすべての script タグを取得 + var scriptElements = document.QuerySelectorAll("script[type='application/ld+json']"); + + foreach (var script in scriptElements) + { + string jsonText = script.TextContent?.Trim() ?? string.Empty; + if (string.IsNullOrEmpty(jsonText)) + { + continue; + } + + try + { + // メモリ割り当てを最小限に抑えるため JsonDocument を利用 + using var jsonDoc = JsonDocument.Parse(jsonText); + var root = jsonDoc.RootElement; + + // 単一オブジェクトの場合 + if (root.ValueKind == JsonValueKind.Object) + { + if (TryExtractDate(root, out var date)) + { + return date; + } + } + // JSON-LD が配列([{...}, {...}])で定義されている場合 + else if (root.ValueKind == JsonValueKind.Array) + { + foreach (var item in root.EnumerateArray()) + { + if (item.ValueKind == JsonValueKind.Object && TryExtractDate(item, out var date)) + { + return date; + } + } + } + } + catch (JsonException) + { + // 不正な JSON 文字列が含まれていた場合は次のタグへスキップ + continue; + } + } + + return null; + } + + /// + /// @type が ImageGallery か判定し、datePublished の値を取り出します。 + /// + private static bool TryExtractDate(JsonElement element, out string? datePublished) + { + datePublished = null; + + // @type が "ImageGallery" かどうか確認 + if (element.TryGetProperty("@type", out var typeProp) && + string.Equals(typeProp.GetString(), "ImageGallery", StringComparison.OrdinalIgnoreCase)) + { + if (element.TryGetProperty("datePublished", out var dateProp)) + { + datePublished = dateProp.GetString(); + return true; + } + } + + return false; + } } diff --git a/rss-generator/Class/RSS/hitomi_la/index_japanese.cs b/rss-generator/Class/RSS/hitomi_la/index_japanese.cs index e6c7e82..590740b 100644 --- a/rss-generator/Class/RSS/hitomi_la/index_japanese.cs +++ b/rss-generator/Class/RSS/hitomi_la/index_japanese.cs @@ -1,4 +1,4 @@ -using AngleSharp.Html.Dom; +using AngleSharp.Html.Dom; using AngleSharp.Html.Parser; using System; using System.Collections.Generic; @@ -27,8 +27,8 @@ internal class index_japanese var results = await Task.WhenAll(tasks); var allItems = results.SelectMany(items => items).ToList(); - // 簡潔なRSSに整形 - var simplifiedItems = allItems.OrderByDescending(x => x.PublishDate).Select(originalItem => + // 簡潔なRSSに非同期で整形 + var itemTasks = allItems.OrderByDescending(x => x.PublishDate).Select(async originalItem => { List getAddCategory() { @@ -134,9 +134,7 @@ internal class index_japanese body.Append(originalSummary); if (link != null) { - var task = CmnUtil.GetDocument(client, link.ToString()); - task.Wait(); - var docContents = task.Result; + var docContents = await CmnUtil.GetDocument(client, link.ToString()); var imgElements = docContents.QuerySelectorAll("#post-hentai img"); foreach (var img in imgElements) { @@ -155,12 +153,15 @@ internal class index_japanese newItem.Summary = new TextSyndicationContent(body.ToString(), TextSyndicationContentKind.Html); return newItem; - }); + }).ToList(); + + var simplifiedItems = await Task.WhenAll(itemTasks); + // 新しい統合フィードを作成 var mergedFeed = new SyndicationFeed( - "momon-ga.com", + "hitomi.la", "", - new Uri("https://momon-ga.com/magazine/"), + new Uri("https://hitomi.la/index-japanese.html"), simplifiedItems ); diff --git a/rss-generator/Class/RSS/missav.cs b/rss-generator/Class/RSS/missav.cs index 83b7f60..46d5451 100644 --- a/rss-generator/Class/RSS/missav.cs +++ b/rss-generator/Class/RSS/missav.cs @@ -1,4 +1,4 @@ -using AngleSharp; +using AngleSharp; using AngleSharp.Dom; using NpgsqlTypes; using System; @@ -18,14 +18,24 @@ internal static class missav var items = new List(); try { - // 複数ページをまとめる場合に先に取得する + // セッションを作成して複数ページ取得を高速化 + string? sessionId = await CmnUtil.CreateFlareSolverrSession(client); var docList = new List(); - for (int pageId = 1; pageId <= 2; pageId++) + try { - var pageURL = $"https://missav.ai/ja/new{(pageId > 1 ? $"?page={pageId}" : "")}"; - var rssUrl = $"https://freshrss.armonica-vps.work/api/misc.php/FreshRss FlareSolverr?raw=1&feed={pageURL}"; - var document = await CmnUtil.GetDocument(client, rssUrl); - docList.Add(document); + for (int pageId = 1; pageId <= 2; pageId++) + { + var pageURL = $"https://missav.ai/ja/new{(pageId > 1 ? $"?page={pageId}" : "")}"; + var document = await CmnUtil.GetDocumentFlareSolverr(client, pageURL, session: sessionId); + docList.Add(document); + } + } + finally + { + if (sessionId != null) + { + await CmnUtil.DestroyFlareSolverrSession(client, sessionId); + } } var linkList = new List(); diff --git a/rss-generator/Class/RSS/missav123.cs b/rss-generator/Class/RSS/missav123.cs index b5f0bf2..721acdf 100644 --- a/rss-generator/Class/RSS/missav123.cs +++ b/rss-generator/Class/RSS/missav123.cs @@ -1,4 +1,4 @@ -using AngleSharp.Dom; +using AngleSharp.Dom; using System; using System.Collections.Generic; using System.ServiceModel.Syndication; @@ -21,7 +21,6 @@ internal class missav123 for (int pageId = 1; pageId <= 2; pageId++) { var pageURL = $"{pageMaster}/ja/recent{(pageId > 1 ? $"?page={pageId}" : "")}"; - var rssUrl = $"https://freshrss.armonica-vps.work/api/misc.php/FreshRss FlareSolverr?raw=1&feed={pageURL}"; var document = await CmnUtil.GetDocument(client, pageURL); docList.Add(document); } diff --git a/rss-generator/Class/RSS/momonga_fanzine.cs b/rss-generator/Class/RSS/momonga_fanzine.cs index 5df628c..ed55ca1 100644 --- a/rss-generator/Class/RSS/momonga_fanzine.cs +++ b/rss-generator/Class/RSS/momonga_fanzine.cs @@ -1,4 +1,4 @@ -using AngleSharp.Html.Parser; +using AngleSharp.Html.Parser; using System.ServiceModel.Syndication; using System.Text; using System.Text.RegularExpressions; @@ -24,8 +24,8 @@ internal class momonga_fanzine var results = await Task.WhenAll(tasks); var allItems = results.SelectMany(items => items).ToList(); - // 簡潔なRSSに整形 - var simplifiedItems = allItems.OrderByDescending(x => x.PublishDate).Select(originalItem => + // 簡潔なRSSに非同期で整形 + var itemTasks = allItems.OrderByDescending(x => x.PublishDate).Select(async originalItem => { List getAddCategory() { @@ -48,38 +48,26 @@ internal class momonga_fanzine // ---------------------------------------------------- // 正規表現で各項目の値を抽出する // ---------------------------------------------------- - // パターン解説: - // 【[^】]+】 : 「【」で始まり、「】以外の文字」が続き、「】」で終わる文字列 (例: 【パロディ】) - // ([^【]*) : その後、「【」以外の文字が続く限り取得する (Group1としてキャプチャ) string pattern = @"【[^】]+】([^【]*)"; var matches = Regex.Matches(targetText, pattern); foreach (Match match in matches) { - // match.Groups[1].Value には "パロ1, パロ 2 " などの値が入ります - // (項目の中身が空の場合は空文字になります) string valuesStr = match.Groups[1].Value; if (string.IsNullOrWhiteSpace(valuesStr)) { - continue; // 中身が空の場合はスキップ + continue; } - // ---------------------------------------------------- - // 3. カンマで分割し、前後の空白を除去してカテゴリに追加 - // ---------------------------------------------------- - // ※もし全角カンマ「、」も考慮したい場合は new[] { ',', '、' } と記述します string[] splitValues = valuesStr.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries); foreach (var val in splitValues) { - // 前後の半角・全角スペースを取り除く string categoryName = val.Trim(); - // 空でなければカテゴリとして追加 if (!string.IsNullOrEmpty(categoryName)) { - // domain属性なし(前回回答の手法)でカテゴリを追加 result.Add(new SyndicationCategory(categoryName)); } } @@ -90,14 +78,9 @@ internal class momonga_fanzine var newItem = new SyndicationItem() { - // タイトルは TextSyndicationContent でラップする必要があります - Title = new TextSyndicationContent(originalItem.Title?.Text ?? "No Title") - , Id = CmnUtil.CreateGuid(originalItem.Id) - , PublishDate = originalItem.PublishDate - , LastUpdatedTime = originalItem.LastUpdatedTime - // 元のコンテンツをそのまま引き継ぐ - // , Content = originalItem.Content - // , Summary = originalItem.Summary + Title = new TextSyndicationContent(originalItem.Title?.Text ?? "No Title"), + PublishDate = originalItem.PublishDate, + LastUpdatedTime = originalItem.LastUpdatedTime }; // リンクを引き継ぐ @@ -122,14 +105,15 @@ internal class momonga_fanzine newItem.Authors.Add(author); } + // ID + newItem.Id = CmnUtil.CreateGuid($"{link}"); + var body = new StringBuilder(); var originalSummary = originalItem.Summary.Text; body.Append(originalSummary); if (link != null) { - var task = CmnUtil.GetDocument(client, link.ToString()); - task.Wait(); - var docContents = task.Result; + var docContents = await CmnUtil.GetDocument(client, link.ToString()); var imgElements = docContents.QuerySelectorAll("#post-hentai img"); foreach (var img in imgElements) { @@ -156,7 +140,10 @@ internal class momonga_fanzine newItem.Summary = new TextSyndicationContent(body.ToString(), TextSyndicationContentKind.Html); return newItem; - }); + }).ToList(); + + var simplifiedItems = await Task.WhenAll(itemTasks); + // 新しい統合フィードを作成 var mergedFeed = new SyndicationFeed( "momon-ga.com", diff --git a/rss-generator/Class/RSS/momonga_magazine.cs b/rss-generator/Class/RSS/momonga_magazine.cs index d2a47ba..0665fe7 100644 --- a/rss-generator/Class/RSS/momonga_magazine.cs +++ b/rss-generator/Class/RSS/momonga_magazine.cs @@ -1,4 +1,4 @@ -using AngleSharp.Html.Parser; +using AngleSharp.Html.Parser; using System.ServiceModel.Syndication; using System.Text; using System.Text.RegularExpressions; @@ -24,8 +24,8 @@ internal class momonga_magazine var results = await Task.WhenAll(tasks); var allItems = results.SelectMany(items => items).ToList(); - // 簡潔なRSSに整形 - var simplifiedItems = allItems.OrderByDescending(x => x.PublishDate).Select(originalItem => + // 簡潔なRSSに非同期で整形 + var itemTasks = allItems.OrderByDescending(x => x.PublishDate).Select(async originalItem => { List getAddCategory() { @@ -48,38 +48,26 @@ internal class momonga_magazine // ---------------------------------------------------- // 正規表現で各項目の値を抽出する // ---------------------------------------------------- - // パターン解説: - // 【[^】]+】 : 「【」で始まり、「】以外の文字」が続き、「】」で終わる文字列 (例: 【パロディ】) - // ([^【]*) : その後、「【」以外の文字が続く限り取得する (Group1としてキャプチャ) string pattern = @"【[^】]+】([^【]*)"; var matches = Regex.Matches(targetText, pattern); foreach (Match match in matches) { - // match.Groups[1].Value には "パロ1, パロ 2 " などの値が入ります - // (項目の中身が空の場合は空文字になります) string valuesStr = match.Groups[1].Value; if (string.IsNullOrWhiteSpace(valuesStr)) { - continue; // 中身が空の場合はスキップ + continue; } - // ---------------------------------------------------- - // 3. カンマで分割し、前後の空白を除去してカテゴリに追加 - // ---------------------------------------------------- - // ※もし全角カンマ「、」も考慮したい場合は new[] { ',', '、' } と記述します string[] splitValues = valuesStr.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries); foreach (var val in splitValues) { - // 前後の半角・全角スペースを取り除く string categoryName = val.Trim(); - // 空でなければカテゴリとして追加 if (!string.IsNullOrEmpty(categoryName)) { - // domain属性なし(前回回答の手法)でカテゴリを追加 result.Add(new SyndicationCategory(categoryName)); } } @@ -90,15 +78,9 @@ internal class momonga_magazine var newItem = new SyndicationItem() { - // タイトルは TextSyndicationContent でラップする必要があります - Title = new TextSyndicationContent(originalItem.Title?.Text ?? "No Title") - , Id = CmnUtil.CreateGuid(originalItem.Id) - , PublishDate = originalItem.PublishDate - , LastUpdatedTime = originalItem.LastUpdatedTime - - // 元のコンテンツをそのまま引き継ぐ - // , Content = originalItem.Content - // , Summary = originalItem.Summary + Title = new TextSyndicationContent(originalItem.Title?.Text ?? "No Title"), + PublishDate = originalItem.PublishDate, + LastUpdatedTime = originalItem.LastUpdatedTime }; // リンクを引き継ぐ @@ -123,14 +105,15 @@ internal class momonga_magazine newItem.Authors.Add(author); } + // ID + newItem.Id = CmnUtil.CreateGuid($"{link}"); + var body = new StringBuilder(); var originalSummary = originalItem.Summary.Text; body.Append(originalSummary); if (link != null) { - var task = CmnUtil.GetDocument(client, link.ToString()); - task.Wait(); - var docContents = task.Result; + var docContents = await CmnUtil.GetDocument(client, link.ToString()); var imgElements = docContents.QuerySelectorAll("#post-hentai img"); foreach (var img in imgElements) { @@ -140,7 +123,15 @@ internal class momonga_magazine // 相対URLの場合も考慮して、絶対URLに解決する if (Uri.TryCreate(link, src, out var absoluteUri)) { + string encoded = ToBase64Url(link.AbsoluteUri); + string downloadUrl = $"https://apicien.armonica-vps.work/fanzine/{encoded}"; + + body.Append($"
"); + body.Append($"[Download]"); + body.Append($"
"); + body.Append($"
"); body.Append($""); + body.Append($"
"); break; } } @@ -149,7 +140,10 @@ internal class momonga_magazine newItem.Summary = new TextSyndicationContent(body.ToString(), TextSyndicationContentKind.Html); return newItem; - }); + }).ToList(); + + var simplifiedItems = await Task.WhenAll(itemTasks); + // 新しい統合フィードを作成 var mergedFeed = new SyndicationFeed( "momon-ga.com", @@ -165,4 +159,17 @@ internal class momonga_magazine LogManager.AddLog($"[エラー] momonga-magazine:{ex.Message}"); } } + + /// + /// 文字列をURLセーフなBase64(Base64URL)に変換します (RFC 4648 準拠) + /// + static string ToBase64Url(string plainText) + { + byte[] bytes = Encoding.UTF8.GetBytes(plainText); + return Convert.ToBase64String(bytes) + .Replace('+', '-') // '+' を '-' に置換 + .Replace('/', '_') // '/' を '_' に置換 + .TrimEnd('='); // 末尾のパディング '=' を削除 + } + } diff --git a/rss-generator/Class/RSS/npb_6.cs b/rss-generator/Class/RSS/npb_6.cs index 96626fa..339014a 100644 --- a/rss-generator/Class/RSS/npb_6.cs +++ b/rss-generator/Class/RSS/npb_6.cs @@ -1,4 +1,5 @@ -using AngleSharp.Dom; +using AngleSharp.Dom; +using PuppeteerSharp; using System; using System.Collections.Generic; using System.ServiceModel.Syndication; @@ -15,8 +16,19 @@ internal class npb_6 try { var pageURL = $"https://baseball.yahoo.co.jp/npb/teams/6/top"; - var rssUrl = $"https://freshrss.armonica-vps.work/api/misc.php/FreshRss FlareSolverr?raw=1&feed={pageURL}"; - var document = await CmnUtil.GetDocument(client, rssUrl); + IDocument document; + string? sessionId = await CmnUtil.CreateFlareSolverrSession(client); + try + { + document = await CmnUtil.GetDocumentFlareSolverr(client, pageURL, session: sessionId); + } + finally + { + if (sessionId != null) + { + await CmnUtil.DestroyFlareSolverrSession(client, sessionId); + } + } var linkList = new List(); if (document != null) diff --git a/rss-generator/Program.cs b/rss-generator/Program.cs index 87323cb..9ee665d 100644 --- a/rss-generator/Program.cs +++ b/rss-generator/Program.cs @@ -6,6 +6,27 @@ using rss_generator.Class.RSS.hitomi_la; using rss_generator.Class.RSS.X; using System.Diagnostics; + +// 1. 同期スレッド等の未処理例外 +AppDomain.CurrentDomain.UnhandledException += (sender, e) => +{ + var ex = e.ExceptionObject as Exception; + if (ex != null) + { + CmnUtil.PrintCallStackToEvent(ex); + } +}; + +// 2. async/await や Task 内で放置された未処理例外 (GC回収時に発火) +TaskScheduler.UnobservedTaskException += (sender, e) => +{ + if (e != null) + { + CmnUtil.PrintCallStackToEvent(e.Exception); + e.SetObserved(); // クラッシュを防止する場合に指定 + } +}; + var services = new ServiceCollection(); // 2. IHttpClientFactory の登録とクッキーの有効化設定(名前付きクライアント) services.AddHttpClient("webClient"); @@ -57,7 +78,7 @@ while (true) tasks.Add(momonga_fanzine.CreateRSSAsync(client)); tasks.Add(momonga_magazine.CreateRSSAsync(client)); } - tasks.Add(Youtube.CreateRSSAsync(client)); + //tasks.Add(Youtube.CreateRSSAsync(client)); tasks.Add(x_list.CreateRSSAsync(client)); tasks.Add(javtwi.CreateRSSAsync(client)); tasks.Add(kimochiinfo.CreateRSSAsync(client)); @@ -72,13 +93,13 @@ while (true) await Task.WhenAll(tasks); sw.Stop(); - LogManager.AddLog($"[情報] RSS生成完了 実行時間:{sw.Elapsed.TotalSeconds}秒"); + LogManager.AddLog($"[情報] RSS生成完了 実行時間:{sw.Elapsed.TotalSeconds:0.################}秒"); sw.Restart(); // /bin/bash /usr/local/bin/update_rss.sh #if DEBUG sw.Stop(); - LogManager.AddLog($"[情報] 実行完了 実行時間:{sw.Elapsed.TotalSeconds}秒"); + LogManager.AddLog($"[情報] デバッグ完了"); await LogManager.FlushAndCloseAsync(); break; @@ -87,7 +108,7 @@ while (true) await CmnUtil.RunCommandAsync("/bin/bash", "/usr/local/bin/update_rss.sh", ""); sw.Stop(); - LogManager.AddLog($"[情報] FreshRSS 更新完了 実行時間:{sw.Elapsed.TotalSeconds}秒"); + LogManager.AddLog($"[情報] FreshRSS 更新完了 実行時間:{sw.Elapsed.TotalSeconds:0.################}秒"); #endif }