From 0a5a612760b79ceff2a5d46f0ec4ad92136639d6 Mon Sep 17 00:00:00 2001 From: "KOUJIRI-WIN11\\koujiri" Date: Thu, 3 Sep 2026 14:22:09 +0900 Subject: [PATCH] =?UTF-8?q?aurahentai=E3=82=92=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rss-generator/Class/CmnUtil.cs | 154 ++++++++++------ rss-generator/Class/RSS/aurahentai.cs | 253 ++++++++++---------------- rss-generator/Program.cs | 11 +- 3 files changed, 203 insertions(+), 215 deletions(-) diff --git a/rss-generator/Class/CmnUtil.cs b/rss-generator/Class/CmnUtil.cs index b4479cc..426ab00 100644 --- a/rss-generator/Class/CmnUtil.cs +++ b/rss-generator/Class/CmnUtil.cs @@ -265,9 +265,14 @@ public class CmnUtil DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull }; + /// + /// FlareSolverr への同時アクセスによる過負荷・Chromium競合・クラッシュを防止するためのセマフォ(直列化) + /// + private static readonly SemaphoreSlim FlareSolverrSemaphore = new(1, 1); + /// /// FlareSolverr 経由で対象 URL の HTML を取得し、AngleSharp の IDocument を生成して返します。 - /// メモリ割り当ての最小化およびストリーム直接デシリアライズにより、メモリリーク防止と高速化を行っています。 + /// 排他制御および最大3回の自動リトライ(セッション無効化フォールバック・待機付き)を備えています。 /// /// HttpClient インスタンス /// 取得対象のWebページURL @@ -275,64 +280,107 @@ public class CmnUtil /// FlareSolverr側の最大タイムアウトミリ秒(デフォルト: 60000ms) /// セッションID(セッションを再利用して高速化する場合に指定) /// AngleSharpのIDocument - public static async Task GetDocumentFlareSolverr(HttpClient client, string pageURL, string flareSolverrEndpoint = "https://udukipc-flaresolverr.armonica-vps.work/v1", int maxTimeout = 60000, string? session = null) + public static async Task GetDocumentFlareSolverr( + HttpClient client, + string pageURL, + string flareSolverrEndpoint = "https://udukipc-flaresolverr.armonica-vps.work/v1", + int maxTimeout = 60000, + string? session = null) { - Console.WriteLine($"Fetching via FlareSolverr: {pageURL}"); - - var payload = new FlareSolverrRequest - { - Cmd = "request.get", - Url = pageURL, - MaxTimeout = maxTimeout, - Session = session - }; - - using var request = new HttpRequestMessage(HttpMethod.Post, flareSolverrEndpoint) - { - Content = JsonContent.Create(payload, options: FlareSolverrJsonOptions) - }; - - // レスポンスヘッダー受信時点で制御を戻し、ストリームから直接デシリアライズしてメモリ消費を抑制 - using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead); + await FlareSolverrSemaphore.WaitAsync(); try { - response.EnsureSuccessStatusCode(); + const int maxRetries = 3; + string? currentSession = session; + + for (int attempt = 1; attempt <= maxRetries; attempt++) + { + Console.WriteLine($"Fetching via FlareSolverr (試行 {attempt}/{maxRetries}): {pageURL}"); + + var payload = new FlareSolverrRequest + { + Cmd = "request.get", + Url = pageURL, + MaxTimeout = maxTimeout, + Session = currentSession + }; + + using var request = new HttpRequestMessage(HttpMethod.Post, flareSolverrEndpoint) + { + Content = JsonContent.Create(payload, options: FlareSolverrJsonOptions) + }; + + try + { + using var response = await client.SendAsync(request); + var responseContent = await response.Content.ReadAsStringAsync(); + + if (!response.IsSuccessStatusCode) + { + LogManager.AddLog($"[警告] FlareSolverr HTTP失敗 (Status: {(int)response.StatusCode} {response.ReasonPhrase}, URL: {pageURL}, Detail: {responseContent})"); + + // セッション不整合による500の可能性があるため、次回はセッションなしでリトライ + if (currentSession != null) + { + LogManager.AddLog($"[情報] セッション {currentSession} を解除して次回セッションなしで再試行します。"); + currentSession = null; + } + + if (attempt < maxRetries) + { + await Task.Delay(attempt * 2500); + continue; + } + + throw new HttpRequestException($"FlareSolverr サーバーエラー (Status: {(int)response.StatusCode}, Detail: {responseContent})"); + } + + var flareResponse = JsonSerializer.Deserialize(responseContent, FlareSolverrJsonOptions); + if (flareResponse == null) + { + throw new InvalidOperationException($"FlareSolverr レスポンスのデシリアライズ失敗: {pageURL}"); + } + + if (!string.Equals(flareResponse.Status, "ok", StringComparison.OrdinalIgnoreCase)) + { + string errorMsg = $"FlareSolverr エラー (Status: {flareResponse.Status}, Message: {flareResponse.Message}): {pageURL}"; + LogManager.AddLog($"[警告] {errorMsg}"); + + if (attempt < maxRetries) + { + currentSession = null; + await Task.Delay(attempt * 2500); + continue; + } + + throw new InvalidOperationException(errorMsg); + } + + string htmlContent = flareResponse.Solution?.Response ?? string.Empty; + if (string.IsNullOrWhiteSpace(htmlContent)) + { + Console.WriteLine($"Warning: FlareSolverr から取得した HTML が空です ({pageURL})"); + } + + var config = Configuration.Default; + var context = BrowsingContext.New(config); + IDocument document = await context.OpenAsync(req => req.Content(htmlContent).Address(pageURL)); + return document; + } + catch (Exception ex) when (attempt < maxRetries && ex is not OperationCanceledException) + { + LogManager.AddLog($"[警告] FlareSolverr リクエスト再試行 (試行 {attempt}/{maxRetries}, URL: {pageURL}): {ex.Message}"); + currentSession = null; + await Task.Delay(attempt * 2500); + } + } + + throw new InvalidOperationException($"FlareSolverr での取得が最大試行回数 ({maxRetries}回) を超えて失敗しました: {pageURL}"); } - catch (Exception ex) + finally { - LogManager.AddLog($"[エラー] FlareSolverr {pageURL} {ex.Message}"); - PrintCallStackToEvent(ex); - throw; + FlareSolverrSemaphore.Release(); } - - using var responseStream = await response.Content.ReadAsStreamAsync(); - var flareResponse = await JsonSerializer.DeserializeAsync(responseStream, FlareSolverrJsonOptions); - - if (flareResponse == null) - { - throw new InvalidOperationException($"FlareSolverr からのレスポンスのデシリアライズに失敗しました: {pageURL}"); - } - - if (!string.Equals(flareResponse.Status, "ok", StringComparison.OrdinalIgnoreCase)) - { - string errorMsg = $"FlareSolverr エラー (Status: {flareResponse.Status}, Message: {flareResponse.Message}): {pageURL}"; - LogManager.AddLog($"[エラー] {errorMsg}"); - throw new InvalidOperationException(errorMsg); - } - - string htmlContent = flareResponse.Solution?.Response ?? string.Empty; - - if (string.IsNullOrWhiteSpace(htmlContent)) - { - Console.WriteLine($"Warning: FlareSolverr から取得した HTML が空です ({pageURL})"); - } - - // AngleSharp の軽量コンテキスト生成とメモリ内パース - var config = Configuration.Default; - var context = BrowsingContext.New(config); - IDocument document = await context.OpenAsync(req => req.Content(htmlContent).Address(pageURL)); - - return document; } /// diff --git a/rss-generator/Class/RSS/aurahentai.cs b/rss-generator/Class/RSS/aurahentai.cs index 9a4d8b1..9cd0418 100644 --- a/rss-generator/Class/RSS/aurahentai.cs +++ b/rss-generator/Class/RSS/aurahentai.cs @@ -2,41 +2,46 @@ using AngleSharp.Dom; using AngleSharp.Html.Dom; using System; using System.Collections.Generic; -using System.Reflection.Metadata; +using System.Linq; +using System.Net.Http; using System.ServiceModel.Syndication; using System.Text; -using System.Text.Json; using System.Text.RegularExpressions; -using System.Xml.Linq; -using WebDriverBiDi.Script; +using System.Threading.Tasks; namespace rss_generator.Class.RSS; internal class aurahentai { + /// + /// aurahentai.com の日本語作品一覧から RSS フィード (aurahentai.xml) を生成します。 + /// 一覧ページから作品情報を直接パースすることで、FlareSolverr へのリクエスト数を最小化(全150件をたった5回のリクエストで取得)し、 + /// 500 Internal Server Error や過負荷を根本防止しています。 + /// + /// HttpClient インスタンス public static async Task CreateRSSAsync(HttpClient client) { var baseURL = "https://aurahentai.com/language/japanese"; - var query = ""; - - var feed = new SyndicationFeed("aurahentai.com", "", new Uri($"{baseURL}{query}")); + var feed = new SyndicationFeed("aurahentai.com", "", new Uri(baseURL)); var items = new List(); + try { - // セッションを作成して複数ページおよび詳細ページ取得を高速化 + // セッションを作成して複数ページ取得を高速化 string? sessionId = await CmnUtil.CreateFlareSolverrSession(client); try { for (int pageId = 1; pageId <= 5; pageId++) { - query = $"{pageId}"; - var pageURL = $"{baseURL}?page={query}"; + var pageURL = $"{baseURL}?page={pageId}"; var document = await CmnUtil.GetDocumentFlareSolverr(client, pageURL, session: sessionId); - var newItems = await CreateFeedItem(client, document, sessionId); - foreach (var newItem in newItems) + if (document == null) { - items.Add(newItem); + continue; } + + var newItems = CreateFeedItemsFromListPage(document, pageId); + items.AddRange(newItems); } } finally @@ -46,175 +51,119 @@ internal class aurahentai await CmnUtil.DestroyFlareSolverrSession(client, sessionId); } } - feed.Items = items; + + // ID 重複排除 + var uniqueItems = items + .GroupBy(x => x.Id) + .Select(g => g.First()) + .ToList(); + + feed.Items = uniqueItems; CmnUtil.ExportXMLFile("aurahentai.xml", feed); + LogManager.AddLog($"[情報] aurahentai.xml を正常に出力しました (記事数: {uniqueItems.Count})"); } catch (Exception ex) { - LogManager.AddLog($"[エラー] aurahentai:{ex.Message}"); + LogManager.AddLog($"[エラー] aurahentai: {ex.Message}"); } } - private static async Task> CreateFeedItem(HttpClient client, IDocument docListPage, string? sessionId = null) + /// + /// 一覧ページの IDocument から個別作品の SyndicationItem を生成します。 + /// タイトル、リンク、カバー画像、ページ数、カテゴリは一覧ページの DOM からすべて取得可能です。 + /// + /// AngleSharp の一覧ページ IDocument + /// ページ番号 + /// SyndicationItem のリスト + private static List CreateFeedItemsFromListPage(IDocument docListPage, int pageIndex) { var items = new List(); - var elements = docListPage.QuerySelectorAll("main article>a"); - var linkList = new List(); - foreach (var linkElement in elements) + var articleElements = docListPage.QuerySelectorAll("main article"); + + int itemIndex = 0; + foreach (var article in articleElements) { - var item = new SyndicationItem(); + var linkElement = article.QuerySelector("a.block, a"); + if (linkElement == null) continue; var relLink = linkElement.GetAttribute("href") ?? string.Empty; - var contentsUri = CmnUtil.GetAbsoluteUri(docListPage.BaseUrl!, relLink)!; + if (string.IsNullOrWhiteSpace(relLink)) continue; + + var contentsUri = CmnUtil.GetAbsoluteUri(docListPage.BaseUrl!, relLink); + if (contentsUri == null) continue; var linkHref = contentsUri.AbsoluteUri; + + // 1. タイトルの取得 (h3 または img alt) + var title = article.QuerySelector("h3")?.TextContent?.Trim(); + if (string.IsNullOrWhiteSpace(title)) { - item.Links.Add(new SyndicationLink(new Uri(linkHref))); + title = article.QuerySelector("img")?.GetAttribute("alt")?.Trim() ?? "No Title"; } - var docContents = await CmnUtil.GetDocumentFlareSolverr(client, linkHref, session: sessionId); - var divDetail = docContents.QuerySelector("main>div:nth-of-type(2)")!; - - var title = ""; - { - if (divDetail.QuerySelector("p:nth-of-type(1)") is IHtmlElement element) - { - title = $"{element.TextContent.Trim()}"; - item.Title = new TextSyndicationContent(title); - } - } - { - if (title == "" && divDetail.QuerySelector("h1") is IHtmlElement element) - { - title = $"{element.TextContent.Trim()}"; - item.Title = new TextSyndicationContent(title); - } - } - var imgUrl = ""; - { - var replaceFr = "https://aurahentai.com/gallery/"; - var replaceTo = "https://img.aurahentai.com/g/"; - imgUrl = $"{linkHref.Replace(replaceFr, replaceTo)}/001.webp"; - } - var pubDate = new DateTime(); - { - var pubDateStr = GetDatePublished(docContents); - // 3. DateTimeOffset型としてパース(時差情報を維持するため) - if (DateTimeOffset.TryParse(pubDateStr, out DateTimeOffset parsedOffset)) - { - // 4. 日本時間(UTC+9時間)のオフセットに変換 - DateTimeOffset jstOffset = parsedOffset.ToOffset(TimeSpan.FromHours(9)); + // 2. カバー画像 URL の決定 + var replaceFr = "https://aurahentai.com/gallery/"; + var replaceTo = "https://img.aurahentai.com/g/"; + var imgUrl = $"{linkHref.Replace(replaceFr, replaceTo)}/001.webp"; - // 5. DateTime型(KindはUnspecifiedまたはLocal扱い)として戻す - pubDate = jstOffset.DateTime; - } + // 3. カテゴリ(Doujinshi, Manga 等) + var categorySpan = article.QuerySelector("span.uppercase, span[class*='uppercase']"); + var category = categorySpan?.TextContent?.Trim() ?? string.Empty; - } + // 4. ページ数 (例: 34p) + var pageSpan = article.QuerySelectorAll("span").FirstOrDefault(s => s.TextContent.Trim().EndsWith("p", StringComparison.OrdinalIgnoreCase)); + var pageCount = pageSpan?.TextContent?.Trim() ?? string.Empty; + // 5. 日時 (一覧の掲載順序を維持するために少しずつオフセット) + var pubDate = DateTime.Now.AddMinutes(-((pageIndex - 1) * 30 + itemIndex)); + itemIndex++; + + // 6. 本文 HTML の生成 var body = new StringBuilder(); - { - // ギャラリー表示リンク - var match = Regex.Match(linkHref, @"/gallery/([^/]+)", RegexOptions.IgnoreCase); - if (match.Success) - { - string galleryId = match.Groups[1].Value; - string galleryUrl = $"https://udukipc-web.armonica-vps.work/aurahentai/{galleryId}/"; - body.Append($"
"); - body.Append($"[Gallery]"); - body.Append($"
"); - } - // ギャラリーの1枚目画像 + // ギャラリー表示リンク + var match = Regex.Match(linkHref, @"/gallery/([^/]+)", RegexOptions.IgnoreCase); + if (match.Success) + { + string galleryId = match.Groups[1].Value; + string galleryUrl = $"https://udukipc-web.armonica-vps.work/aurahentai/{galleryId}/"; body.Append($"
"); - body.Append($""); + body.Append($"[Gallery]"); body.Append($"
"); } - item.Summary = new TextSyndicationContent(body.ToString(), TextSyndicationContentKind.Html); - // ID - item.Id = CmnUtil.CreateGuid($"{linkHref}"); - // Date - item.PublishDate = pubDate; - item.LastUpdatedTime = pubDate; + // 1枚目画像 + body.Append($"
"); + body.Append($""); + body.Append($"
"); + + if (!string.IsNullOrWhiteSpace(pageCount) || !string.IsNullOrWhiteSpace(category)) + { + body.Append($"

"); + if (!string.IsNullOrWhiteSpace(category)) body.Append($"[{category}] "); + if (!string.IsNullOrWhiteSpace(pageCount)) body.Append($"ページ数: {pageCount}"); + body.Append($"

"); + } + + var item = new SyndicationItem + { + Title = new TextSyndicationContent(title), + Id = CmnUtil.CreateGuid(linkHref), + PublishDate = pubDate, + LastUpdatedTime = pubDate, + Summary = new TextSyndicationContent(body.ToString(), TextSyndicationContentKind.Html) + }; + + item.Links.Add(new SyndicationLink(new Uri(linkHref))); + + if (!string.IsNullOrWhiteSpace(category)) + { + item.Categories.Add(new SyndicationCategory(category)); + } items.Add(item); } return items; } - - /// - /// IDocument 内の複数ある JSON-LD から、@type が "ImageGallery" の datePublished を取得します。 - /// - /// AngleSharp の IDocument - /// 取得した datePublished 文字列(見つからない場合は null) - private static string? GetDatePublished(IDocument document) - { - // type="application/ld+json" を持つすべての script タグを取得 - var scriptElements = document.QuerySelectorAll("script[type='application/ld+json']"); - - foreach (var script in scriptElements) - { - string jsonText = script.TextContent?.Trim() ?? string.Empty; - if (string.IsNullOrEmpty(jsonText)) - { - continue; - } - - try - { - // メモリ割り当てを最小限に抑えるため JsonDocument を利用 - using var jsonDoc = JsonDocument.Parse(jsonText); - var root = jsonDoc.RootElement; - - // 単一オブジェクトの場合 - if (root.ValueKind == JsonValueKind.Object) - { - if (TryExtractDate(root, out var date)) - { - return date; - } - } - // JSON-LD が配列([{...}, {...}])で定義されている場合 - else if (root.ValueKind == JsonValueKind.Array) - { - foreach (var item in root.EnumerateArray()) - { - if (item.ValueKind == JsonValueKind.Object && TryExtractDate(item, out var date)) - { - return date; - } - } - } - } - catch (JsonException) - { - // 不正な JSON 文字列が含まれていた場合は次のタグへスキップ - continue; - } - } - - return null; - } - - /// - /// @type が ImageGallery か判定し、datePublished の値を取り出します。 - /// - private static bool TryExtractDate(JsonElement element, out string? datePublished) - { - datePublished = null; - - // @type が "ImageGallery" かどうか確認 - if (element.TryGetProperty("@type", out var typeProp) && - string.Equals(typeProp.GetString(), "ImageGallery", StringComparison.OrdinalIgnoreCase)) - { - if (element.TryGetProperty("datePublished", out var dateProp)) - { - datePublished = dateProp.GetString(); - return true; - } - } - - return false; - } } diff --git a/rss-generator/Program.cs b/rss-generator/Program.cs index 4acc443..9f178b6 100644 --- a/rss-generator/Program.cs +++ b/rss-generator/Program.cs @@ -80,16 +80,7 @@ while (true) tasks.Add(momonga_magazine.CreateRSSAsync(client)); } #if DEBUG - //tasks.Add(aurahentai.CreateRSSAsync(client)); - tasks.Add(pixiv_non_r18_non_ai.CreateRSSAsync(client)); - tasks.Add(pixiv_non_r18_ai.CreateRSSAsync(client)); - tasks.Add(pixiv_r18_non_ai.CreateRSSAsync(client)); - tasks.Add(pixiv_r18_ai.CreateRSSAsync(client)); - //tasks.Add(YouTubeFeedManager.CreateRSSAsync(client)); - //tasks.Add(momonga_fanzine.CreateRSSAsync(client)); - //tasks.Add(momonga_magazine.CreateRSSAsync(client)); - //tasks.Add(e_hentai_non_ai.CreateRSSAsync(client)); - //tasks.Add(Youtube.CreateRSSAsync(client)); + tasks.Add(aurahentai.CreateRSSAsync(client)); #else tasks.Add(YouTubeFeedManager.CreateRSSAsync(client));