From f31595ac9f1b9ffd6438ae8ec288e74cc5112a72 Mon Sep 17 00:00:00 2001 From: uduki Date: Fri, 28 Aug 2026 11:39:05 +0900 Subject: [PATCH] =?UTF-8?q?=E5=87=BA=E5=8A=9B=E5=86=85=E5=AE=B9=E3=81=AE?= =?UTF-8?q?=E8=A8=AD=E5=AE=9A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rss-generator/Class/CmnUtil.cs | 87 ++++++ rss-generator/Class/RSS/Danbooru/rank.cs | 369 ++++++++++++++++------- rss-generator/Class/RSS/aurahentai.cs | 3 + rss-generator/Class/RSS/javtwi.cs | 5 +- rss-generator/Program.cs | 33 +- 5 files changed, 367 insertions(+), 130 deletions(-) diff --git a/rss-generator/Class/CmnUtil.cs b/rss-generator/Class/CmnUtil.cs index 9ce6898..b4479cc 100644 --- a/rss-generator/Class/CmnUtil.cs +++ b/rss-generator/Class/CmnUtil.cs @@ -335,6 +335,93 @@ public class CmnUtil return document; } + /// + /// FlareSolverr 経由で対象 URL の RSS/Atom フィードを取得し、SyndicationItem のコレクションを返します。 + /// メモリ割り当ての最小化およびストリーム直接デシリアライズにより、メモリリーク防止と高速化を行っています。 + /// + /// HttpClient インスタンス + /// 取得対象のフィードURL + /// FlareSolverrのエンドポイントURL(デフォルト: https://udukipc-flaresolverr.armonica-vps.work/v1) + /// FlareSolverr側の最大タイムアウトミリ秒(デフォルト: 60000ms) + /// セッションID(セッションを再利用して高速化する場合に指定) + /// SyndicationItem のコレクション + public static async Task> GetFeedItemsFlareSolverr( + HttpClient client, + string url, + string flareSolverrEndpoint = "https://udukipc-flaresolverr.armonica-vps.work/v1", + int maxTimeout = 60000, + string? session = null) + { + Console.WriteLine($"Fetching Feed via FlareSolverr: {url}"); + + var payload = new FlareSolverrRequest + { + Cmd = "request.get", + Url = url, + MaxTimeout = maxTimeout, + Session = session + }; + + using var request = new HttpRequestMessage(HttpMethod.Post, flareSolverrEndpoint) + { + Content = JsonContent.Create(payload, options: FlareSolverrJsonOptions) + }; + + try + { + // レスポンスヘッダー受信時点で制御を戻し、ストリームから直接デシリアライズしてメモリ消費を抑制 + using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead); + response.EnsureSuccessStatusCode(); + + using var responseStream = await response.Content.ReadAsStreamAsync(); + var flareResponse = await JsonSerializer.DeserializeAsync(responseStream, FlareSolverrJsonOptions); + + if (flareResponse == null) + { + throw new InvalidOperationException($"FlareSolverr からのレスポンスのデシリアライズに失敗しました: {url}"); + } + + if (!string.Equals(flareResponse.Status, "ok", StringComparison.OrdinalIgnoreCase)) + { + string errorMsg = $"FlareSolverr エラー (Status: {flareResponse.Status}, Message: {flareResponse.Message}): {url}"; + LogManager.AddLog($"[エラー] {errorMsg}"); + throw new InvalidOperationException(errorMsg); + } + + string xmlContent = flareResponse.Solution?.Response ?? string.Empty; + + if (string.IsNullOrWhiteSpace(xmlContent)) + { + Console.WriteLine($"Warning: FlareSolverr から取得したフィード内容が空です ({url})"); + return Enumerable.Empty(); + } + + // FlareSolverr(Chromium)がXMLをレンダリングして
...
で返してきた場合のアンラップ処理 + if (xmlContent.TrimStart().StartsWith("(); + } + catch (Exception ex) + { + LogManager.AddLog($"[エラー] FlareSolverr フィード {url} の取得に失敗: {ex.Message}"); + PrintCallStackToEvent(ex); + return Enumerable.Empty(); + } + } + /// /// FlareSolverr の新規セッションを作成します。セッションを再利用することで2回目以降のリクエストが大幅に高速化します。 /// diff --git a/rss-generator/Class/RSS/Danbooru/rank.cs b/rss-generator/Class/RSS/Danbooru/rank.cs index 9f6f3e3..3a52b87 100644 --- a/rss-generator/Class/RSS/Danbooru/rank.cs +++ b/rss-generator/Class/RSS/Danbooru/rank.cs @@ -1,168 +1,309 @@ using AngleSharp.Dom; +using AngleSharp.Html.Dom; using AngleSharp.Html.Parser; using System; using System.Collections.Generic; +using System.Linq; +using System.Net; using System.ServiceModel.Syndication; using System.Text; -using System.Text.RegularExpressions; +using System.Threading.Tasks; namespace rss_generator.Class.RSS.Danbooru; internal class rank { + /// + /// Danbooruのランクフィードを取得し、各投稿ページから詳細情報を取得して整形したRSSを出力します。 + /// + /// HttpClient インスタンス public static async Task CreateRSSAsync(HttpClient client) { var pageMaster = "https://danbooru.donmai.us/posts.atom?tags=order%3Arank"; try { - // 簡潔なRSSに整形 - var items = await CmnUtil.GetFeedItems(client, pageMaster); - var simplifiedItems = items.Select(n => n); - foreach (var item in simplifiedItems) + // FlareSolverr セッションを作成して複数リクエストを高速化 + string? sessionId = await CmnUtil.CreateFlareSolverrSession(client); + try { - await Simplified(client, item); + // FlareSolverr 経由で Atom フィードアイテムを取得 + var items = await CmnUtil.GetFeedItemsFlareSolverr(client, pageMaster, session: sessionId); + var simplifiedItems = new List(); + + foreach (var item in items) + { + var simplified = await Simplified(client, item, sessionId); + if (simplified != null) + { + simplifiedItems.Add(simplified); + } + } + + // 新しい統合フィードを作成 + var mergedFeed = new SyndicationFeed( + "Danbooru", + "Danbooru Rank Posts", + new Uri("https://danbooru.donmai.us/posts?tags=order%3Arank"), + simplifiedItems + ); + + CmnUtil.ExportXMLFile("rank.xml", mergedFeed, "danbooru"); + } + finally + { + if (sessionId != null) + { + await CmnUtil.DestroyFlareSolverrSession(client, sessionId); + } } - - - // 新しい統合フィードを作成 - var mergedFeed = new SyndicationFeed( - "Danbooru", - "description", - new Uri("https://danbooru.donmai.us/posts?tags=order%3Arank"), - simplifiedItems - ); - - CmnUtil.ExportXMLFile("rank.xml", mergedFeed, "danbooru"); } catch (Exception ex) { LogManager.AddLog($"[エラー] danbooru:{ex.Message}"); + CmnUtil.PrintCallStackToEvent(ex); } } - public static async Task Simplified(HttpClient client, SyndicationItem originalItem) + /// + /// 元のフィードアイテムとDanbooru投稿詳細ページからコンテンツ(画像・タグ・作者・元ソース等)を取得・整形します。 + /// + /// HttpClient インスタンス + /// 元のフィードアイテム + /// FlareSolverr セッションID + /// 整形後のSyndicationItem + public static async Task Simplified(HttpClient client, SyndicationItem originalItem, string? sessionId = null) { - List getAddCategory() + var link = originalItem.Links.FirstOrDefault(l => l.RelationshipType == "alternate" || string.IsNullOrEmpty(l.RelationshipType))?.Uri + ?? originalItem.Links.FirstOrDefault()?.Uri; + + IDocument? docContents = null; + if (link != null) { - List result = new List(); - - string orgText = originalItem.Summary.Text; - var parser = new HtmlParser(); - var document = parser.ParseDocument(orgText); - - var firstP = document.QuerySelector("p"); - if (firstP != null) + try { - var targetText = firstP.TextContent; - int tagIndex = targetText.IndexOf("【タグ】"); - if (tagIndex >= 0) - { - // 【タグ】が見つかった場合、そこから前の文字列だけを残す - targetText = targetText.Substring(0, tagIndex); - } - // ---------------------------------------------------- - // 正規表現で各項目の値を抽出する - // ---------------------------------------------------- - // パターン解説: - // 【[^】]+】 : 「【」で始まり、「】以外の文字」が続き、「】」で終わる文字列 (例: 【パロディ】) - // ([^【]*) : その後、「【」以外の文字が続く限り取得する (Group1としてキャプチャ) - string pattern = @"【[^】]+】([^【]*)"; - var matches = Regex.Matches(targetText, pattern); - - foreach (Match match in matches) - { - // match.Groups[1].Value には "パロ1, パロ 2 " などの値が入ります - // (項目の中身が空の場合は空文字になります) - string valuesStr = match.Groups[1].Value; - - if (string.IsNullOrWhiteSpace(valuesStr)) - { - continue; // 中身が空の場合はスキップ - } - - // ---------------------------------------------------- - // 3. カンマで分割し、前後の空白を除去してカテゴリに追加 - // ---------------------------------------------------- - // ※もし全角カンマ「、」も考慮したい場合は new[] { ',', '、' } と記述します - string[] splitValues = valuesStr.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries); - - foreach (var val in splitValues) - { - // 前後の半角・全角スペースを取り除く - string categoryName = val.Trim(); - - // 空でなければカテゴリとして追加 - if (!string.IsNullOrEmpty(categoryName)) - { - // domain属性なし(前回回答の手法)でカテゴリを追加 - result.Add(new SyndicationCategory(categoryName)); - } - } - } + docContents = await CmnUtil.GetDocumentFlareSolverr(client, link.ToString(), session: sessionId); + } + catch (Exception ex) + { + LogManager.AddLog($"[警告] Danbooru 投稿ページ取得失敗 ({link}): {ex.Message}"); } - return result; } - var newItem = new SyndicationItem() + // 1. タイトルの決定(個別ページのog:titleまたはAtomのTitle) + var title = originalItem.Title?.Text ?? "Danbooru Post"; + if (docContents != null) { - // タイトルは TextSyndicationContent でラップする必要があります - Title = new TextSyndicationContent(originalItem.Title?.Text ?? "No Title") - , - Id = CmnUtil.CreateGuid(originalItem.Id) - , - PublishDate = originalItem.PublishDate - , - LastUpdatedTime = originalItem.LastUpdatedTime + var ogTitle = docContents.QuerySelector("meta[property='og:title']")?.GetAttribute("content"); + if (!string.IsNullOrWhiteSpace(ogTitle)) + { + title = ogTitle; + if (title.EndsWith(" | Danbooru", StringComparison.OrdinalIgnoreCase)) + { + title = title[..^11].Trim(); + } + } + } - // 元のコンテンツをそのまま引き継ぐ - // , Content = originalItem.Content - // , Summary = originalItem.Summary + var newItem = new SyndicationItem + { + Title = new TextSyndicationContent(title), + Id = CmnUtil.CreateGuid(originalItem.Id ?? link?.ToString() ?? Guid.NewGuid().ToString()), + PublishDate = originalItem.PublishDate, + LastUpdatedTime = originalItem.LastUpdatedTime }; - // リンクを引き継ぐ - var link = originalItem.Links.FirstOrDefault()?.Uri; if (link != null) { newItem.Links.Add(new SyndicationLink(link)); } - // カテゴリに本文内の作者などの情報を追加する - foreach (var category in getAddCategory()) + + // 2. タグ分類(Artist / Copyright / Character / General / Meta)の取得 + List GetTagsBySelector(string selector) { - newItem.Categories.Add(category); - } - // 【追加】元のRSSが持っていたカテゴリ(タグ)をそのまま引き継ぐ - foreach (var category in originalItem.Categories) - { - category.Scheme = null; - newItem.Categories.Add(category); - } - foreach (var author in originalItem.Authors) - { - newItem.Authors.Add(author); + var tagList = new List(); + if (docContents == null) return tagList; + + var elements = docContents.QuerySelectorAll(selector); + foreach (var el in elements) + { + var tagName = el.GetAttribute("data-tag-name"); + if (string.IsNullOrWhiteSpace(tagName)) + { + tagName = el.QuerySelector(".search-tag")?.TextContent?.Trim(); + } + if (!string.IsNullOrWhiteSpace(tagName) && !tagList.Contains(tagName)) + { + tagList.Add(tagName); + } + } + return tagList; } - var body = new StringBuilder(); - var originalSummary = originalItem.Summary.Text; - body.Append(originalSummary); - if (link != null) + var artistTags = GetTagsBySelector("#tag-list .artist-tag-list li"); + var copyrightTags = GetTagsBySelector("#tag-list .copyright-tag-list li"); + var characterTags = GetTagsBySelector("#tag-list .character-tag-list li"); + var generalTags = GetTagsBySelector("#tag-list .general-tag-list li"); + var metaTags = GetTagsBySelector("#tag-list .meta-tag-list li"); + + // 3. カテゴリの登録(重複防止) + var addedCategories = new HashSet(StringComparer.OrdinalIgnoreCase); + + void AddCategories(IEnumerable tags) { - var docContents = await CmnUtil.GetDocument(client, link.ToString()); - var imgElements = docContents.QuerySelectorAll("#post-hentai img"); - foreach (var img in imgElements) + foreach (var tag in tags) { - var src = img.GetAttribute("src"); - if (!string.IsNullOrEmpty(src)) + if (!string.IsNullOrWhiteSpace(tag) && addedCategories.Add(tag)) { - // 相対URLの場合も考慮して、絶対URLに解決する - if (Uri.TryCreate(link, src, out var absoluteUri)) + newItem.Categories.Add(new SyndicationCategory(tag)); + } + } + } + + AddCategories(artistTags); + AddCategories(copyrightTags); + AddCategories(characterTags); + AddCategories(generalTags); + AddCategories(metaTags); + + // 元のアイテムのカテゴリも引き継ぐ + foreach (var category in originalItem.Categories) + { + if (!string.IsNullOrWhiteSpace(category.Name) && addedCategories.Add(category.Name)) + { + category.Scheme = null; + newItem.Categories.Add(category); + } + } + + // 4. 作者(Authors)の設定 + if (artistTags.Count > 0) + { + foreach (var artist in artistTags) + { + newItem.Authors.Add(new SyndicationPerson { Name = artist }); + } + } + else + { + foreach (var author in originalItem.Authors) + { + newItem.Authors.Add(author); + } + } + + // 5. 画像URLの取得(最高画質 / オリジナル画像) + string? imageUrl = null; + if (docContents != null) + { + // og:image はオリジナル画像URLであることが多い + imageUrl = docContents.QuerySelector("meta[property='og:image']")?.GetAttribute("content") + ?? docContents.QuerySelector("#image-container img")?.GetAttribute("src") + ?? docContents.QuerySelector("#image")?.GetAttribute("src") + ?? docContents.QuerySelector("meta[name='twitter:image']")?.GetAttribute("content"); + } + + // HTMLのフォールバック画像抽出 + if (string.IsNullOrEmpty(imageUrl) && originalItem.Content is TextSyndicationContent textContent) + { + var contentParser = new HtmlParser(); + var contentDoc = contentParser.ParseDocument(textContent.Text); + imageUrl = contentDoc.QuerySelector("img")?.GetAttribute("src"); + } + + // 6. 元ソースURLの取得 + string? sourceUrl = null; + if (docContents != null) + { + var sourceElem = docContents.QuerySelector("#post-info-source a") ?? docContents.QuerySelector("#post-information a[href^='http']"); + sourceUrl = sourceElem?.GetAttribute("href"); + } + if (string.IsNullOrEmpty(sourceUrl)) + { + var relatedLink = originalItem.Links.FirstOrDefault(l => l.RelationshipType == "related")?.Uri; + if (relatedLink != null) + { + sourceUrl = relatedLink.ToString(); + } + } + + // 7. 作者コメント(artist-commentary)の抽出 + var commentaryHtml = new StringBuilder(); + if (docContents != null) + { + var commentaryContainer = docContents.QuerySelector("#artist-commentary"); + if (commentaryContainer != null) + { + var sections = commentaryContainer.QuerySelectorAll("section"); + foreach (var sec in sections) + { + var isTranslated = sec.Id?.Contains("translated", StringComparison.OrdinalIgnoreCase) == true; + var label = isTranslated ? "翻訳コメント" : "作者コメント"; + + var h3Title = sec.QuerySelector("h3")?.TextContent?.Trim(); + var proseElem = sec.QuerySelector(".prose") ?? sec.QuerySelector("div"); + var contentHtml = proseElem?.InnerHtml?.Trim(); + + // 空文字やプレースホルダー("????")の除外判定 + bool hasTitle = !string.IsNullOrWhiteSpace(h3Title); + bool hasContent = !string.IsNullOrWhiteSpace(contentHtml) && contentHtml != "

????

" && contentHtml != "????"; + + if (hasTitle || hasContent) { - body.Append($""); - break; + commentaryHtml.Append($@"
"); + commentaryHtml.Append($@"
【{label}】{(hasTitle ? $" {WebUtility.HtmlEncode(h3Title)}" : "")}
"); + if (hasContent) + { + commentaryHtml.Append($@"
{contentHtml}
"); + } + commentaryHtml.Append("
"); } } } } + + // 8. HTML本文(Summary)の生成 + var body = new StringBuilder(); + + // 画像の配置 + if (!string.IsNullOrEmpty(imageUrl)) + { + body.Append($@"
"); + } + + // 画像のすぐ下に作者コメントを追加 + if (commentaryHtml.Length > 0) + { + body.Append(commentaryHtml); + } + + body.Append($@"
タイトル: {WebUtility.HtmlEncode(title)}
"); + + if (copyrightTags.Count > 0) + { + body.Append($@"
作品名: {WebUtility.HtmlEncode(string.Join(", ", copyrightTags))}
"); + } + + if (characterTags.Count > 0) + { + body.Append($@"
キャラクター: {WebUtility.HtmlEncode(string.Join(", ", characterTags))}
"); + } + + if (artistTags.Count > 0) + { + body.Append($@"
作者: {WebUtility.HtmlEncode(string.Join(", ", artistTags))}
"); + } + + if (!string.IsNullOrEmpty(sourceUrl)) + { + body.Append($@""); + } + + if (generalTags.Count > 0) + { + body.Append($@"
タグ: {WebUtility.HtmlEncode(string.Join(", ", generalTags))}
"); + } + newItem.Summary = new TextSyndicationContent(body.ToString(), TextSyndicationContentKind.Html); return newItem; diff --git a/rss-generator/Class/RSS/aurahentai.cs b/rss-generator/Class/RSS/aurahentai.cs index e6aea58..142b313 100644 --- a/rss-generator/Class/RSS/aurahentai.cs +++ b/rss-generator/Class/RSS/aurahentai.cs @@ -68,6 +68,9 @@ internal class aurahentai var relLink = linkElement.GetAttribute("href") ?? string.Empty; var contentsUri = CmnUtil.GetAbsoluteUri(docListPage.BaseUrl!, relLink)!; var linkHref = contentsUri.AbsoluteUri; + { + item.Links.Add(new SyndicationLink(new Uri(linkHref))); + } var docContents = await CmnUtil.GetDocumentFlareSolverr(client, linkHref, session: sessionId); var divDetail = docContents.QuerySelector("main>div:nth-of-type(2)")!; diff --git a/rss-generator/Class/RSS/javtwi.cs b/rss-generator/Class/RSS/javtwi.cs index f19deb3..1882fa7 100644 --- a/rss-generator/Class/RSS/javtwi.cs +++ b/rss-generator/Class/RSS/javtwi.cs @@ -64,8 +64,11 @@ internal class javtwi var body = new StringBuilder(); body.Append($@""); - body.Append($@"
"); + body.Append($@""); var item = new SyndicationItem( title, diff --git a/rss-generator/Program.cs b/rss-generator/Program.cs index 9ee665d..bcda4d3 100644 --- a/rss-generator/Program.cs +++ b/rss-generator/Program.cs @@ -1,16 +1,13 @@ -using AngleSharp.Dom; -using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection; using rss_generator.Class; using rss_generator.Class.RSS; -using rss_generator.Class.RSS.hitomi_la; -using rss_generator.Class.RSS.X; using System.Diagnostics; // 1. 同期スレッド等の未処理例外 AppDomain.CurrentDomain.UnhandledException += (sender, e) => { - var ex = e.ExceptionObject as Exception; + Exception? ex = e.ExceptionObject as Exception; if (ex != null) { CmnUtil.PrintCallStackToEvent(ex); @@ -27,7 +24,7 @@ TaskScheduler.UnobservedTaskException += (sender, e) => } }; -var services = new ServiceCollection(); +ServiceCollection services = new ServiceCollection(); // 2. IHttpClientFactory の登録とクッキーの有効化設定(名前付きクライアント) services.AddHttpClient("webClient"); // コンテナをビルド @@ -58,7 +55,7 @@ while (true) #endif // 指定時刻になったら処理を実行 - var sw = new Stopwatch(); + Stopwatch sw = new Stopwatch(); sw.Restart(); try { @@ -69,26 +66,32 @@ while (true) // サイトによってはAcceptヘッダーがないと弾く場合もあるため、念のため追加 client.DefaultRequestHeaders.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"); - var tasks = new List(); + List tasks = new List(); - var from = new DateTime(nowDate.Year, nowDate.Month, nowDate.Day, 5, 0, 0); - var to = new DateTime(nowDate.Year, nowDate.Month, nowDate.Day, 7, 15, 0); + DateTime from = new DateTime(nowDate.Year, nowDate.Month, nowDate.Day, 5, 0, 0); + DateTime to = new DateTime(nowDate.Year, nowDate.Month, nowDate.Day, 7, 15, 0); if (from < nowDate && nowDate < to) { tasks.Add(momonga_fanzine.CreateRSSAsync(client)); tasks.Add(momonga_magazine.CreateRSSAsync(client)); } - //tasks.Add(Youtube.CreateRSSAsync(client)); - tasks.Add(x_list.CreateRSSAsync(client)); - tasks.Add(javtwi.CreateRSSAsync(client)); +#if DEBUG + //tasks.Add(index_japanese.CreateRSSAsync(client)); +#else + tasks.Add(Youtube.CreateRSSAsync(client)); + + tasks.Add(rss_generator.Class.RSS.X.x_list.CreateRSSAsync(client)); tasks.Add(kimochiinfo.CreateRSSAsync(client)); tasks.Add(missav.CreateRSSAsync(client)); tasks.Add(missav123.CreateRSSAsync(client)); tasks.Add(npb_6.CreateRSSAsync(client)); tasks.Add(ci_en.CreateRSSAsync(client)); - - //tasks.Add(index_japanese.CreateRSSAsync(client)); + tasks.Add(aurahentai.CreateRSSAsync(client)); + tasks.Add(javtwi.CreateRSSAsync(client)); + tasks.Add(rss_generator.Class.RSS.Danbooru.rank.CreateRSSAsync(client)); +#endif + await Task.WhenAll(tasks);