出力内容の設定

This commit is contained in:
armonica committed 2026-08-28 11:39:05 +09:00
1 parent b01665423e
commit f31595ac9f
5 files changed
+366 -129

No files matched your search

+87
View File
@@ -335,6 +335,93 @@ public class CmnUtil
return document;
}
/// <summary>
/// FlareSolverr 経由で対象 URL の RSS/Atom フィードを取得し、SyndicationItem のコレクションを返します。
/// メモリ割り当ての最小化およびストリーム直接デシリアライズにより、メモリリーク防止と高速化を行っています。
/// </summary>
/// <param name="client">HttpClient インスタンス</param>
/// <param name="url">取得対象のフィードURL</param>
/// <param name="flareSolverrEndpoint">FlareSolverrのエンドポイントURL(デフォルト: https://udukipc-flaresolverr.armonica-vps.work/v1)</param>
/// <param name="maxTimeout">FlareSolverr側の最大タイムアウトミリ秒(デフォルト: 60000ms)</param>
/// <param name="session">セッションID(セッションを再利用して高速化する場合に指定)</param>
/// <returns>SyndicationItem のコレクション</returns>
public static async Task<IEnumerable<SyndicationItem>> GetFeedItemsFlareSolverr(
HttpClient client,
string url,
string flareSolverrEndpoint = "https://udukipc-flaresolverr.armonica-vps.work/v1",
int maxTimeout = 60000,
string? session = null)
{
Console.WriteLine($"Fetching Feed via FlareSolverr: {url}");
var payload = new FlareSolverrRequest
{
Cmd = "request.get",
Url = url,
MaxTimeout = maxTimeout,
Session = session
};
using var request = new HttpRequestMessage(HttpMethod.Post, flareSolverrEndpoint)
{
Content = JsonContent.Create(payload, options: FlareSolverrJsonOptions)
};
try
{
// レスポンスヘッダー受信時点で制御を戻し、ストリームから直接デシリアライズしてメモリ消費を抑制
using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
response.EnsureSuccessStatusCode();
using var responseStream = await response.Content.ReadAsStreamAsync();
var flareResponse = await JsonSerializer.DeserializeAsync<FlareSolverrResponse>(responseStream, FlareSolverrJsonOptions);
if (flareResponse == null)
{
throw new InvalidOperationException($"FlareSolverr からのレスポンスのデシリアライズに失敗しました: {url}");
}
if (!string.Equals(flareResponse.Status, "ok", StringComparison.OrdinalIgnoreCase))
{
string errorMsg = $"FlareSolverr エラー (Status: {flareResponse.Status}, Message: {flareResponse.Message}): {url}";
LogManager.AddLog($"[エラー] {errorMsg}");
throw new InvalidOperationException(errorMsg);
}
string xmlContent = flareResponse.Solution?.Response ?? string.Empty;
if (string.IsNullOrWhiteSpace(xmlContent))
{
Console.WriteLine($"Warning: FlareSolverr から取得したフィード内容が空です ({url})");
return Enumerable.Empty<SyndicationItem>();
}
// FlareSolverr(Chromium)がXMLをレンダリングして <pre>...</pre> で返してきた場合のアンラップ処理
if (xmlContent.TrimStart().StartsWith("<html", StringComparison.OrdinalIgnoreCase))
{
var parser = new HtmlParser();
var doc = parser.ParseDocument(xmlContent);
var pre = doc.QuerySelector("pre");
if (pre != null && !string.IsNullOrWhiteSpace(pre.TextContent))
{
xmlContent = pre.TextContent;
}
}
using var stringReader = new StringReader(xmlContent);
using var xmlReader = XmlReader.Create(stringReader);
var feed = SyndicationFeed.Load(xmlReader);
return feed?.Items ?? Enumerable.Empty<SyndicationItem>();
}
catch (Exception ex)
{
LogManager.AddLog($"[エラー] FlareSolverr フィード {url} の取得に失敗: {ex.Message}");
PrintCallStackToEvent(ex);
return Enumerable.Empty<SyndicationItem>();
}
}
/// <summary>
/// FlareSolverr の新規セッションを作成します。セッションを再利用することで2回目以降のリクエストが大幅に高速化します。
/// </summary>
+255 -114
View File
@@ -1,168 +1,309 @@
using AngleSharp.Dom;
using AngleSharp.Html.Dom;
using AngleSharp.Html.Parser;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.ServiceModel.Syndication;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
namespace rss_generator.Class.RSS.Danbooru;
internal class rank
{
/// <summary>
/// Danbooruのランクフィードを取得し、各投稿ページから詳細情報を取得して整形したRSSを出力します。
/// </summary>
/// <param name="client">HttpClient インスタンス</param>
public static async Task CreateRSSAsync(HttpClient client)
{
var pageMaster = "https://danbooru.donmai.us/posts.atom?tags=order%3Arank";
try
{
// 簡潔なRSSに整形
var items = await CmnUtil.GetFeedItems(client, pageMaster);
var simplifiedItems = items.Select(n => n);
foreach (var item in simplifiedItems)
// FlareSolverr セッションを作成して複数リクエストを高速化
string? sessionId = await CmnUtil.CreateFlareSolverrSession(client);
try
{
await Simplified(client, item);
// FlareSolverr 経由で Atom フィードアイテムを取得
var items = await CmnUtil.GetFeedItemsFlareSolverr(client, pageMaster, session: sessionId);
var simplifiedItems = new List<SyndicationItem>();
foreach (var item in items)
{
var simplified = await Simplified(client, item, sessionId);
if (simplified != null)
{
simplifiedItems.Add(simplified);
}
}
// 新しい統合フィードを作成
var mergedFeed = new SyndicationFeed(
"Danbooru",
"Danbooru Rank Posts",
new Uri("https://danbooru.donmai.us/posts?tags=order%3Arank"),
simplifiedItems
);
CmnUtil.ExportXMLFile("rank.xml", mergedFeed, "danbooru");
}
finally
{
if (sessionId != null)
{
await CmnUtil.DestroyFlareSolverrSession(client, sessionId);
}
}
// 新しい統合フィードを作成
var mergedFeed = new SyndicationFeed(
"Danbooru",
"description",
new Uri("https://danbooru.donmai.us/posts?tags=order%3Arank"),
simplifiedItems
);
CmnUtil.ExportXMLFile("rank.xml", mergedFeed, "danbooru");
}
catch (Exception ex)
{
LogManager.AddLog($"[エラー] danbooru:{ex.Message}");
CmnUtil.PrintCallStackToEvent(ex);
}
}
public static async Task<SyndicationItem> Simplified(HttpClient client, SyndicationItem originalItem)
/// <summary>
/// 元のフィードアイテムとDanbooru投稿詳細ページからコンテンツ(画像・タグ・作者・元ソース等)を取得・整形します。
/// </summary>
/// <param name="client">HttpClient インスタンス</param>
/// <param name="originalItem">元のフィードアイテム</param>
/// <param name="sessionId">FlareSolverr セッションID</param>
/// <returns>整形後のSyndicationItem</returns>
public static async Task<SyndicationItem> Simplified(HttpClient client, SyndicationItem originalItem, string? sessionId = null)
{
List<SyndicationCategory> getAddCategory()
var link = originalItem.Links.FirstOrDefault(l => l.RelationshipType == "alternate" || string.IsNullOrEmpty(l.RelationshipType))?.Uri
?? originalItem.Links.FirstOrDefault()?.Uri;
IDocument? docContents = null;
if (link != null)
{
List<SyndicationCategory> result = new List<SyndicationCategory>();
string orgText = originalItem.Summary.Text;
var parser = new HtmlParser();
var document = parser.ParseDocument(orgText);
var firstP = document.QuerySelector("p");
if (firstP != null)
try
{
var targetText = firstP.TextContent;
int tagIndex = targetText.IndexOf("【タグ】");
if (tagIndex >= 0)
{
// 【タグ】が見つかった場合、そこから前の文字列だけを残す
targetText = targetText.Substring(0, tagIndex);
}
// ----------------------------------------------------
// 正規表現で各項目の値を抽出する
// ----------------------------------------------------
// パターン解説:
// 【[^】]+】 : 「【」で始まり、「】以外の文字」が続き、「】」で終わる文字列 (例: 【パロディ】)
// ([^【]*) : その後、「【」以外の文字が続く限り取得する (Group1としてキャプチャ)
string pattern = @"【[^】]+】([^【]*)";
var matches = Regex.Matches(targetText, pattern);
foreach (Match match in matches)
{
// match.Groups[1].Value には "パロ1, パロ 2 " などの値が入ります
// (項目の中身が空の場合は空文字になります)
string valuesStr = match.Groups[1].Value;
if (string.IsNullOrWhiteSpace(valuesStr))
{
continue; // 中身が空の場合はスキップ
}
// ----------------------------------------------------
// 3. カンマで分割し、前後の空白を除去してカテゴリに追加
// ----------------------------------------------------
// ※もし全角カンマ「、」も考慮したい場合は new[] { ',', '、' } と記述します
string[] splitValues = valuesStr.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
foreach (var val in splitValues)
{
// 前後の半角・全角スペースを取り除く
string categoryName = val.Trim();
// 空でなければカテゴリとして追加
if (!string.IsNullOrEmpty(categoryName))
{
// domain属性なし(前回回答の手法)でカテゴリを追加
result.Add(new SyndicationCategory(categoryName));
}
}
}
docContents = await CmnUtil.GetDocumentFlareSolverr(client, link.ToString(), session: sessionId);
}
catch (Exception ex)
{
LogManager.AddLog($"[警告] Danbooru 投稿ページ取得失敗 ({link}): {ex.Message}");
}
return result;
}
var newItem = new SyndicationItem()
// 1. タイトルの決定(個別ページのog:titleまたはAtomのTitle)
var title = originalItem.Title?.Text ?? "Danbooru Post";
if (docContents != null)
{
// タイトルは TextSyndicationContent でラップする必要があります
Title = new TextSyndicationContent(originalItem.Title?.Text ?? "No Title")
,
Id = CmnUtil.CreateGuid(originalItem.Id)
,
PublishDate = originalItem.PublishDate
,
LastUpdatedTime = originalItem.LastUpdatedTime
var ogTitle = docContents.QuerySelector("meta[property='og:title']")?.GetAttribute("content");
if (!string.IsNullOrWhiteSpace(ogTitle))
{
title = ogTitle;
if (title.EndsWith(" | Danbooru", StringComparison.OrdinalIgnoreCase))
{
title = title[..^11].Trim();
}
}
}
// 元のコンテンツをそのまま引き継ぐ
// , Content = originalItem.Content
// , Summary = originalItem.Summary
var newItem = new SyndicationItem
{
Title = new TextSyndicationContent(title),
Id = CmnUtil.CreateGuid(originalItem.Id ?? link?.ToString() ?? Guid.NewGuid().ToString()),
PublishDate = originalItem.PublishDate,
LastUpdatedTime = originalItem.LastUpdatedTime
};
// リンクを引き継ぐ
var link = originalItem.Links.FirstOrDefault()?.Uri;
if (link != null)
{
newItem.Links.Add(new SyndicationLink(link));
}
// カテゴリに本文内の作者などの情報を追加する
foreach (var category in getAddCategory())
// 2. タグ分類(Artist / Copyright / Character / General / Meta)の取得
List<string> GetTagsBySelector(string selector)
{
newItem.Categories.Add(category);
}
// 【追加】元のRSSが持っていたカテゴリ(タグ)をそのまま引き継ぐ
foreach (var category in originalItem.Categories)
{
category.Scheme = null;
newItem.Categories.Add(category);
}
foreach (var author in originalItem.Authors)
{
newItem.Authors.Add(author);
var tagList = new List<string>();
if (docContents == null) return tagList;
var elements = docContents.QuerySelectorAll(selector);
foreach (var el in elements)
{
var tagName = el.GetAttribute("data-tag-name");
if (string.IsNullOrWhiteSpace(tagName))
{
tagName = el.QuerySelector(".search-tag")?.TextContent?.Trim();
}
if (!string.IsNullOrWhiteSpace(tagName) && !tagList.Contains(tagName))
{
tagList.Add(tagName);
}
}
return tagList;
}
var body = new StringBuilder();
var originalSummary = originalItem.Summary.Text;
body.Append(originalSummary);
if (link != null)
var artistTags = GetTagsBySelector("#tag-list .artist-tag-list li");
var copyrightTags = GetTagsBySelector("#tag-list .copyright-tag-list li");
var characterTags = GetTagsBySelector("#tag-list .character-tag-list li");
var generalTags = GetTagsBySelector("#tag-list .general-tag-list li");
var metaTags = GetTagsBySelector("#tag-list .meta-tag-list li");
// 3. カテゴリの登録(重複防止)
var addedCategories = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
void AddCategories(IEnumerable<string> tags)
{
var docContents = await CmnUtil.GetDocument(client, link.ToString());
var imgElements = docContents.QuerySelectorAll("#post-hentai img");
foreach (var img in imgElements)
foreach (var tag in tags)
{
var src = img.GetAttribute("src");
if (!string.IsNullOrEmpty(src))
if (!string.IsNullOrWhiteSpace(tag) && addedCategories.Add(tag))
{
// 相対URLの場合も考慮して、絶対URLに解決する
if (Uri.TryCreate(link, src, out var absoluteUri))
newItem.Categories.Add(new SyndicationCategory(tag));
}
}
}
AddCategories(artistTags);
AddCategories(copyrightTags);
AddCategories(characterTags);
AddCategories(generalTags);
AddCategories(metaTags);
// 元のアイテムのカテゴリも引き継ぐ
foreach (var category in originalItem.Categories)
{
if (!string.IsNullOrWhiteSpace(category.Name) && addedCategories.Add(category.Name))
{
category.Scheme = null;
newItem.Categories.Add(category);
}
}
// 4. 作者(Authors)の設定
if (artistTags.Count > 0)
{
foreach (var artist in artistTags)
{
newItem.Authors.Add(new SyndicationPerson { Name = artist });
}
}
else
{
foreach (var author in originalItem.Authors)
{
newItem.Authors.Add(author);
}
}
// 5. 画像URLの取得(最高画質 / オリジナル画像)
string? imageUrl = null;
if (docContents != null)
{
// og:image はオリジナル画像URLであることが多い
imageUrl = docContents.QuerySelector("meta[property='og:image']")?.GetAttribute("content")
?? docContents.QuerySelector("#image-container img")?.GetAttribute("src")
?? docContents.QuerySelector("#image")?.GetAttribute("src")
?? docContents.QuerySelector("meta[name='twitter:image']")?.GetAttribute("content");
}
// HTMLのフォールバック画像抽出
if (string.IsNullOrEmpty(imageUrl) && originalItem.Content is TextSyndicationContent textContent)
{
var contentParser = new HtmlParser();
var contentDoc = contentParser.ParseDocument(textContent.Text);
imageUrl = contentDoc.QuerySelector("img")?.GetAttribute("src");
}
// 6. 元ソースURLの取得
string? sourceUrl = null;
if (docContents != null)
{
var sourceElem = docContents.QuerySelector("#post-info-source a") ?? docContents.QuerySelector("#post-information a[href^='http']");
sourceUrl = sourceElem?.GetAttribute("href");
}
if (string.IsNullOrEmpty(sourceUrl))
{
var relatedLink = originalItem.Links.FirstOrDefault(l => l.RelationshipType == "related")?.Uri;
if (relatedLink != null)
{
sourceUrl = relatedLink.ToString();
}
}
// 7. 作者コメント(artist-commentary)の抽出
var commentaryHtml = new StringBuilder();
if (docContents != null)
{
var commentaryContainer = docContents.QuerySelector("#artist-commentary");
if (commentaryContainer != null)
{
var sections = commentaryContainer.QuerySelectorAll("section");
foreach (var sec in sections)
{
var isTranslated = sec.Id?.Contains("translated", StringComparison.OrdinalIgnoreCase) == true;
var label = isTranslated ? "翻訳コメント" : "作者コメント";
var h3Title = sec.QuerySelector("h3")?.TextContent?.Trim();
var proseElem = sec.QuerySelector(".prose") ?? sec.QuerySelector("div");
var contentHtml = proseElem?.InnerHtml?.Trim();
// 空文字やプレースホルダー("????")の除外判定
bool hasTitle = !string.IsNullOrWhiteSpace(h3Title);
bool hasContent = !string.IsNullOrWhiteSpace(contentHtml) && contentHtml != "<p>????</p>" && contentHtml != "????";
if (hasTitle || hasContent)
{
body.Append($"<img src=\"{absoluteUri}\" />");
break;
commentaryHtml.Append($@"<div style=""margin: 8px 0; padding: 8px 12px; background: #f8f9fa; border-left: 4px solid #4a90e2; border-radius: 4px;"">");
commentaryHtml.Append($@"<div style=""font-weight: bold; margin-bottom: 4px;"">【{label}】{(hasTitle ? $" {WebUtility.HtmlEncode(h3Title)}" : "")}</div>");
if (hasContent)
{
commentaryHtml.Append($@"<div>{contentHtml}</div>");
}
commentaryHtml.Append("</div>");
}
}
}
}
// 8. HTML本文(Summary)の生成
var body = new StringBuilder();
// 画像の配置
if (!string.IsNullOrEmpty(imageUrl))
{
body.Append($@"<div><a href=""{link}""><img src=""{imageUrl}"" alt=""{WebUtility.HtmlEncode(title)}"" style=""max-width: 100%; height: auto;"" /></a></div>");
}
// 画像のすぐ下に作者コメントを追加
if (commentaryHtml.Length > 0)
{
body.Append(commentaryHtml);
}
body.Append($@"<div><strong>タイトル:</strong> {WebUtility.HtmlEncode(title)}</div>");
if (copyrightTags.Count > 0)
{
body.Append($@"<div><strong>作品名:</strong> {WebUtility.HtmlEncode(string.Join(", ", copyrightTags))}</div>");
}
if (characterTags.Count > 0)
{
body.Append($@"<div><strong>キャラクター:</strong> {WebUtility.HtmlEncode(string.Join(", ", characterTags))}</div>");
}
if (artistTags.Count > 0)
{
body.Append($@"<div><strong>作者:</strong> {WebUtility.HtmlEncode(string.Join(", ", artistTags))}</div>");
}
if (!string.IsNullOrEmpty(sourceUrl))
{
body.Append($@"<div><strong>元ソース:</strong> <a href=""{sourceUrl}"" target=""_blank"">{WebUtility.HtmlEncode(sourceUrl)}</a></div>");
}
if (generalTags.Count > 0)
{
body.Append($@"<div><strong>タグ:</strong> {WebUtility.HtmlEncode(string.Join(", ", generalTags))}</div>");
}
newItem.Summary = new TextSyndicationContent(body.ToString(), TextSyndicationContentKind.Html);
return newItem;
+3
View File
@@ -68,6 +68,9 @@ internal class aurahentai
var relLink = linkElement.GetAttribute("href") ?? string.Empty;
var contentsUri = CmnUtil.GetAbsoluteUri(docListPage.BaseUrl!, relLink)!;
var linkHref = contentsUri.AbsoluteUri;
{
item.Links.Add(new SyndicationLink(new Uri(linkHref)));
}
var docContents = await CmnUtil.GetDocumentFlareSolverr(client, linkHref, session: sessionId);
var divDetail = docContents.QuerySelector("main>div:nth-of-type(2)")!;
+4 -1
View File
@@ -64,8 +64,11 @@ internal class javtwi
var body = new StringBuilder();
body.Append($@"<div><a href=""{itemUri}"">{title}</a></div>");
body.Append($@"<div><img src=""{imageUri}"" alt=""{title}"" style=""max-width: 100%; height: auto;"" /></div>");
body.Append($@"<video controls poster=""{imageUri}"" preload=""metadata"" style=""max-width: 100%; height: auto;"">");
body.Append($@"<source src=""{itemUri}"" type=""video/mp4"">");
body.Append($@"<p><a href=""{itemUri}"" target=""_blank"" rel=""noopener noreferrer"">動画を開く (.mp4)</a></p>");
body.Append($@"</video>");
var item = new SyndicationItem(
title,
+17 -14
View File
@@ -1,16 +1,13 @@
using AngleSharp.Dom;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection;
using rss_generator.Class;
using rss_generator.Class.RSS;
using rss_generator.Class.RSS.hitomi_la;
using rss_generator.Class.RSS.X;
using System.Diagnostics;
// 1. 同期スレッド等の未処理例外
AppDomain.CurrentDomain.UnhandledException += (sender, e) =>
{
var ex = e.ExceptionObject as Exception;
Exception? ex = e.ExceptionObject as Exception;
if (ex != null)
{
CmnUtil.PrintCallStackToEvent(ex);
@@ -27,7 +24,7 @@ TaskScheduler.UnobservedTaskException += (sender, e) =>
}
};
var services = new ServiceCollection();
ServiceCollection services = new ServiceCollection();
// 2. IHttpClientFactory の登録とクッキーの有効化設定(名前付きクライアント)
services.AddHttpClient("webClient");
// コンテナをビルド
@@ -58,7 +55,7 @@ while (true)
#endif
// 指定時刻になったら処理を実行
var sw = new Stopwatch();
Stopwatch sw = new Stopwatch();
sw.Restart();
try
{
@@ -69,26 +66,32 @@ while (true)
// サイトによってはAcceptヘッダーがないと弾く場合もあるため、念のため追加
client.DefaultRequestHeaders.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
var tasks = new List<Task>();
List<Task> tasks = new List<Task>();
var from = new DateTime(nowDate.Year, nowDate.Month, nowDate.Day, 5, 0, 0);
var to = new DateTime(nowDate.Year, nowDate.Month, nowDate.Day, 7, 15, 0);
DateTime from = new DateTime(nowDate.Year, nowDate.Month, nowDate.Day, 5, 0, 0);
DateTime to = new DateTime(nowDate.Year, nowDate.Month, nowDate.Day, 7, 15, 0);
if (from < nowDate && nowDate < to)
{
tasks.Add(momonga_fanzine.CreateRSSAsync(client));
tasks.Add(momonga_magazine.CreateRSSAsync(client));
}
//tasks.Add(Youtube.CreateRSSAsync(client));
tasks.Add(x_list.CreateRSSAsync(client));
tasks.Add(javtwi.CreateRSSAsync(client));
#if DEBUG
//tasks.Add(index_japanese.CreateRSSAsync(client));
#else
tasks.Add(Youtube.CreateRSSAsync(client));
tasks.Add(rss_generator.Class.RSS.X.x_list.CreateRSSAsync(client));
tasks.Add(kimochiinfo.CreateRSSAsync(client));
tasks.Add(missav.CreateRSSAsync(client));
tasks.Add(missav123.CreateRSSAsync(client));
tasks.Add(npb_6.CreateRSSAsync(client));
tasks.Add(ci_en.CreateRSSAsync(client));
//tasks.Add(index_japanese.CreateRSSAsync(client));
tasks.Add(aurahentai.CreateRSSAsync(client));
tasks.Add(javtwi.CreateRSSAsync(client));
tasks.Add(rss_generator.Class.RSS.Danbooru.rank.CreateRSSAsync(client));
#endif
await Task.WhenAll(tasks);