全体的に可能な限りbase64で表示するように

This commit is contained in:
armonica committed 2026-09-05 03:00:45 +09:00
1 parent f5230cb0ab
commit 63188d87f4
23 files changed
+307 -34

No files matched your search

+255 -1
View File
@@ -8,12 +8,15 @@ using System.Diagnostics;
using System.Net;
using System.Net.Http.Headers;
using System.Net.Http.Json;
using System.Collections.Concurrent;
using System.Security.Cryptography;
using System.ServiceModel.Syndication;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using System.Xml;
namespace rss_generator.Class;
@@ -167,7 +170,258 @@ public class CmnUtil
}
/// <summary>
/// Feedをファイルとして出力
/// 全ての RSS 出力処理において、記事内の画像(img src / video poster)を自動で Base64 形式(Data URI)に変換して埋め込むかどうか。
/// 環境変数 EMBED_ALL_IMAGES_BASE64 (true/false) での上書きも可能です。
/// </summary>
public static bool EmbedAllImagesBase64
{
get
{
var envVal = Environment.GetEnvironmentVariable("EMBED_ALL_IMAGES_BASE64");
if (!string.IsNullOrWhiteSpace(envVal) && bool.TryParse(envVal, out var parsed))
{
return parsed;
}
return _embedAllImagesBase64;
}
set => _embedAllImagesBase64 = value;
}
private static bool _embedAllImagesBase64 = true;
/// <summary>
/// 画像URLから画像データを取得し、Base64 (Data URI: data:{mime};base64,{data}) 形式の文字列として返します。
/// サイト固有の Referer ヘッダーの自動付与やタイムアウト(6秒)制御を行います。
/// </summary>
public static async Task<string?> FetchImageAsBase64Async(HttpClient client, string imageUrl, int timeoutSeconds = 6)
{
if (string.IsNullOrWhiteSpace(imageUrl)) return null;
if (imageUrl.StartsWith("data:", StringComparison.OrdinalIgnoreCase)) return imageUrl;
var targetUrl = imageUrl.Trim();
// // で始まるスキーム相対URLを https: に補完
if (targetUrl.StartsWith("//"))
{
targetUrl = "https:" + targetUrl;
}
// HTML エンティティの多重エンコード(&amp; や &amp;amp; など)を完全に解決
while (targetUrl.Contains("&amp;", StringComparison.OrdinalIgnoreCase))
{
targetUrl = WebUtility.HtmlDecode(targetUrl);
}
if (!targetUrl.StartsWith("http://", StringComparison.OrdinalIgnoreCase) && !targetUrl.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) return null;
try
{
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds));
using var request = new HttpRequestMessage(HttpMethod.Get, targetUrl);
// User-Agent の設定
request.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36");
// ドメインに応じた適切な Referer の自動判定と付与
if (Uri.TryCreate(targetUrl, UriKind.Absolute, out var uri))
{
var host = uri.Host.ToLowerInvariant();
if (host.Contains("pximg.net") || host.Contains("pixiv"))
{
request.Headers.Referrer = new Uri("https://www.pixiv.net/");
}
else if (host.Contains("hitomi.la") || host.Contains("gold-usergeneratedcontent.net"))
{
request.Headers.Referrer = new Uri("https://hitomi.la/");
}
else if (host.Contains("ci-en.dlsite.com"))
{
request.Headers.Referrer = new Uri("https://ci-en.dlsite.com/");
}
else if (host.Contains("momon-ga.com"))
{
request.Headers.Referrer = new Uri("https://momon-ga.com/");
}
else
{
request.Headers.Referrer = new Uri($"{uri.Scheme}://{uri.Authority}/");
}
}
using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cts.Token);
if (!response.IsSuccessStatusCode)
{
return null;
}
// 15MB 以上の巨大画像は安全のためスキップ
if (response.Content.Headers.ContentLength.HasValue && response.Content.Headers.ContentLength.Value > 15 * 1024 * 1024)
{
return null;
}
var contentType = response.Content.Headers.ContentType?.MediaType;
if (string.IsNullOrWhiteSpace(contentType) || !contentType.StartsWith("image/", StringComparison.OrdinalIgnoreCase))
{
contentType = "image/jpeg";
}
var bytes = await response.Content.ReadAsByteArrayAsync(cts.Token);
if (bytes == null || bytes.Length == 0) return null;
var base64 = Convert.ToBase64String(bytes);
return $"data:{contentType};base64,{base64}";
}
catch
{
// タイムアウトや取得失敗時は元のURLのままフォールバックするため静かにnull返却
return null;
}
}
/// <summary>
/// SyndicationFeed に含まれる全記事(24時間フィルタ適用後)の HTML 本文から画像 URL を抽出し、
/// 並列ダウンロードして Base64 形式(Data URI)に置換します。
/// </summary>
public static async Task EmbedImagesAsBase64InFeedAsync(SyndicationFeed feed, HttpClient client, int maxParallel = 8)
{
if (feed.Items == null) return;
var itemsList = feed.Items.ToList();
if (itemsList.Count == 0) return;
// 1. 全アイテムの HTML から画像 URL を抽出(重複排除)
var urlPattern = new Regex(@"(?:<img\b[^>]*?\bsrc=[""'](?<url>[^""'\s>]+)[""']|<video\b[^>]*?\bposter=[""'](?<url>[^""'\s>]+)[""'])", RegexOptions.IgnoreCase | RegexOptions.Compiled);
// rawUrl -> decodedUrl のマッピング
var targetUrls = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
foreach (var item in itemsList)
{
var html = item.Summary?.Text ?? (item.Content as TextSyndicationContent)?.Text;
if (string.IsNullOrWhiteSpace(html)) continue;
var matches = urlPattern.Matches(html);
foreach (Match match in matches)
{
var rawUrl = match.Groups["url"].Value;
if (!string.IsNullOrWhiteSpace(rawUrl) && !rawUrl.StartsWith("data:", StringComparison.OrdinalIgnoreCase))
{
var decoded = rawUrl;
while (decoded.Contains("&amp;", StringComparison.OrdinalIgnoreCase))
{
decoded = WebUtility.HtmlDecode(decoded);
}
if (decoded.StartsWith("//"))
{
decoded = "https:" + decoded;
}
targetUrls[rawUrl] = decoded;
}
}
}
if (targetUrls.Count == 0) return;
// 2. 並列ダウンロードして Base64 辞書を構築
var cacheMap = new ConcurrentDictionary<string, string>(StringComparer.OrdinalIgnoreCase);
using var semaphore = new SemaphoreSlim(maxParallel);
var downloadTasks = targetUrls.Select(async kvp =>
{
var rawUrl = kvp.Key;
var fetchUrl = kvp.Value;
await semaphore.WaitAsync();
try
{
var dataUri = await FetchImageAsBase64Async(client, fetchUrl);
if (!string.IsNullOrWhiteSpace(dataUri))
{
cacheMap[rawUrl] = dataUri;
cacheMap[fetchUrl] = dataUri;
}
}
finally
{
semaphore.Release();
}
});
await Task.WhenAll(downloadTasks);
// 3. 各アイテムの HTML 内の画像 URL を Base64 文字列に置換
if (cacheMap.Count > 0)
{
foreach (var item in itemsList)
{
if (item.Summary != null && !string.IsNullOrWhiteSpace(item.Summary.Text))
{
var newHtml = item.Summary.Text;
bool modified = false;
foreach (var kvp in cacheMap)
{
if (newHtml.Contains(kvp.Key, StringComparison.OrdinalIgnoreCase))
{
newHtml = newHtml.Replace(kvp.Key, kvp.Value, StringComparison.OrdinalIgnoreCase);
modified = true;
}
}
if (modified)
{
item.Summary = new TextSyndicationContent(newHtml, TextSyndicationContentKind.Html);
}
}
}
}
int distinctSuccess = targetUrls.Keys.Count(k => cacheMap.ContainsKey(k));
LogManager.AddLog($"[情報] [CmnUtil] 画像の Base64 変換完了: {distinctSuccess}/{targetUrls.Count} 件成功 (記事数: {itemsList.Count})");
}
/// <summary>
/// Feed を非同期でファイルに出力します。画像 Base64 変換が有効な場合は自動で変換を行います。
/// </summary>
public static async Task ExportXMLFileAsync(string fileName, SyndicationFeed feed, HttpClient? client = null, string addDir = "")
{
// 24時間以上前の古い記事を除外
if (feed.Items != null)
{
feed.Items = FilterItemsWithinHours(feed.Items, 24);
}
// 画像の Base64 埋め込み処理
if (EmbedAllImagesBase64 && client != null)
{
try
{
await EmbedImagesAsBase64InFeedAsync(feed, client);
}
catch (Exception ex)
{
LogManager.AddLog($"[警告] [CmnUtil] 画像 Base64 変換中に例外発生 ({fileName}): {ex.Message}");
}
}
var filePath = GetExportXmlPath(fileName, addDir);
var settings = new XmlWriterSettings
{
Encoding = new UTF8Encoding(false), // BOMなしUTF-8
Indent = true,
Async = true
};
using var xmlWriter = XmlWriter.Create(filePath, settings);
var formatter = new Rss20FeedFormatter(feed);
formatter.WriteTo(xmlWriter);
int count = feed.Items?.Count() ?? 0;
string displayFileName = System.IO.Path.GetFileName(filePath);
LogManager.AddLog($"[情報] {displayFileName} を正常に出力しました (記事数: {count})");
}
/// <summary>
/// Feedをファイルとして出力(同期版・後方互換用)
/// </summary>
/// <param name="fileName"></param>
/// <param name="feed"></param>
+1 -1
View File
@@ -59,7 +59,7 @@ internal class rank
simplifiedItems
);
CmnUtil.ExportXMLFile("rank.xml", mergedFeed, "danbooru");
await CmnUtil.ExportXMLFileAsync("rank.xml", mergedFeed, client, "danbooru");
}
finally
{
@@ -575,6 +575,21 @@ internal static class pixiv_common
return items;
}
/// <summary>
/// SyndicationFeed を作成し、指定されたファイル名の XML として非同期で出力します。
/// </summary>
public static async Task CreateAndExportFeedAsync(string feedTitle, string feedDescription, string feedLink, string fileName, List<SyndicationItem> items, HttpClient? client = null)
{
var feed = new SyndicationFeed(
feedTitle,
feedDescription,
new Uri(feedLink),
items
);
await CmnUtil.ExportXMLFileAsync(fileName, feed, client);
}
/// <summary>
/// SyndicationFeed を作成し、指定されたファイル名の XML として出力します。
/// </summary>
@@ -36,12 +36,13 @@ internal class pixiv_non_r18_ai
var items = await pixiv_common.ConvertToSyndicationItemsAsync(client, illusts);
// フィード作成と XML 出力
pixiv_common.CreateAndExportFeed(
await pixiv_common.CreateAndExportFeedAsync(
FEED_TITLE,
FEED_DESCRIPTION,
FEED_LINK,
XML_FILE_NAME,
items
items,
client
);
}
catch (Exception ex)
@@ -36,12 +36,13 @@ internal class pixiv_non_r18_non_ai
var items = await pixiv_common.ConvertToSyndicationItemsAsync(client, illusts);
// フィード作成と XML 出力
pixiv_common.CreateAndExportFeed(
await pixiv_common.CreateAndExportFeedAsync(
FEED_TITLE,
FEED_DESCRIPTION,
FEED_LINK,
XML_FILE_NAME,
items
items,
client
);
}
catch (Exception ex)
@@ -36,12 +36,13 @@ internal class pixiv_r18_ai
var items = await pixiv_common.ConvertToSyndicationItemsAsync(client, illusts);
// フィード作成と XML 出力
pixiv_common.CreateAndExportFeed(
await pixiv_common.CreateAndExportFeedAsync(
FEED_TITLE,
FEED_DESCRIPTION,
FEED_LINK,
XML_FILE_NAME,
items
items,
client
);
}
catch (Exception ex)
@@ -36,12 +36,13 @@ internal class pixiv_r18_non_ai
var items = await pixiv_common.ConvertToSyndicationItemsAsync(client, illusts);
// フィード作成と XML 出力
pixiv_common.CreateAndExportFeed(
await pixiv_common.CreateAndExportFeedAsync(
FEED_TITLE,
FEED_DESCRIPTION,
FEED_LINK,
XML_FILE_NAME,
items
items,
client
);
}
catch (Exception ex)
+3 -3
View File
@@ -1,4 +1,4 @@
using System.ServiceModel.Syndication;
using System.ServiceModel.Syndication;
namespace rss_generator.Class.RSS.X;
@@ -21,7 +21,7 @@ internal class x_list
, CreateSub1(client, "2070670309846655252", "NormalIllust")
};
Task.WaitAll(tasks);
await Task.WhenAll(tasks);
}
public static async Task CreateSub1(HttpClient client, string listId, string fileName)
@@ -79,7 +79,7 @@ internal class x_list
simplifiedItems
);
CmnUtil.ExportXMLFile($"{fileName}.xml", mergedFeed);
await CmnUtil.ExportXMLFileAsync($"{fileName}.xml", mergedFeed, client);
}
catch (Exception ex)
{
@@ -72,9 +72,9 @@ public static class YouTubeFeedManager
string catTitleSuffix = $" - {category}";
ExportMergedFeed(catLive, YouTubeFeedType.Live, liveFile, $"YouTube Live{catTitleSuffix}");
ExportMergedFeed(catVideo, YouTubeFeedType.Video, videoFile, $"YouTube Videos{catTitleSuffix}");
ExportMergedFeed(catShorts, YouTubeFeedType.Shorts, shortsFile, $"YouTube Shorts{catTitleSuffix}");
await ExportMergedFeedAsync(client, catLive, YouTubeFeedType.Live, liveFile, $"YouTube Live{catTitleSuffix}");
await ExportMergedFeedAsync(client, catVideo, YouTubeFeedType.Video, videoFile, $"YouTube Videos{catTitleSuffix}");
await ExportMergedFeedAsync(client, catShorts, YouTubeFeedType.Shorts, shortsFile, $"YouTube Shorts{catTitleSuffix}");
}
catch (Exception ex)
{
@@ -323,7 +323,8 @@ public static class YouTubeFeedManager
/// <summary>
/// 全チャンネル統合フィードを出力します。
/// </summary>
private static void ExportMergedFeed(
private static async Task ExportMergedFeedAsync(
HttpClient client,
List<YouTubeVideoItem> items,
YouTubeFeedType feedType,
string fileName,
@@ -353,7 +354,7 @@ public static class YouTubeFeedManager
distinctItems
);
CmnUtil.ExportXMLFile(fileName, feed, YouTubeConfig.OutputSubDir);
await CmnUtil.ExportXMLFileAsync(fileName, feed, client, YouTubeConfig.OutputSubDir);
}
/// <summary>
+2 -2
View File
@@ -1,4 +1,4 @@
using AngleSharp.Html.Dom;
using AngleSharp.Html.Dom;
using System;
using System.Collections.Generic;
using System.ServiceModel.Syndication;
@@ -50,7 +50,7 @@ internal class Youtube
items
);
CmnUtil.ExportXMLFile($"{fileName}.xml", mergedFeed, "youtube");
await CmnUtil.ExportXMLFileAsync($"{fileName}.xml", mergedFeed, client, "youtube");
}
}
catch (Exception ex)
+1 -1
View File
@@ -82,7 +82,7 @@ internal class aurahentai
feed.Items = uniqueItems;
CmnUtil.ExportXMLFile("aurahentai.xml", feed);
await CmnUtil.ExportXMLFileAsync("aurahentai.xml", feed, client);
}
catch (Exception ex)
{
+1 -1
View File
@@ -183,7 +183,7 @@ internal class ci_en
}
feed.Items = items;
CmnUtil.ExportXMLFile("ci_en.xml", feed);
await CmnUtil.ExportXMLFileAsync("ci_en.xml", feed, client);
}
catch (Exception ex)
{
@@ -34,7 +34,7 @@ internal class e_hentai_ai
);
// XML ファイルとして出力
CmnUtil.ExportXMLFile("e_hentai_ai.xml", feed);
await CmnUtil.ExportXMLFileAsync("e_hentai_ai.xml", feed, client);
}
catch (Exception ex)
{
@@ -34,7 +34,7 @@ internal class e_hentai_non_ai
);
// XML ファイルとして出力
CmnUtil.ExportXMLFile("e_hentai_non_ai.xml", feed);
await CmnUtil.ExportXMLFileAsync("e_hentai_non_ai.xml", feed, client);
}
catch (Exception ex)
{
@@ -68,7 +68,7 @@ internal class index_japanese
feed.Items = uniqueItems;
CmnUtil.ExportXMLFile("hitomi-japanese.xml", feed);
await CmnUtil.ExportXMLFileAsync("hitomi-japanese.xml", feed, client);
}
catch (Exception ex)
{
+1 -1
View File
@@ -115,7 +115,7 @@ internal class javtwi
feed.Items = items;
CmnUtil.ExportXMLFile("javtwi.xml", feed);
await CmnUtil.ExportXMLFileAsync("javtwi.xml", feed, client);
}
catch (Exception ex)
{
+1 -1
View File
@@ -122,7 +122,7 @@ internal class kimochiinfo
}
feed.Items = items;
CmnUtil.ExportXMLFile("kimochiinfo.xml", feed);
await CmnUtil.ExportXMLFileAsync("kimochiinfo.xml", feed, client);
}
catch (Exception ex)
{
+1 -2
View File
@@ -191,8 +191,7 @@ internal class mangalear
simplifiedItems
);
// 5. XML ファイルとして出力
CmnUtil.ExportXMLFile("mangalear.xml", mergedFeed);
await CmnUtil.ExportXMLFileAsync("mangalear.xml", mergedFeed, client);
}
catch (Exception ex)
{
+1 -1
View File
@@ -94,7 +94,7 @@ internal static class missav
}
feed.Items = items;
CmnUtil.ExportXMLFile("missav.xml", feed);
await CmnUtil.ExportXMLFileAsync("missav.xml", feed, client);
}
catch (Exception ex)
{
+1 -1
View File
@@ -209,7 +209,7 @@ internal class missav123
}
feed.Items = items;
CmnUtil.ExportXMLFile("123av.xml", feed);
await CmnUtil.ExportXMLFileAsync("123av.xml", feed, client);
}
finally
{
+1 -1
View File
@@ -169,7 +169,7 @@ internal class momonga_fanzine
simplifiedItems
);
CmnUtil.ExportXMLFile("momonga-fanzine.xml", mergedFeed);
await CmnUtil.ExportXMLFileAsync("momonga-fanzine.xml", mergedFeed, client);
}
catch (Exception ex)
{
+1 -1
View File
@@ -169,7 +169,7 @@ internal class momonga_magazine
simplifiedItems
);
CmnUtil.ExportXMLFile("momonga-magazine.xml", mergedFeed);
await CmnUtil.ExportXMLFileAsync("momonga-magazine.xml", mergedFeed, client);
}
catch (Exception ex)
{
+1 -1
View File
@@ -81,7 +81,7 @@ internal class npb_6
}
feed.Items = items;
CmnUtil.ExportXMLFile("npb_6.xml", feed);
await CmnUtil.ExportXMLFileAsync("npb_6.xml", feed, client);
}
catch (Exception ex)
{