From ef730787dd0304cb81682c8dc76af62dbf78abb8 Mon Sep 17 00:00:00 2001 From: uduki Date: Sat, 5 Sep 2026 13:08:17 +0900 Subject: [PATCH] =?UTF-8?q?=E3=83=97=E3=83=AD=E3=82=B0=E3=83=A9=E3=83=A0?= =?UTF-8?q?=E3=81=AE=E8=A8=AD=E7=BD=AE=E5=85=88=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 19 ++++ compose.yaml | 26 +++++ rss-generator/Class/CmnUtil.cs | 12 +- rss-generator/Class/RSS/X/x_list.cs | 31 ++++-- .../Class/RSS/YouTube/YouTubeFeedManager.cs | 4 +- rss-generator/Program.cs | 105 ++++++++++++------ rss-generator/rss-generator.csproj | 23 +++- 7 files changed, 161 insertions(+), 59 deletions(-) create mode 100644 Dockerfile create mode 100644 compose.yaml diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..261b7e0 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,19 @@ +# .NET 10.0 ランタイムベースイメージ +FROM mcr.microsoft.com/dotnet/runtime:10.0 AS runtime +WORKDIR /app + +# タイムゾーン設定 (JST) および必要なツールのインストール +RUN apt-get update && apt-get install -y --no-install-recommends \ + tzdata \ + ca-certificates \ + curl \ + && rm -rf /var/lib/apt/lists/* + +# 日本時間に設定 +ENV TZ=Asia/Tokyo + +# 発行済みバイナリをコンテナ内へ配置 +COPY ./app . + +# アプリケーション実行 +ENTRYPOINT ["dotnet", "rss-generator.dll"] diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..d2ee898 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,26 @@ +services: + rss-generator: + build: . + image: local-rss-generator:latest + container_name: rss-generator + restart: always + mem_limit: 512m + memswap_limit: 512m + deploy: + resources: + limits: + memory: 512M + environment: + - TZ=Asia/Tokyo + # .NET GC設定: Workstation GC強制、ヒープ制限70% + - DOTNET_gcServer=0 + - DOTNET_GCHeapHardLimitPercent=70 + volumes: + # 生成されたXMLフィードをホストの C:\feeds に出力 + - C:\feeds:/app/output + networks: + - common-network + +networks: + common-network: + external: true diff --git a/rss-generator/Class/CmnUtil.cs b/rss-generator/Class/CmnUtil.cs index 17ebef5..4c33f59 100644 --- a/rss-generator/Class/CmnUtil.cs +++ b/rss-generator/Class/CmnUtil.cs @@ -252,8 +252,8 @@ public class CmnUtil return null; } - // 15MB 以上の巨大画像は安全のためスキップ - if (response.Content.Headers.ContentLength.HasValue && response.Content.Headers.ContentLength.Value > 15 * 1024 * 1024) + // 2MB 以上の画像はメモリ保護のためスキップ(元のURLを参照させる) + if (response.Content.Headers.ContentLength.HasValue && response.Content.Headers.ContentLength.Value > 2 * 1024 * 1024) { return null; } @@ -281,7 +281,7 @@ public class CmnUtil /// SyndicationFeed に含まれる全記事(24時間フィルタ適用後)の HTML 本文から画像 URL を抽出し、 /// 並列ダウンロードして Base64 形式(Data URI)に置換します。 /// - public static async Task EmbedImagesAsBase64InFeedAsync(SyndicationFeed feed, HttpClient client, int maxParallel = 8) + public static async Task EmbedImagesAsBase64InFeedAsync(SyndicationFeed feed, HttpClient client, int maxParallel = 2) { if (feed.Items == null) return; @@ -321,7 +321,7 @@ public class CmnUtil if (targetUrls.Count == 0) return; - // 2. 並列ダウンロードして Base64 辞書を構築 + // 2. 並列ダウンロードして Base64 辞書を構築(メモリ節約のため同時並行数は2に制御) var cacheMap = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); using var semaphore = new SemaphoreSlim(maxParallel); @@ -377,6 +377,10 @@ public class CmnUtil int distinctSuccess = targetUrls.Keys.Count(k => cacheMap.ContainsKey(k)); LogManager.AddLog($"[情報] [CmnUtil] 画像の Base64 変換完了: {distinctSuccess}/{targetUrls.Count} 件成功 (記事数: {itemsList.Count})"); + + // 使用済みメモリの即時解放 + cacheMap.Clear(); + targetUrls.Clear(); } /// diff --git a/rss-generator/Class/RSS/X/x_list.cs b/rss-generator/Class/RSS/X/x_list.cs index 625c7e7..4b05a67 100644 --- a/rss-generator/Class/RSS/X/x_list.cs +++ b/rss-generator/Class/RSS/X/x_list.cs @@ -9,19 +9,30 @@ internal class x_list private const string QUERY2 = "showEmojiForRetweetAndReply=1?filter_title=%5E%28🔁%7CRT%29"; public static async Task CreateRSSAsync(HttpClient client) { - var tasks = new List() + // メモリ消費を最小化するため、各リストを1つずつ順次処理 + var targets = new (string ListId, string FileName)[] { - CreateSub1(client, "2076742238231818243", "granbluefantasy") - , CreateSub1(client, "1695011723005280535", "AI") - , CreateSub1(client, "1573473287333560321", "R18") - , CreateSub1(client, "2070669317667217577", "VR") - , CreateSub1(client, "1573916143969976320", "VTuber") - , CreateSub1(client, "2070673943057539507", "AnimeReal") - , CreateSub1(client, "1599409403743961088", "AnimeOther") - , CreateSub1(client, "2070670309846655252", "NormalIllust") + ("2076742238231818243", "granbluefantasy"), + ("1695011723005280535", "AI"), + ("1573473287333560321", "R18"), + ("2070669317667217577", "VR"), + ("1573916143969976320", "VTuber"), + ("2070673943057539507", "AnimeReal"), + ("1599409403743961088", "AnimeOther"), + ("2070670309846655252", "NormalIllust") }; - await Task.WhenAll(tasks); + foreach (var (listId, fileName) in targets) + { + try + { + await CreateSub1(client, listId, fileName); + } + catch (Exception ex) + { + LogManager.AddLog($"[エラー] Xリスト生成失敗 ({fileName}): {ex.Message}"); + } + } } public static async Task CreateSub1(HttpClient client, string listId, string fileName) diff --git a/rss-generator/Class/RSS/YouTube/YouTubeFeedManager.cs b/rss-generator/Class/RSS/YouTube/YouTubeFeedManager.cs index 1e08635..d3cae02 100644 --- a/rss-generator/Class/RSS/YouTube/YouTubeFeedManager.cs +++ b/rss-generator/Class/RSS/YouTube/YouTubeFeedManager.cs @@ -29,8 +29,8 @@ public static class YouTubeFeedManager var allVideoItems = new List(); var allShortsItems = new List(); - // YouTubeからのレート制限を防ぐため、同時並行数を安全な値(6)に制御 - using var semaphore = new System.Threading.SemaphoreSlim(6); + // メモリ消費とレート制限を抑えるため、同時並行数を2に制御 + using var semaphore = new System.Threading.SemaphoreSlim(2); var channelTasks = YouTubeConfig.Channels.Select(async channel => { diff --git a/rss-generator/Program.cs b/rss-generator/Program.cs index 7114505..ffd67d9 100644 --- a/rss-generator/Program.cs +++ b/rss-generator/Program.cs @@ -79,51 +79,72 @@ while (true) // サイトによってはAcceptヘッダーがないと弾く場合もあるため、念のため追加 client.DefaultRequestHeaders.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"); - List tasks = new List(); + // メモリ消費を最小化(512MB制限内)するため、各フィードを並行ではなく順次実行し、タスクごとにGCを実行 + var feedActions = new List<(string Name, Func Action)>(); DateTime from = new DateTime(nowDate.Year, nowDate.Month, nowDate.Day, 5, 0, 0); DateTime to = new DateTime(nowDate.Year, nowDate.Month, nowDate.Day, 7, 15, 0); if (from < nowDate && nowDate < to) { - tasks.Add(momonga_fanzine.CreateRSSAsync(client)); - tasks.Add(momonga_magazine.CreateRSSAsync(client)); + feedActions.Add(("momonga_fanzine", () => momonga_fanzine.CreateRSSAsync(client))); + feedActions.Add(("momonga_magazine", () => momonga_magazine.CreateRSSAsync(client))); } #if DEBUG - tasks.Add(momonga_fanzine.CreateRSSAsync(client)); - tasks.Add(momonga_magazine.CreateRSSAsync(client)); - tasks.Add(YouTubeFeedManager.CreateRSSAsync(client)); - tasks.Add(rss_generator.Class.RSS.X.x_list.CreateRSSAsync(client)); - tasks.Add(kimochiinfo.CreateRSSAsync(client)); - tasks.Add(ci_en.CreateRSSAsync(client)); - tasks.Add(javtwi.CreateRSSAsync(client)); - tasks.Add(mangalear.CreateRSSAsync(client)); - tasks.Add(e_hentai_ai.CreateRSSAsync(client)); - tasks.Add(e_hentai_non_ai.CreateRSSAsync(client)); - tasks.Add(pixiv_non_r18_non_ai.CreateRSSAsync(client)); - tasks.Add(pixiv_non_r18_ai.CreateRSSAsync(client)); - tasks.Add(pixiv_r18_non_ai.CreateRSSAsync(client)); - tasks.Add(pixiv_r18_ai.CreateRSSAsync(client)); - tasks.Add(FlareSolverrFeedManager.CreateAllRSSAsync(client)); + feedActions.Add(("momonga_fanzine", () => momonga_fanzine.CreateRSSAsync(client))); + feedActions.Add(("momonga_magazine", () => momonga_magazine.CreateRSSAsync(client))); + feedActions.Add(("YouTube", () => YouTubeFeedManager.CreateRSSAsync(client))); + feedActions.Add(("X (Twitter)", () => rss_generator.Class.RSS.X.x_list.CreateRSSAsync(client))); + feedActions.Add(("kimochiinfo", () => kimochiinfo.CreateRSSAsync(client))); + feedActions.Add(("ci_en", () => ci_en.CreateRSSAsync(client))); + feedActions.Add(("javtwi", () => javtwi.CreateRSSAsync(client))); + feedActions.Add(("mangalear", () => mangalear.CreateRSSAsync(client))); + feedActions.Add(("e_hentai_ai", () => e_hentai_ai.CreateRSSAsync(client))); + feedActions.Add(("e_hentai_non_ai", () => e_hentai_non_ai.CreateRSSAsync(client))); + feedActions.Add(("pixiv_non_r18_non_ai", () => pixiv_non_r18_non_ai.CreateRSSAsync(client))); + feedActions.Add(("pixiv_non_r18_ai", () => pixiv_non_r18_ai.CreateRSSAsync(client))); + feedActions.Add(("pixiv_r18_non_ai", () => pixiv_r18_non_ai.CreateRSSAsync(client))); + feedActions.Add(("pixiv_r18_ai", () => pixiv_r18_ai.CreateRSSAsync(client))); + feedActions.Add(("FlareSolverr", () => FlareSolverrFeedManager.CreateAllRSSAsync(client))); #else - tasks.Add(YouTubeFeedManager.CreateRSSAsync(client)); - tasks.Add(rss_generator.Class.RSS.X.x_list.CreateRSSAsync(client)); - tasks.Add(kimochiinfo.CreateRSSAsync(client)); - tasks.Add(ci_en.CreateRSSAsync(client)); - tasks.Add(javtwi.CreateRSSAsync(client)); - tasks.Add(mangalear.CreateRSSAsync(client)); - tasks.Add(e_hentai_ai.CreateRSSAsync(client)); - tasks.Add(e_hentai_non_ai.CreateRSSAsync(client)); - tasks.Add(pixiv_non_r18_non_ai.CreateRSSAsync(client)); - tasks.Add(pixiv_non_r18_ai.CreateRSSAsync(client)); - tasks.Add(pixiv_r18_non_ai.CreateRSSAsync(client)); - tasks.Add(pixiv_r18_ai.CreateRSSAsync(client)); + feedActions.Add(("YouTube", () => YouTubeFeedManager.CreateRSSAsync(client))); + feedActions.Add(("X (Twitter)", () => rss_generator.Class.RSS.X.x_list.CreateRSSAsync(client))); + feedActions.Add(("kimochiinfo", () => kimochiinfo.CreateRSSAsync(client))); + feedActions.Add(("ci_en", () => ci_en.CreateRSSAsync(client))); + feedActions.Add(("javtwi", () => javtwi.CreateRSSAsync(client))); + feedActions.Add(("mangalear", () => mangalear.CreateRSSAsync(client))); + feedActions.Add(("e_hentai_ai", () => e_hentai_ai.CreateRSSAsync(client))); + feedActions.Add(("e_hentai_non_ai", () => e_hentai_non_ai.CreateRSSAsync(client))); + feedActions.Add(("pixiv_non_r18_non_ai", () => pixiv_non_r18_non_ai.CreateRSSAsync(client))); + feedActions.Add(("pixiv_non_r18_ai", () => pixiv_non_r18_ai.CreateRSSAsync(client))); + feedActions.Add(("pixiv_r18_non_ai", () => pixiv_r18_non_ai.CreateRSSAsync(client))); + feedActions.Add(("pixiv_r18_ai", () => pixiv_r18_ai.CreateRSSAsync(client))); // FlareSolverr 利用タスク群(単一セッションで順次実行し、完了後に破棄してメモリ最小化) - tasks.Add(FlareSolverrFeedManager.CreateAllRSSAsync(client)); + feedActions.Add(("FlareSolverr", () => FlareSolverrFeedManager.CreateAllRSSAsync(client))); #endif - - await Task.WhenAll(tasks); + foreach (var (name, action) in feedActions) + { + try + { + var taskSw = Stopwatch.StartNew(); + LogManager.AddLog($"[情報] 【{name}】 取得開始"); + await action(); + taskSw.Stop(); + LogManager.AddLog($"[情報] 【{name}】 完了 ({taskSw.Elapsed.TotalSeconds:0.##}秒)"); + } + catch (Exception ex) + { + LogManager.AddLog($"[エラー] 【{name}】 実行時エラー: {ex.Message}"); + } + finally + { + // 各タスク完了後にメモリを積極回収してOSに返却 + GC.Collect(2, GCCollectionMode.Aggressive, true, true); + GC.WaitForPendingFinalizers(); + GC.Collect(2, GCCollectionMode.Aggressive, true, true); + } + } sw.Stop(); LogManager.AddLog($"[情報] RSS生成完了 実行時間:{sw.Elapsed.TotalSeconds:0.##}秒"); @@ -139,11 +160,21 @@ while (true) await LogManager.FlushAndCloseAsync(); break; #else - LogManager.AddLog($"[情報] FreshRSS 更新開始"); - await CmnUtil.RunCommandAsync("/bin/bash", "/usr/local/bin/update_rss.sh", ""); + // FreshRSS更新スクリプトが存在する場合のみ実行(ローカル実行時は安全にスキップ) + const string updateScriptPath = "/usr/local/bin/update_rss.sh"; + if (File.Exists(updateScriptPath)) + { + LogManager.AddLog($"[情報] FreshRSS 更新開始"); + await CmnUtil.RunCommandAsync("/bin/bash", updateScriptPath, ""); - sw.Stop(); - LogManager.AddLog($"[情報] FreshRSS 更新完了 実行時間:{sw.Elapsed.TotalSeconds:0.##}秒"); + sw.Stop(); + LogManager.AddLog($"[情報] FreshRSS 更新完了 実行時間:{sw.Elapsed.TotalSeconds:0.##}秒"); + } + else + { + sw.Stop(); + LogManager.AddLog($"[情報] FreshRSS 更新スクリプトが存在しないためスキップしました"); + } totalSw.Stop(); LogManager.AddLog($"[情報] 全処理完了 トータル実行時間:{totalSw.Elapsed.TotalSeconds:0.##}秒"); diff --git a/rss-generator/rss-generator.csproj b/rss-generator/rss-generator.csproj index 933b076..16816b4 100644 --- a/rss-generator/rss-generator.csproj +++ b/rss-generator/rss-generator.csproj @@ -1,4 +1,4 @@ - + Exe @@ -6,6 +6,11 @@ rss_generator enable enable + + false + true + + false @@ -24,11 +29,17 @@ + - - - - - + + + + + + + + + +