From fe5e3335af071476bce3c95993cb9c6f8a370763 Mon Sep 17 00:00:00 2001 From: uduki Date: Wed, 2 Sep 2026 09:54:56 +0900 Subject: [PATCH] feat: implement initial RSS generation modules for e-hentai, hitomi.la, and YouTube integrations --- .../Class/RSS/YouTube/YouTubeConfig.cs | 51 +++ .../Class/RSS/YouTube/YouTubeFeedManager.cs | 247 ++++++++++ .../Class/RSS/YouTube/YouTubeModels.cs | 137 ++++++ .../Class/RSS/YouTube/YouTubeParser.cs | 431 ++++++++++++++++++ .../Class/RSS/e_hentai/e_hentai_common.cs | 74 +-- .../Class/RSS/hitomi_la/index_japanese.cs | 319 +++++++------ rss-generator/Program.cs | 14 +- 7 files changed, 1070 insertions(+), 203 deletions(-) create mode 100644 rss-generator/Class/RSS/YouTube/YouTubeConfig.cs create mode 100644 rss-generator/Class/RSS/YouTube/YouTubeFeedManager.cs create mode 100644 rss-generator/Class/RSS/YouTube/YouTubeModels.cs create mode 100644 rss-generator/Class/RSS/YouTube/YouTubeParser.cs diff --git a/rss-generator/Class/RSS/YouTube/YouTubeConfig.cs b/rss-generator/Class/RSS/YouTube/YouTubeConfig.cs new file mode 100644 index 0000000..47e133a --- /dev/null +++ b/rss-generator/Class/RSS/YouTube/YouTubeConfig.cs @@ -0,0 +1,51 @@ +using System.Collections.Generic; + +namespace rss_generator.Class.RSS.YouTube; + +/// +/// YouTubeフィード生成の設定情報 +/// +public static class YouTubeConfig +{ + /// + /// 出力対象とするYouTubeチャンネル・ユーザーの固定一覧。 + /// ここにチャンネルID(UC...)またはハンドル(@...)を追加してください。 + /// + public static readonly List Channels = new() + { + new YouTubeChannelTarget + { + IdOrHandle = "@ShirakamiFubuki", + Name = "白上フブキ", + FilePrefix = "shirakami_fubuki" + }, + // 必要に応じてチャンネルを追加・編集してください。 + // 例: + // new YouTubeChannelTarget + // { + // IdOrHandle = "UCdn5BQ06XqgXoAxIhbqw5Rg", + // Name = "白上フブキ", + // FilePrefix = "fubuki_ch" + // }, + }; + + /// + /// チャンネルごとの個別XMLファイルを出力するかどうか(例: shirakami_fubuki_live.xml) + /// + public static bool ExportIndividualFeeds { get; set; } = true; + + /// + /// 全チャンネルを統合したXMLファイルを出力するかどうか(all_live.xml, all_video.xml, all_shorts.xml) + /// + public static bool ExportMergedFeeds { get; set; } = true; + + /// + /// 出力先のサブディレクトリ名(output/youtube/ 配下に出力されます) + /// + public static string OutputSubDir { get; set; } = "youtube"; + + /// + /// 各フィードの最大保持件数 + /// + public static int MaxItemsPerFeed { get; set; } = 30; +} diff --git a/rss-generator/Class/RSS/YouTube/YouTubeFeedManager.cs b/rss-generator/Class/RSS/YouTube/YouTubeFeedManager.cs new file mode 100644 index 0000000..dd8a1bc --- /dev/null +++ b/rss-generator/Class/RSS/YouTube/YouTubeFeedManager.cs @@ -0,0 +1,247 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.ServiceModel.Syndication; +using System.Text; +using System.Threading.Tasks; + +namespace rss_generator.Class.RSS.YouTube; + +/// +/// YouTubeのライブ配信・通常動画・ショート動画を分離してRSS(XML)を生成・出力するマネージャークラス +/// +public static class YouTubeFeedManager +{ + /// + /// YouTubeの各チャンネルからフィードを生成してXMLファイルに出力します。 + /// + /// HttpClient インスタンス + public static async Task CreateRSSAsync(HttpClient client) + { + LogManager.AddLog("[情報] YouTubeフィード生成を開始します"); + + try + { + var allLiveItems = new List(); + var allVideoItems = new List(); + var allShortsItems = new List(); + + var channelTasks = YouTubeConfig.Channels.Select(async channel => + { + try + { + await ProcessChannelAsync(client, channel, allLiveItems, allVideoItems, allShortsItems); + } + catch (Exception ex) + { + LogManager.AddLog($"[エラー] チャンネル処理失敗 ({channel.Name ?? channel.IdOrHandle}): {ex.Message}"); + } + }); + + await Task.WhenAll(channelTasks); + + // 全チャンネル統合フィードの出力 + if (YouTubeConfig.ExportMergedFeeds) + { + ExportMergedFeed(allLiveItems, YouTubeFeedType.Live, "all_live.xml", "YouTube Live (All Channels)"); + ExportMergedFeed(allVideoItems, YouTubeFeedType.Video, "all_video.xml", "YouTube Videos (All Channels)"); + ExportMergedFeed(allShortsItems, YouTubeFeedType.Shorts, "all_shorts.xml", "YouTube Shorts (All Channels)"); + } + + LogManager.AddLog("[情報] YouTubeフィード生成が完了しました"); + } + catch (Exception ex) + { + LogManager.AddLog($"[エラー] YouTubeFeedManager.CreateRSSAsync: {ex.Message}"); + CmnUtil.PrintCallStackToEvent(ex); + } + } + + /// + /// 1つのチャンネルについて各タブ(Live, Video, Shorts)を取得・処理します。 + /// + private static async Task ProcessChannelAsync( + HttpClient client, + YouTubeChannelTarget channel, + List allLiveItems, + List allVideoItems, + List allShortsItems) + { + // 3つのタブを並行取得 + var liveTask = FetchTabItemsAsync(client, channel, YouTubeFeedType.Live); + var videoTask = FetchTabItemsAsync(client, channel, YouTubeFeedType.Video); + var shortsTask = FetchTabItemsAsync(client, channel, YouTubeFeedType.Shorts); + + await Task.WhenAll(liveTask, videoTask, shortsTask); + + var liveItems = await liveTask; + var videoItems = await videoTask; + var shortsItems = await shortsTask; + + // 個別フィードの出力 + if (YouTubeConfig.ExportIndividualFeeds) + { + string prefix = channel.GetSafeFilePrefix(); + string channelTitle = channel.Name ?? channel.IdOrHandle; + + ExportFeed(liveItems, channelTitle, $"{prefix}_live.xml", YouTubeFeedType.Live); + ExportFeed(videoItems, channelTitle, $"{prefix}_video.xml", YouTubeFeedType.Video); + ExportFeed(shortsItems, channelTitle, $"{prefix}_shorts.xml", YouTubeFeedType.Shorts); + } + + // 統合フィード用に蓄積 + lock (allLiveItems) { allLiveItems.AddRange(liveItems); } + lock (allVideoItems) { allVideoItems.AddRange(videoItems); } + lock (allShortsItems) { allShortsItems.AddRange(shortsItems); } + } + + /// + /// 指定されたタブ種別のHTMLを取得してパースします。 + /// + private static async Task> FetchTabItemsAsync( + HttpClient client, + YouTubeChannelTarget channel, + YouTubeFeedType feedType) + { + string url = channel.GetTabUrl(feedType); + + try + { + // HttpRequestMessageを作成してリクエスト送信 + using var request = new HttpRequestMessage(HttpMethod.Get, url); + request.Headers.Add("Accept-Language", "ja-JP,ja;q=0.9"); + + using var response = await client.SendAsync(request); + if (!response.IsSuccessStatusCode) + { + LogManager.AddLog($"[警告] HTTPステータス異常 ({response.StatusCode}): {url}"); + return new List(); + } + + string html = await response.Content.ReadAsStringAsync(); + var items = YouTubeParser.ParseTabContent(html, feedType, channel); + + // 最大件数の制御 + return items.Take(YouTubeConfig.MaxItemsPerFeed).ToList(); + } + catch (Exception ex) + { + LogManager.AddLog($"[エラー] タブ取得失敗 ({url}): {ex.Message}"); + return new List(); + } + } + + /// + /// チャンネル個別フィードを出力します。 + /// + private static void ExportFeed( + List items, + string channelTitle, + string fileName, + YouTubeFeedType feedType) + { + string typeLabel = feedType switch + { + YouTubeFeedType.Live => "ライブ配信", + YouTubeFeedType.Video => "動画", + YouTubeFeedType.Shorts => "ショート動画", + _ => "フィード" + }; + + var syndicationItems = items.Select(CreateSyndicationItem).ToList(); + + var feed = new SyndicationFeed( + $"{channelTitle} - {typeLabel}", + $"{channelTitle} の {typeLabel} RSSフィード", + new Uri(items.FirstOrDefault()?.ChannelUrl ?? "https://www.youtube.com"), + syndicationItems + ); + + CmnUtil.ExportXMLFile(fileName, feed, YouTubeConfig.OutputSubDir); + } + + /// + /// 全チャンネル統合フィードを出力します。 + /// + private static void ExportMergedFeed( + List items, + YouTubeFeedType feedType, + string fileName, + string feedTitle) + { + string typeLabel = feedType switch + { + YouTubeFeedType.Live => "ライブ配信", + YouTubeFeedType.Video => "動画", + YouTubeFeedType.Shorts => "ショート動画", + _ => "フィード" + }; + + // 公開日時の新しい順にソートし、重複を除外 + var distinctItems = items + .GroupBy(i => i.VideoId) + .Select(g => g.First()) + .OrderByDescending(i => i.PublishedDate) + .Take(YouTubeConfig.MaxItemsPerFeed * 3) + .Select(CreateSyndicationItem) + .ToList(); + + var feed = new SyndicationFeed( + $"{feedTitle} ({typeLabel})", + $"登録チャンネル全件の {typeLabel} 統合RSSフィード", + new Uri("https://www.youtube.com"), + distinctItems + ); + + CmnUtil.ExportXMLFile(fileName, feed, YouTubeConfig.OutputSubDir); + } + + /// + /// YouTubeVideoItemからSyndicationItemを生成します。 + /// + private static SyndicationItem CreateSyndicationItem(YouTubeVideoItem video) + { + var item = new SyndicationItem + { + Title = new TextSyndicationContent(video.Title), + Id = CmnUtil.CreateGuid(video.Url), + PublishDate = video.PublishedDate, + LastUpdatedTime = video.PublishedDate + }; + + item.Links.Add(new SyndicationLink(new Uri(video.Url))); + + if (!string.IsNullOrWhiteSpace(video.ChannelTitle)) + { + item.Authors.Add(new SyndicationPerson { Name = video.ChannelTitle }); + } + + item.Categories.Add(new SyndicationCategory(video.FeedType.ToString())); + + // HTML形式のSummaryを生成 + var body = new StringBuilder(); + + if (!string.IsNullOrWhiteSpace(video.ThumbnailUrl)) + { + body.Append($@"
"); + } + + body.Append($@""); + body.Append($@"
チャンネル: {WebUtility.HtmlEncode(video.ChannelTitle)}
"); + + if (!string.IsNullOrWhiteSpace(video.PublishedText)) + { + body.Append($@"
ステータス/日時: {WebUtility.HtmlEncode(video.PublishedText)}
"); + } + + if (!string.IsNullOrWhiteSpace(video.ViewCountText)) + { + body.Append($@"
視聴情報: {WebUtility.HtmlEncode(video.ViewCountText)}
"); + } + + item.Summary = new TextSyndicationContent(body.ToString(), TextSyndicationContentKind.Html); + + return item; + } +} diff --git a/rss-generator/Class/RSS/YouTube/YouTubeModels.cs b/rss-generator/Class/RSS/YouTube/YouTubeModels.cs new file mode 100644 index 0000000..c20cf66 --- /dev/null +++ b/rss-generator/Class/RSS/YouTube/YouTubeModels.cs @@ -0,0 +1,137 @@ +using System; + +namespace rss_generator.Class.RSS.YouTube; + +/// +/// YouTubeフィードの種類(ライブ配信・通常動画・ショート動画) +/// +public enum YouTubeFeedType +{ + /// ライブ配信(配信中またはアーカイブ) + Live, + /// 通常の動画 + Video, + /// ショート動画 + Shorts +} + +/// +/// 監視対象チャンネル・ユーザーの設定情報 +/// +public class YouTubeChannelTarget +{ + /// + /// チャンネルID(例: "UCdn5BQ06XqgXoAxIhbqw5Rg")またはハンドル名(例: "@ShirakamiFubuki") + /// + public string IdOrHandle { get; set; } = ""; + + /// + /// チャンネル名(未指定の場合は自動取得またはIdOrHandleを使用) + /// + public string? Name { get; set; } + + /// + /// 出力ファイル名のプレフィックス(例: "fubuki" とすると "fubuki_live.xml" などが出力されます) + /// 未指定の場合はIdOrHandleから安全な文字列を生成します。 + /// + public string? FilePrefix { get; set; } + + /// + /// 指定したタブ種別に応じたURLを取得します。 + /// + public string GetTabUrl(YouTubeFeedType type) + { + string baseTarget = IdOrHandle.Trim(); + string baseUrl; + + if (baseTarget.StartsWith("@")) + { + baseUrl = $"https://www.youtube.com/{baseTarget}"; + } + else if (baseTarget.StartsWith("UC", StringComparison.OrdinalIgnoreCase)) + { + baseUrl = $"https://www.youtube.com/channel/{baseTarget}"; + } + else if (baseTarget.StartsWith("http://", StringComparison.OrdinalIgnoreCase) || + baseTarget.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) + { + baseUrl = baseTarget.TrimEnd('/'); + } + else + { + // @なしのハンドルまたはチャンネルID + baseUrl = $"https://www.youtube.com/@{baseTarget}"; + } + + return type switch + { + YouTubeFeedType.Live => $"{baseUrl}/streams", + YouTubeFeedType.Video => $"{baseUrl}/videos", + YouTubeFeedType.Shorts => $"{baseUrl}/shorts", + _ => baseUrl + }; + } + + /// + /// ファイル名用の安全なプレフィックスを取得します。 + /// + public string GetSafeFilePrefix() + { + if (!string.IsNullOrWhiteSpace(FilePrefix)) + { + return FilePrefix.Trim(); + } + + var prefix = IdOrHandle.Trim().TrimStart('@'); + foreach (char c in System.IO.Path.GetInvalidFileNameChars()) + { + prefix = prefix.Replace(c, '_'); + } + return prefix; + } +} + +/// +/// 抽出されたYouTube動画アイテム情報 +/// +public class YouTubeVideoItem +{ + /// 動画ID + public string VideoId { get; set; } = ""; + + /// 動画タイトル + public string Title { get; set; } = ""; + + /// 動画URL + public string Url { get; set; } = ""; + + /// サムネイル画像URL + public string ThumbnailUrl { get; set; } = ""; + + /// チャンネル名 + public string ChannelTitle { get; set; } = ""; + + /// チャンネルURL + public string ChannelUrl { get; set; } = ""; + + /// 公開日時(推定または取得日時) + public DateTimeOffset PublishedDate { get; set; } = DateTimeOffset.Now; + + /// YouTube上に表示されていた日時文字列(例: "2 日前 に配信済み", "3 時間前") + public string PublishedText { get; set; } = ""; + + /// フィード種別 + public YouTubeFeedType FeedType { get; set; } + + /// 現在ライブ配信中かどうか + public bool IsLiveNow { get; set; } + + /// 予約枠(公開予定・待機所)かどうか + public bool IsUpcoming { get; set; } + + /// メンバー限定かどうか + public bool IsMembersOnly { get; set; } + + /// 視聴回数や同接数の表示文字列 + public string ViewCountText { get; set; } = ""; +} diff --git a/rss-generator/Class/RSS/YouTube/YouTubeParser.cs b/rss-generator/Class/RSS/YouTube/YouTubeParser.cs new file mode 100644 index 0000000..fac6600 --- /dev/null +++ b/rss-generator/Class/RSS/YouTube/YouTubeParser.cs @@ -0,0 +1,431 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text.Json; +using System.Text.RegularExpressions; + +namespace rss_generator.Class.RSS.YouTube; + +/// +/// YouTubeのHTMLからytInitialDataを抽出・解析するパーサークラス +/// +public static class YouTubeParser +{ + private static readonly Regex YtInitialDataRegex = new( + @"var ytInitialData\s*=\s*(\{.+?\});|ytInitialData\s*=\s*(\{.+?\});", + RegexOptions.Compiled | RegexOptions.Singleline); + + /// + /// YouTubeチャンネルタブのHTMLから動画アイテム一覧を抽出します。 + /// + /// 取得したHTML文字列 + /// タブ種別(Live, Video, Shorts) + /// チャンネル設定 + /// 条件に合致する動画アイテムリスト + public static List ParseTabContent(string html, YouTubeFeedType feedType, YouTubeChannelTarget channel) + { + var items = new List(); + if (string.IsNullOrWhiteSpace(html)) return items; + + var match = YtInitialDataRegex.Match(html); + if (!match.Success) return items; + + // グループ1またはグループ2からJSON文字列を取得 + string jsonText = match.Groups[1].Success ? match.Groups[1].Value : match.Groups[2].Value; + + try + { + using var doc = JsonDocument.Parse(jsonText); + var root = doc.RootElement; + + // チャンネル名の抽出(取得できれば設定モデルに反映) + string channelTitle = ExtractChannelTitle(root) ?? channel.Name ?? channel.IdOrHandle; + string channelUrl = channel.GetTabUrl(feedType); + + // tabs から選択中のタブ(または全タブからrichGridRenderer)を走査 + if (root.TryGetProperty("contents", out var contents) && + contents.TryGetProperty("twoColumnBrowseResultsRenderer", out var twoCol) && + twoCol.TryGetProperty("tabs", out var tabs)) + { + foreach (var tab in tabs.EnumerateArray()) + { + if (!tab.TryGetProperty("tabRenderer", out var tabRenderer)) continue; + + // 選択されているタブ、あるいはコンテンツを持つタブを探す + bool isSelected = tabRenderer.TryGetProperty("selected", out var sel) && sel.GetBoolean(); + if (!isSelected) continue; + + if (!tabRenderer.TryGetProperty("content", out var tabContent)) continue; + if (!tabContent.TryGetProperty("richGridRenderer", out var grid)) continue; + if (!grid.TryGetProperty("contents", out var gridItems)) continue; + + foreach (var itemElem in gridItems.EnumerateArray()) + { + if (itemElem.TryGetProperty("richItemRenderer", out var richItem) && + richItem.TryGetProperty("content", out var richContent)) + { + var videoItem = ParseRichContent(richContent, feedType, channelTitle, channelUrl); + if (videoItem != null) + { + // フィルタリング判定: + // 1. ライブ配信の場合: 予約枠(Upcoming)は除外 + if (feedType == YouTubeFeedType.Live && videoItem.IsUpcoming) + { + continue; + } + + // 2. 通常の動画の場合: メンバー限定は除外 + if (feedType == YouTubeFeedType.Video && videoItem.IsMembersOnly) + { + continue; + } + + // 3. ライブ配信でもメンバー限定を除外(ユーザー要望: 動画についてはメンバー限定のものは除外) + if (feedType == YouTubeFeedType.Live && videoItem.IsMembersOnly) + { + continue; + } + + items.Add(videoItem); + } + } + } + } + } + } + catch (Exception ex) + { + LogManager.AddLog($"[エラー] YouTubeParser.ParseTabContent: {ex.Message}"); + } + + return items; + } + + /// + /// richItemRenderer内のcontent要素から動画情報をパースします。 + /// + private static YouTubeVideoItem? ParseRichContent(JsonElement richContent, YouTubeFeedType feedType, string channelTitle, string channelUrl) + { + // 最新UI: lockupViewModel(動画・ライブ配信) + if (richContent.TryGetProperty("lockupViewModel", out var lockup)) + { + return ParseLockupViewModel(lockup, feedType, channelTitle, channelUrl); + } + + // 最新UI: shortsLockupViewModel(ショート動画) + if (richContent.TryGetProperty("shortsLockupViewModel", out var shortsLockup)) + { + return ParseShortsLockupViewModel(shortsLockup, channelTitle, channelUrl); + } + + // 従来UI: videoRenderer + if (richContent.TryGetProperty("videoRenderer", out var vr)) + { + return ParseLegacyVideoRenderer(vr, feedType, channelTitle, channelUrl); + } + + return null; + } + + /// + /// lockupViewModel(動画・ライブ配信)の解析 + /// + private static YouTubeVideoItem? ParseLockupViewModel(JsonElement lockup, YouTubeFeedType feedType, string channelTitle, string channelUrl) + { + string contentId = lockup.TryGetProperty("contentId", out var cid) ? cid.GetString() ?? "" : ""; + if (string.IsNullOrWhiteSpace(contentId)) return null; + + string title = ""; + string publishedText = ""; + string viewCountText = ""; + bool isUpcoming = false; + bool isMembersOnly = false; + bool isLiveNow = false; + + // メタデータ情報の取得 + if (lockup.TryGetProperty("metadata", out var metadata) && + metadata.TryGetProperty("lockupMetadataViewModel", out var lvm)) + { + // タイトル + if (lvm.TryGetProperty("title", out var titleObj) && + titleObj.TryGetProperty("content", out var titleContent)) + { + title = titleContent.GetString() ?? ""; + } + + // メタデータ行(公開日・予定・視聴回数・バッジ) + if (lvm.TryGetProperty("metadata", out var metaInner) && + metaInner.TryGetProperty("contentMetadataViewModel", out var cmvm) && + cmvm.TryGetProperty("metadataRows", out var rows)) + { + foreach (var row in rows.EnumerateArray()) + { + // テキストパーツ + if (row.TryGetProperty("metadataParts", out var parts)) + { + foreach (var part in parts.EnumerateArray()) + { + if (part.TryGetProperty("text", out var pt) && pt.TryGetProperty("content", out var pc)) + { + string text = pc.GetString() ?? ""; + if (string.IsNullOrWhiteSpace(text)) continue; + + if (text.Contains("公開予定") || text.Contains("待機中") || text.Contains("upcoming", StringComparison.OrdinalIgnoreCase)) + { + isUpcoming = true; + publishedText = text; + } + else if (text.Contains("前 に配信済み") || text.Contains("配信済み") || text.Contains("前")) + { + publishedText = text; + } + else if (text.Contains("視聴中") || text.Contains("ライブ") || text.Contains("LIVE")) + { + isLiveNow = true; + viewCountText = text; + } + else if (text.Contains("回視聴")) + { + viewCountText = text; + } + } + } + } + + // バッジ判定(メンバー限定) + if (row.TryGetProperty("badges", out var badges)) + { + foreach (var badge in badges.EnumerateArray()) + { + if (badge.TryGetProperty("badgeViewModel", out var bvm)) + { + string bStyle = bvm.TryGetProperty("badgeStyle", out var bs) ? bs.GetString() ?? "" : ""; + string bText = bvm.TryGetProperty("badgeText", out var bt) ? bt.GetString() ?? "" : ""; + string iconName = bvm.TryGetProperty("iconName", out var ic) ? ic.GetString() ?? "" : ""; + + if (bStyle.Contains("MEMBERS_ONLY", StringComparison.OrdinalIgnoreCase) || + bText.Contains("メンバー限定") || + iconName.Contains("SPONSORSHIP_STAR", StringComparison.OrdinalIgnoreCase)) + { + isMembersOnly = true; + } + + if (bStyle.Contains("LIVE", StringComparison.OrdinalIgnoreCase) || bText.Contains("ライブ") || bText.Contains("LIVE")) + { + isLiveNow = true; + } + } + } + } + } + } + } + + // タイトルによるメンバー限定補助判定 + if (title.Contains("【メン限】") || title.Contains("[メン限]") || title.Contains("(メン限)") || title.Contains("メンバー限定")) + { + isMembersOnly = true; + } + + var item = new YouTubeVideoItem + { + VideoId = contentId, + Title = title, + Url = $"https://www.youtube.com/watch?v={contentId}", + ThumbnailUrl = $"https://i.ytimg.com/vi/{contentId}/hqdefault.jpg", + ChannelTitle = channelTitle, + ChannelUrl = channelUrl, + PublishedText = publishedText, + PublishedDate = ParseRelativeDate(publishedText), + FeedType = feedType, + IsLiveNow = isLiveNow, + IsUpcoming = isUpcoming, + IsMembersOnly = isMembersOnly, + ViewCountText = viewCountText + }; + + return item; + } + + /// + /// shortsLockupViewModel(ショート動画)の解析 + /// + private static YouTubeVideoItem? ParseShortsLockupViewModel(JsonElement shortsLockup, string channelTitle, string channelUrl) + { + string videoId = ""; + string title = ""; + string viewCountText = ""; + + // videoId の抽出(onTap や entityId から) + if (shortsLockup.TryGetProperty("onTap", out var onTap) && + onTap.TryGetProperty("innertubeCommand", out var cmd) && + cmd.TryGetProperty("reelWatchEndpoint", out var rwe) && + rwe.TryGetProperty("videoId", out var vid)) + { + videoId = vid.GetString() ?? ""; + } + + if (string.IsNullOrWhiteSpace(videoId) && shortsLockup.TryGetProperty("entityId", out var eid)) + { + string entityStr = eid.GetString() ?? ""; + // 例: "shorts-shelf-item--XZmpizZcLo" + int lastDash = entityStr.LastIndexOf('-'); + if (lastDash >= 0 && entityStr.Length > lastDash + 1) + { + videoId = entityStr[(lastDash + 1)..]; + } + } + + // オーバーレイメタデータ(タイトル・視聴回数) + if (shortsLockup.TryGetProperty("overlayMetadata", out var overlayMeta)) + { + if (overlayMeta.TryGetProperty("primaryText", out var pt) && + pt.TryGetProperty("content", out var tc)) + { + title = tc.GetString() ?? ""; + } + if (overlayMeta.TryGetProperty("secondaryText", out var st) && + st.TryGetProperty("content", out var sc)) + { + viewCountText = sc.GetString() ?? ""; + } + } + + if (string.IsNullOrWhiteSpace(videoId)) return null; + + var item = new YouTubeVideoItem + { + VideoId = videoId, + Title = title, + Url = $"https://www.youtube.com/shorts/{videoId}", + ThumbnailUrl = $"https://i.ytimg.com/vi/{videoId}/hqdefault.jpg", + ChannelTitle = channelTitle, + ChannelUrl = channelUrl, + PublishedText = viewCountText, + PublishedDate = DateTimeOffset.Now, + FeedType = YouTubeFeedType.Shorts, + IsLiveNow = false, + IsUpcoming = false, + IsMembersOnly = false, + ViewCountText = viewCountText + }; + + return item; + } + + /// + /// 旧来の videoRenderer の解析(フォールバック) + /// + private static YouTubeVideoItem? ParseLegacyVideoRenderer(JsonElement vr, YouTubeFeedType feedType, string channelTitle, string channelUrl) + { + string videoId = vr.TryGetProperty("videoId", out var vid) ? vid.GetString() ?? "" : ""; + if (string.IsNullOrWhiteSpace(videoId)) return null; + + string title = ""; + if (vr.TryGetProperty("title", out var titleObj)) + { + if (titleObj.TryGetProperty("runs", out var runs) && runs.GetArrayLength() > 0) + { + title = runs[0].GetProperty("text").GetString() ?? ""; + } + else if (titleObj.TryGetProperty("simpleText", out var st)) + { + title = st.GetString() ?? ""; + } + } + + bool isUpcoming = vr.TryGetProperty("upcomingEventData", out _); + string publishedText = ""; + if (vr.TryGetProperty("publishedTimeText", out var ptt) && ptt.TryGetProperty("simpleText", out var pst)) + { + publishedText = pst.GetString() ?? ""; + } + + bool isMembersOnly = false; + if (vr.TryGetProperty("badges", out var badges)) + { + string raw = badges.GetRawText(); + if (raw.Contains("MEMBERS_ONLY", StringComparison.OrdinalIgnoreCase) || raw.Contains("メンバー限定")) + { + isMembersOnly = true; + } + } + if (title.Contains("【メン限】") || title.Contains("メンバー限定")) + { + isMembersOnly = true; + } + + return new YouTubeVideoItem + { + VideoId = videoId, + Title = title, + Url = $"https://www.youtube.com/watch?v={videoId}", + ThumbnailUrl = $"https://i.ytimg.com/vi/{videoId}/hqdefault.jpg", + ChannelTitle = channelTitle, + ChannelUrl = channelUrl, + PublishedText = publishedText, + PublishedDate = ParseRelativeDate(publishedText), + FeedType = feedType, + IsLiveNow = false, + IsUpcoming = isUpcoming, + IsMembersOnly = isMembersOnly, + ViewCountText = "" + }; + } + + /// + /// rootからチャンネル名を抽出します。 + /// + private static string? ExtractChannelTitle(JsonElement root) + { + try + { + if (root.TryGetProperty("header", out var header)) + { + if (header.TryGetProperty("pageHeaderRenderer", out var phr) && + phr.TryGetProperty("pageTitle", out var pt)) + { + return pt.GetString(); + } + if (header.TryGetProperty("c4TabbedHeaderRenderer", out var c4) && + c4.TryGetProperty("title", out var ct)) + { + return ct.GetString(); + } + } + } + catch + { + // 失敗時はnullを返す + } + return null; + } + + /// + /// 「2 日前 に配信済み」「3 時間前」などの日本語相対文字列から DateTimeOffset を推定算出します。 + /// + public static DateTimeOffset ParseRelativeDate(string text) + { + var now = DateTimeOffset.Now; + if (string.IsNullOrWhiteSpace(text)) return now; + + var match = Regex.Match(text, @"(\d+)\s*(秒|分|時間|日|週間|か月|年)前"); + if (match.Success && int.TryParse(match.Groups[1].Value, out int value)) + { + string unit = match.Groups[2].Value; + return unit switch + { + "秒" => now.AddSeconds(-value), + "分" => now.AddMinutes(-value), + "時間" => now.AddHours(-value), + "日" => now.AddDays(-value), + "週間" => now.AddDays(-value * 7), + "か月" => now.AddMonths(-value), + "年" => now.AddYears(-value), + _ => now + }; + } + + return now; + } +} diff --git a/rss-generator/Class/RSS/e_hentai/e_hentai_common.cs b/rss-generator/Class/RSS/e_hentai/e_hentai_common.cs index 3d3178c..23a3ead 100644 --- a/rss-generator/Class/RSS/e_hentai/e_hentai_common.cs +++ b/rss-generator/Class/RSS/e_hentai/e_hentai_common.cs @@ -200,11 +200,9 @@ internal static class e_hentai_common } /// - /// 1作品(1記事)あたりに RSS 本文へ展開する最大画像数。 - /// 大量(数百枚以上)の画像を一斉に読み込むと Hath サーバーのレートリミット(403 Forbidden)が発生するため、 - /// 上限を設けてプレビュー表示し、上限を超える場合は元サイトへの全ページ閲覧リンクを表示します。 + /// 1作品(1記事)あたりに RSS 本文へ展開する画像数(先頭1枚)。 /// - public const int MaxImagesPerGallery = 30; + public const int MaxImagesPerGallery = 1; /// /// 対象ギャラリーの詳細ページ・ビューアページを巡回し、日本語タイトルおよび画像の直接URLを取得します。 @@ -392,63 +390,31 @@ internal static class e_hentai_common // 本文 HTML の整形 var body = new StringBuilder(); - body.Append($@""); - body.Append("
"); - if (!string.IsNullOrWhiteSpace(gallery.Category)) + // ギャラリー表示リンク + var match = System.Text.RegularExpressions.Regex.Match(gallery.Url, @"/g/(\d+)/([a-zA-Z0-9]+)", System.Text.RegularExpressions.RegexOptions.IgnoreCase); + if (match.Success) { - body.Append($"Category: {WebUtility.HtmlEncode(gallery.Category)} "); - } - if (!string.IsNullOrWhiteSpace(gallery.Pages)) - { - body.Append($"| Pages: {WebUtility.HtmlEncode(gallery.Pages)} "); - } - if (!string.IsNullOrWhiteSpace(gallery.Uploader)) - { - body.Append($"| Uploader: {WebUtility.HtmlEncode(gallery.Uploader)}"); - } - body.Append("
"); - - if (gallery.Tags.Count > 0) - { - body.Append($"
Tags: {WebUtility.HtmlEncode(string.Join(", ", gallery.Tags))}
"); + string galleryId = match.Groups[1].Value; + string token = match.Groups[2].Value; + string galleryUrl = $"https://udukipc-web.armonica-vps.work/ehentai/{galleryId}/{token}"; + body.Append($"
"); + body.Append($"[Gallery]"); + body.Append($"
"); } - // ---------------------------------------------------- - // 403 Forbidden 対策: - // 1. referrerpolicy="no-referrer" によるリファラー送信防止 - // 2. loading="lazy" による順次遅延読み込み - // 3. 大量画像(数百枚)によるレートリミット防止(上限30枚+全ページ閲覧リンク) - // ---------------------------------------------------- - if (gallery.ImageUrls.Count > 0) + // ギャラリーの1枚目画像 + string? firstImg = gallery.ImageUrls.FirstOrDefault(); + if (string.IsNullOrEmpty(firstImg)) { - int pageNum = 1; - foreach (var imgUrl in gallery.ImageUrls) - { - body.Append($@"
"); - pageNum++; - } - - // ページ数表記(例: "804 pages")から総ページ数を抽出し、上限を超えている場合は案内リンクを表示 - int totalPages = 0; - if (!string.IsNullOrEmpty(gallery.Pages)) - { - var match = System.Text.RegularExpressions.Regex.Match(gallery.Pages, @"\d+"); - if (match.Success) int.TryParse(match.Value, out totalPages); - } - - if (totalPages > gallery.ImageUrls.Count) - { - body.Append($@"
"); - body.Append($@"

※ 全 {totalPages} ページ中、最初の {gallery.ImageUrls.Count} ページを表示しています。

"); - body.Append($@"E-Hentai で全ページを見る({totalPages}P)"); - body.Append("
"); - } + firstImg = gallery.ThumbnailUrl; } - else if (!string.IsNullOrWhiteSpace(gallery.ThumbnailUrl)) + + if (!string.IsNullOrEmpty(firstImg)) { - // フォールバック: 全画像が取得できなかった場合はサムネイルを表示 - body.Append($@"
"); + body.Append($"
"); + body.Append($""); + body.Append($"
"); } item.Summary = new TextSyndicationContent(body.ToString(), TextSyndicationContentKind.Html); diff --git a/rss-generator/Class/RSS/hitomi_la/index_japanese.cs b/rss-generator/Class/RSS/hitomi_la/index_japanese.cs index 7528790..e7d4b6f 100644 --- a/rss-generator/Class/RSS/hitomi_la/index_japanese.cs +++ b/rss-generator/Class/RSS/hitomi_la/index_japanese.cs @@ -1,3 +1,4 @@ +using AngleSharp.Dom; using AngleSharp.Html.Dom; using AngleSharp.Html.Parser; using System; @@ -7,7 +8,6 @@ using System.Net.Http; using System.ServiceModel.Syndication; using System.Text; using System.Text.RegularExpressions; -using System.Threading; using System.Threading.Tasks; namespace rss_generator.Class.RSS.hitomi_la; @@ -16,160 +16,193 @@ internal class index_japanese { public static async Task CreateRSSAsync(HttpClient client) { - var pageMaster = "https://hitomi.la/index-japanese.html"; + var baseURL = "https://hitomi.la/index-japanese.html"; + + var feed = new SyndicationFeed("hitomi.la", "", new Uri(baseURL)); + var items = new List(); try { - var linkList = new List(); - for (int pageId = 1; pageId <= 10; pageId++) + // セッションを作成して複数ページ取得を高速化 + string? sessionId = await CmnUtil.CreateFlareSolverrSession(client); + try { - var pageURL = $"{pageMaster}/feed/atom/{(pageId > 1 ? $"?paged={pageId}" : "")}"; - linkList.Add(pageURL); + for (int pageId = 1; pageId <= 5; pageId++) + { + var pageURL = pageId == 1 ? baseURL : $"{baseURL}?page={pageId}"; + var document = await CmnUtil.GetDocumentFlareSolverr(client, pageURL, session: sessionId); + if (document == null) + { + continue; + } + + var newItems = ParseGalleryItems(document); + items.AddRange(newItems); + } + } + finally + { + if (sessionId != null) + { + await CmnUtil.DestroyFlareSolverrSession(client, sessionId); + } } - var tasks = linkList.Select(url => CmnUtil.GetFeedItems(client, url)).ToList(); - var results = await Task.WhenAll(tasks); - var allItems = results.SelectMany(items => items).ToList(); + // 重複除外(IDベース) + var uniqueItems = items + .GroupBy(x => x.Id) + .Select(g => g.First()) + .ToList(); - var semaphore = new SemaphoreSlim(5); - // 簡潔なRSSに非同期で整形 - var itemTasks = allItems.OrderByDescending(x => x.PublishDate).Select(async originalItem => - { - await semaphore.WaitAsync(); - try - { - List getAddCategory() - { - List result = new List(); + feed.Items = uniqueItems; - string orgText = originalItem.Summary?.Text ?? ""; - var parser = new HtmlParser(); - var document = parser.ParseDocument(orgText); - - var firstP = document.QuerySelector("p"); - if (firstP != null) - { - var targetText = firstP.TextContent; - int tagIndex = targetText.IndexOf("【タグ】"); - if (tagIndex >= 0) - { - targetText = targetText.Substring(0, tagIndex); - } - string pattern = @"【[^】]+】([^【]*)"; - var matches = Regex.Matches(targetText, pattern); - - foreach (Match match in matches) - { - string valuesStr = match.Groups[1].Value; - - if (string.IsNullOrWhiteSpace(valuesStr)) - { - continue; - } - - string[] splitValues = valuesStr.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries); - - foreach (var val in splitValues) - { - string categoryName = val.Trim(); - - if (!string.IsNullOrEmpty(categoryName)) - { - result.Add(new SyndicationCategory(categoryName)); - } - } - } - } - return result; - } - - var newItem = new SyndicationItem() - { - Title = new TextSyndicationContent(originalItem.Title?.Text ?? "No Title"), - Id = CmnUtil.CreateGuid(originalItem.Id), - PublishDate = originalItem.PublishDate, - LastUpdatedTime = originalItem.LastUpdatedTime - }; - - var link = originalItem.Links.FirstOrDefault()?.Uri; - if (link != null) - { - newItem.Links.Add(new SyndicationLink(link)); - } - foreach (var category in getAddCategory()) - { - newItem.Categories.Add(category); - } - foreach (var category in originalItem.Categories) - { - category.Scheme = null; - newItem.Categories.Add(category); - } - foreach (var author in originalItem.Authors) - { - newItem.Authors.Add(author); - } - - var body = new StringBuilder(); - var originalSummary = originalItem.Summary?.Text ?? ""; - body.Append(originalSummary); - if (link != null) - { - var docContents = await CmnUtil.GetDocument(client, link.ToString()); - var imgElements = docContents.QuerySelectorAll("#post-hentai img, img"); - const int maxImages = 30; - int addedCount = 0; - - foreach (var img in imgElements) - { - var src = img.GetAttribute("src"); - if (!string.IsNullOrEmpty(src)) - { - if (Uri.TryCreate(link, src, out var absoluteUri)) - { - body.Append($@"
"); - addedCount++; - if (addedCount >= maxImages) - { - break; - } - } - } - } - - if (imgElements.Length > maxImages) - { - body.Append($@"
"); - body.Append($@"

※ 全 {imgElements.Length} ページ中、最初の {addedCount} ページを表示しています。

"); - body.Append($@"hitomi.la で全ページを見る({imgElements.Length}P)"); - body.Append("
"); - } - } - newItem.Summary = new TextSyndicationContent(body.ToString(), TextSyndicationContentKind.Html); - - return newItem; - } - finally - { - semaphore.Release(); - } - }).ToList(); - - var simplifiedItems = await Task.WhenAll(itemTasks); - - var mergedFeed = new SyndicationFeed( - "hitomi.la", - "", - new Uri("https://hitomi.la/index-japanese.html"), - simplifiedItems - ); - - CmnUtil.ExportXMLFile("hitomi-japanese.xml", mergedFeed); - LogManager.AddLog($"[情報] hitomi-japanese.xml を正常に出力しました (記事数: {simplifiedItems.Length})"); + CmnUtil.ExportXMLFile("hitomi-japanese.xml", feed); + LogManager.AddLog($"[情報] hitomi-japanese.xml を正常に出力しました (記事数: {uniqueItems.Count})"); } catch (Exception ex) { LogManager.AddLog($"[エラー] hitomi-japanese:{ex.Message}"); + CmnUtil.PrintCallStackToEvent(ex); } } + + private static List ParseGalleryItems(IDocument document) + { + var result = new List(); + var cards = document.QuerySelectorAll(".gallery-content > div"); + + foreach (var card in cards) + { + var titleElem = card.QuerySelector("h1.lillie a, h1 a"); + if (titleElem == null) + { + continue; + } + + var title = titleElem.TextContent?.Trim() ?? "No Title"; + var relLink = titleElem.GetAttribute("href") ?? string.Empty; + var linkHref = !string.IsNullOrEmpty(relLink) + ? (relLink.StartsWith("http") ? relLink : $"https://hitomi.la{(relLink.StartsWith("/") ? "" : "/")}{relLink}") + : string.Empty; + + if (string.IsNullOrEmpty(linkHref)) + { + continue; + } + + // 作品IDの抽出 + string galleryId = ""; + var matchId = Regex.Match(linkHref, @"(\d+)\.html", RegexOptions.IgnoreCase); + if (!matchId.Success) + { + matchId = Regex.Match(linkHref, @"(\d{5,})"); + } + if (matchId.Success) + { + galleryId = matchId.Groups[1].Value; + } + + var item = new SyndicationItem + { + Title = new TextSyndicationContent(title), + Id = CmnUtil.CreateGuid(linkHref) + }; + + item.Links.Add(new SyndicationLink(new Uri(linkHref))); + + // 投稿日時の取得 + var dateElem = card.QuerySelector(".date"); + var dateStr = dateElem?.GetAttribute("data-posted") ?? dateElem?.TextContent?.Trim(); + if (!string.IsNullOrEmpty(dateStr) && DateTimeOffset.TryParse(dateStr, out var parsedOffset)) + { + var jstDate = parsedOffset.ToOffset(TimeSpan.FromHours(9)).DateTime; + item.PublishDate = jstDate; + item.LastUpdatedTime = jstDate; + } + else + { + item.PublishDate = DateTime.Now; + item.LastUpdatedTime = DateTime.Now; + } + + // 作家の取得 + var artistElems = card.QuerySelectorAll(".artist-list li a"); + foreach (var artistElem in artistElems) + { + var artist = artistElem.TextContent?.Trim(); + if (!string.IsNullOrEmpty(artist) && !string.Equals(artist, "N/A", StringComparison.OrdinalIgnoreCase)) + { + item.Authors.Add(new SyndicationPerson { Name = artist }); + item.Categories.Add(new SyndicationCategory(artist)); + } + } + + // シリーズの取得 + var seriesElems = card.QuerySelectorAll(".series-list li a"); + foreach (var seriesElem in seriesElems) + { + var series = seriesElem.TextContent?.Trim(); + if (!string.IsNullOrEmpty(series) && !string.Equals(series, "N/A", StringComparison.OrdinalIgnoreCase)) + { + item.Categories.Add(new SyndicationCategory(series)); + } + } + + // タイプの取得 + var typeElem = card.QuerySelector(".dj-desc a[href*='/type/']"); + if (typeElem != null) + { + var typeName = typeElem.TextContent?.Trim(); + if (!string.IsNullOrEmpty(typeName) && !string.Equals(typeName, "N/A", StringComparison.OrdinalIgnoreCase)) + { + item.Categories.Add(new SyndicationCategory(typeName)); + } + } + + // タグの取得 + var tagElems = card.QuerySelectorAll(".relatedtags li a"); + foreach (var tagElem in tagElems) + { + var tag = tagElem.TextContent?.Trim(); + if (!string.IsNullOrEmpty(tag)) + { + // 記号(♀ ♂ など)をトリム + tag = Regex.Replace(tag, @"[\s♀♂]+$", "").Trim(); + if (!string.IsNullOrEmpty(tag)) + { + item.Categories.Add(new SyndicationCategory(tag)); + } + } + } + + // サムネイル画像の取得 + var imgElem = card.QuerySelector("img"); + var imgSrc = imgElem?.GetAttribute("data-src") ?? imgElem?.GetAttribute("src") ?? string.Empty; + if (!string.IsNullOrEmpty(imgSrc) && imgSrc.StartsWith("//")) + { + imgSrc = $"https:{imgSrc}"; + } + + // HTML構築 + var body = new StringBuilder(); + if (!string.IsNullOrEmpty(galleryId)) + { + string galleryUrl = $"https://udukipc-web.armonica-vps.work/hitomi/{galleryId}"; + body.Append($"
"); + body.Append($"[Gallery]"); + body.Append($"
"); + } + if (!string.IsNullOrEmpty(imgSrc)) + { + body.Append($"
"); + body.Append($""); + body.Append($"
"); + } + + item.Summary = new TextSyndicationContent(body.ToString(), TextSyndicationContentKind.Html); + result.Add(item); + } + + return result; + } } diff --git a/rss-generator/Program.cs b/rss-generator/Program.cs index 84c8415..c0e5af2 100644 --- a/rss-generator/Program.cs +++ b/rss-generator/Program.cs @@ -3,6 +3,7 @@ using rss_generator.Class; using rss_generator.Class.RSS; using rss_generator.Class.RSS.e_hentai; using rss_generator.Class.RSS.hitomi_la; +using rss_generator.Class.RSS.YouTube; using System.Diagnostics; @@ -78,14 +79,13 @@ while (true) tasks.Add(momonga_magazine.CreateRSSAsync(client)); } #if DEBUG - //tasks.Add(index_japanese.CreateRSSAsync(client)); - tasks.Add(mangalear.CreateRSSAsync(client)); - tasks.Add(momonga_fanzine.CreateRSSAsync(client)); - tasks.Add(momonga_magazine.CreateRSSAsync(client)); - tasks.Add(e_hentai_ai.CreateRSSAsync(client)); - tasks.Add(e_hentai_non_ai.CreateRSSAsync(client)); + tasks.Add(YouTubeFeedManager.CreateRSSAsync(client)); + //tasks.Add(momonga_fanzine.CreateRSSAsync(client)); + //tasks.Add(momonga_magazine.CreateRSSAsync(client)); + //tasks.Add(e_hentai_non_ai.CreateRSSAsync(client)); #else tasks.Add(Youtube.CreateRSSAsync(client)); + tasks.Add(YouTubeFeedManager.CreateRSSAsync(client)); tasks.Add(rss_generator.Class.RSS.X.x_list.CreateRSSAsync(client)); tasks.Add(kimochiinfo.CreateRSSAsync(client)); @@ -100,6 +100,8 @@ while (true) tasks.Add(mangalear.CreateRSSAsync(client)); tasks.Add(e_hentai_ai.CreateRSSAsync(client)); tasks.Add(e_hentai_non_ai.CreateRSSAsync(client)); + + tasks.Add(index_japanese.CreateRSSAsync(client)); #endif