feat: implement initial RSS generation modules for e-hentai, hitomi.la, and YouTube integrations

This commit is contained in:
armonica committed 2026-09-02 09:54:56 +09:00
1 parent a0c94f3563
commit fe5e3335af
7 files changed
+1070 -203

No files matched your search

@@ -0,0 +1,51 @@
using System.Collections.Generic;
namespace rss_generator.Class.RSS.YouTube;
/// <summary>
/// YouTubeフィード生成の設定情報
/// </summary>
public static class YouTubeConfig
{
/// <summary>
/// 出力対象とするYouTubeチャンネル・ユーザーの固定一覧。
/// ここにチャンネルID(UC...)またはハンドル(@...)を追加してください。
/// </summary>
public static readonly List<YouTubeChannelTarget> Channels = new()
{
new YouTubeChannelTarget
{
IdOrHandle = "@ShirakamiFubuki",
Name = "白上フブキ",
FilePrefix = "shirakami_fubuki"
},
// 必要に応じてチャンネルを追加・編集してください。
// 例:
// new YouTubeChannelTarget
// {
// IdOrHandle = "UCdn5BQ06XqgXoAxIhbqw5Rg",
// Name = "白上フブキ",
// FilePrefix = "fubuki_ch"
// },
};
/// <summary>
/// チャンネルごとの個別XMLファイルを出力するかどうか(例: shirakami_fubuki_live.xml)
/// </summary>
public static bool ExportIndividualFeeds { get; set; } = true;
/// <summary>
/// 全チャンネルを統合したXMLファイルを出力するかどうか(all_live.xml, all_video.xml, all_shorts.xml)
/// </summary>
public static bool ExportMergedFeeds { get; set; } = true;
/// <summary>
/// 出力先のサブディレクトリ名(output/youtube/ 配下に出力されます)
/// </summary>
public static string OutputSubDir { get; set; } = "youtube";
/// <summary>
/// 各フィードの最大保持件数
/// </summary>
public static int MaxItemsPerFeed { get; set; } = 30;
}
@@ -0,0 +1,247 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.ServiceModel.Syndication;
using System.Text;
using System.Threading.Tasks;
namespace rss_generator.Class.RSS.YouTube;
/// <summary>
/// YouTubeのライブ配信・通常動画・ショート動画を分離してRSS(XML)を生成・出力するマネージャークラス
/// </summary>
public static class YouTubeFeedManager
{
/// <summary>
/// YouTubeの各チャンネルからフィードを生成してXMLファイルに出力します。
/// </summary>
/// <param name="client">HttpClient インスタンス</param>
public static async Task CreateRSSAsync(HttpClient client)
{
LogManager.AddLog("[情報] YouTubeフィード生成を開始します");
try
{
var allLiveItems = new List<YouTubeVideoItem>();
var allVideoItems = new List<YouTubeVideoItem>();
var allShortsItems = new List<YouTubeVideoItem>();
var channelTasks = YouTubeConfig.Channels.Select(async channel =>
{
try
{
await ProcessChannelAsync(client, channel, allLiveItems, allVideoItems, allShortsItems);
}
catch (Exception ex)
{
LogManager.AddLog($"[エラー] チャンネル処理失敗 ({channel.Name ?? channel.IdOrHandle}): {ex.Message}");
}
});
await Task.WhenAll(channelTasks);
// 全チャンネル統合フィードの出力
if (YouTubeConfig.ExportMergedFeeds)
{
ExportMergedFeed(allLiveItems, YouTubeFeedType.Live, "all_live.xml", "YouTube Live (All Channels)");
ExportMergedFeed(allVideoItems, YouTubeFeedType.Video, "all_video.xml", "YouTube Videos (All Channels)");
ExportMergedFeed(allShortsItems, YouTubeFeedType.Shorts, "all_shorts.xml", "YouTube Shorts (All Channels)");
}
LogManager.AddLog("[情報] YouTubeフィード生成が完了しました");
}
catch (Exception ex)
{
LogManager.AddLog($"[エラー] YouTubeFeedManager.CreateRSSAsync: {ex.Message}");
CmnUtil.PrintCallStackToEvent(ex);
}
}
/// <summary>
/// 1つのチャンネルについて各タブ(Live, Video, Shorts)を取得・処理します。
/// </summary>
private static async Task ProcessChannelAsync(
HttpClient client,
YouTubeChannelTarget channel,
List<YouTubeVideoItem> allLiveItems,
List<YouTubeVideoItem> allVideoItems,
List<YouTubeVideoItem> allShortsItems)
{
// 3つのタブを並行取得
var liveTask = FetchTabItemsAsync(client, channel, YouTubeFeedType.Live);
var videoTask = FetchTabItemsAsync(client, channel, YouTubeFeedType.Video);
var shortsTask = FetchTabItemsAsync(client, channel, YouTubeFeedType.Shorts);
await Task.WhenAll(liveTask, videoTask, shortsTask);
var liveItems = await liveTask;
var videoItems = await videoTask;
var shortsItems = await shortsTask;
// 個別フィードの出力
if (YouTubeConfig.ExportIndividualFeeds)
{
string prefix = channel.GetSafeFilePrefix();
string channelTitle = channel.Name ?? channel.IdOrHandle;
ExportFeed(liveItems, channelTitle, $"{prefix}_live.xml", YouTubeFeedType.Live);
ExportFeed(videoItems, channelTitle, $"{prefix}_video.xml", YouTubeFeedType.Video);
ExportFeed(shortsItems, channelTitle, $"{prefix}_shorts.xml", YouTubeFeedType.Shorts);
}
// 統合フィード用に蓄積
lock (allLiveItems) { allLiveItems.AddRange(liveItems); }
lock (allVideoItems) { allVideoItems.AddRange(videoItems); }
lock (allShortsItems) { allShortsItems.AddRange(shortsItems); }
}
/// <summary>
/// 指定されたタブ種別のHTMLを取得してパースします。
/// </summary>
private static async Task<List<YouTubeVideoItem>> FetchTabItemsAsync(
HttpClient client,
YouTubeChannelTarget channel,
YouTubeFeedType feedType)
{
string url = channel.GetTabUrl(feedType);
try
{
// HttpRequestMessageを作成してリクエスト送信
using var request = new HttpRequestMessage(HttpMethod.Get, url);
request.Headers.Add("Accept-Language", "ja-JP,ja;q=0.9");
using var response = await client.SendAsync(request);
if (!response.IsSuccessStatusCode)
{
LogManager.AddLog($"[警告] HTTPステータス異常 ({response.StatusCode}): {url}");
return new List<YouTubeVideoItem>();
}
string html = await response.Content.ReadAsStringAsync();
var items = YouTubeParser.ParseTabContent(html, feedType, channel);
// 最大件数の制御
return items.Take(YouTubeConfig.MaxItemsPerFeed).ToList();
}
catch (Exception ex)
{
LogManager.AddLog($"[エラー] タブ取得失敗 ({url}): {ex.Message}");
return new List<YouTubeVideoItem>();
}
}
/// <summary>
/// チャンネル個別フィードを出力します。
/// </summary>
private static void ExportFeed(
List<YouTubeVideoItem> items,
string channelTitle,
string fileName,
YouTubeFeedType feedType)
{
string typeLabel = feedType switch
{
YouTubeFeedType.Live => "ライブ配信",
YouTubeFeedType.Video => "動画",
YouTubeFeedType.Shorts => "ショート動画",
_ => "フィード"
};
var syndicationItems = items.Select(CreateSyndicationItem).ToList();
var feed = new SyndicationFeed(
$"{channelTitle} - {typeLabel}",
$"{channelTitle} の {typeLabel} RSSフィード",
new Uri(items.FirstOrDefault()?.ChannelUrl ?? "https://www.youtube.com"),
syndicationItems
);
CmnUtil.ExportXMLFile(fileName, feed, YouTubeConfig.OutputSubDir);
}
/// <summary>
/// 全チャンネル統合フィードを出力します。
/// </summary>
private static void ExportMergedFeed(
List<YouTubeVideoItem> items,
YouTubeFeedType feedType,
string fileName,
string feedTitle)
{
string typeLabel = feedType switch
{
YouTubeFeedType.Live => "ライブ配信",
YouTubeFeedType.Video => "動画",
YouTubeFeedType.Shorts => "ショート動画",
_ => "フィード"
};
// 公開日時の新しい順にソートし、重複を除外
var distinctItems = items
.GroupBy(i => i.VideoId)
.Select(g => g.First())
.OrderByDescending(i => i.PublishedDate)
.Take(YouTubeConfig.MaxItemsPerFeed * 3)
.Select(CreateSyndicationItem)
.ToList();
var feed = new SyndicationFeed(
$"{feedTitle} ({typeLabel})",
$"登録チャンネル全件の {typeLabel} 統合RSSフィード",
new Uri("https://www.youtube.com"),
distinctItems
);
CmnUtil.ExportXMLFile(fileName, feed, YouTubeConfig.OutputSubDir);
}
/// <summary>
/// YouTubeVideoItemからSyndicationItemを生成します。
/// </summary>
private static SyndicationItem CreateSyndicationItem(YouTubeVideoItem video)
{
var item = new SyndicationItem
{
Title = new TextSyndicationContent(video.Title),
Id = CmnUtil.CreateGuid(video.Url),
PublishDate = video.PublishedDate,
LastUpdatedTime = video.PublishedDate
};
item.Links.Add(new SyndicationLink(new Uri(video.Url)));
if (!string.IsNullOrWhiteSpace(video.ChannelTitle))
{
item.Authors.Add(new SyndicationPerson { Name = video.ChannelTitle });
}
item.Categories.Add(new SyndicationCategory(video.FeedType.ToString()));
// HTML形式のSummaryを生成
var body = new StringBuilder();
if (!string.IsNullOrWhiteSpace(video.ThumbnailUrl))
{
body.Append($@"<div><a href=""{video.Url}""><img src=""{video.ThumbnailUrl}"" alt=""{WebUtility.HtmlEncode(video.Title)}"" style=""max-width: 100%; height: auto; margin-bottom: 8px; border-radius: 6px;"" /></a></div>");
}
body.Append($@"<div style=""margin-bottom: 4px;""><strong>タイトル:</strong> <a href=""{video.Url}"">{WebUtility.HtmlEncode(video.Title)}</a></div>");
body.Append($@"<div style=""margin-bottom: 4px;""><strong>チャンネル:</strong> {WebUtility.HtmlEncode(video.ChannelTitle)}</div>");
if (!string.IsNullOrWhiteSpace(video.PublishedText))
{
body.Append($@"<div style=""margin-bottom: 4px;""><strong>ステータス/日時:</strong> {WebUtility.HtmlEncode(video.PublishedText)}</div>");
}
if (!string.IsNullOrWhiteSpace(video.ViewCountText))
{
body.Append($@"<div style=""margin-bottom: 4px;""><strong>視聴情報:</strong> {WebUtility.HtmlEncode(video.ViewCountText)}</div>");
}
item.Summary = new TextSyndicationContent(body.ToString(), TextSyndicationContentKind.Html);
return item;
}
}
@@ -0,0 +1,137 @@
using System;
namespace rss_generator.Class.RSS.YouTube;
/// <summary>
/// YouTubeフィードの種類(ライブ配信・通常動画・ショート動画)
/// </summary>
public enum YouTubeFeedType
{
/// <summary>ライブ配信(配信中またはアーカイブ)</summary>
Live,
/// <summary>通常の動画</summary>
Video,
/// <summary>ショート動画</summary>
Shorts
}
/// <summary>
/// 監視対象チャンネル・ユーザーの設定情報
/// </summary>
public class YouTubeChannelTarget
{
/// <summary>
/// チャンネルID(例: "UCdn5BQ06XqgXoAxIhbqw5Rg")またはハンドル名(例: "@ShirakamiFubuki")
/// </summary>
public string IdOrHandle { get; set; } = "";
/// <summary>
/// チャンネル名(未指定の場合は自動取得またはIdOrHandleを使用)
/// </summary>
public string? Name { get; set; }
/// <summary>
/// 出力ファイル名のプレフィックス(例: "fubuki" とすると "fubuki_live.xml" などが出力されます)
/// 未指定の場合はIdOrHandleから安全な文字列を生成します。
/// </summary>
public string? FilePrefix { get; set; }
/// <summary>
/// 指定したタブ種別に応じたURLを取得します。
/// </summary>
public string GetTabUrl(YouTubeFeedType type)
{
string baseTarget = IdOrHandle.Trim();
string baseUrl;
if (baseTarget.StartsWith("@"))
{
baseUrl = $"https://www.youtube.com/{baseTarget}";
}
else if (baseTarget.StartsWith("UC", StringComparison.OrdinalIgnoreCase))
{
baseUrl = $"https://www.youtube.com/channel/{baseTarget}";
}
else if (baseTarget.StartsWith("http://", StringComparison.OrdinalIgnoreCase) ||
baseTarget.StartsWith("https://", StringComparison.OrdinalIgnoreCase))
{
baseUrl = baseTarget.TrimEnd('/');
}
else
{
// @なしのハンドルまたはチャンネルID
baseUrl = $"https://www.youtube.com/@{baseTarget}";
}
return type switch
{
YouTubeFeedType.Live => $"{baseUrl}/streams",
YouTubeFeedType.Video => $"{baseUrl}/videos",
YouTubeFeedType.Shorts => $"{baseUrl}/shorts",
_ => baseUrl
};
}
/// <summary>
/// ファイル名用の安全なプレフィックスを取得します。
/// </summary>
public string GetSafeFilePrefix()
{
if (!string.IsNullOrWhiteSpace(FilePrefix))
{
return FilePrefix.Trim();
}
var prefix = IdOrHandle.Trim().TrimStart('@');
foreach (char c in System.IO.Path.GetInvalidFileNameChars())
{
prefix = prefix.Replace(c, '_');
}
return prefix;
}
}
/// <summary>
/// 抽出されたYouTube動画アイテム情報
/// </summary>
public class YouTubeVideoItem
{
/// <summary>動画ID</summary>
public string VideoId { get; set; } = "";
/// <summary>動画タイトル</summary>
public string Title { get; set; } = "";
/// <summary>動画URL</summary>
public string Url { get; set; } = "";
/// <summary>サムネイル画像URL</summary>
public string ThumbnailUrl { get; set; } = "";
/// <summary>チャンネル名</summary>
public string ChannelTitle { get; set; } = "";
/// <summary>チャンネルURL</summary>
public string ChannelUrl { get; set; } = "";
/// <summary>公開日時(推定または取得日時)</summary>
public DateTimeOffset PublishedDate { get; set; } = DateTimeOffset.Now;
/// <summary>YouTube上に表示されていた日時文字列(例: "2 日前 に配信済み", "3 時間前")</summary>
public string PublishedText { get; set; } = "";
/// <summary>フィード種別</summary>
public YouTubeFeedType FeedType { get; set; }
/// <summary>現在ライブ配信中かどうか</summary>
public bool IsLiveNow { get; set; }
/// <summary>予約枠(公開予定・待機所)かどうか</summary>
public bool IsUpcoming { get; set; }
/// <summary>メンバー限定かどうか</summary>
public bool IsMembersOnly { get; set; }
/// <summary>視聴回数や同接数の表示文字列</summary>
public string ViewCountText { get; set; } = "";
}
@@ -0,0 +1,431 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Text.Json;
using System.Text.RegularExpressions;
namespace rss_generator.Class.RSS.YouTube;
/// <summary>
/// YouTubeのHTMLからytInitialDataを抽出・解析するパーサークラス
/// </summary>
public static class YouTubeParser
{
private static readonly Regex YtInitialDataRegex = new(
@"var ytInitialData\s*=\s*(\{.+?\});</script>|ytInitialData\s*=\s*(\{.+?\});",
RegexOptions.Compiled | RegexOptions.Singleline);
/// <summary>
/// YouTubeチャンネルタブのHTMLから動画アイテム一覧を抽出します。
/// </summary>
/// <param name="html">取得したHTML文字列</param>
/// <param name="feedType">タブ種別(Live, Video, Shorts)</param>
/// <param name="channel">チャンネル設定</param>
/// <returns>条件に合致する動画アイテムリスト</returns>
public static List<YouTubeVideoItem> ParseTabContent(string html, YouTubeFeedType feedType, YouTubeChannelTarget channel)
{
var items = new List<YouTubeVideoItem>();
if (string.IsNullOrWhiteSpace(html)) return items;
var match = YtInitialDataRegex.Match(html);
if (!match.Success) return items;
// グループ1またはグループ2からJSON文字列を取得
string jsonText = match.Groups[1].Success ? match.Groups[1].Value : match.Groups[2].Value;
try
{
using var doc = JsonDocument.Parse(jsonText);
var root = doc.RootElement;
// チャンネル名の抽出(取得できれば設定モデルに反映)
string channelTitle = ExtractChannelTitle(root) ?? channel.Name ?? channel.IdOrHandle;
string channelUrl = channel.GetTabUrl(feedType);
// tabs から選択中のタブ(または全タブからrichGridRenderer)を走査
if (root.TryGetProperty("contents", out var contents) &&
contents.TryGetProperty("twoColumnBrowseResultsRenderer", out var twoCol) &&
twoCol.TryGetProperty("tabs", out var tabs))
{
foreach (var tab in tabs.EnumerateArray())
{
if (!tab.TryGetProperty("tabRenderer", out var tabRenderer)) continue;
// 選択されているタブ、あるいはコンテンツを持つタブを探す
bool isSelected = tabRenderer.TryGetProperty("selected", out var sel) && sel.GetBoolean();
if (!isSelected) continue;
if (!tabRenderer.TryGetProperty("content", out var tabContent)) continue;
if (!tabContent.TryGetProperty("richGridRenderer", out var grid)) continue;
if (!grid.TryGetProperty("contents", out var gridItems)) continue;
foreach (var itemElem in gridItems.EnumerateArray())
{
if (itemElem.TryGetProperty("richItemRenderer", out var richItem) &&
richItem.TryGetProperty("content", out var richContent))
{
var videoItem = ParseRichContent(richContent, feedType, channelTitle, channelUrl);
if (videoItem != null)
{
// フィルタリング判定:
// 1. ライブ配信の場合: 予約枠(Upcoming)は除外
if (feedType == YouTubeFeedType.Live && videoItem.IsUpcoming)
{
continue;
}
// 2. 通常の動画の場合: メンバー限定は除外
if (feedType == YouTubeFeedType.Video && videoItem.IsMembersOnly)
{
continue;
}
// 3. ライブ配信でもメンバー限定を除外(ユーザー要望: 動画についてはメンバー限定のものは除外)
if (feedType == YouTubeFeedType.Live && videoItem.IsMembersOnly)
{
continue;
}
items.Add(videoItem);
}
}
}
}
}
}
catch (Exception ex)
{
LogManager.AddLog($"[エラー] YouTubeParser.ParseTabContent: {ex.Message}");
}
return items;
}
/// <summary>
/// richItemRenderer内のcontent要素から動画情報をパースします。
/// </summary>
private static YouTubeVideoItem? ParseRichContent(JsonElement richContent, YouTubeFeedType feedType, string channelTitle, string channelUrl)
{
// 最新UI: lockupViewModel(動画・ライブ配信)
if (richContent.TryGetProperty("lockupViewModel", out var lockup))
{
return ParseLockupViewModel(lockup, feedType, channelTitle, channelUrl);
}
// 最新UI: shortsLockupViewModel(ショート動画)
if (richContent.TryGetProperty("shortsLockupViewModel", out var shortsLockup))
{
return ParseShortsLockupViewModel(shortsLockup, channelTitle, channelUrl);
}
// 従来UI: videoRenderer
if (richContent.TryGetProperty("videoRenderer", out var vr))
{
return ParseLegacyVideoRenderer(vr, feedType, channelTitle, channelUrl);
}
return null;
}
/// <summary>
/// lockupViewModel(動画・ライブ配信)の解析
/// </summary>
private static YouTubeVideoItem? ParseLockupViewModel(JsonElement lockup, YouTubeFeedType feedType, string channelTitle, string channelUrl)
{
string contentId = lockup.TryGetProperty("contentId", out var cid) ? cid.GetString() ?? "" : "";
if (string.IsNullOrWhiteSpace(contentId)) return null;
string title = "";
string publishedText = "";
string viewCountText = "";
bool isUpcoming = false;
bool isMembersOnly = false;
bool isLiveNow = false;
// メタデータ情報の取得
if (lockup.TryGetProperty("metadata", out var metadata) &&
metadata.TryGetProperty("lockupMetadataViewModel", out var lvm))
{
// タイトル
if (lvm.TryGetProperty("title", out var titleObj) &&
titleObj.TryGetProperty("content", out var titleContent))
{
title = titleContent.GetString() ?? "";
}
// メタデータ行(公開日・予定・視聴回数・バッジ)
if (lvm.TryGetProperty("metadata", out var metaInner) &&
metaInner.TryGetProperty("contentMetadataViewModel", out var cmvm) &&
cmvm.TryGetProperty("metadataRows", out var rows))
{
foreach (var row in rows.EnumerateArray())
{
// テキストパーツ
if (row.TryGetProperty("metadataParts", out var parts))
{
foreach (var part in parts.EnumerateArray())
{
if (part.TryGetProperty("text", out var pt) && pt.TryGetProperty("content", out var pc))
{
string text = pc.GetString() ?? "";
if (string.IsNullOrWhiteSpace(text)) continue;
if (text.Contains("公開予定") || text.Contains("待機中") || text.Contains("upcoming", StringComparison.OrdinalIgnoreCase))
{
isUpcoming = true;
publishedText = text;
}
else if (text.Contains("前 に配信済み") || text.Contains("配信済み") || text.Contains("前"))
{
publishedText = text;
}
else if (text.Contains("視聴中") || text.Contains("ライブ") || text.Contains("LIVE"))
{
isLiveNow = true;
viewCountText = text;
}
else if (text.Contains("回視聴"))
{
viewCountText = text;
}
}
}
}
// バッジ判定(メンバー限定)
if (row.TryGetProperty("badges", out var badges))
{
foreach (var badge in badges.EnumerateArray())
{
if (badge.TryGetProperty("badgeViewModel", out var bvm))
{
string bStyle = bvm.TryGetProperty("badgeStyle", out var bs) ? bs.GetString() ?? "" : "";
string bText = bvm.TryGetProperty("badgeText", out var bt) ? bt.GetString() ?? "" : "";
string iconName = bvm.TryGetProperty("iconName", out var ic) ? ic.GetString() ?? "" : "";
if (bStyle.Contains("MEMBERS_ONLY", StringComparison.OrdinalIgnoreCase) ||
bText.Contains("メンバー限定") ||
iconName.Contains("SPONSORSHIP_STAR", StringComparison.OrdinalIgnoreCase))
{
isMembersOnly = true;
}
if (bStyle.Contains("LIVE", StringComparison.OrdinalIgnoreCase) || bText.Contains("ライブ") || bText.Contains("LIVE"))
{
isLiveNow = true;
}
}
}
}
}
}
}
// タイトルによるメンバー限定補助判定
if (title.Contains("【メン限】") || title.Contains("[メン限]") || title.Contains("(メン限)") || title.Contains("メンバー限定"))
{
isMembersOnly = true;
}
var item = new YouTubeVideoItem
{
VideoId = contentId,
Title = title,
Url = $"https://www.youtube.com/watch?v={contentId}",
ThumbnailUrl = $"https://i.ytimg.com/vi/{contentId}/hqdefault.jpg",
ChannelTitle = channelTitle,
ChannelUrl = channelUrl,
PublishedText = publishedText,
PublishedDate = ParseRelativeDate(publishedText),
FeedType = feedType,
IsLiveNow = isLiveNow,
IsUpcoming = isUpcoming,
IsMembersOnly = isMembersOnly,
ViewCountText = viewCountText
};
return item;
}
/// <summary>
/// shortsLockupViewModel(ショート動画)の解析
/// </summary>
private static YouTubeVideoItem? ParseShortsLockupViewModel(JsonElement shortsLockup, string channelTitle, string channelUrl)
{
string videoId = "";
string title = "";
string viewCountText = "";
// videoId の抽出(onTap や entityId から)
if (shortsLockup.TryGetProperty("onTap", out var onTap) &&
onTap.TryGetProperty("innertubeCommand", out var cmd) &&
cmd.TryGetProperty("reelWatchEndpoint", out var rwe) &&
rwe.TryGetProperty("videoId", out var vid))
{
videoId = vid.GetString() ?? "";
}
if (string.IsNullOrWhiteSpace(videoId) && shortsLockup.TryGetProperty("entityId", out var eid))
{
string entityStr = eid.GetString() ?? "";
// 例: "shorts-shelf-item--XZmpizZcLo"
int lastDash = entityStr.LastIndexOf('-');
if (lastDash >= 0 && entityStr.Length > lastDash + 1)
{
videoId = entityStr[(lastDash + 1)..];
}
}
// オーバーレイメタデータ(タイトル・視聴回数)
if (shortsLockup.TryGetProperty("overlayMetadata", out var overlayMeta))
{
if (overlayMeta.TryGetProperty("primaryText", out var pt) &&
pt.TryGetProperty("content", out var tc))
{
title = tc.GetString() ?? "";
}
if (overlayMeta.TryGetProperty("secondaryText", out var st) &&
st.TryGetProperty("content", out var sc))
{
viewCountText = sc.GetString() ?? "";
}
}
if (string.IsNullOrWhiteSpace(videoId)) return null;
var item = new YouTubeVideoItem
{
VideoId = videoId,
Title = title,
Url = $"https://www.youtube.com/shorts/{videoId}",
ThumbnailUrl = $"https://i.ytimg.com/vi/{videoId}/hqdefault.jpg",
ChannelTitle = channelTitle,
ChannelUrl = channelUrl,
PublishedText = viewCountText,
PublishedDate = DateTimeOffset.Now,
FeedType = YouTubeFeedType.Shorts,
IsLiveNow = false,
IsUpcoming = false,
IsMembersOnly = false,
ViewCountText = viewCountText
};
return item;
}
/// <summary>
/// 旧来の videoRenderer の解析(フォールバック)
/// </summary>
private static YouTubeVideoItem? ParseLegacyVideoRenderer(JsonElement vr, YouTubeFeedType feedType, string channelTitle, string channelUrl)
{
string videoId = vr.TryGetProperty("videoId", out var vid) ? vid.GetString() ?? "" : "";
if (string.IsNullOrWhiteSpace(videoId)) return null;
string title = "";
if (vr.TryGetProperty("title", out var titleObj))
{
if (titleObj.TryGetProperty("runs", out var runs) && runs.GetArrayLength() > 0)
{
title = runs[0].GetProperty("text").GetString() ?? "";
}
else if (titleObj.TryGetProperty("simpleText", out var st))
{
title = st.GetString() ?? "";
}
}
bool isUpcoming = vr.TryGetProperty("upcomingEventData", out _);
string publishedText = "";
if (vr.TryGetProperty("publishedTimeText", out var ptt) && ptt.TryGetProperty("simpleText", out var pst))
{
publishedText = pst.GetString() ?? "";
}
bool isMembersOnly = false;
if (vr.TryGetProperty("badges", out var badges))
{
string raw = badges.GetRawText();
if (raw.Contains("MEMBERS_ONLY", StringComparison.OrdinalIgnoreCase) || raw.Contains("メンバー限定"))
{
isMembersOnly = true;
}
}
if (title.Contains("【メン限】") || title.Contains("メンバー限定"))
{
isMembersOnly = true;
}
return new YouTubeVideoItem
{
VideoId = videoId,
Title = title,
Url = $"https://www.youtube.com/watch?v={videoId}",
ThumbnailUrl = $"https://i.ytimg.com/vi/{videoId}/hqdefault.jpg",
ChannelTitle = channelTitle,
ChannelUrl = channelUrl,
PublishedText = publishedText,
PublishedDate = ParseRelativeDate(publishedText),
FeedType = feedType,
IsLiveNow = false,
IsUpcoming = isUpcoming,
IsMembersOnly = isMembersOnly,
ViewCountText = ""
};
}
/// <summary>
/// rootからチャンネル名を抽出します。
/// </summary>
private static string? ExtractChannelTitle(JsonElement root)
{
try
{
if (root.TryGetProperty("header", out var header))
{
if (header.TryGetProperty("pageHeaderRenderer", out var phr) &&
phr.TryGetProperty("pageTitle", out var pt))
{
return pt.GetString();
}
if (header.TryGetProperty("c4TabbedHeaderRenderer", out var c4) &&
c4.TryGetProperty("title", out var ct))
{
return ct.GetString();
}
}
}
catch
{
// 失敗時はnullを返す
}
return null;
}
/// <summary>
/// 「2 日前 に配信済み」「3 時間前」などの日本語相対文字列から DateTimeOffset を推定算出します。
/// </summary>
public static DateTimeOffset ParseRelativeDate(string text)
{
var now = DateTimeOffset.Now;
if (string.IsNullOrWhiteSpace(text)) return now;
var match = Regex.Match(text, @"(\d+)\s*(秒|分|時間|日|週間|か月|年)前");
if (match.Success && int.TryParse(match.Groups[1].Value, out int value))
{
string unit = match.Groups[2].Value;
return unit switch
{
"秒" => now.AddSeconds(-value),
"分" => now.AddMinutes(-value),
"時間" => now.AddHours(-value),
"日" => now.AddDays(-value),
"週間" => now.AddDays(-value * 7),
"か月" => now.AddMonths(-value),
"年" => now.AddYears(-value),
_ => now
};
}
return now;
}
}
@@ -200,11 +200,9 @@ internal static class e_hentai_common
}
/// <summary>
/// 1作品(1記事)あたりに RSS 本文へ展開する最大画像数。
/// 大量(数百枚以上)の画像を一斉に読み込むと Hath サーバーのレートリミット(403 Forbidden)が発生するため、
/// 上限を設けてプレビュー表示し、上限を超える場合は元サイトへの全ページ閲覧リンクを表示します。
/// 1作品(1記事)あたりに RSS 本文へ展開する画像数(先頭1枚)。
/// </summary>
public const int MaxImagesPerGallery = 30;
public const int MaxImagesPerGallery = 1;
/// <summary>
/// 対象ギャラリーの詳細ページ・ビューアページを巡回し、日本語タイトルおよび画像の直接URLを取得します。
@@ -392,63 +390,31 @@ internal static class e_hentai_common
// 本文 HTML の整形
var body = new StringBuilder();
body.Append($@"<div style=""margin-bottom: 8px;""><a href=""{gallery.Url}"" target=""_blank"" rel=""noopener noreferrer""><b>{WebUtility.HtmlEncode(gallery.Title)}</b></a></div>");
body.Append("<div style=\"font-size: 0.9em; color: #555; margin-bottom: 6px;\">");
if (!string.IsNullOrWhiteSpace(gallery.Category))
// ギャラリー表示リンク
var match = System.Text.RegularExpressions.Regex.Match(gallery.Url, @"/g/(\d+)/([a-zA-Z0-9]+)", System.Text.RegularExpressions.RegexOptions.IgnoreCase);
if (match.Success)
{
body.Append($"<span><b>Category:</b> {WebUtility.HtmlEncode(gallery.Category)}</span> ");
}
if (!string.IsNullOrWhiteSpace(gallery.Pages))
{
body.Append($"| <span><b>Pages:</b> {WebUtility.HtmlEncode(gallery.Pages)}</span> ");
}
if (!string.IsNullOrWhiteSpace(gallery.Uploader))
{
body.Append($"| <span><b>Uploader:</b> {WebUtility.HtmlEncode(gallery.Uploader)}</span>");
}
body.Append("</div>");
if (gallery.Tags.Count > 0)
{
body.Append($"<div style=\"font-size: 0.85em; color: #777; margin-bottom: 12px;\"><b>Tags:</b> {WebUtility.HtmlEncode(string.Join(", ", gallery.Tags))}</div>");
string galleryId = match.Groups[1].Value;
string token = match.Groups[2].Value;
string galleryUrl = $"https://udukipc-web.armonica-vps.work/ehentai/{galleryId}/{token}";
body.Append($"<div>");
body.Append($"<a href=\"{galleryUrl}\">[Gallery]</a>");
body.Append($"</div>");
}
// ----------------------------------------------------
// 403 Forbidden 対策:
// 1. referrerpolicy="no-referrer" によるリファラー送信防止
// 2. loading="lazy" による順次遅延読み込み
// 3. 大量画像(数百枚)によるレートリミット防止(上限30枚+全ページ閲覧リンク)
// ----------------------------------------------------
if (gallery.ImageUrls.Count > 0)
// ギャラリーの1枚目画像
string? firstImg = gallery.ImageUrls.FirstOrDefault();
if (string.IsNullOrEmpty(firstImg))
{
int pageNum = 1;
foreach (var imgUrl in gallery.ImageUrls)
{
body.Append($@"<div style=""margin-bottom: 8px;""><a href=""{gallery.Url}"" target=""_blank"" rel=""noopener noreferrer""><img src=""{imgUrl}"" alt=""{WebUtility.HtmlEncode(gallery.Title)} - {pageNum}"" referrerpolicy=""no-referrer"" loading=""lazy"" style=""max-width: 100%; height: auto; display: block; margin: 8px 0;"" /></a></div>");
pageNum++;
}
// ページ数表記(例: "804 pages")から総ページ数を抽出し、上限を超えている場合は案内リンクを表示
int totalPages = 0;
if (!string.IsNullOrEmpty(gallery.Pages))
{
var match = System.Text.RegularExpressions.Regex.Match(gallery.Pages, @"\d+");
if (match.Success) int.TryParse(match.Value, out totalPages);
}
if (totalPages > gallery.ImageUrls.Count)
{
body.Append($@"<div style=""margin: 16px 0; padding: 12px; background-color: #f5f5f5; border: 1px solid #ddd; border-radius: 4px; text-align: center;"">");
body.Append($@"<p style=""margin: 0 0 8px 0; font-size: 0.95em; color: #333;"">※ 全 {totalPages} ページ中、最初の {gallery.ImageUrls.Count} ページを表示しています。</p>");
body.Append($@"<a href=""{gallery.Url}"" target=""_blank"" rel=""noopener noreferrer"" style=""display: inline-block; padding: 8px 16px; background-color: #007bff; color: #fff; text-decoration: none; border-radius: 4px; font-weight: bold;"">E-Hentai で全ページを見る({totalPages}P)</a>");
body.Append("</div>");
}
firstImg = gallery.ThumbnailUrl;
}
else if (!string.IsNullOrWhiteSpace(gallery.ThumbnailUrl))
if (!string.IsNullOrEmpty(firstImg))
{
// フォールバック: 全画像が取得できなかった場合はサムネイルを表示
body.Append($@"<div style=""margin-bottom: 8px;""><a href=""{gallery.Url}"" target=""_blank"" rel=""noopener noreferrer""><img src=""{gallery.ThumbnailUrl}"" alt=""{WebUtility.HtmlEncode(gallery.Title)}"" referrerpolicy=""no-referrer"" loading=""lazy"" style=""max-width: 100%; height: auto;"" /></a></div>");
body.Append($"<div>");
body.Append($"<img src=\"{firstImg}\" />");
body.Append($"</div>");
}
item.Summary = new TextSyndicationContent(body.ToString(), TextSyndicationContentKind.Html);
@@ -1,3 +1,4 @@
using AngleSharp.Dom;
using AngleSharp.Html.Dom;
using AngleSharp.Html.Parser;
using System;
@@ -7,7 +8,6 @@ using System.Net.Http;
using System.ServiceModel.Syndication;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
namespace rss_generator.Class.RSS.hitomi_la;
@@ -16,160 +16,193 @@ internal class index_japanese
{
public static async Task CreateRSSAsync(HttpClient client)
{
var pageMaster = "https://hitomi.la/index-japanese.html";
var baseURL = "https://hitomi.la/index-japanese.html";
var feed = new SyndicationFeed("hitomi.la", "", new Uri(baseURL));
var items = new List<SyndicationItem>();
try
{
var linkList = new List<string>();
for (int pageId = 1; pageId <= 10; pageId++)
// セッションを作成して複数ページ取得を高速化
string? sessionId = await CmnUtil.CreateFlareSolverrSession(client);
try
{
var pageURL = $"{pageMaster}/feed/atom/{(pageId > 1 ? $"?paged={pageId}" : "")}";
linkList.Add(pageURL);
for (int pageId = 1; pageId <= 5; pageId++)
{
var pageURL = pageId == 1 ? baseURL : $"{baseURL}?page={pageId}";
var document = await CmnUtil.GetDocumentFlareSolverr(client, pageURL, session: sessionId);
if (document == null)
{
continue;
}
var newItems = ParseGalleryItems(document);
items.AddRange(newItems);
}
}
finally
{
if (sessionId != null)
{
await CmnUtil.DestroyFlareSolverrSession(client, sessionId);
}
}
var tasks = linkList.Select(url => CmnUtil.GetFeedItems(client, url)).ToList();
var results = await Task.WhenAll(tasks);
var allItems = results.SelectMany(items => items).ToList();
// 重複除外(IDベース)
var uniqueItems = items
.GroupBy(x => x.Id)
.Select(g => g.First())
.ToList();
var semaphore = new SemaphoreSlim(5);
// 簡潔なRSSに非同期で整形
var itemTasks = allItems.OrderByDescending(x => x.PublishDate).Select(async originalItem =>
{
await semaphore.WaitAsync();
try
{
List<SyndicationCategory> getAddCategory()
{
List<SyndicationCategory> result = new List<SyndicationCategory>();
feed.Items = uniqueItems;
string orgText = originalItem.Summary?.Text ?? "";
var parser = new HtmlParser();
var document = parser.ParseDocument(orgText);
var firstP = document.QuerySelector("p");
if (firstP != null)
{
var targetText = firstP.TextContent;
int tagIndex = targetText.IndexOf("【タグ】");
if (tagIndex >= 0)
{
targetText = targetText.Substring(0, tagIndex);
}
string pattern = @"【[^】]+】([^【]*)";
var matches = Regex.Matches(targetText, pattern);
foreach (Match match in matches)
{
string valuesStr = match.Groups[1].Value;
if (string.IsNullOrWhiteSpace(valuesStr))
{
continue;
}
string[] splitValues = valuesStr.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
foreach (var val in splitValues)
{
string categoryName = val.Trim();
if (!string.IsNullOrEmpty(categoryName))
{
result.Add(new SyndicationCategory(categoryName));
}
}
}
}
return result;
}
var newItem = new SyndicationItem()
{
Title = new TextSyndicationContent(originalItem.Title?.Text ?? "No Title"),
Id = CmnUtil.CreateGuid(originalItem.Id),
PublishDate = originalItem.PublishDate,
LastUpdatedTime = originalItem.LastUpdatedTime
};
var link = originalItem.Links.FirstOrDefault()?.Uri;
if (link != null)
{
newItem.Links.Add(new SyndicationLink(link));
}
foreach (var category in getAddCategory())
{
newItem.Categories.Add(category);
}
foreach (var category in originalItem.Categories)
{
category.Scheme = null;
newItem.Categories.Add(category);
}
foreach (var author in originalItem.Authors)
{
newItem.Authors.Add(author);
}
var body = new StringBuilder();
var originalSummary = originalItem.Summary?.Text ?? "";
body.Append(originalSummary);
if (link != null)
{
var docContents = await CmnUtil.GetDocument(client, link.ToString());
var imgElements = docContents.QuerySelectorAll("#post-hentai img, img");
const int maxImages = 30;
int addedCount = 0;
foreach (var img in imgElements)
{
var src = img.GetAttribute("src");
if (!string.IsNullOrEmpty(src))
{
if (Uri.TryCreate(link, src, out var absoluteUri))
{
body.Append($@"<div style=""margin-bottom: 8px;""><a href=""{link}"" target=""_blank"" rel=""noopener noreferrer""><img src=""{absoluteUri}"" referrerpolicy=""no-referrer"" loading=""lazy"" style=""max-width: 100%; height: auto; display: block; margin: 8px 0;"" /></a></div>");
addedCount++;
if (addedCount >= maxImages)
{
break;
}
}
}
}
if (imgElements.Length > maxImages)
{
body.Append($@"<div style=""margin: 16px 0; padding: 12px; background-color: #f5f5f5; border: 1px solid #ddd; border-radius: 4px; text-align: center;"">");
body.Append($@"<p style=""margin: 0 0 8px 0; font-size: 0.95em; color: #333;"">※ 全 {imgElements.Length} ページ中、最初の {addedCount} ページを表示しています。</p>");
body.Append($@"<a href=""{link}"" target=""_blank"" rel=""noopener noreferrer"" style=""display: inline-block; padding: 8px 16px; background-color: #007bff; color: #fff; text-decoration: none; border-radius: 4px; font-weight: bold;"">hitomi.la で全ページを見る({imgElements.Length}P)</a>");
body.Append("</div>");
}
}
newItem.Summary = new TextSyndicationContent(body.ToString(), TextSyndicationContentKind.Html);
return newItem;
}
finally
{
semaphore.Release();
}
}).ToList();
var simplifiedItems = await Task.WhenAll(itemTasks);
var mergedFeed = new SyndicationFeed(
"hitomi.la",
"",
new Uri("https://hitomi.la/index-japanese.html"),
simplifiedItems
);
CmnUtil.ExportXMLFile("hitomi-japanese.xml", mergedFeed);
LogManager.AddLog($"[情報] hitomi-japanese.xml を正常に出力しました (記事数: {simplifiedItems.Length})");
CmnUtil.ExportXMLFile("hitomi-japanese.xml", feed);
LogManager.AddLog($"[情報] hitomi-japanese.xml を正常に出力しました (記事数: {uniqueItems.Count})");
}
catch (Exception ex)
{
LogManager.AddLog($"[エラー] hitomi-japanese:{ex.Message}");
CmnUtil.PrintCallStackToEvent(ex);
}
}
private static List<SyndicationItem> ParseGalleryItems(IDocument document)
{
var result = new List<SyndicationItem>();
var cards = document.QuerySelectorAll(".gallery-content > div");
foreach (var card in cards)
{
var titleElem = card.QuerySelector("h1.lillie a, h1 a");
if (titleElem == null)
{
continue;
}
var title = titleElem.TextContent?.Trim() ?? "No Title";
var relLink = titleElem.GetAttribute("href") ?? string.Empty;
var linkHref = !string.IsNullOrEmpty(relLink)
? (relLink.StartsWith("http") ? relLink : $"https://hitomi.la{(relLink.StartsWith("/") ? "" : "/")}{relLink}")
: string.Empty;
if (string.IsNullOrEmpty(linkHref))
{
continue;
}
// 作品IDの抽出
string galleryId = "";
var matchId = Regex.Match(linkHref, @"(\d+)\.html", RegexOptions.IgnoreCase);
if (!matchId.Success)
{
matchId = Regex.Match(linkHref, @"(\d{5,})");
}
if (matchId.Success)
{
galleryId = matchId.Groups[1].Value;
}
var item = new SyndicationItem
{
Title = new TextSyndicationContent(title),
Id = CmnUtil.CreateGuid(linkHref)
};
item.Links.Add(new SyndicationLink(new Uri(linkHref)));
// 投稿日時の取得
var dateElem = card.QuerySelector(".date");
var dateStr = dateElem?.GetAttribute("data-posted") ?? dateElem?.TextContent?.Trim();
if (!string.IsNullOrEmpty(dateStr) && DateTimeOffset.TryParse(dateStr, out var parsedOffset))
{
var jstDate = parsedOffset.ToOffset(TimeSpan.FromHours(9)).DateTime;
item.PublishDate = jstDate;
item.LastUpdatedTime = jstDate;
}
else
{
item.PublishDate = DateTime.Now;
item.LastUpdatedTime = DateTime.Now;
}
// 作家の取得
var artistElems = card.QuerySelectorAll(".artist-list li a");
foreach (var artistElem in artistElems)
{
var artist = artistElem.TextContent?.Trim();
if (!string.IsNullOrEmpty(artist) && !string.Equals(artist, "N/A", StringComparison.OrdinalIgnoreCase))
{
item.Authors.Add(new SyndicationPerson { Name = artist });
item.Categories.Add(new SyndicationCategory(artist));
}
}
// シリーズの取得
var seriesElems = card.QuerySelectorAll(".series-list li a");
foreach (var seriesElem in seriesElems)
{
var series = seriesElem.TextContent?.Trim();
if (!string.IsNullOrEmpty(series) && !string.Equals(series, "N/A", StringComparison.OrdinalIgnoreCase))
{
item.Categories.Add(new SyndicationCategory(series));
}
}
// タイプの取得
var typeElem = card.QuerySelector(".dj-desc a[href*='/type/']");
if (typeElem != null)
{
var typeName = typeElem.TextContent?.Trim();
if (!string.IsNullOrEmpty(typeName) && !string.Equals(typeName, "N/A", StringComparison.OrdinalIgnoreCase))
{
item.Categories.Add(new SyndicationCategory(typeName));
}
}
// タグの取得
var tagElems = card.QuerySelectorAll(".relatedtags li a");
foreach (var tagElem in tagElems)
{
var tag = tagElem.TextContent?.Trim();
if (!string.IsNullOrEmpty(tag))
{
// 記号(♀ ♂ など)をトリム
tag = Regex.Replace(tag, @"[\s♀♂]+$", "").Trim();
if (!string.IsNullOrEmpty(tag))
{
item.Categories.Add(new SyndicationCategory(tag));
}
}
}
// サムネイル画像の取得
var imgElem = card.QuerySelector("img");
var imgSrc = imgElem?.GetAttribute("data-src") ?? imgElem?.GetAttribute("src") ?? string.Empty;
if (!string.IsNullOrEmpty(imgSrc) && imgSrc.StartsWith("//"))
{
imgSrc = $"https:{imgSrc}";
}
// HTML構築
var body = new StringBuilder();
if (!string.IsNullOrEmpty(galleryId))
{
string galleryUrl = $"https://udukipc-web.armonica-vps.work/hitomi/{galleryId}";
body.Append($"<div>");
body.Append($"<a href=\"{galleryUrl}\">[Gallery]</a>");
body.Append($"</div>");
}
if (!string.IsNullOrEmpty(imgSrc))
{
body.Append($"<div>");
body.Append($"<img src=\"{imgSrc}\" />");
body.Append($"</div>");
}
item.Summary = new TextSyndicationContent(body.ToString(), TextSyndicationContentKind.Html);
result.Add(item);
}
return result;
}
}
+8 -6
View File
@@ -3,6 +3,7 @@ using rss_generator.Class;
using rss_generator.Class.RSS;
using rss_generator.Class.RSS.e_hentai;
using rss_generator.Class.RSS.hitomi_la;
using rss_generator.Class.RSS.YouTube;
using System.Diagnostics;
@@ -78,14 +79,13 @@ while (true)
tasks.Add(momonga_magazine.CreateRSSAsync(client));
}
#if DEBUG
//tasks.Add(index_japanese.CreateRSSAsync(client));
tasks.Add(mangalear.CreateRSSAsync(client));
tasks.Add(momonga_fanzine.CreateRSSAsync(client));
tasks.Add(momonga_magazine.CreateRSSAsync(client));
tasks.Add(e_hentai_ai.CreateRSSAsync(client));
tasks.Add(e_hentai_non_ai.CreateRSSAsync(client));
tasks.Add(YouTubeFeedManager.CreateRSSAsync(client));
//tasks.Add(momonga_fanzine.CreateRSSAsync(client));
//tasks.Add(momonga_magazine.CreateRSSAsync(client));
//tasks.Add(e_hentai_non_ai.CreateRSSAsync(client));
#else
tasks.Add(Youtube.CreateRSSAsync(client));
tasks.Add(YouTubeFeedManager.CreateRSSAsync(client));
tasks.Add(rss_generator.Class.RSS.X.x_list.CreateRSSAsync(client));
tasks.Add(kimochiinfo.CreateRSSAsync(client));
@@ -100,6 +100,8 @@ while (true)
tasks.Add(mangalear.CreateRSSAsync(client));
tasks.Add(e_hentai_ai.CreateRSSAsync(client));
tasks.Add(e_hentai_non_ai.CreateRSSAsync(client));
tasks.Add(index_japanese.CreateRSSAsync(client));
#endif