From d18ed28eda0202cdea13d8e53a1796bb10546542 Mon Sep 17 00:00:00 2001 From: uduki Date: Sat, 5 Sep 2026 22:33:39 +0900 Subject: [PATCH] feat: add scrapers for missav.ai and missav123.com with video streaming integration --- rss-generator/Class/RSS/missav.cs | 290 +++++++++++++++++++++------ rss-generator/Class/RSS/missav123.cs | 29 +-- rss-generator/Program.cs | 9 +- 3 files changed, 246 insertions(+), 82 deletions(-) diff --git a/rss-generator/Class/RSS/missav.cs b/rss-generator/Class/RSS/missav.cs index 318075f..7ea4118 100644 --- a/rss-generator/Class/RSS/missav.cs +++ b/rss-generator/Class/RSS/missav.cs @@ -1,21 +1,27 @@ -using AngleSharp; using AngleSharp.Dom; -using NpgsqlTypes; +using AngleSharp.Html.Dom; using System; using System.Collections.Generic; -using System.Diagnostics; +using System.Linq; +using System.Net; using System.ServiceModel.Syndication; using System.Text; -using System.Xml; +using System.Text.RegularExpressions; +using System.Threading.Tasks; namespace rss_generator.Class.RSS; +/// +/// missav.ai の新着動画を取得し、WebImageViewer (https://udukipc-web.armonica-vps.work/) で動画を視聴できるリンクを含むRSSを生成します。 +/// internal static class missav { public static async Task CreateRSSAsync(HttpClient client, string? sessionId = null) { - var feed = new SyndicationFeed("missav.ai", "", new Uri("https://missav.ai/ja/new")); + var pageMaster = "https://missav.ai"; + var feed = new SyndicationFeed("missav.ai", "missav Recent Videos", new Uri($"{pageMaster}/ja/new")); var items = new List(); + try { // 外部からセッションが渡されなかった場合は自前で作成 @@ -24,15 +30,228 @@ internal static class missav { sessionId = await CmnUtil.CreateFlareSolverrSession(client); } - var docList = new List(); + try { + var docList = new List(); for (int pageId = 1; pageId <= 2; pageId++) { - var pageURL = $"https://missav.ai/ja/new{(pageId > 1 ? $"?page={pageId}" : "")}"; + var pageURL = $"{pageMaster}/ja/new{(pageId > 1 ? $"?page={pageId}" : "")}"; var document = await CmnUtil.GetDocumentFlareSolverr(client, pageURL, session: sessionId); - docList.Add(document); + if (document != null) + { + docList.Add(document); + } } + + var linkList = new List(); + foreach (var document in docList) + { + var elements = document.QuerySelectorAll("div.thumbnail.group"); + var targetDivs = elements.Where(e => e.ClassList.Length == 2).ToList(); + + foreach (var div in targetDivs) + { + var titleElement = div.QuerySelector(".text-secondary") ?? div.QuerySelector("a[alt]"); + var linkElement = div.QuerySelector(".text-secondary") ?? div.QuerySelector("a"); + var imgElement = div.QuerySelector("img"); + + var title = titleElement?.TextContent?.Trim() ?? "タイトルなし"; + var relHref = linkElement?.GetAttribute("href") ?? string.Empty; + if (string.IsNullOrWhiteSpace(relHref)) + { + continue; + } + + var linkHref = relHref.StartsWith("http", StringComparison.OrdinalIgnoreCase) + ? relHref + : $"{pageMaster.TrimEnd('/')}/{relHref.TrimStart('/')}"; + + var imageSrc = imgElement?.GetAttribute("data-src") ?? imgElement?.GetAttribute("src"); + if (!linkList.Contains(linkHref)) + { + linkList.Add(linkHref); + } + else + { + continue; + } + + var itemUri = new Uri(linkHref); + var imageUri = !string.IsNullOrEmpty(imageSrc) ? new Uri(imageSrc) : null; + + // 詳細ページから動画ストリーム情報およびメタ情報を取得 + string? streamUrl = null; + string? descText = null; + string? maker = null; + var actresses = new List(); + var genres = new List(); + var tags = new List(); + + try + { + var docDetail = await CmnUtil.GetDocumentFlareSolverr(client, linkHref, session: sessionId); + if (docDetail != null) + { + var sourceText = docDetail.Source.Text; + + // 1. surrit.com の m3u8 URL を正規表現で抽出 + var surritMatch = Regex.Match(sourceText, @"https?:\\?/\\?/[^""'\s\\]*surrit\.com/[^""'\s\\]+?\.m3u8", RegexOptions.IgnoreCase); + if (surritMatch.Success) + { + streamUrl = surritMatch.Value.Replace(@"\/", "/"); + } + else + { + // 2. UUID から surrit.com の playlist.m3u8 を構築 + var uuidMatch = Regex.Match(sourceText, @"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", RegexOptions.IgnoreCase); + if (uuidMatch.Success) + { + streamUrl = $"https://surrit.com/{uuidMatch.Value}/playlist.m3u8"; + } + else + { + // 3. 汎用 .m3u8 のフォールバック抽出 + var m3u8Match = Regex.Match(sourceText, @"https?:\\?/\\?/[^""'\s\\]+?\.m3u8", RegexOptions.IgnoreCase); + if (m3u8Match.Success) + { + streamUrl = m3u8Match.Value.Replace(@"\/", "/"); + } + } + } + + // 詳細テキスト(説明文) + descText = docDetail.QuerySelector("meta[name='description']")?.GetAttribute("content") + ?? docDetail.QuerySelector("meta[property='og:description']")?.GetAttribute("content") + ?? docDetail.QuerySelector(".text-secondary")?.TextContent?.Trim(); + + // メーカー / レーベル + maker = docDetail.QuerySelector("a[href*='/makers/'], a[href*='/maker/'], a[href*='/labels/'], a[href*='/label/']")?.TextContent?.Trim(); + + // 女優 + var actressElements = docDetail.QuerySelectorAll("a[href*='/actresses/'], a[href*='/actress/']"); + foreach (var a in actressElements) + { + var aName = a.TextContent?.Trim(); + if (!string.IsNullOrEmpty(aName) && !actresses.Contains(aName)) + { + actresses.Add(aName); + } + } + + // ジャンル + var genreElements = docDetail.QuerySelectorAll("a[href*='/genres/'], a[href*='/genre/']"); + foreach (var g in genreElements) + { + var gName = g.TextContent?.Trim(); + if (!string.IsNullOrEmpty(gName) && !genres.Contains(gName)) + { + genres.Add(gName); + } + } + + // タグ + var tagElements = docDetail.QuerySelectorAll("a[href*='/tags/'], a[href*='/tag/']"); + foreach (var t in tagElements) + { + var tName = t.TextContent?.Trim(); + if (!string.IsNullOrEmpty(tName) && !tags.Contains(tName)) + { + tags.Add(tName); + } + } + } + } + catch (Exception ex) + { + LogManager.AddLog($"[警告] missav 詳細ページ取得失敗 ({linkHref}): {ex.Message}"); + } + + // 動画IDの抽出とWebImageViewerリンクの生成 + var videoIdMatch = Regex.Match(linkHref, @"/ja/([^/?#]+)", RegexOptions.IgnoreCase); + string videoId = videoIdMatch.Success ? videoIdMatch.Groups[1].Value : linkHref.TrimEnd('/').Split('/').Last(); + string viewerUrl = $"https://udukipc-web.armonica-vps.work/missav/{Uri.EscapeDataString(videoId)}"; + if (!string.IsNullOrEmpty(streamUrl)) + { + viewerUrl += $"?stream={Uri.EscapeDataString(streamUrl)}"; + } + + // 本文 HTML(Summary)の生成(123avと同様の構成) + var body = new StringBuilder(); + + // 1. WebImageViewer への再生リンク & 外部リンク(記事への直接埋め込みは行わない) + body.Append($@"
"); + body.Append($@"

▶ WebImageViewerで動画を再生する

"); + body.Append($@"

missav公式ページで開く

"); + body.Append($@"
"); + + // 2. サムネイル画像 + if (imageUri != null) + { + body.Append($@"
"); + } + + // 3. メタ情報(詳細、メーカー、女優、ジャンル、タグ) + if (!string.IsNullOrWhiteSpace(descText)) + { + body.Append($@"
詳細: {WebUtility.HtmlEncode(descText)}
"); + } + if (!string.IsNullOrWhiteSpace(maker)) + { + body.Append($@"
メーカー: {WebUtility.HtmlEncode(maker)}
"); + } + if (actresses.Count > 0) + { + body.Append($@"
女優: {WebUtility.HtmlEncode(string.Join(", ", actresses))}
"); + } + if (genres.Count > 0) + { + body.Append($@"
ジャンル: {WebUtility.HtmlEncode(string.Join(", ", genres))}
"); + } + if (tags.Count > 0) + { + body.Append($@"
タグ: {WebUtility.HtmlEncode(string.Join(", ", tags))}
"); + } + + var item = new SyndicationItem( + title, + string.Empty, + itemUri, + CmnUtil.CreateGuid(linkHref), + DateTime.Now + ); + + // カテゴリ登録(ジャンル、タグ、女優) + foreach (var g in genres) + { + item.Categories.Add(new SyndicationCategory(g)); + } + foreach (var t in tags) + { + item.Categories.Add(new SyndicationCategory(t)); + } + foreach (var a in actresses) + { + item.Categories.Add(new SyndicationCategory(a)); + } + + // 作者/メーカー登録 + if (!string.IsNullOrEmpty(maker)) + { + item.Authors.Add(new SyndicationPerson { Name = maker }); + } + else if (actresses.Count > 0) + { + item.Authors.Add(new SyndicationPerson { Name = actresses[0] }); + } + + item.Summary = new TextSyndicationContent(body.ToString(), TextSyndicationContentKind.Html); + items.Add(item); + } + } + + feed.Items = items; + await CmnUtil.ExportXMLFileAsync("missav.xml", feed, client); } finally { @@ -41,64 +260,11 @@ internal static class missav await CmnUtil.DestroyFlareSolverrSession(client, sessionId); } } - - var linkList = new List(); - foreach (var document in docList) - { - if (document == null) - { - continue; - } - - var elements = document.QuerySelectorAll("div.thumbnail.group"); - var targetDivs = elements.Where(e => e.ClassList.Length == 2).ToList(); - foreach (var div in targetDivs) - { - var titleElement = div.QuerySelector(".text-secondary"); - var linkElement = div.QuerySelector(".text-secondary"); - var imgElement = div.QuerySelector("img"); - - var title = titleElement?.TextContent?.Trim() ?? "タイトルなし"; - var linkHref = linkElement?.GetAttribute("href") ?? string.Empty; - var imageSrc = imgElement?.GetAttribute("data-src"); - if (linkHref != string.Empty && !linkList.Contains(linkHref)) - { - linkList.Add(linkHref); - } - else - { - continue; - } - - var itemUri = !string.IsNullOrEmpty(linkHref) ? new Uri(linkHref) : null; - var imageUri = !string.IsNullOrEmpty(imageSrc) ? new Uri(imageSrc) : null; - - var body = new StringBuilder(); - body.Append($@""); - body.Append($@"
"); - - var item = new SyndicationItem( - title, - string.Empty, - itemUri, - CmnUtil.CreateGuid(linkHref), - DateTime.Now // 取得できた公開日時があればそれをセット - ); - // item.Content = SyndicationContent.CreateHtmlContent(body.ToString()); - // ※ RSS 2.0の場合は Content よりも Summary(タグ)が使われることが多いため、 - // 両方に設定しておくか、Summary側にHTMLを設定すると大半のリーダーで確実です。 - item.Summary = new TextSyndicationContent(body.ToString(), TextSyndicationContentKind.Html); - - items.Add(item); - } - } - feed.Items = items; - - await CmnUtil.ExportXMLFileAsync("missav.xml", feed, client); } catch (Exception ex) { LogManager.AddLog($"[エラー] missav:{ex.Message}"); + CmnUtil.PrintCallStackToEvent(ex); } } } \ No newline at end of file diff --git a/rss-generator/Class/RSS/missav123.cs b/rss-generator/Class/RSS/missav123.cs index 66cbe2d..2627810 100644 --- a/rss-generator/Class/RSS/missav123.cs +++ b/rss-generator/Class/RSS/missav123.cs @@ -15,7 +15,7 @@ namespace rss_generator.Class.RSS; internal class missav123 { /// - /// 123av.com の新着動画を取得し、FreshRSS等のRSSリーダー内で動画を直接再生できるiframeを埋め込んだRSSを生成します。 + /// 123av.com の新着動画を取得し、WebImageViewer (https://udukipc-web.armonica-vps.work/) で動画を視聴できるリンクを含むRSSを生成します。 /// /// HttpClient インスタンス public static async Task CreateRSSAsync(HttpClient client, string? sessionId = null) @@ -136,24 +136,27 @@ internal class missav123 LogManager.AddLog($"[警告] 123av 詳細ページ取得失敗 ({linkHref}): {ex.Message}"); } + // 動画IDの抽出とWebImageViewerリンクの生成 + var videoIdMatch = Regex.Match(linkHref, @"/video/([^/?#]+)", RegexOptions.IgnoreCase); + string videoId = videoIdMatch.Success ? videoIdMatch.Groups[1].Value : linkHref.TrimEnd('/').Split('/').Last(); + string viewerUrl = $"https://udukipc-web.armonica-vps.work/123av/{Uri.EscapeDataString(videoId)}"; + if (!string.IsNullOrEmpty(playerUrl)) + { + viewerUrl += $"?play={Uri.EscapeDataString(playerUrl)}"; + } + // 本文 HTML(Summary)の生成 var body = new StringBuilder(); - // 1. タイトルリンク - //body.Append($@""); - - // 2. FreshRSS等で直接再生できる動画埋め込み iframe + // 1. WebImageViewer への再生リンク & 外部リンク(記事への直接埋め込みは行わない) + body.Append($@"
"); + body.Append($@"

▶ WebImageViewerで動画を再生する

"); + body.Append($@"

123av公式ページで開く"); if (!string.IsNullOrEmpty(playerUrl)) { - body.Append($@"

"); - body.Append($@""); - body.Append($@"
"); - body.Append($@""); - } - else - { - body.Append($@""); + body.Append($@" | 直接動画プレイヤーを開く"); } + body.Append($@"

"); // 3. サムネイル画像 if (imageUri != null) diff --git a/rss-generator/Program.cs b/rss-generator/Program.cs index fb8c384..034c74e 100644 --- a/rss-generator/Program.cs +++ b/rss-generator/Program.cs @@ -82,13 +82,6 @@ while (true) // メモリ消費を最小化(512MB制限内)するため、各フィードを並行ではなく順次実行し、タスクごとにGCを実行 var feedActions = new List<(string Name, Func Action)>(); - DateTime from = new DateTime(nowDate.Year, nowDate.Month, nowDate.Day, 5, 0, 0); - DateTime to = new DateTime(nowDate.Year, nowDate.Month, nowDate.Day, 7, 15, 0); - if (from < nowDate && nowDate < to) - { - feedActions.Add(("momonga_fanzine", () => momonga_fanzine.CreateRSSAsync(client))); - feedActions.Add(("momonga_magazine", () => momonga_magazine.CreateRSSAsync(client))); - } #if DEBUG feedActions.Add(("momonga_fanzine", () => momonga_fanzine.CreateRSSAsync(client))); feedActions.Add(("momonga_magazine", () => momonga_magazine.CreateRSSAsync(client))); @@ -118,6 +111,8 @@ while (true) feedActions.Add(("pixiv_non_r18_ai", () => pixiv_non_r18_ai.CreateRSSAsync(client))); feedActions.Add(("pixiv_r18_non_ai", () => pixiv_r18_non_ai.CreateRSSAsync(client))); feedActions.Add(("pixiv_r18_ai", () => pixiv_r18_ai.CreateRSSAsync(client))); + feedActions.Add(("momonga_fanzine", () => momonga_fanzine.CreateRSSAsync(client))); + feedActions.Add(("momonga_magazine", () => momonga_magazine.CreateRSSAsync(client))); // FlareSolverr 利用タスク群(単一セッションで順次実行し、完了後に破棄してメモリ最小化) feedActions.Add(("FlareSolverr", () => FlareSolverrFeedManager.CreateAllRSSAsync(client)));