feat: add scrapers for missav.ai and missav123.com with video streaming integration

This commit is contained in:
armonica committed 2026-09-05 22:33:39 +09:00
1 parent b182316504
commit d18ed28eda
3 files changed
+246 -82

No files matched your search

+228 -62
View File
@@ -1,21 +1,27 @@
using AngleSharp;
using AngleSharp.Dom; using AngleSharp.Dom;
using NpgsqlTypes; using AngleSharp.Html.Dom;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics; using System.Linq;
using System.Net;
using System.ServiceModel.Syndication; using System.ServiceModel.Syndication;
using System.Text; using System.Text;
using System.Xml; using System.Text.RegularExpressions;
using System.Threading.Tasks;
namespace rss_generator.Class.RSS; namespace rss_generator.Class.RSS;
/// <summary>
/// missav.ai の新着動画を取得し、WebImageViewer (https://udukipc-web.armonica-vps.work/) で動画を視聴できるリンクを含むRSSを生成します。
/// </summary>
internal static class missav internal static class missav
{ {
public static async Task CreateRSSAsync(HttpClient client, string? sessionId = null) public static async Task CreateRSSAsync(HttpClient client, string? sessionId = null)
{ {
var feed = new SyndicationFeed("missav.ai", "", new Uri("https://missav.ai/ja/new")); var pageMaster = "https://missav.ai";
var feed = new SyndicationFeed("missav.ai", "missav Recent Videos", new Uri($"{pageMaster}/ja/new"));
var items = new List<SyndicationItem>(); var items = new List<SyndicationItem>();
try try
{ {
// 外部からセッションが渡されなかった場合は自前で作成 // 外部からセッションが渡されなかった場合は自前で作成
@@ -24,15 +30,228 @@ internal static class missav
{ {
sessionId = await CmnUtil.CreateFlareSolverrSession(client); sessionId = await CmnUtil.CreateFlareSolverrSession(client);
} }
var docList = new List<IDocument>();
try try
{ {
var docList = new List<IDocument>();
for (int pageId = 1; pageId <= 2; pageId++) for (int pageId = 1; pageId <= 2; pageId++)
{ {
var pageURL = $"https://missav.ai/ja/new{(pageId > 1 ? $"?page={pageId}" : "")}"; var pageURL = $"{pageMaster}/ja/new{(pageId > 1 ? $"?page={pageId}" : "")}";
var document = await CmnUtil.GetDocumentFlareSolverr(client, pageURL, session: sessionId); var document = await CmnUtil.GetDocumentFlareSolverr(client, pageURL, session: sessionId);
docList.Add(document); if (document != null)
{
docList.Add(document);
}
} }
var linkList = new List<string>();
foreach (var document in docList)
{
var elements = document.QuerySelectorAll("div.thumbnail.group");
var targetDivs = elements.Where(e => e.ClassList.Length == 2).ToList();
foreach (var div in targetDivs)
{
var titleElement = div.QuerySelector(".text-secondary") ?? div.QuerySelector("a[alt]");
var linkElement = div.QuerySelector(".text-secondary") ?? div.QuerySelector("a");
var imgElement = div.QuerySelector("img");
var title = titleElement?.TextContent?.Trim() ?? "タイトルなし";
var relHref = linkElement?.GetAttribute("href") ?? string.Empty;
if (string.IsNullOrWhiteSpace(relHref))
{
continue;
}
var linkHref = relHref.StartsWith("http", StringComparison.OrdinalIgnoreCase)
? relHref
: $"{pageMaster.TrimEnd('/')}/{relHref.TrimStart('/')}";
var imageSrc = imgElement?.GetAttribute("data-src") ?? imgElement?.GetAttribute("src");
if (!linkList.Contains(linkHref))
{
linkList.Add(linkHref);
}
else
{
continue;
}
var itemUri = new Uri(linkHref);
var imageUri = !string.IsNullOrEmpty(imageSrc) ? new Uri(imageSrc) : null;
// 詳細ページから動画ストリーム情報およびメタ情報を取得
string? streamUrl = null;
string? descText = null;
string? maker = null;
var actresses = new List<string>();
var genres = new List<string>();
var tags = new List<string>();
try
{
var docDetail = await CmnUtil.GetDocumentFlareSolverr(client, linkHref, session: sessionId);
if (docDetail != null)
{
var sourceText = docDetail.Source.Text;
// 1. surrit.com の m3u8 URL を正規表現で抽出
var surritMatch = Regex.Match(sourceText, @"https?:\\?/\\?/[^""'\s\\]*surrit\.com/[^""'\s\\]+?\.m3u8", RegexOptions.IgnoreCase);
if (surritMatch.Success)
{
streamUrl = surritMatch.Value.Replace(@"\/", "/");
}
else
{
// 2. UUID から surrit.com の playlist.m3u8 を構築
var uuidMatch = Regex.Match(sourceText, @"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", RegexOptions.IgnoreCase);
if (uuidMatch.Success)
{
streamUrl = $"https://surrit.com/{uuidMatch.Value}/playlist.m3u8";
}
else
{
// 3. 汎用 .m3u8 のフォールバック抽出
var m3u8Match = Regex.Match(sourceText, @"https?:\\?/\\?/[^""'\s\\]+?\.m3u8", RegexOptions.IgnoreCase);
if (m3u8Match.Success)
{
streamUrl = m3u8Match.Value.Replace(@"\/", "/");
}
}
}
// 詳細テキスト(説明文)
descText = docDetail.QuerySelector("meta[name='description']")?.GetAttribute("content")
?? docDetail.QuerySelector("meta[property='og:description']")?.GetAttribute("content")
?? docDetail.QuerySelector(".text-secondary")?.TextContent?.Trim();
// メーカー / レーベル
maker = docDetail.QuerySelector("a[href*='/makers/'], a[href*='/maker/'], a[href*='/labels/'], a[href*='/label/']")?.TextContent?.Trim();
// 女優
var actressElements = docDetail.QuerySelectorAll("a[href*='/actresses/'], a[href*='/actress/']");
foreach (var a in actressElements)
{
var aName = a.TextContent?.Trim();
if (!string.IsNullOrEmpty(aName) && !actresses.Contains(aName))
{
actresses.Add(aName);
}
}
// ジャンル
var genreElements = docDetail.QuerySelectorAll("a[href*='/genres/'], a[href*='/genre/']");
foreach (var g in genreElements)
{
var gName = g.TextContent?.Trim();
if (!string.IsNullOrEmpty(gName) && !genres.Contains(gName))
{
genres.Add(gName);
}
}
// タグ
var tagElements = docDetail.QuerySelectorAll("a[href*='/tags/'], a[href*='/tag/']");
foreach (var t in tagElements)
{
var tName = t.TextContent?.Trim();
if (!string.IsNullOrEmpty(tName) && !tags.Contains(tName))
{
tags.Add(tName);
}
}
}
}
catch (Exception ex)
{
LogManager.AddLog($"[警告] missav 詳細ページ取得失敗 ({linkHref}): {ex.Message}");
}
// 動画IDの抽出とWebImageViewerリンクの生成
var videoIdMatch = Regex.Match(linkHref, @"/ja/([^/?#]+)", RegexOptions.IgnoreCase);
string videoId = videoIdMatch.Success ? videoIdMatch.Groups[1].Value : linkHref.TrimEnd('/').Split('/').Last();
string viewerUrl = $"https://udukipc-web.armonica-vps.work/missav/{Uri.EscapeDataString(videoId)}";
if (!string.IsNullOrEmpty(streamUrl))
{
viewerUrl += $"?stream={Uri.EscapeDataString(streamUrl)}";
}
// 本文 HTML(Summary)の生成(123avと同様の構成)
var body = new StringBuilder();
// 1. WebImageViewer への再生リンク & 外部リンク(記事への直接埋め込みは行わない)
body.Append($@"<div style=""margin-bottom: 12px; padding: 10px 14px; background: rgba(236, 72, 153, 0.08); border-radius: 8px; border-left: 4px solid #ec4899;"">");
body.Append($@"<p style=""margin: 0 0 6px 0;""><a href=""{viewerUrl}"" target=""_blank"" rel=""noopener noreferrer"" style=""font-weight: bold; font-size: 1.1em; color: #db2777; text-decoration: none;"">▶ WebImageViewerで動画を再生する</a></p>");
body.Append($@"<p style=""margin: 0; font-size: 0.9em; color: #64748b;""><a href=""{itemUri}"" target=""_blank"" rel=""noopener noreferrer"" referrerpolicy=""no-referrer"">missav公式ページで開く</a></p>");
body.Append($@"</div>");
// 2. サムネイル画像
if (imageUri != null)
{
body.Append($@"<div><img src=""{imageUri}"" referrerpolicy=""no-referrer"" alt=""{WebUtility.HtmlEncode(title)}"" style=""max-width: 100%; height: auto; margin-top: 8px; border-radius: 6px;"" /></div>");
}
// 3. メタ情報(詳細、メーカー、女優、ジャンル、タグ)
if (!string.IsNullOrWhiteSpace(descText))
{
body.Append($@"<div style=""margin-top: 8px;""><strong>詳細:</strong> {WebUtility.HtmlEncode(descText)}</div>");
}
if (!string.IsNullOrWhiteSpace(maker))
{
body.Append($@"<div><strong>メーカー:</strong> {WebUtility.HtmlEncode(maker)}</div>");
}
if (actresses.Count > 0)
{
body.Append($@"<div><strong>女優:</strong> {WebUtility.HtmlEncode(string.Join(", ", actresses))}</div>");
}
if (genres.Count > 0)
{
body.Append($@"<div><strong>ジャンル:</strong> {WebUtility.HtmlEncode(string.Join(", ", genres))}</div>");
}
if (tags.Count > 0)
{
body.Append($@"<div><strong>タグ:</strong> {WebUtility.HtmlEncode(string.Join(", ", tags))}</div>");
}
var item = new SyndicationItem(
title,
string.Empty,
itemUri,
CmnUtil.CreateGuid(linkHref),
DateTime.Now
);
// カテゴリ登録(ジャンル、タグ、女優)
foreach (var g in genres)
{
item.Categories.Add(new SyndicationCategory(g));
}
foreach (var t in tags)
{
item.Categories.Add(new SyndicationCategory(t));
}
foreach (var a in actresses)
{
item.Categories.Add(new SyndicationCategory(a));
}
// 作者/メーカー登録
if (!string.IsNullOrEmpty(maker))
{
item.Authors.Add(new SyndicationPerson { Name = maker });
}
else if (actresses.Count > 0)
{
item.Authors.Add(new SyndicationPerson { Name = actresses[0] });
}
item.Summary = new TextSyndicationContent(body.ToString(), TextSyndicationContentKind.Html);
items.Add(item);
}
}
feed.Items = items;
await CmnUtil.ExportXMLFileAsync("missav.xml", feed, client);
} }
finally finally
{ {
@@ -41,64 +260,11 @@ internal static class missav
await CmnUtil.DestroyFlareSolverrSession(client, sessionId); await CmnUtil.DestroyFlareSolverrSession(client, sessionId);
} }
} }
var linkList = new List<string>();
foreach (var document in docList)
{
if (document == null)
{
continue;
}
var elements = document.QuerySelectorAll("div.thumbnail.group");
var targetDivs = elements.Where(e => e.ClassList.Length == 2).ToList();
foreach (var div in targetDivs)
{
var titleElement = div.QuerySelector(".text-secondary");
var linkElement = div.QuerySelector(".text-secondary");
var imgElement = div.QuerySelector("img");
var title = titleElement?.TextContent?.Trim() ?? "タイトルなし";
var linkHref = linkElement?.GetAttribute("href") ?? string.Empty;
var imageSrc = imgElement?.GetAttribute("data-src");
if (linkHref != string.Empty && !linkList.Contains(linkHref))
{
linkList.Add(linkHref);
}
else
{
continue;
}
var itemUri = !string.IsNullOrEmpty(linkHref) ? new Uri(linkHref) : null;
var imageUri = !string.IsNullOrEmpty(imageSrc) ? new Uri(imageSrc) : null;
var body = new StringBuilder();
body.Append($@"<div><a href=""{itemUri}"" target=""_blank"" rel=""noopener noreferrer"" referrerpolicy=""no-referrer"">{title}</a></div>");
body.Append($@"<div><img src=""{imageUri}"" referrerpolicy=""no-referrer"" alt=""{title}"" style=""max-width: 100%; height: auto;"" /></div>");
var item = new SyndicationItem(
title,
string.Empty,
itemUri,
CmnUtil.CreateGuid(linkHref),
DateTime.Now // 取得できた公開日時があればそれをセット
);
// item.Content = SyndicationContent.CreateHtmlContent(body.ToString());
// ※ RSS 2.0の場合は Content よりも Summary(<description>タグ)が使われることが多いため、
// 両方に設定しておくか、Summary側にHTMLを設定すると大半のリーダーで確実です。
item.Summary = new TextSyndicationContent(body.ToString(), TextSyndicationContentKind.Html);
items.Add(item);
}
}
feed.Items = items;
await CmnUtil.ExportXMLFileAsync("missav.xml", feed, client);
} }
catch (Exception ex) catch (Exception ex)
{ {
LogManager.AddLog($"[エラー] missav:{ex.Message}"); LogManager.AddLog($"[エラー] missav:{ex.Message}");
CmnUtil.PrintCallStackToEvent(ex);
} }
} }
} }
+16 -13
View File
@@ -15,7 +15,7 @@ namespace rss_generator.Class.RSS;
internal class missav123 internal class missav123
{ {
/// <summary> /// <summary>
/// 123av.com の新着動画を取得し、FreshRSS等のRSSリーダー内で動画を直接再生できるiframeを埋め込んだRSSを生成します。 /// 123av.com の新着動画を取得し、WebImageViewer (https://udukipc-web.armonica-vps.work/) で動画を視聴できるリンクを含むRSSを生成します。
/// </summary> /// </summary>
/// <param name="client">HttpClient インスタンス</param> /// <param name="client">HttpClient インスタンス</param>
public static async Task CreateRSSAsync(HttpClient client, string? sessionId = null) public static async Task CreateRSSAsync(HttpClient client, string? sessionId = null)
@@ -136,24 +136,27 @@ internal class missav123
LogManager.AddLog($"[警告] 123av 詳細ページ取得失敗 ({linkHref}): {ex.Message}"); LogManager.AddLog($"[警告] 123av 詳細ページ取得失敗 ({linkHref}): {ex.Message}");
} }
// 動画IDの抽出とWebImageViewerリンクの生成
var videoIdMatch = Regex.Match(linkHref, @"/video/([^/?#]+)", RegexOptions.IgnoreCase);
string videoId = videoIdMatch.Success ? videoIdMatch.Groups[1].Value : linkHref.TrimEnd('/').Split('/').Last();
string viewerUrl = $"https://udukipc-web.armonica-vps.work/123av/{Uri.EscapeDataString(videoId)}";
if (!string.IsNullOrEmpty(playerUrl))
{
viewerUrl += $"?play={Uri.EscapeDataString(playerUrl)}";
}
// 本文 HTML(Summary)の生成 // 本文 HTML(Summary)の生成
var body = new StringBuilder(); var body = new StringBuilder();
// 1. タイトルリンク // 1. WebImageViewer への再生リンク & 外部リンク(記事への直接埋め込みは行わない)
//body.Append($@"<div><a href=""{itemUri}""><strong>{WebUtility.HtmlEncode(title)}</strong></a></div>"); body.Append($@"<div style=""margin-bottom: 12px; padding: 10px 14px; background: rgba(99, 102, 241, 0.08); border-radius: 8px; border-left: 4px solid #6366f1;"">");
body.Append($@"<p style=""margin: 0 0 6px 0;""><a href=""{viewerUrl}"" target=""_blank"" rel=""noopener noreferrer"" style=""font-weight: bold; font-size: 1.1em; color: #4f46e5; text-decoration: none;"">▶ WebImageViewerで動画を再生する</a></p>");
// 2. FreshRSS等で直接再生できる動画埋め込み iframe body.Append($@"<p style=""margin: 0; font-size: 0.9em; color: #64748b;""><a href=""{itemUri}"" target=""_blank"" rel=""noopener noreferrer"" referrerpolicy=""no-referrer"">123av公式ページで開く</a>");
if (!string.IsNullOrEmpty(playerUrl)) if (!string.IsNullOrEmpty(playerUrl))
{ {
body.Append($@"<div style=""position: relative; width: 100%; max-width: 800px; aspect-ratio: 16/9; margin: 12px 0;"">"); body.Append($@" | <a href=""{playerUrl}"" target=""_blank"" rel=""noopener noreferrer"" referrerpolicy=""no-referrer"">直接動画プレイヤーを開く</a>");
body.Append($@"<iframe src=""about:blank"" data-src=""{playerUrl}"" style=""width: 100%; height: 100%; border: 0;"" allow=""autoplay; fullscreen; encrypted-media"" allowfullscreen=""true"" scrolling=""no"" frameborder=""0""></iframe>");
body.Append($@"</div>");
body.Append($@"<div><p><a href=""{itemUri}"" target=""_blank"" rel=""noopener noreferrer"" referrerpolicy=""no-referrer"">123avで開く</a> | <a href=""{playerUrl}"" target=""_blank"" rel=""noopener noreferrer"" referrerpolicy=""no-referrer"">直接動画プレイヤーを開く</a></p></div>");
}
else
{
body.Append($@"<div><p><a href=""{itemUri}"" target=""_blank"" rel=""noopener noreferrer"" referrerpolicy=""no-referrer"">123avで動画を再生する</a></p></div>");
} }
body.Append($@"</p></div>");
// 3. サムネイル画像 // 3. サムネイル画像
if (imageUri != null) if (imageUri != null)
+2 -7
View File
@@ -82,13 +82,6 @@ while (true)
// メモリ消費を最小化(512MB制限内)するため、各フィードを並行ではなく順次実行し、タスクごとにGCを実行 // メモリ消費を最小化(512MB制限内)するため、各フィードを並行ではなく順次実行し、タスクごとにGCを実行
var feedActions = new List<(string Name, Func<Task> Action)>(); var feedActions = new List<(string Name, Func<Task> Action)>();
DateTime from = new DateTime(nowDate.Year, nowDate.Month, nowDate.Day, 5, 0, 0);
DateTime to = new DateTime(nowDate.Year, nowDate.Month, nowDate.Day, 7, 15, 0);
if (from < nowDate && nowDate < to)
{
feedActions.Add(("momonga_fanzine", () => momonga_fanzine.CreateRSSAsync(client)));
feedActions.Add(("momonga_magazine", () => momonga_magazine.CreateRSSAsync(client)));
}
#if DEBUG #if DEBUG
feedActions.Add(("momonga_fanzine", () => momonga_fanzine.CreateRSSAsync(client))); feedActions.Add(("momonga_fanzine", () => momonga_fanzine.CreateRSSAsync(client)));
feedActions.Add(("momonga_magazine", () => momonga_magazine.CreateRSSAsync(client))); feedActions.Add(("momonga_magazine", () => momonga_magazine.CreateRSSAsync(client)));
@@ -118,6 +111,8 @@ while (true)
feedActions.Add(("pixiv_non_r18_ai", () => pixiv_non_r18_ai.CreateRSSAsync(client))); feedActions.Add(("pixiv_non_r18_ai", () => pixiv_non_r18_ai.CreateRSSAsync(client)));
feedActions.Add(("pixiv_r18_non_ai", () => pixiv_r18_non_ai.CreateRSSAsync(client))); feedActions.Add(("pixiv_r18_non_ai", () => pixiv_r18_non_ai.CreateRSSAsync(client)));
feedActions.Add(("pixiv_r18_ai", () => pixiv_r18_ai.CreateRSSAsync(client))); feedActions.Add(("pixiv_r18_ai", () => pixiv_r18_ai.CreateRSSAsync(client)));
feedActions.Add(("momonga_fanzine", () => momonga_fanzine.CreateRSSAsync(client)));
feedActions.Add(("momonga_magazine", () => momonga_magazine.CreateRSSAsync(client)));
// FlareSolverr 利用タスク群(単一セッションで順次実行し、完了後に破棄してメモリ最小化) // FlareSolverr 利用タスク群(単一セッションで順次実行し、完了後に破棄してメモリ最小化)
feedActions.Add(("FlareSolverr", () => FlareSolverrFeedManager.CreateAllRSSAsync(client))); feedActions.Add(("FlareSolverr", () => FlareSolverrFeedManager.CreateAllRSSAsync(client)));