fix: クローラーのページカウント・探索打ち切り不具合の修正、WebPサムネイル正規化、Senzuri等の動画直リンク即時解決
This commit is contained in:
1 parent
838ff22c5e
commit
1e3cac0452
8 files changed
+217
-61
No files matched your search
@@ -550,10 +550,13 @@ public class CmnUtil
|
|||||||
}
|
}
|
||||||
|
|
||||||
// 5. ストリームによるレスポンス受信と直接パース(ResponseHeadersRead で中間バッファリングと巨大文字列生成を排除)
|
// 5. ストリームによるレスポンス受信と直接パース(ResponseHeadersRead で中間バッファリングと巨大文字列生成を排除)
|
||||||
using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
|
using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||||
|
linkedCts.CancelAfter(TimeSpan.FromSeconds(15)); // ハング防止の最大15秒タイムアウト
|
||||||
|
|
||||||
|
using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, linkedCts.Token);
|
||||||
response.EnsureSuccessStatusCode();
|
response.EnsureSuccessStatusCode();
|
||||||
|
|
||||||
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken);
|
await using var stream = await response.Content.ReadAsStreamAsync(linkedCts.Token);
|
||||||
|
|
||||||
// Content-Type (charset 等) を取得して AngleSharp に通知
|
// Content-Type (charset 等) を取得して AngleSharp に通知
|
||||||
var contentType = response.Content.Headers.ContentType?.ToString();
|
var contentType = response.Content.Headers.ContentType?.ToString();
|
||||||
|
|||||||
@@ -61,22 +61,52 @@ internal class Blog2ntSiteResolver : IEroterestSiteResolver
|
|||||||
if (!seen.Add(href)) continue;
|
if (!seen.Add(href)) continue;
|
||||||
|
|
||||||
// 動画リンクを優先
|
// 動画リンクを優先
|
||||||
if (href.Contains("txxx.com", StringComparison.OrdinalIgnoreCase) ||
|
bool isVideo = href.Contains("txxx.com", StringComparison.OrdinalIgnoreCase) ||
|
||||||
href.Contains("senzuri.tube", StringComparison.OrdinalIgnoreCase) ||
|
href.Contains("senzuri.tube", StringComparison.OrdinalIgnoreCase) ||
|
||||||
href.Contains("upornia.com", StringComparison.OrdinalIgnoreCase) ||
|
href.Contains("upornia.com", StringComparison.OrdinalIgnoreCase) ||
|
||||||
href.Contains("do-ga.eroterest.net", StringComparison.OrdinalIgnoreCase) ||
|
href.Contains("do-ga.eroterest.net", StringComparison.OrdinalIgnoreCase) ||
|
||||||
href.Contains("laxd.com", StringComparison.OrdinalIgnoreCase) ||
|
href.Contains("laxd.com", StringComparison.OrdinalIgnoreCase) ||
|
||||||
href.Contains("fc2.com", StringComparison.OrdinalIgnoreCase) ||
|
href.Contains("fc2.com", StringComparison.OrdinalIgnoreCase) ||
|
||||||
href.Contains("youjizz.com", StringComparison.OrdinalIgnoreCase))
|
href.Contains("youjizz.com", StringComparison.OrdinalIgnoreCase);
|
||||||
|
|
||||||
|
if (isVideo)
|
||||||
{
|
{
|
||||||
result.NextCandidateUrls.Insert(0, href);
|
result.NextCandidateUrls.Insert(0, href);
|
||||||
}
|
}
|
||||||
else
|
else if (result.NextCandidateUrls.Count < 5)
|
||||||
{
|
{
|
||||||
result.NextCandidateUrls.Add(href);
|
result.NextCandidateUrls.Add(href);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 動画ホスト直リンクがある場合は即座に特定
|
||||||
|
var firstVideo = result.NextCandidateUrls.Find(u =>
|
||||||
|
u.Contains("txxx.com", StringComparison.OrdinalIgnoreCase) ||
|
||||||
|
u.Contains("senzuri.tube", StringComparison.OrdinalIgnoreCase) ||
|
||||||
|
u.Contains("upornia.com", StringComparison.OrdinalIgnoreCase) ||
|
||||||
|
u.Contains("do-ga.eroterest.net", StringComparison.OrdinalIgnoreCase) ||
|
||||||
|
u.Contains("laxd.com", StringComparison.OrdinalIgnoreCase) ||
|
||||||
|
u.Contains("fc2.com", StringComparison.OrdinalIgnoreCase) ||
|
||||||
|
u.Contains("youjizz.com", StringComparison.OrdinalIgnoreCase));
|
||||||
|
|
||||||
|
if (!string.IsNullOrWhiteSpace(firstVideo))
|
||||||
|
{
|
||||||
|
var targetResolver = SiteResolverRegistry.GetResolver(firstVideo);
|
||||||
|
var directResolve = await targetResolver.ResolveAsync(client, firstVideo, null, context, ct);
|
||||||
|
if (directResolve.HasFinalVideo)
|
||||||
|
{
|
||||||
|
result.VideoPageUrl = directResolve.VideoPageUrl ?? firstVideo;
|
||||||
|
result.VideoSiteName = directResolve.VideoSiteName;
|
||||||
|
result.VideoSiteId = directResolve.VideoSiteId;
|
||||||
|
result.StreamUrl = directResolve.StreamUrl;
|
||||||
|
result.VideoUploadDate = directResolve.VideoUploadDate;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
result.VideoPageUrl = firstVideo;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -73,12 +73,32 @@ internal class BlogterestSiteResolver : IEroterestSiteResolver
|
|||||||
{
|
{
|
||||||
result.NextCandidateUrls.Insert(0, rawHref);
|
result.NextCandidateUrls.Insert(0, rawHref);
|
||||||
}
|
}
|
||||||
else
|
else if (result.NextCandidateUrls.Count < 5)
|
||||||
{
|
{
|
||||||
result.NextCandidateUrls.Add(rawHref);
|
result.NextCandidateUrls.Add(rawHref);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 動画ホスト直リンクがある場合は即座に特定
|
||||||
|
var firstVideoLink = result.NextCandidateUrls.FirstOrDefault(IsDirectVideoHostingUrl);
|
||||||
|
if (!string.IsNullOrWhiteSpace(firstVideoLink))
|
||||||
|
{
|
||||||
|
var targetResolver = SiteResolverRegistry.GetResolver(firstVideoLink);
|
||||||
|
var directResolve = await targetResolver.ResolveAsync(client, firstVideoLink, null, context, ct);
|
||||||
|
if (directResolve.HasFinalVideo)
|
||||||
|
{
|
||||||
|
result.VideoPageUrl = directResolve.VideoPageUrl ?? firstVideoLink;
|
||||||
|
result.VideoSiteName = directResolve.VideoSiteName;
|
||||||
|
result.VideoSiteId = directResolve.VideoSiteId;
|
||||||
|
result.StreamUrl = directResolve.StreamUrl;
|
||||||
|
result.VideoUploadDate = directResolve.VideoUploadDate;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
result.VideoPageUrl = firstVideoLink;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ internal static class EroterestCrawler
|
|||||||
// 探索キュー: (URL, Depth)
|
// 探索キュー: (URL, Depth)
|
||||||
var queue = new Queue<(string Url, int Depth)>();
|
var queue = new Queue<(string Url, int Depth)>();
|
||||||
queue.Enqueue((startUrl, 1));
|
queue.Enqueue((startUrl, 1));
|
||||||
context.VisitedUrls.Add(startUrl);
|
context.EnqueuedUrls.Add(startUrl);
|
||||||
|
|
||||||
SiteResolveResult? bestResult = null;
|
SiteResolveResult? bestResult = null;
|
||||||
|
|
||||||
@@ -45,12 +45,17 @@ internal static class EroterestCrawler
|
|||||||
|
|
||||||
var (currentUrl, depth) = queue.Dequeue();
|
var (currentUrl, depth) = queue.Dequeue();
|
||||||
context.CurrentDepth = depth;
|
context.CurrentDepth = depth;
|
||||||
|
context.VisitedPageCount++;
|
||||||
|
context.VisitedUrls.Add(currentUrl);
|
||||||
context.TraversedRoute.Add(currentUrl);
|
context.TraversedRoute.Add(currentUrl);
|
||||||
|
|
||||||
|
using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||||
|
timeoutCts.CancelAfter(TimeSpan.FromSeconds(8));
|
||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
var resolver = SiteResolverRegistry.GetResolver(currentUrl);
|
var resolver = SiteResolverRegistry.GetResolver(currentUrl);
|
||||||
var result = await resolver.ResolveAsync(client, currentUrl, null, context, ct);
|
var result = await resolver.ResolveAsync(client, currentUrl, null, context, timeoutCts.Token);
|
||||||
|
|
||||||
// 記事公開日の補完
|
// 記事公開日の補完
|
||||||
if (bestResult == null || (!bestResult.ArticlePublishDate.HasValue && result.ArticlePublishDate.HasValue))
|
if (bestResult == null || (!bestResult.ArticlePublishDate.HasValue && result.ArticlePublishDate.HasValue))
|
||||||
@@ -67,18 +72,18 @@ internal static class EroterestCrawler
|
|||||||
}
|
}
|
||||||
|
|
||||||
// 2. 確定した動画ページが見つかった場合
|
// 2. 確定した動画ページが見つかった場合
|
||||||
if (!string.IsNullOrWhiteSpace(result.VideoPageUrl))
|
if (result.HasFinalVideo)
|
||||||
{
|
{
|
||||||
bestResult = result;
|
bestResult = result;
|
||||||
|
|
||||||
// もしその動画ページ自体がストリームを解析できるサイト(例: do-ga.eroterest.net 等)であれば、さらに1回だけ追跡
|
// もしその動画ページ自体がストリームを解析できるサイト(例: do-ga.eroterest.net 等)であれば、さらに1回だけ追跡
|
||||||
if (depth < MovieEroterestConfig.MaxCrawlDepth && !context.VisitedUrls.Contains(result.VideoPageUrl))
|
if (depth < MovieEroterestConfig.MaxCrawlDepth && !context.VisitedUrls.Contains(result.VideoPageUrl!))
|
||||||
{
|
{
|
||||||
var targetResolver = SiteResolverRegistry.GetResolver(result.VideoPageUrl);
|
var targetResolver = SiteResolverRegistry.GetResolver(result.VideoPageUrl!);
|
||||||
if (targetResolver.SiteName == "doga")
|
if (targetResolver.SiteName == "doga")
|
||||||
{
|
{
|
||||||
context.VisitedUrls.Add(result.VideoPageUrl);
|
context.VisitedUrls.Add(result.VideoPageUrl!);
|
||||||
var deepVideoResult = await targetResolver.ResolveAsync(client, result.VideoPageUrl, null, context, ct);
|
var deepVideoResult = await targetResolver.ResolveAsync(client, result.VideoPageUrl!, null, context, timeoutCts.Token);
|
||||||
if (!string.IsNullOrWhiteSpace(deepVideoResult.StreamUrl))
|
if (!string.IsNullOrWhiteSpace(deepVideoResult.StreamUrl))
|
||||||
{
|
{
|
||||||
deepVideoResult.TraversedRoute = context.TraversedRoute;
|
deepVideoResult.TraversedRoute = context.TraversedRoute;
|
||||||
@@ -95,19 +100,26 @@ internal static class EroterestCrawler
|
|||||||
// 3. 動画が見つからず、次へ進む候補リンクがある場合
|
// 3. 動画が見つからず、次へ進む候補リンクがある場合
|
||||||
if (depth < MovieEroterestConfig.MaxCrawlDepth)
|
if (depth < MovieEroterestConfig.MaxCrawlDepth)
|
||||||
{
|
{
|
||||||
|
int enqueuedCount = 0;
|
||||||
foreach (var nextUrl in result.NextCandidateUrls)
|
foreach (var nextUrl in result.NextCandidateUrls)
|
||||||
{
|
{
|
||||||
if (string.IsNullOrWhiteSpace(nextUrl)) continue;
|
if (string.IsNullOrWhiteSpace(nextUrl)) continue;
|
||||||
|
if (IsJunkArchiveUrl(nextUrl)) continue;
|
||||||
|
|
||||||
// 同一URLの重複探索防止
|
// 同一URLの重複探索防止
|
||||||
if (context.VisitedUrls.Add(nextUrl))
|
if (context.EnqueuedUrls.Add(nextUrl))
|
||||||
{
|
{
|
||||||
// 動画サイトの可能性が高いリンクを優先的にキューの前方に配置
|
|
||||||
queue.Enqueue((nextUrl, depth + 1));
|
queue.Enqueue((nextUrl, depth + 1));
|
||||||
|
enqueuedCount++;
|
||||||
|
if (enqueuedCount >= 3) break; // 1ページあたり最大3候補に制限
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
catch (OperationCanceledException)
|
||||||
|
{
|
||||||
|
LogManager.AddLog($"[情報] [EroterestCrawler] URLタイムアウトスキップ ({currentUrl})");
|
||||||
|
}
|
||||||
catch (Exception ex)
|
catch (Exception ex)
|
||||||
{
|
{
|
||||||
LogManager.AddLog($"[警告] [EroterestCrawler] URL探索失敗 ({currentUrl}, 階層 {depth}): {ex.Message}");
|
LogManager.AddLog($"[警告] [EroterestCrawler] URL探索失敗 ({currentUrl}, 階層 {depth}): {ex.Message}");
|
||||||
@@ -116,4 +128,29 @@ internal static class EroterestCrawler
|
|||||||
|
|
||||||
return bestResult ?? SiteResolveResult.Empty();
|
return bestResult ?? SiteResolveResult.Empty();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// クロール不要なアーカイブ・カテゴリ・トップページ等のジャンクURLを判定
|
||||||
|
/// </summary>
|
||||||
|
private static bool IsJunkArchiveUrl(string url)
|
||||||
|
{
|
||||||
|
if (string.IsNullOrWhiteSpace(url)) return true;
|
||||||
|
var u = url.ToLowerInvariant();
|
||||||
|
return u.Contains("blog-date-") ||
|
||||||
|
u.Contains("blog-category-") ||
|
||||||
|
u.Contains("/category/") ||
|
||||||
|
u.Contains("/tag/") ||
|
||||||
|
u.Contains("/author/") ||
|
||||||
|
u.Contains("/archive/") ||
|
||||||
|
u.Contains("/date/") ||
|
||||||
|
u.Contains("/archives/category/") ||
|
||||||
|
u.Contains("/archives/tag/") ||
|
||||||
|
u.EndsWith(".com/") ||
|
||||||
|
u.EndsWith(".net/") ||
|
||||||
|
u.EndsWith(".jp/") ||
|
||||||
|
u.EndsWith(".site/") ||
|
||||||
|
u.EndsWith(".xyz/") ||
|
||||||
|
u.Contains("movie.eroterest.net");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -18,6 +18,11 @@ internal class SiteResolveContext
|
|||||||
/// </summary>
|
/// </summary>
|
||||||
public DateTimeOffset ReferenceTime { get; set; } = DateTimeOffset.Now;
|
public DateTimeOffset ReferenceTime { get; set; } = DateTimeOffset.Now;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// キューに追加済みURLの集合(同一URLの二重キュー投入防止)
|
||||||
|
/// </summary>
|
||||||
|
public HashSet<string> EnqueuedUrls { get; set; } = new(StringComparer.OrdinalIgnoreCase);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// 訪問済みURLの集合(同一URLの二重取得・無限ループ防止)
|
/// 訪問済みURLの集合(同一URLの二重取得・無限ループ防止)
|
||||||
/// </summary>
|
/// </summary>
|
||||||
@@ -28,8 +33,13 @@ internal class SiteResolveContext
|
|||||||
/// </summary>
|
/// </summary>
|
||||||
public List<string> TraversedRoute { get; set; } = new();
|
public List<string> TraversedRoute { get; set; } = new();
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// 実際にHTTPリクエストを行って訪問したページ数
|
||||||
|
/// </summary>
|
||||||
|
public int VisitedPageCount { get; set; } = 0;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// 現在までに訪問した総ページ数
|
/// 現在までに訪問した総ページ数
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public int TotalVisitedPages => VisitedUrls.Count;
|
public int TotalVisitedPages => VisitedPageCount;
|
||||||
}
|
}
|
||||||
@@ -149,18 +149,15 @@ internal static class MovieEroterestFeedManager
|
|||||||
?? itemEl.QuerySelector(".itemTitle a")?.TextContent?.Trim()
|
?? itemEl.QuerySelector(".itemTitle a")?.TextContent?.Trim()
|
||||||
?? "動画エロタレスト";
|
?? "動画エロタレスト";
|
||||||
|
|
||||||
// サムネイル画像
|
// サムネイル画像(picture source[srcset] の webp を最優先)
|
||||||
var imgEl = itemEl.QuerySelector(".itemImage img");
|
var sourceEl = itemEl.QuerySelector(".itemImage picture source[srcset]");
|
||||||
var thumbSrc = imgEl?.GetAttribute("src");
|
var thumbSrc = sourceEl?.GetAttribute("srcset");
|
||||||
if (string.IsNullOrWhiteSpace(thumbSrc) || thumbSrc.StartsWith("data:"))
|
if (string.IsNullOrWhiteSpace(thumbSrc))
|
||||||
{
|
{
|
||||||
var sourceEl = itemEl.QuerySelector(".itemImage picture source");
|
var imgEl = itemEl.QuerySelector(".itemImage img");
|
||||||
thumbSrc = sourceEl?.GetAttribute("srcset");
|
thumbSrc = imgEl?.GetAttribute("src");
|
||||||
}
|
|
||||||
if (!string.IsNullOrWhiteSpace(thumbSrc) && thumbSrc.StartsWith("//"))
|
|
||||||
{
|
|
||||||
thumbSrc = "https:" + thumbSrc;
|
|
||||||
}
|
}
|
||||||
|
var normalizedThumb = MovieEroterestVideoParser.NormalizeThumbnailUrl(thumbSrc, pageUrl);
|
||||||
|
|
||||||
// 再生時間
|
// 再生時間
|
||||||
var duration = itemEl.QuerySelector(".movieTime, .duration")?.TextContent?.Trim();
|
var duration = itemEl.QuerySelector(".movieTime, .duration")?.TextContent?.Trim();
|
||||||
@@ -174,7 +171,7 @@ internal static class MovieEroterestFeedManager
|
|||||||
// 記事公開時間(相対時間表記: 例 "4分前", "1時間前")
|
// 記事公開時間(相対時間表記: 例 "4分前", "1時間前")
|
||||||
var itemTime = itemEl.QuerySelector(".itemTime")?.TextContent?.Trim();
|
var itemTime = itemEl.QuerySelector(".itemTime")?.TextContent?.Trim();
|
||||||
|
|
||||||
preliminaryItems.Add((fullUrl, title, thumbSrc, duration, provider, siteName, itemTime, orderIndex++));
|
preliminaryItems.Add((fullUrl, title, normalizedThumb, duration, provider, siteName, itemTime, orderIndex++));
|
||||||
|
|
||||||
if (preliminaryItems.Count >= MovieEroterestConfig.MaxArticlesPerFeed)
|
if (preliminaryItems.Count >= MovieEroterestConfig.MaxArticlesPerFeed)
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -69,7 +69,7 @@ internal static class MovieEroterestVideoParser
|
|||||||
{
|
{
|
||||||
ArticlePageUrl = articleUrl,
|
ArticlePageUrl = articleUrl,
|
||||||
Title = preliminaryTitle ?? string.Empty,
|
Title = preliminaryTitle ?? string.Empty,
|
||||||
ThumbnailUrl = preliminaryThumbnail,
|
ThumbnailUrl = NormalizeThumbnailUrl(preliminaryThumbnail),
|
||||||
Duration = preliminaryDuration,
|
Duration = preliminaryDuration,
|
||||||
Provider = preliminaryProvider,
|
Provider = preliminaryProvider,
|
||||||
DestinationBlogSiteName = preliminarySiteName,
|
DestinationBlogSiteName = preliminarySiteName,
|
||||||
@@ -107,19 +107,20 @@ internal static class MovieEroterestVideoParser
|
|||||||
info.Title = h3Title;
|
info.Title = h3Title;
|
||||||
}
|
}
|
||||||
|
|
||||||
// サムネイル画像
|
// サムネイル画像(picture source[srcset] の webp を最優先)
|
||||||
if (string.IsNullOrWhiteSpace(info.ThumbnailUrl))
|
var sourceEl = articleDoc.QuerySelector(".itemImage picture source[srcset]");
|
||||||
|
var thumbSrc = sourceEl?.GetAttribute("srcset");
|
||||||
|
if (string.IsNullOrWhiteSpace(thumbSrc))
|
||||||
{
|
{
|
||||||
var imgEl = articleDoc.QuerySelector(".itemImage img");
|
var imgEl = articleDoc.QuerySelector(".itemImage img");
|
||||||
var src = imgEl?.GetAttribute("src");
|
thumbSrc = imgEl?.GetAttribute("src");
|
||||||
if (string.IsNullOrWhiteSpace(src) || src.StartsWith("data:"))
|
}
|
||||||
|
if (!string.IsNullOrWhiteSpace(thumbSrc) && !thumbSrc.StartsWith("data:"))
|
||||||
|
{
|
||||||
|
var normalizedThumb = NormalizeThumbnailUrl(thumbSrc, articleUrl);
|
||||||
|
if (!string.IsNullOrWhiteSpace(normalizedThumb))
|
||||||
{
|
{
|
||||||
var sourceEl = articleDoc.QuerySelector(".itemImage picture source");
|
info.ThumbnailUrl = normalizedThumb;
|
||||||
src = sourceEl?.GetAttribute("srcset");
|
|
||||||
}
|
|
||||||
if (!string.IsNullOrWhiteSpace(src))
|
|
||||||
{
|
|
||||||
info.ThumbnailUrl = NormalizeUrl(src, articleUrl);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -565,4 +566,28 @@ internal static class MovieEroterestVideoParser
|
|||||||
|
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// サムネイル画像URLを正規化します。
|
||||||
|
/// 特に eroimg.net は jpeg への外部アクセスが 403 で遮断され webp のみが許可されているため、
|
||||||
|
/// /images/get/ -> /webp/get/ および .jpeg / .jpg -> .webp へ自動変換します。
|
||||||
|
/// </summary>
|
||||||
|
public static string? NormalizeThumbnailUrl(string? rawUrl, string? baseUrl = null)
|
||||||
|
{
|
||||||
|
if (string.IsNullOrWhiteSpace(rawUrl)) return null;
|
||||||
|
|
||||||
|
var url = NormalizeUrl(rawUrl, baseUrl ?? MovieEroterestConfig.BaseUrl);
|
||||||
|
if (string.IsNullOrWhiteSpace(url)) return null;
|
||||||
|
|
||||||
|
if (url.Contains("eroimg.net", StringComparison.OrdinalIgnoreCase))
|
||||||
|
{
|
||||||
|
// /images/get/ -> /webp/get/
|
||||||
|
url = Regex.Replace(url, @"/images/get/", "/webp/get/", RegexOptions.IgnoreCase);
|
||||||
|
// .jpeg / .jpg -> .webp
|
||||||
|
url = Regex.Replace(url, @"\.(?:jpeg|jpg)$", ".webp", RegexOptions.IgnoreCase);
|
||||||
|
}
|
||||||
|
|
||||||
|
return url;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -61,10 +61,14 @@ internal class WordPressBlogSiteResolver : IEroterestSiteResolver
|
|||||||
}
|
}
|
||||||
|
|
||||||
// 2. 本文および周辺リンクの走査
|
// 2. 本文および周辺リンクの走査
|
||||||
var links = doc.QuerySelectorAll(".entry-content a, .post-content a, article a, main a, a[href]");
|
// 特に主要な動画枠(#kabuse #douga, .entry-content, article)を優先
|
||||||
|
var contentLinks = doc.QuerySelectorAll("#kabuse a, a#douga, .entry-content a, .post-content a, article a, main a");
|
||||||
|
var allLinks = contentLinks.Length > 0 ? contentLinks : doc.QuerySelectorAll("a[href]");
|
||||||
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||||
|
|
||||||
foreach (var a in links)
|
string? foundDirectVideoUrl = null;
|
||||||
|
|
||||||
|
foreach (var a in allLinks)
|
||||||
{
|
{
|
||||||
var href = a.GetAttribute("href")?.Trim();
|
var href = a.GetAttribute("href")?.Trim();
|
||||||
if (string.IsNullOrWhiteSpace(href) || href.Equals(url, StringComparison.OrdinalIgnoreCase)) continue;
|
if (string.IsNullOrWhiteSpace(href) || href.Equals(url, StringComparison.OrdinalIgnoreCase)) continue;
|
||||||
@@ -73,37 +77,67 @@ internal class WordPressBlogSiteResolver : IEroterestSiteResolver
|
|||||||
href.EndsWith(".jpeg", StringComparison.OrdinalIgnoreCase) ||
|
href.EndsWith(".jpeg", StringComparison.OrdinalIgnoreCase) ||
|
||||||
href.EndsWith(".png", StringComparison.OrdinalIgnoreCase) ||
|
href.EndsWith(".png", StringComparison.OrdinalIgnoreCase) ||
|
||||||
href.EndsWith(".gif", StringComparison.OrdinalIgnoreCase) ||
|
href.EndsWith(".gif", StringComparison.OrdinalIgnoreCase) ||
|
||||||
|
href.EndsWith(".webp", StringComparison.OrdinalIgnoreCase) ||
|
||||||
href.EndsWith(".css", StringComparison.OrdinalIgnoreCase) ||
|
href.EndsWith(".css", StringComparison.OrdinalIgnoreCase) ||
|
||||||
href.EndsWith(".js", StringComparison.OrdinalIgnoreCase) ||
|
href.EndsWith(".js", StringComparison.OrdinalIgnoreCase) ||
|
||||||
href.Contains("/tag/", StringComparison.OrdinalIgnoreCase) ||
|
href.Contains("/tag/", StringComparison.OrdinalIgnoreCase) ||
|
||||||
href.Contains("/category/", StringComparison.OrdinalIgnoreCase) ||
|
href.Contains("/category/", StringComparison.OrdinalIgnoreCase) ||
|
||||||
href.Contains("movie.eroterest.net/popular", StringComparison.OrdinalIgnoreCase) ||
|
href.Contains("/author/", StringComparison.OrdinalIgnoreCase) ||
|
||||||
href.Contains("movie.eroterest.net/index/", StringComparison.OrdinalIgnoreCase))
|
href.Contains("movie.eroterest.net", StringComparison.OrdinalIgnoreCase))
|
||||||
{
|
{
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!seen.Add(href)) continue;
|
if (!seen.Add(href)) continue;
|
||||||
|
|
||||||
// 動画リンクは最優先で先頭に追加
|
// 動画リンクかどうかの判定
|
||||||
if (href.Contains("txxx.com", StringComparison.OrdinalIgnoreCase) ||
|
bool isVideoUrl = href.Contains("txxx.com", StringComparison.OrdinalIgnoreCase) ||
|
||||||
href.Contains("hclips.com", StringComparison.OrdinalIgnoreCase) ||
|
href.Contains("hclips.com", StringComparison.OrdinalIgnoreCase) ||
|
||||||
href.Contains("hdzog.com", StringComparison.OrdinalIgnoreCase) ||
|
href.Contains("hdzog.com", StringComparison.OrdinalIgnoreCase) ||
|
||||||
href.Contains("senzuri.tube", StringComparison.OrdinalIgnoreCase) ||
|
href.Contains("senzuri.tube", StringComparison.OrdinalIgnoreCase) ||
|
||||||
href.Contains("upornia.com", StringComparison.OrdinalIgnoreCase) ||
|
href.Contains("upornia.com", StringComparison.OrdinalIgnoreCase) ||
|
||||||
href.Contains("do-ga.eroterest.net", StringComparison.OrdinalIgnoreCase) ||
|
href.Contains("do-ga.eroterest.net", StringComparison.OrdinalIgnoreCase) ||
|
||||||
href.Contains("video.laxd.com", StringComparison.OrdinalIgnoreCase) ||
|
href.Contains("video.laxd.com", StringComparison.OrdinalIgnoreCase) ||
|
||||||
href.Contains("adult.contents.fc2.com", StringComparison.OrdinalIgnoreCase) ||
|
href.Contains("adult.contents.fc2.com", StringComparison.OrdinalIgnoreCase) ||
|
||||||
href.Contains("vjav.com", StringComparison.OrdinalIgnoreCase) ||
|
href.Contains("vjav.com", StringComparison.OrdinalIgnoreCase) ||
|
||||||
href.Contains("youjizz.com", StringComparison.OrdinalIgnoreCase) ||
|
href.Contains("youjizz.com", StringComparison.OrdinalIgnoreCase) ||
|
||||||
href.Contains("spankbang.com", StringComparison.OrdinalIgnoreCase) ||
|
href.Contains("spankbang.com", StringComparison.OrdinalIgnoreCase) ||
|
||||||
href.Contains("xvideos.com", StringComparison.OrdinalIgnoreCase))
|
href.Contains("xvideos.com", StringComparison.OrdinalIgnoreCase);
|
||||||
|
|
||||||
|
if (isVideoUrl)
|
||||||
{
|
{
|
||||||
|
if (foundDirectVideoUrl == null)
|
||||||
|
{
|
||||||
|
foundDirectVideoUrl = href;
|
||||||
|
}
|
||||||
result.NextCandidateUrls.Insert(0, href);
|
result.NextCandidateUrls.Insert(0, href);
|
||||||
}
|
}
|
||||||
|
else if (result.NextCandidateUrls.Count < 5) // 不要な大量リンクのエンキューを抑制
|
||||||
|
{
|
||||||
|
// 別アンテナ・まとめブログへのリンク候補
|
||||||
|
if (href.Contains("archives/") || href.Contains("/entry/") || href.Contains(".html") || href.Contains("/page/"))
|
||||||
|
{
|
||||||
|
result.NextCandidateUrls.Add(href);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 動画リンクが発見された場合は即座にリゾルバーで動画情報を解決
|
||||||
|
if (!string.IsNullOrWhiteSpace(foundDirectVideoUrl))
|
||||||
|
{
|
||||||
|
var targetResolver = SiteResolverRegistry.GetResolver(foundDirectVideoUrl);
|
||||||
|
var directResolve = await targetResolver.ResolveAsync(client, foundDirectVideoUrl, null, context, ct);
|
||||||
|
if (directResolve.HasFinalVideo)
|
||||||
|
{
|
||||||
|
result.VideoPageUrl = directResolve.VideoPageUrl ?? foundDirectVideoUrl;
|
||||||
|
result.VideoSiteName = directResolve.VideoSiteName;
|
||||||
|
result.VideoSiteId = directResolve.VideoSiteId;
|
||||||
|
result.StreamUrl = directResolve.StreamUrl;
|
||||||
|
result.VideoUploadDate = directResolve.VideoUploadDate;
|
||||||
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
result.NextCandidateUrls.Add(href);
|
result.VideoPageUrl = foundDirectVideoUrl;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user