diff --git a/rss-generator/Class/Common/CmnUtil.cs b/rss-generator/Class/Common/CmnUtil.cs index eac0695..d65f135 100644 --- a/rss-generator/Class/Common/CmnUtil.cs +++ b/rss-generator/Class/Common/CmnUtil.cs @@ -550,10 +550,13 @@ public class CmnUtil } // 5. ストリームによるレスポンス受信と直接パース(ResponseHeadersRead で中間バッファリングと巨大文字列生成を排除) - using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken); + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + linkedCts.CancelAfter(TimeSpan.FromSeconds(15)); // ハング防止の最大15秒タイムアウト + + using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, linkedCts.Token); response.EnsureSuccessStatusCode(); - await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken); + await using var stream = await response.Content.ReadAsStreamAsync(linkedCts.Token); // Content-Type (charset 等) を取得して AngleSharp に通知 var contentType = response.Content.Headers.ContentType?.ToString(); diff --git a/rss-generator/Class/Eroterest/Blog2nt/Blog2ntSiteResolver.cs b/rss-generator/Class/Eroterest/Blog2nt/Blog2ntSiteResolver.cs index e7dbf67..45ac6b2 100644 --- a/rss-generator/Class/Eroterest/Blog2nt/Blog2ntSiteResolver.cs +++ b/rss-generator/Class/Eroterest/Blog2nt/Blog2ntSiteResolver.cs @@ -61,22 +61,52 @@ internal class Blog2ntSiteResolver : IEroterestSiteResolver if (!seen.Add(href)) continue; // 動画リンクを優先 - if (href.Contains("txxx.com", StringComparison.OrdinalIgnoreCase) || - href.Contains("senzuri.tube", StringComparison.OrdinalIgnoreCase) || - href.Contains("upornia.com", StringComparison.OrdinalIgnoreCase) || - href.Contains("do-ga.eroterest.net", StringComparison.OrdinalIgnoreCase) || - href.Contains("laxd.com", StringComparison.OrdinalIgnoreCase) || - href.Contains("fc2.com", StringComparison.OrdinalIgnoreCase) || - href.Contains("youjizz.com", StringComparison.OrdinalIgnoreCase)) + bool isVideo = href.Contains("txxx.com", StringComparison.OrdinalIgnoreCase) || + href.Contains("senzuri.tube", StringComparison.OrdinalIgnoreCase) || + href.Contains("upornia.com", StringComparison.OrdinalIgnoreCase) || + href.Contains("do-ga.eroterest.net", StringComparison.OrdinalIgnoreCase) || + href.Contains("laxd.com", StringComparison.OrdinalIgnoreCase) || + href.Contains("fc2.com", StringComparison.OrdinalIgnoreCase) || + href.Contains("youjizz.com", StringComparison.OrdinalIgnoreCase); + + if (isVideo) { result.NextCandidateUrls.Insert(0, href); } - else + else if (result.NextCandidateUrls.Count < 5) { result.NextCandidateUrls.Add(href); } } + // 動画ホスト直リンクがある場合は即座に特定 + var firstVideo = result.NextCandidateUrls.Find(u => + u.Contains("txxx.com", StringComparison.OrdinalIgnoreCase) || + u.Contains("senzuri.tube", StringComparison.OrdinalIgnoreCase) || + u.Contains("upornia.com", StringComparison.OrdinalIgnoreCase) || + u.Contains("do-ga.eroterest.net", StringComparison.OrdinalIgnoreCase) || + u.Contains("laxd.com", StringComparison.OrdinalIgnoreCase) || + u.Contains("fc2.com", StringComparison.OrdinalIgnoreCase) || + u.Contains("youjizz.com", StringComparison.OrdinalIgnoreCase)); + + if (!string.IsNullOrWhiteSpace(firstVideo)) + { + var targetResolver = SiteResolverRegistry.GetResolver(firstVideo); + var directResolve = await targetResolver.ResolveAsync(client, firstVideo, null, context, ct); + if (directResolve.HasFinalVideo) + { + result.VideoPageUrl = directResolve.VideoPageUrl ?? firstVideo; + result.VideoSiteName = directResolve.VideoSiteName; + result.VideoSiteId = directResolve.VideoSiteId; + result.StreamUrl = directResolve.StreamUrl; + result.VideoUploadDate = directResolve.VideoUploadDate; + } + else + { + result.VideoPageUrl = firstVideo; + } + } + return result; } } diff --git a/rss-generator/Class/Eroterest/Blogterest/BlogterestSiteResolver.cs b/rss-generator/Class/Eroterest/Blogterest/BlogterestSiteResolver.cs index 5dddc42..570312c 100644 --- a/rss-generator/Class/Eroterest/Blogterest/BlogterestSiteResolver.cs +++ b/rss-generator/Class/Eroterest/Blogterest/BlogterestSiteResolver.cs @@ -73,12 +73,32 @@ internal class BlogterestSiteResolver : IEroterestSiteResolver { result.NextCandidateUrls.Insert(0, rawHref); } - else + else if (result.NextCandidateUrls.Count < 5) { result.NextCandidateUrls.Add(rawHref); } } + // 動画ホスト直リンクがある場合は即座に特定 + var firstVideoLink = result.NextCandidateUrls.FirstOrDefault(IsDirectVideoHostingUrl); + if (!string.IsNullOrWhiteSpace(firstVideoLink)) + { + var targetResolver = SiteResolverRegistry.GetResolver(firstVideoLink); + var directResolve = await targetResolver.ResolveAsync(client, firstVideoLink, null, context, ct); + if (directResolve.HasFinalVideo) + { + result.VideoPageUrl = directResolve.VideoPageUrl ?? firstVideoLink; + result.VideoSiteName = directResolve.VideoSiteName; + result.VideoSiteId = directResolve.VideoSiteId; + result.StreamUrl = directResolve.StreamUrl; + result.VideoUploadDate = directResolve.VideoUploadDate; + } + else + { + result.VideoPageUrl = firstVideoLink; + } + } + return result; } diff --git a/rss-generator/Class/Eroterest/Core/EroterestCrawler.cs b/rss-generator/Class/Eroterest/Core/EroterestCrawler.cs index bb4199d..663995d 100644 --- a/rss-generator/Class/Eroterest/Core/EroterestCrawler.cs +++ b/rss-generator/Class/Eroterest/Core/EroterestCrawler.cs @@ -35,7 +35,7 @@ internal static class EroterestCrawler // 探索キュー: (URL, Depth) var queue = new Queue<(string Url, int Depth)>(); queue.Enqueue((startUrl, 1)); - context.VisitedUrls.Add(startUrl); + context.EnqueuedUrls.Add(startUrl); SiteResolveResult? bestResult = null; @@ -45,12 +45,17 @@ internal static class EroterestCrawler var (currentUrl, depth) = queue.Dequeue(); context.CurrentDepth = depth; + context.VisitedPageCount++; + context.VisitedUrls.Add(currentUrl); context.TraversedRoute.Add(currentUrl); + using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(ct); + timeoutCts.CancelAfter(TimeSpan.FromSeconds(8)); + try { var resolver = SiteResolverRegistry.GetResolver(currentUrl); - var result = await resolver.ResolveAsync(client, currentUrl, null, context, ct); + var result = await resolver.ResolveAsync(client, currentUrl, null, context, timeoutCts.Token); // 記事公開日の補完 if (bestResult == null || (!bestResult.ArticlePublishDate.HasValue && result.ArticlePublishDate.HasValue)) @@ -67,18 +72,18 @@ internal static class EroterestCrawler } // 2. 確定した動画ページが見つかった場合 - if (!string.IsNullOrWhiteSpace(result.VideoPageUrl)) + if (result.HasFinalVideo) { bestResult = result; // もしその動画ページ自体がストリームを解析できるサイト(例: do-ga.eroterest.net 等)であれば、さらに1回だけ追跡 - if (depth < MovieEroterestConfig.MaxCrawlDepth && !context.VisitedUrls.Contains(result.VideoPageUrl)) + if (depth < MovieEroterestConfig.MaxCrawlDepth && !context.VisitedUrls.Contains(result.VideoPageUrl!)) { - var targetResolver = SiteResolverRegistry.GetResolver(result.VideoPageUrl); + var targetResolver = SiteResolverRegistry.GetResolver(result.VideoPageUrl!); if (targetResolver.SiteName == "doga") { - context.VisitedUrls.Add(result.VideoPageUrl); - var deepVideoResult = await targetResolver.ResolveAsync(client, result.VideoPageUrl, null, context, ct); + context.VisitedUrls.Add(result.VideoPageUrl!); + var deepVideoResult = await targetResolver.ResolveAsync(client, result.VideoPageUrl!, null, context, timeoutCts.Token); if (!string.IsNullOrWhiteSpace(deepVideoResult.StreamUrl)) { deepVideoResult.TraversedRoute = context.TraversedRoute; @@ -95,19 +100,26 @@ internal static class EroterestCrawler // 3. 動画が見つからず、次へ進む候補リンクがある場合 if (depth < MovieEroterestConfig.MaxCrawlDepth) { + int enqueuedCount = 0; foreach (var nextUrl in result.NextCandidateUrls) { if (string.IsNullOrWhiteSpace(nextUrl)) continue; + if (IsJunkArchiveUrl(nextUrl)) continue; // 同一URLの重複探索防止 - if (context.VisitedUrls.Add(nextUrl)) + if (context.EnqueuedUrls.Add(nextUrl)) { - // 動画サイトの可能性が高いリンクを優先的にキューの前方に配置 queue.Enqueue((nextUrl, depth + 1)); + enqueuedCount++; + if (enqueuedCount >= 3) break; // 1ページあたり最大3候補に制限 } } } } + catch (OperationCanceledException) + { + LogManager.AddLog($"[情報] [EroterestCrawler] URLタイムアウトスキップ ({currentUrl})"); + } catch (Exception ex) { LogManager.AddLog($"[警告] [EroterestCrawler] URL探索失敗 ({currentUrl}, 階層 {depth}): {ex.Message}"); @@ -116,4 +128,29 @@ internal static class EroterestCrawler return bestResult ?? SiteResolveResult.Empty(); } + + /// + /// クロール不要なアーカイブ・カテゴリ・トップページ等のジャンクURLを判定 + /// + private static bool IsJunkArchiveUrl(string url) + { + if (string.IsNullOrWhiteSpace(url)) return true; + var u = url.ToLowerInvariant(); + return u.Contains("blog-date-") || + u.Contains("blog-category-") || + u.Contains("/category/") || + u.Contains("/tag/") || + u.Contains("/author/") || + u.Contains("/archive/") || + u.Contains("/date/") || + u.Contains("/archives/category/") || + u.Contains("/archives/tag/") || + u.EndsWith(".com/") || + u.EndsWith(".net/") || + u.EndsWith(".jp/") || + u.EndsWith(".site/") || + u.EndsWith(".xyz/") || + u.Contains("movie.eroterest.net"); + } } + diff --git a/rss-generator/Class/Eroterest/Core/SiteResolveContext.cs b/rss-generator/Class/Eroterest/Core/SiteResolveContext.cs index 0e6265d..b896a6c 100644 --- a/rss-generator/Class/Eroterest/Core/SiteResolveContext.cs +++ b/rss-generator/Class/Eroterest/Core/SiteResolveContext.cs @@ -18,6 +18,11 @@ internal class SiteResolveContext /// public DateTimeOffset ReferenceTime { get; set; } = DateTimeOffset.Now; + /// + /// キューに追加済みURLの集合(同一URLの二重キュー投入防止) + /// + public HashSet EnqueuedUrls { get; set; } = new(StringComparer.OrdinalIgnoreCase); + /// /// 訪問済みURLの集合(同一URLの二重取得・無限ループ防止) /// @@ -28,8 +33,13 @@ internal class SiteResolveContext /// public List TraversedRoute { get; set; } = new(); + /// + /// 実際にHTTPリクエストを行って訪問したページ数 + /// + public int VisitedPageCount { get; set; } = 0; + /// /// 現在までに訪問した総ページ数 /// - public int TotalVisitedPages => VisitedUrls.Count; + public int TotalVisitedPages => VisitedPageCount; } diff --git a/rss-generator/Class/Eroterest/MovieEroterestFeedManager.cs b/rss-generator/Class/Eroterest/MovieEroterestFeedManager.cs index 26cce34..e06cd1f 100644 --- a/rss-generator/Class/Eroterest/MovieEroterestFeedManager.cs +++ b/rss-generator/Class/Eroterest/MovieEroterestFeedManager.cs @@ -149,18 +149,15 @@ internal static class MovieEroterestFeedManager ?? itemEl.QuerySelector(".itemTitle a")?.TextContent?.Trim() ?? "動画エロタレスト"; - // サムネイル画像 - var imgEl = itemEl.QuerySelector(".itemImage img"); - var thumbSrc = imgEl?.GetAttribute("src"); - if (string.IsNullOrWhiteSpace(thumbSrc) || thumbSrc.StartsWith("data:")) + // サムネイル画像(picture source[srcset] の webp を最優先) + var sourceEl = itemEl.QuerySelector(".itemImage picture source[srcset]"); + var thumbSrc = sourceEl?.GetAttribute("srcset"); + if (string.IsNullOrWhiteSpace(thumbSrc)) { - var sourceEl = itemEl.QuerySelector(".itemImage picture source"); - thumbSrc = sourceEl?.GetAttribute("srcset"); - } - if (!string.IsNullOrWhiteSpace(thumbSrc) && thumbSrc.StartsWith("//")) - { - thumbSrc = "https:" + thumbSrc; + var imgEl = itemEl.QuerySelector(".itemImage img"); + thumbSrc = imgEl?.GetAttribute("src"); } + var normalizedThumb = MovieEroterestVideoParser.NormalizeThumbnailUrl(thumbSrc, pageUrl); // 再生時間 var duration = itemEl.QuerySelector(".movieTime, .duration")?.TextContent?.Trim(); @@ -174,7 +171,7 @@ internal static class MovieEroterestFeedManager // 記事公開時間(相対時間表記: 例 "4分前", "1時間前") var itemTime = itemEl.QuerySelector(".itemTime")?.TextContent?.Trim(); - preliminaryItems.Add((fullUrl, title, thumbSrc, duration, provider, siteName, itemTime, orderIndex++)); + preliminaryItems.Add((fullUrl, title, normalizedThumb, duration, provider, siteName, itemTime, orderIndex++)); if (preliminaryItems.Count >= MovieEroterestConfig.MaxArticlesPerFeed) { diff --git a/rss-generator/Class/Eroterest/MovieEroterestVideoParser.cs b/rss-generator/Class/Eroterest/MovieEroterestVideoParser.cs index 27a2f1a..7025376 100644 --- a/rss-generator/Class/Eroterest/MovieEroterestVideoParser.cs +++ b/rss-generator/Class/Eroterest/MovieEroterestVideoParser.cs @@ -69,7 +69,7 @@ internal static class MovieEroterestVideoParser { ArticlePageUrl = articleUrl, Title = preliminaryTitle ?? string.Empty, - ThumbnailUrl = preliminaryThumbnail, + ThumbnailUrl = NormalizeThumbnailUrl(preliminaryThumbnail), Duration = preliminaryDuration, Provider = preliminaryProvider, DestinationBlogSiteName = preliminarySiteName, @@ -107,19 +107,20 @@ internal static class MovieEroterestVideoParser info.Title = h3Title; } - // サムネイル画像 - if (string.IsNullOrWhiteSpace(info.ThumbnailUrl)) + // サムネイル画像(picture source[srcset] の webp を最優先) + var sourceEl = articleDoc.QuerySelector(".itemImage picture source[srcset]"); + var thumbSrc = sourceEl?.GetAttribute("srcset"); + if (string.IsNullOrWhiteSpace(thumbSrc)) { var imgEl = articleDoc.QuerySelector(".itemImage img"); - var src = imgEl?.GetAttribute("src"); - if (string.IsNullOrWhiteSpace(src) || src.StartsWith("data:")) + thumbSrc = imgEl?.GetAttribute("src"); + } + if (!string.IsNullOrWhiteSpace(thumbSrc) && !thumbSrc.StartsWith("data:")) + { + var normalizedThumb = NormalizeThumbnailUrl(thumbSrc, articleUrl); + if (!string.IsNullOrWhiteSpace(normalizedThumb)) { - var sourceEl = articleDoc.QuerySelector(".itemImage picture source"); - src = sourceEl?.GetAttribute("srcset"); - } - if (!string.IsNullOrWhiteSpace(src)) - { - info.ThumbnailUrl = NormalizeUrl(src, articleUrl); + info.ThumbnailUrl = normalizedThumb; } } @@ -565,4 +566,28 @@ internal static class MovieEroterestVideoParser return null; } + + /// + /// サムネイル画像URLを正規化します。 + /// 特に eroimg.net は jpeg への外部アクセスが 403 で遮断され webp のみが許可されているため、 + /// /images/get/ -> /webp/get/ および .jpeg / .jpg -> .webp へ自動変換します。 + /// + public static string? NormalizeThumbnailUrl(string? rawUrl, string? baseUrl = null) + { + if (string.IsNullOrWhiteSpace(rawUrl)) return null; + + var url = NormalizeUrl(rawUrl, baseUrl ?? MovieEroterestConfig.BaseUrl); + if (string.IsNullOrWhiteSpace(url)) return null; + + if (url.Contains("eroimg.net", StringComparison.OrdinalIgnoreCase)) + { + // /images/get/ -> /webp/get/ + url = Regex.Replace(url, @"/images/get/", "/webp/get/", RegexOptions.IgnoreCase); + // .jpeg / .jpg -> .webp + url = Regex.Replace(url, @"\.(?:jpeg|jpg)$", ".webp", RegexOptions.IgnoreCase); + } + + return url; + } } + diff --git a/rss-generator/Class/Eroterest/WordPressBlog/WordPressBlogSiteResolver.cs b/rss-generator/Class/Eroterest/WordPressBlog/WordPressBlogSiteResolver.cs index b0971f1..bb88de4 100644 --- a/rss-generator/Class/Eroterest/WordPressBlog/WordPressBlogSiteResolver.cs +++ b/rss-generator/Class/Eroterest/WordPressBlog/WordPressBlogSiteResolver.cs @@ -61,10 +61,14 @@ internal class WordPressBlogSiteResolver : IEroterestSiteResolver } // 2. 本文および周辺リンクの走査 - var links = doc.QuerySelectorAll(".entry-content a, .post-content a, article a, main a, a[href]"); + // 特に主要な動画枠(#kabuse #douga, .entry-content, article)を優先 + var contentLinks = doc.QuerySelectorAll("#kabuse a, a#douga, .entry-content a, .post-content a, article a, main a"); + var allLinks = contentLinks.Length > 0 ? contentLinks : doc.QuerySelectorAll("a[href]"); var seen = new HashSet(StringComparer.OrdinalIgnoreCase); - foreach (var a in links) + string? foundDirectVideoUrl = null; + + foreach (var a in allLinks) { var href = a.GetAttribute("href")?.Trim(); if (string.IsNullOrWhiteSpace(href) || href.Equals(url, StringComparison.OrdinalIgnoreCase)) continue; @@ -73,37 +77,67 @@ internal class WordPressBlogSiteResolver : IEroterestSiteResolver href.EndsWith(".jpeg", StringComparison.OrdinalIgnoreCase) || href.EndsWith(".png", StringComparison.OrdinalIgnoreCase) || href.EndsWith(".gif", StringComparison.OrdinalIgnoreCase) || + href.EndsWith(".webp", StringComparison.OrdinalIgnoreCase) || href.EndsWith(".css", StringComparison.OrdinalIgnoreCase) || href.EndsWith(".js", StringComparison.OrdinalIgnoreCase) || href.Contains("/tag/", StringComparison.OrdinalIgnoreCase) || href.Contains("/category/", StringComparison.OrdinalIgnoreCase) || - href.Contains("movie.eroterest.net/popular", StringComparison.OrdinalIgnoreCase) || - href.Contains("movie.eroterest.net/index/", StringComparison.OrdinalIgnoreCase)) + href.Contains("/author/", StringComparison.OrdinalIgnoreCase) || + href.Contains("movie.eroterest.net", StringComparison.OrdinalIgnoreCase)) { continue; } if (!seen.Add(href)) continue; - // 動画リンクは最優先で先頭に追加 - if (href.Contains("txxx.com", StringComparison.OrdinalIgnoreCase) || - href.Contains("hclips.com", StringComparison.OrdinalIgnoreCase) || - href.Contains("hdzog.com", StringComparison.OrdinalIgnoreCase) || - href.Contains("senzuri.tube", StringComparison.OrdinalIgnoreCase) || - href.Contains("upornia.com", StringComparison.OrdinalIgnoreCase) || - href.Contains("do-ga.eroterest.net", StringComparison.OrdinalIgnoreCase) || - href.Contains("video.laxd.com", StringComparison.OrdinalIgnoreCase) || - href.Contains("adult.contents.fc2.com", StringComparison.OrdinalIgnoreCase) || - href.Contains("vjav.com", StringComparison.OrdinalIgnoreCase) || - href.Contains("youjizz.com", StringComparison.OrdinalIgnoreCase) || - href.Contains("spankbang.com", StringComparison.OrdinalIgnoreCase) || - href.Contains("xvideos.com", StringComparison.OrdinalIgnoreCase)) + // 動画リンクかどうかの判定 + bool isVideoUrl = href.Contains("txxx.com", StringComparison.OrdinalIgnoreCase) || + href.Contains("hclips.com", StringComparison.OrdinalIgnoreCase) || + href.Contains("hdzog.com", StringComparison.OrdinalIgnoreCase) || + href.Contains("senzuri.tube", StringComparison.OrdinalIgnoreCase) || + href.Contains("upornia.com", StringComparison.OrdinalIgnoreCase) || + href.Contains("do-ga.eroterest.net", StringComparison.OrdinalIgnoreCase) || + href.Contains("video.laxd.com", StringComparison.OrdinalIgnoreCase) || + href.Contains("adult.contents.fc2.com", StringComparison.OrdinalIgnoreCase) || + href.Contains("vjav.com", StringComparison.OrdinalIgnoreCase) || + href.Contains("youjizz.com", StringComparison.OrdinalIgnoreCase) || + href.Contains("spankbang.com", StringComparison.OrdinalIgnoreCase) || + href.Contains("xvideos.com", StringComparison.OrdinalIgnoreCase); + + if (isVideoUrl) { + if (foundDirectVideoUrl == null) + { + foundDirectVideoUrl = href; + } result.NextCandidateUrls.Insert(0, href); } + else if (result.NextCandidateUrls.Count < 5) // 不要な大量リンクのエンキューを抑制 + { + // 別アンテナ・まとめブログへのリンク候補 + if (href.Contains("archives/") || href.Contains("/entry/") || href.Contains(".html") || href.Contains("/page/")) + { + result.NextCandidateUrls.Add(href); + } + } + } + + // 動画リンクが発見された場合は即座にリゾルバーで動画情報を解決 + if (!string.IsNullOrWhiteSpace(foundDirectVideoUrl)) + { + var targetResolver = SiteResolverRegistry.GetResolver(foundDirectVideoUrl); + var directResolve = await targetResolver.ResolveAsync(client, foundDirectVideoUrl, null, context, ct); + if (directResolve.HasFinalVideo) + { + result.VideoPageUrl = directResolve.VideoPageUrl ?? foundDirectVideoUrl; + result.VideoSiteName = directResolve.VideoSiteName; + result.VideoSiteId = directResolve.VideoSiteId; + result.StreamUrl = directResolve.StreamUrl; + result.VideoUploadDate = directResolve.VideoUploadDate; + } else { - result.NextCandidateUrls.Add(href); + result.VideoPageUrl = foundDirectVideoUrl; } }