diff --git a/WebImageViewer/Class/Unified/Scrapers/UnifiedEHentaiScraper.cs b/WebImageViewer/Class/Unified/Scrapers/UnifiedEHentaiScraper.cs index 8af4e6c..1c019eb 100644 --- a/WebImageViewer/Class/Unified/Scrapers/UnifiedEHentaiScraper.cs +++ b/WebImageViewer/Class/Unified/Scrapers/UnifiedEHentaiScraper.cs @@ -1,3 +1,4 @@ +using System.Collections.Concurrent; using System.Text.Json; using System.Text.RegularExpressions; using HtmlAgilityPack; @@ -9,13 +10,14 @@ namespace WebImageViewer.Class.Unified.Scrapers; /// /// E-Hentai (e-hentai.org) 固有の統一画像スクレイパー -/// 公式JSON API (gdata) とギャラリーHTML解析を組み合わせた画像取得を実装 +/// 公式JSON API (gdata) とギャラリーHTML解析、showpage API を組み合わせた画像取得を実装 /// public class UnifiedEHentaiScraper : BaseUnifiedScraper { private const string BaseDomain = "e-hentai.org"; private const string BaseUrl = "https://e-hentai.org"; private const string ApiEndpoint = "https://api.e-hentai.org/api.php"; + private static readonly HttpClient ApiClient = new() { Timeout = TimeSpan.FromSeconds(20) }; public override string SiteName => "E-Hentai"; public override string SiteKey => "ehentai"; @@ -28,7 +30,7 @@ public class UnifiedEHentaiScraper : BaseUnifiedScraper public override async Task ScrapeAsync(string urlOrPath, CancellationToken ct = default) { - var (gid, token) = ResolveGidAndToken(urlOrPath); + var (gid, token) = await ResolveGidAndTokenAsync(urlOrPath, ct); if (gid <= 0 || string.IsNullOrWhiteSpace(token)) { Logger.LogWarning("[{SiteName}] GIDとTokenを抽出できませんでした: {Input}", SiteName, urlOrPath); @@ -40,23 +42,28 @@ public class UnifiedEHentaiScraper : BaseUnifiedScraper // 1. 公式API (gdata) を呼び出してメタデータを取得 UnifiedGalleryItem? item = null; + int fileCount = 0; try { - item = await FetchMetadataViaApiAsync(gid, token, ct); + var meta = await FetchMetadataViaApiAsync(gid, token, ct); + if (meta != null) + { + item = meta.Value.Item; + fileCount = meta.Value.FileCount; + } } catch (Exception ex) { - Logger.LogWarning(ex, "[{SiteName}] 公式APIからのメタデータ取得に失敗しました。HTMLからフォールバックします", SiteName); + Logger.LogWarning(ex, "[{SiteName}] 公式APIからのメタデータ取得に失敗しました", SiteName); } - // 2. ギャラリーHTMLから画像ページリストを抽出 - var doc = await LoadHtmlDocumentAsync(galleryUrl, ct: ct); - if (doc == null && item == null) return null; - if (item == null) { - var title = doc?.DocumentNode.SelectSingleNode("//h1[@id='gn']")?.InnerText?.Trim() - ?? doc?.DocumentNode.SelectSingleNode("//h1[@id='gj']")?.InnerText?.Trim() + var doc = await LoadHtmlDocumentAsync(galleryUrl, ct: ct); + if (doc == null) return null; + + var title = doc.DocumentNode.SelectSingleNode("//h1[@id='gn']")?.InnerText?.Trim() + ?? doc.DocumentNode.SelectSingleNode("//h1[@id='gj']")?.InnerText?.Trim() ?? $"E-Hentai ギャラリー #{gid}"; item = new UnifiedGalleryItem { @@ -69,73 +76,111 @@ public class UnifiedEHentaiScraper : BaseUnifiedScraper }; } - if (doc != null) + // 2. ギャラリーページから各ページのリンクを収集 + var pageEntries = await CollectPageEntriesAsync(gid, token, fileCount, ct); + if (pageEntries.Count == 0) { - // サムネイルコンテナ (gdtm / gdtl) から各画像の詳細ページURLまたはサムネイルを抽出 - var imgNodes = doc.DocumentNode.SelectNodes("//div[@id='gdt']//div[contains(@class, 'gdtm') or contains(@class, 'gdtl')]//a"); - if (imgNodes != null) - { - int pageNum = 1; - foreach (var a in imgNodes) - { - var href = a.GetAttributeValue("href", "")?.Trim(); - var img = a.SelectSingleNode(".//img"); - var src = img?.GetAttributeValue("src", "")?.Trim(); + Logger.LogWarning("[{SiteName}] ギャラリーページからページリンクを取得できませんでした: {Url}", SiteName, galleryUrl); + return item; + } - if (!string.IsNullOrEmpty(href)) + // 3. 1ページ目から showkey と 1ページ目画像 URL を取得 + var firstEntry = pageEntries[0]; + var (showkey, firstImgUrl, firstWidth, firstHeight) = await FetchPage1DetailsAsync(firstEntry.Url, ct); + + var images = new ConcurrentDictionary(); + if (!string.IsNullOrWhiteSpace(firstImgUrl)) + { + images[1] = new UnifiedImageInfo + { + PageNumber = 1, + ImageUrl = firstImgUrl, + ThumbnailUrl = firstEntry.ThumbUrl ?? item.ThumbnailUrl, + AltText = $"{item.Title} 1ページ", + Width = firstWidth, + Height = firstHeight + }; + } + + // 4. 2ページ目以降の画像 URL を showpage API で並列取得 + var remainingEntries = pageEntries.Skip(1).ToList(); + if (remainingEntries.Count > 0 && !string.IsNullOrWhiteSpace(showkey)) + { + using var semaphore = new SemaphoreSlim(10); + var tasks = remainingEntries.Select(async entry => + { + await semaphore.WaitAsync(ct); + try + { + var imgInfo = await FetchImageViaShowpageApiAsync(gid, entry.Page, entry.ImgKey, showkey, entry.ThumbUrl, item.Title, ct); + if (imgInfo != null) { - item.Images.Add(new UnifiedImageInfo - { - PageNumber = pageNum++, - ImageUrl = href, // 閲覧ビューア側で表示またはプロキシ - ThumbnailUrl = src, - AltText = $"{item.Title} - {pageNum - 1}ページ" - }); + images[entry.Page] = imgInfo; } } - } + catch (Exception ex) + { + Logger.LogWarning(ex, "[{SiteName}] ページ {Page} の画像URL取得に失敗しました", SiteName, entry.Page); + } + finally + { + semaphore.Release(); + } + }); + + await Task.WhenAll(tasks); } - return item; + item.Images = images.Values.OrderBy(img => img.PageNumber).ToList(); + if (item.Images.Count > 0 && string.IsNullOrEmpty(item.ThumbnailUrl)) + { + item.ThumbnailUrl = item.Images[0].ThumbnailUrl; + } + + return item.Images.Count > 0 ? item : null; } - private static (long Gid, string Token) ResolveGidAndToken(string input) + private async Task<(long Gid, string Token)> ResolveGidAndTokenAsync(string input, CancellationToken ct) { var trimmed = input.Trim(); - var match = Regex.Match(trimmed, @"/(?:g|s)/(?\d+)/(?[a-z0-9]+)", RegexOptions.IgnoreCase); - if (match.Success && long.TryParse(match.Groups["gid"].Value, out var gid)) + + var gMatch = Regex.Match(trimmed, @"/g/(\d+)/([a-z0-9]{10})", RegexOptions.IgnoreCase); + if (gMatch.Success) return (long.Parse(gMatch.Groups[1].Value), gMatch.Groups[2].Value.ToLowerInvariant()); + + var pairMatch = Regex.Match(trimmed, @"^(\d+)[/_ -]([a-z0-9]{10})$", RegexOptions.IgnoreCase); + if (pairMatch.Success) return (long.Parse(pairMatch.Groups[1].Value), pairMatch.Groups[2].Value.ToLowerInvariant()); + + var sMatch = Regex.Match(trimmed, @"/s/([a-z0-9]+)/(\d+)-(\d+)", RegexOptions.IgnoreCase); + if (sMatch.Success) { - return (gid, match.Groups["token"].Value); + var pageUrl = trimmed.StartsWith("http") ? trimmed : $"{BaseUrl}/s/{sMatch.Groups[1].Value}/{sMatch.Groups[2].Value}-{sMatch.Groups[3].Value}"; + try + { + var doc = await LoadHtmlDocumentAsync(pageUrl, ct: ct); + var href = doc?.DocumentNode.SelectSingleNode("//a[contains(@href, '/g/')]")?.GetAttributeValue("href", ""); + if (!string.IsNullOrEmpty(href)) + { + var m = Regex.Match(href, @"/g/(\d+)/([a-z0-9]{10})", RegexOptions.IgnoreCase); + if (m.Success) return (long.Parse(m.Groups[1].Value), m.Groups[2].Value.ToLowerInvariant()); + } + } + catch { } } - var matchDirect = Regex.Match(trimmed, @"^(?\d+)/(?[a-z0-9]+)$", RegexOptions.IgnoreCase); - if (matchDirect.Success && long.TryParse(matchDirect.Groups["gid"].Value, out var gidDirect)) - { - return (gidDirect, matchDirect.Groups["token"].Value); - } - - return (0, ""); + return (0, string.Empty); } - private async Task FetchMetadataViaApiAsync(long gid, string token, CancellationToken ct) + private async Task<(UnifiedGalleryItem Item, int FileCount)?> FetchMetadataViaApiAsync(long gid, string token, CancellationToken ct) { - using var client = new HttpClient(); - client.Timeout = TimeSpan.FromSeconds(15); - client.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"); - var payload = new { method = "gdata", - gidlist = new[] { new object[] { gid, token } }, - namespace_ = 1 + gidlist = new object[] { new object[] { gid, token } }, + @namespace = 1 }; - var content = new StringContent( - JsonSerializer.Serialize(payload).Replace("namespace_", "namespace"), - System.Text.Encoding.UTF8, - "application/json"); - - using var resp = await client.PostAsync(ApiEndpoint, content, ct); + var content = new StringContent(JsonSerializer.Serialize(payload), System.Text.Encoding.UTF8, "application/json"); + using var resp = await ApiClient.PostAsync(ApiEndpoint, content, ct); resp.EnsureSuccessStatusCode(); var json = await resp.Content.ReadAsStringAsync(ct); @@ -147,9 +192,9 @@ public class UnifiedEHentaiScraper : BaseUnifiedScraper var titleJpn = g.TryGetProperty("title_jpn", out var tj) ? tj.GetString() : null; var category = g.TryGetProperty("category", out var c) ? c.GetString() ?? "general" : "general"; var thumb = g.TryGetProperty("thumb", out var th) ? th.GetString() : null; + int fileCount = g.TryGetProperty("filecount", out var fc) && int.TryParse(fc.GetString(), out var count) ? count : 0; var displayTitle = !string.IsNullOrWhiteSpace(titleJpn) ? titleJpn : title; - var item = new UnifiedGalleryItem { Id = $"{gid}/{token}", @@ -175,6 +220,113 @@ public class UnifiedEHentaiScraper : BaseUnifiedScraper } } - return item; + return (item, fileCount); } + + private async Task> CollectPageEntriesAsync(long gid, string token, int totalFiles, CancellationToken ct) + { + var entries = new List(); + int pageIndex = 0; + int maxPages = totalFiles > 0 ? (int)Math.Ceiling(totalFiles / 20.0) : 5; + + while (pageIndex < maxPages && pageIndex < 20) + { + var pageUrl = $"{BaseUrl}/g/{gid}/{token}/?p={pageIndex}"; + var doc = await LoadHtmlDocumentAsync(pageUrl, ct: ct); + if (doc == null) break; + + var linkNodes = doc.DocumentNode.SelectNodes("//div[@id='gdt']//a[contains(@href, '/s/')]"); + if (linkNodes == null || linkNodes.Count == 0) break; + + int addedInThisPage = 0; + foreach (var node in linkNodes) + { + var href = node.GetAttributeValue("href", ""); + var match = Regex.Match(href, @"/s/([a-z0-9]+)/(\d+)-(\d+)", RegexOptions.IgnoreCase); + if (match.Success) + { + var imgKey = match.Groups[1].Value; + var pNum = int.Parse(match.Groups[3].Value); + + string? thumbUrl = node.SelectSingleNode(".//img")?.GetAttributeValue("src", null); + entries.Add(new EHPageEntry(pNum, imgKey, href, thumbUrl)); + addedInThisPage++; + } + } + + if (addedInThisPage == 0 || (totalFiles > 0 && entries.Count >= totalFiles)) break; + pageIndex++; + } + + return entries.OrderBy(e => e.Page).ToList(); + } + + private async Task<(string? Showkey, string? ImageUrl, int? Width, int? Height)> FetchPage1DetailsAsync(string page1Url, CancellationToken ct) + { + try + { + var doc = await LoadHtmlDocumentAsync(page1Url, ct: ct); + if (doc == null) return (null, null, null, null); + + var rawHtml = doc.DocumentNode.OuterHtml; + var showkeyMatch = Regex.Match(rawHtml, @"var showkey\s*=\s*""([^""]+)"";"); + var showkey = showkeyMatch.Success ? showkeyMatch.Groups[1].Value : null; + + var imgNode = doc.DocumentNode.SelectSingleNode("//img[@id='img']"); + if (imgNode != null) + { + var imgUrl = imgNode.GetAttributeValue("src", null); + return (showkey, imgUrl, null, null); + } + + return (showkey, null, null, null); + } + catch + { + return (null, null, null, null); + } + } + + private async Task FetchImageViaShowpageApiAsync(long gid, int page, string imgKey, string showkey, string? thumbUrl, string title, CancellationToken ct) + { + var requestObj = new + { + method = "showpage", + gid = gid, + page = page, + imgkey = imgKey, + showkey = showkey + }; + + var content = new StringContent(JsonSerializer.Serialize(requestObj), System.Text.Encoding.UTF8, "application/json"); + using var resp = await ApiClient.PostAsync(ApiEndpoint, content, ct); + resp.EnsureSuccessStatusCode(); + + var resJson = await resp.Content.ReadAsStringAsync(ct); + using var doc = JsonDocument.Parse(resJson); + var root = doc.RootElement; + + if (root.TryGetProperty("i3", out var i3Prop)) + { + var i3Html = i3Prop.GetString(); + if (!string.IsNullOrWhiteSpace(i3Html)) + { + var imgMatch = Regex.Match(i3Html, @" ScrapeAsync(string urlOrPath, CancellationToken ct = default) { - var match = Regex.Match(urlOrPath, @"(?:momon-ga\.com/|momonga/)?(?[a-z0-9_-]+)(?:\.html)?", RegexOptions.IgnoreCase); - var id = match.Success ? match.Groups["id"].Value : urlOrPath.Trim(); + var (targetUrl, normalizedId, resolvedCategory) = ResolveUrlAndId(urlOrPath); - var pageUrl = urlOrPath.StartsWith("http") ? urlOrPath : $"{BaseUrl}/{id}.html"; - var doc = await LoadHtmlDocumentAsync(pageUrl, ct: ct); - if (doc == null) return null; + Logger.LogInformation("[{SiteName}] スクレイピング開始 - ID: {Id}, Category: {Category}, URL: {Url}", + SiteName, normalizedId, resolvedCategory, targetUrl); - var title = doc.DocumentNode.SelectSingleNode("//h1")?.InnerText?.Trim() ?? $"モモンガ作品 ({id})"; + var doc = await LoadHtmlDocumentAsync(targetUrl, ct: ct); + if (doc == null || doc.DocumentNode == null) + { + Logger.LogWarning("[{SiteName}] ドキュメントの読み込みに失敗しました: {Url}", SiteName, targetUrl); + return null; + } + + // タイトルの抽出(div#post-data h1、h1、titleタグ) + var titleNode = doc.DocumentNode.SelectSingleNode("//div[@id='post-data']//h1") ?? + doc.DocumentNode.SelectSingleNode("//h1") ?? + doc.DocumentNode.SelectSingleNode("//title"); + var title = CleanText(titleNode?.InnerText); + if (string.IsNullOrWhiteSpace(title)) title = normalizedId; + + // 余分なサイト名表記を除去(例: "新妻契約 - 同人誌 - エロ漫画 momon:GA(モモンガッ!!)" -> "新妻契約") + if (title.Contains(" - ")) + { + title = title.Split(" - ")[0].Trim(); + } var item = new UnifiedGalleryItem { - Id = id, - Title = CleanText(title), + Id = normalizedId, + Title = string.IsNullOrEmpty(title) ? normalizedId : title, + Category = resolvedCategory, SourceSite = SiteName, - SourceUrl = pageUrl, + SourceUrl = targetUrl, FetchedAt = DateTime.UtcNow }; - var imgNodes = doc.DocumentNode.SelectNodes("//div[contains(@class, 'entry-content')]//img | //div[contains(@class, 'post-content')]//img"); + // サークル、作者、タグの抽出 + ExtractTagsAndMetadata(doc, item); + + // 画像一覧の抽出(#post-hentai img, .entry-content img, article img) + var imgNodes = doc.DocumentNode.SelectNodes("//div[@id='post-hentai']//img") ?? + doc.DocumentNode.SelectNodes("//div[contains(@class, 'entry-content')]//img") ?? + doc.DocumentNode.SelectNodes("//article//img"); + if (imgNodes != null) { int pageNum = 1; foreach (var img in imgNodes) { - var src = img.GetAttributeValue("src", "")?.Trim(); - if (!string.IsNullOrEmpty(src)) + var src = img.GetAttributeValue("src", null) ?? + img.GetAttributeValue("data-src", null) ?? + img.GetAttributeValue("data-lazy-src", null); + + if (string.IsNullOrWhiteSpace(src)) continue; + + // 広告画像やアイコン等を除外 + if (src.Contains("themes", StringComparison.OrdinalIgnoreCase) || + src.Contains("plugins", StringComparison.OrdinalIgnoreCase) || + src.Contains("banner", StringComparison.OrdinalIgnoreCase) || + src.Contains("ad_", StringComparison.OrdinalIgnoreCase)) { - item.Images.Add(new UnifiedImageInfo - { - PageNumber = pageNum++, - ImageUrl = src, - ThumbnailUrl = src, - AltText = $"{item.Title} - {pageNum - 1}ページ" - }); + continue; } + + var alt = img.GetAttributeValue("alt", null); + int? width = int.TryParse(img.GetAttributeValue("width", null), out var w) ? w : null; + int? height = int.TryParse(img.GetAttributeValue("height", null), out var h) ? h : null; + + item.Images.Add(new UnifiedImageInfo + { + PageNumber = pageNum++, + ImageUrl = src.Trim(), + ThumbnailUrl = src.Trim(), + AltText = CleanText(alt) ?? $"{item.Title} - {pageNum - 1}ページ", + Width = width, + Height = height + }); } } - if (item.Images.Count > 0) item.ThumbnailUrl = item.Images[0].ThumbnailUrl; - return item; + // サムネイルの決定(data-thumb属性、または1枚目の画像) + var historyNode = doc.DocumentNode.SelectSingleNode("//div[contains(@class, 'history')]"); + var dataThumb = historyNode?.GetAttributeValue("data-thumb", null); + if (!string.IsNullOrWhiteSpace(dataThumb)) + { + item.ThumbnailUrl = dataThumb.Trim(); + } + else if (item.Images.Count > 0) + { + item.ThumbnailUrl = item.Images[0].ImageUrl; + } + + Logger.LogInformation("[{SiteName}] スクレイピング完了: 『{Title}』, 画像数: {Count}枚", + SiteName, item.Title, item.Images.Count); + + return item.Images.Count > 0 ? item : null; + } + + private static (string TargetUrl, string NormalizedId, string ResolvedCategory) ResolveUrlAndId(string idOrUrl) + { + var trimmed = idOrUrl.Trim(); + string resolvedCategory = "fanzine"; + string normalizedId; + + if (trimmed.StartsWith("http://", StringComparison.OrdinalIgnoreCase) || + trimmed.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) + { + var uri = new Uri(trimmed); + var segments = uri.AbsolutePath.Split('/', StringSplitOptions.RemoveEmptyEntries); + if (segments.Length >= 2) + { + resolvedCategory = segments[0]; + normalizedId = segments[1]; + } + else if (segments.Length == 1) + { + normalizedId = segments[0]; + } + else + { + normalizedId = trimmed; + } + + return ($"{BaseUrl}/{resolvedCategory}/{normalizedId}/", normalizedId, resolvedCategory); + } + + if (trimmed.Contains('/')) + { + var parts = trimmed.Split('/', StringSplitOptions.RemoveEmptyEntries); + if (parts.Length >= 2) + { + resolvedCategory = parts[0]; + normalizedId = parts[1]; + } + else + { + normalizedId = parts[0]; + } + } + else + { + normalizedId = trimmed; + } + + return ($"{BaseUrl}/{resolvedCategory}/{normalizedId}/", normalizedId, resolvedCategory); + } + + private static void ExtractTagsAndMetadata(HtmlDocument doc, UnifiedGalleryItem item) + { + var tagTables = doc.DocumentNode.SelectNodes("//div[@id='post-tag']//div[contains(@class, 'post-tag-table')]"); + if (tagTables == null) return; + + foreach (var table in tagTables) + { + var titleNode = table.SelectSingleNode(".//div[contains(@class, 'post-tag-title')]"); + var titleText = CleanText(titleNode?.InnerText); + + var linkNodes = table.SelectNodes(".//div[contains(@class, 'post-tags')]//a"); + if (linkNodes == null) continue; + + var tagValues = linkNodes.Select(n => CleanText(n.InnerText)).Where(s => !string.IsNullOrEmpty(s)).ToList(); + + if (titleText.Contains("サークル", StringComparison.OrdinalIgnoreCase) || titleText.Contains("グループ", StringComparison.OrdinalIgnoreCase)) + { + item.Circle = string.Join(", ", tagValues); + } + else if (titleText.Contains("作者", StringComparison.OrdinalIgnoreCase) || titleText.Contains("漫画家", StringComparison.OrdinalIgnoreCase)) + { + item.Author = string.Join(", ", tagValues); + } + else if (titleText.Contains("内容", StringComparison.OrdinalIgnoreCase) || titleText.Contains("タグ", StringComparison.OrdinalIgnoreCase) || titleText.Contains("キャラクター", StringComparison.OrdinalIgnoreCase)) + { + foreach (var tag in tagValues) + { + if (!item.Tags.Contains(tag)) item.Tags.Add(tag); + } + } + } } } @@ -73,6 +210,7 @@ public class UnifiedMomongaScraper : BaseUnifiedScraper public class UnifiedAuraHentaiScraper : BaseUnifiedScraper { private const string BaseUrl = "https://aurahentai.com"; + private const string ImageBaseUrl = "https://img.aurahentai.com"; public override string SiteName => "AuraHentai"; public override string SiteKey => "aurahentai"; @@ -85,48 +223,181 @@ public class UnifiedAuraHentaiScraper : BaseUnifiedScraper public override async Task ScrapeAsync(string urlOrPath, CancellationToken ct = default) { - var match = Regex.Match(urlOrPath, @"(?:g/|manga/|aurahentai/)?(?[a-z0-9_-]+)", RegexOptions.IgnoreCase); - var id = match.Success ? match.Groups["id"].Value : urlOrPath.Trim(); + var (targetUrl, normalizedId) = ResolveUrlAndId(urlOrPath); - var pageUrl = urlOrPath.StartsWith("http") ? urlOrPath : $"{BaseUrl}/g/{id}"; - var doc = await LoadHtmlDocumentAsync(pageUrl, ct: ct); - if (doc == null) return null; + Logger.LogInformation("[{SiteName}] スクレイピング開始 - ID: {Id}, URL: {Url}", SiteName, normalizedId, targetUrl); - var title = doc.DocumentNode.SelectSingleNode("//h1")?.InnerText?.Trim() ?? $"AuraHentai 作品 ({id})"; + var doc = await LoadHtmlDocumentAsync(targetUrl, useFlareSolverr: true, ct: ct); + if (doc == null || doc.DocumentNode == null) + { + Logger.LogWarning("[{SiteName}] ドキュメントの読み込みに失敗しました: {Url}", SiteName, targetUrl); + return null; + } + + var titleNode = doc.DocumentNode.SelectSingleNode("//main//h1[not(@id='age-gate-title')]") ?? + doc.DocumentNode.SelectSingleNode("//h1[not(@id='age-gate-title')]") ?? + doc.DocumentNode.SelectSingleNode("//title"); + var title = CleanText(titleNode?.InnerText) ?? normalizedId; + + if (title.Contains(" ? ")) + { + title = title.Split(" ? ")[0].Trim(); + } + else if (title.Contains(" | ")) + { + title = title.Split(" | ")[0].Trim(); + } var item = new UnifiedGalleryItem { - Id = id, - Title = CleanText(title), + Id = normalizedId, + Title = string.IsNullOrEmpty(title) ? normalizedId : title, SourceSite = SiteName, - SourceUrl = pageUrl, + SourceUrl = targetUrl, FetchedAt = DateTime.UtcNow }; - var imgNodes = doc.DocumentNode.SelectNodes("//div[contains(@class, 'reader-area')]//img | //div[@id='reader']//img | //div[contains(@class, 'gallery-thumb')]//img"); + int pageCount = ExtractMetadata(doc, item); + + // 画像一覧の抽出 + var imgNodes = doc.DocumentNode.SelectNodes("//div[contains(@class, 'page-grid')]//img") ?? + doc.DocumentNode.SelectNodes("//main//img[contains(@src, '/g/')]"); + if (imgNodes != null) { int pageNum = 1; foreach (var img in imgNodes) { - var src = img.GetAttributeValue("data-src", "")?.Trim(); - if (string.IsNullOrEmpty(src)) src = img.GetAttributeValue("src", "")?.Trim(); + var src = img.GetAttributeValue("src", null) ?? + img.GetAttributeValue("data-src", null); - if (!string.IsNullOrEmpty(src)) + if (string.IsNullOrWhiteSpace(src) || src.Contains("cover.webp") || src.Contains("logo")) { - item.Images.Add(new UnifiedImageInfo - { - PageNumber = pageNum++, - ImageUrl = src, - ThumbnailUrl = src, - AltText = $"{item.Title} - {pageNum - 1}ページ" - }); + continue; } + + var thumbUrl = src.StartsWith("http") ? src : $"{ImageBaseUrl}{src}"; + var fullImageUrl = thumbUrl.Replace("/thumb/g/", "/g/"); + var alt = img.GetAttributeValue("alt", null); + + item.Images.Add(new UnifiedImageInfo + { + PageNumber = pageNum++, + ImageUrl = fullImageUrl, + ThumbnailUrl = thumbUrl, + AltText = CleanText(alt) ?? $"{item.Title} {pageNum - 1}ページ" + }); + } + } + + // HTMLから取得できなかった場合、ページ数情報があれば連番URLを補完生成 + if (item.Images.Count == 0 && pageCount > 0) + { + for (int i = 1; i <= pageCount; i++) + { + var pageStr = i.ToString("D3"); + item.Images.Add(new UnifiedImageInfo + { + PageNumber = i, + ImageUrl = $"{ImageBaseUrl}/g/{normalizedId}/{pageStr}.webp", + ThumbnailUrl = $"{ImageBaseUrl}/thumb/g/{normalizedId}/{pageStr}.webp", + AltText = $"{item.Title} {i}ページ" + }); } } if (item.Images.Count > 0) item.ThumbnailUrl = item.Images[0].ThumbnailUrl; - return item; + return item.Images.Count > 0 ? item : null; + } + + private static (string TargetUrl, string NormalizedId) ResolveUrlAndId(string idOrUrl) + { + var trimmed = idOrUrl.Trim(); + string normalizedId; + + if (trimmed.StartsWith("http://", StringComparison.OrdinalIgnoreCase) || + trimmed.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) + { + var uri = new Uri(trimmed); + var segments = uri.AbsolutePath.Split('/', StringSplitOptions.RemoveEmptyEntries); + normalizedId = segments.Length > 0 ? segments[^1] : trimmed; + } + else if (trimmed.Contains('/')) + { + var parts = trimmed.Split('/', StringSplitOptions.RemoveEmptyEntries); + normalizedId = parts.Length > 0 ? parts[^1] : trimmed; + } + else + { + normalizedId = trimmed; + } + + if (int.TryParse(normalizedId, out var num) && normalizedId.Length < 6) + { + normalizedId = num.ToString("D6"); + } + + var fullUrl = $"{BaseUrl}/gallery/{normalizedId}"; + return (fullUrl, normalizedId); + } + + private static int ExtractMetadata(HtmlDocument doc, UnifiedGalleryItem item) + { + int detectedPageCount = 0; + var dlNode = doc.DocumentNode.SelectSingleNode("//main//dl"); + if (dlNode == null) return detectedPageCount; + + var dtNodes = dlNode.SelectNodes(".//dt"); + if (dtNodes == null) return detectedPageCount; + + foreach (var dt in dtNodes) + { + var label = CleanText(dt.InnerText).ToLowerInvariant(); + var dd = dt.ParentNode?.SelectSingleNode(".//dd"); + if (dd == null) continue; + + var links = dd.SelectNodes(".//a"); + var values = links != null + ? links.Select(a => CleanText(a.InnerText)).Where(s => !string.IsNullOrEmpty(s)).ToList() + : new List { CleanText(dd.InnerText) }; + + switch (label) + { + case "artists": + case "artist": + item.Author = string.Join(", ", values); + break; + case "groups": + case "group": + case "circles": + case "circle": + item.Circle = string.Join(", ", values); + break; + case "category": + if (values.Count > 0) item.Category = values[0]; + break; + case "tags": + case "characters": + case "character": + case "parodies": + case "parody": + foreach (var val in values) + { + if (!item.Tags.Contains(val)) item.Tags.Add(val); + } + break; + case "pages": + var rawPages = CleanText(dd.InnerText); + var match = Regex.Match(rawPages, @"\d+"); + if (match.Success && int.TryParse(match.Value, out var count)) + { + detectedPageCount = count; + } + break; + } + } + + return detectedPageCount; } } @@ -148,47 +419,183 @@ public class UnifiedMangalearScraper : BaseUnifiedScraper public override async Task ScrapeAsync(string urlOrPath, CancellationToken ct = default) { - var match = Regex.Match(urlOrPath, @"(?:manga/|mangalear/)?(?[a-z0-9_-]+)", RegexOptions.IgnoreCase); - var id = match.Success ? match.Groups["id"].Value : urlOrPath.Trim(); + var (targetUrl, normalizedId, resolvedCategory) = ResolveUrlAndId(urlOrPath); - var pageUrl = urlOrPath.StartsWith("http") ? urlOrPath : $"{BaseUrl}/manga/{id}"; - var doc = await LoadHtmlDocumentAsync(pageUrl, ct: ct); - if (doc == null) return null; + Logger.LogInformation("[{SiteName}] スクレイピング開始 - ID: {Id}, URL: {Url}", SiteName, normalizedId, targetUrl); - var title = doc.DocumentNode.SelectSingleNode("//h1")?.InnerText?.Trim() ?? $"Mangalear 作品 ({id})"; + var doc = await LoadHtmlDocumentAsync(targetUrl, ct: ct); + if (doc == null || doc.DocumentNode == null) + { + Logger.LogWarning("[{SiteName}] ドキュメントの読み込みに失敗しました: {Url}", SiteName, targetUrl); + return null; + } + + var titleNode = doc.DocumentNode.SelectSingleNode("//h1[contains(@class, 'entry-title')]") ?? + doc.DocumentNode.SelectSingleNode("//h1") ?? + doc.DocumentNode.SelectSingleNode("//title"); + var title = CleanText(titleNode?.InnerText) ?? normalizedId; + + // タイトルの整形(先頭の「▲」記号や末尾のサイト名を除去) + title = Regex.Replace(title, @"^[▲\s]+", string.Empty); + if (title.Contains(" - ")) + { + title = title.Split(" - ")[0].Trim(); + } var item = new UnifiedGalleryItem { - Id = id, - Title = CleanText(title), + Id = normalizedId, + Title = string.IsNullOrEmpty(title) ? normalizedId : title, + Category = resolvedCategory, SourceSite = SiteName, - SourceUrl = pageUrl, + SourceUrl = targetUrl, FetchedAt = DateTime.UtcNow }; - var imgNodes = doc.DocumentNode.SelectNodes("//div[contains(@class, 'page-break')]//img | //div[contains(@class, 'reading-content')]//img"); - if (imgNodes != null) - { - int pageNum = 1; - foreach (var img in imgNodes) - { - var src = img.GetAttributeValue("data-src", "")?.Trim(); - if (string.IsNullOrEmpty(src)) src = img.GetAttributeValue("src", "")?.Trim(); + ExtractMetadata(doc, item); - if (!string.IsNullOrEmpty(src)) + // 画像一覧の抽出 + var contentNode = doc.DocumentNode.SelectSingleNode("//div[contains(@class, 'entry-content')]") ?? + doc.DocumentNode.SelectSingleNode("//article"); + + if (contentNode != null) + { + var imgNodes = contentNode.SelectNodes(".//img"); + if (imgNodes != null) + { + var seenUrls = new HashSet(StringComparer.OrdinalIgnoreCase); + int pageNum = 1; + + foreach (var img in imgNodes) { + var cssClass = img.GetAttributeValue("class", string.Empty); + if (cssClass.Contains("wp-smiley") || cssClass.Contains("avatar")) continue; + + string? highResUrl = null; + string? thumbUrl = null; + + var parentAnchor = img.ParentNode?.Name.Equals("a", StringComparison.OrdinalIgnoreCase) == true + ? img.ParentNode + : img.Ancestors("a").FirstOrDefault(); + + if (parentAnchor != null) + { + var href = parentAnchor.GetAttributeValue("href", null)?.Trim(); + if (!string.IsNullOrEmpty(href) && IsImageExtension(href)) + { + highResUrl = href; + } + } + + var src = img.GetAttributeValue("src", null)?.Trim(); + var dataSrc = img.GetAttributeValue("data-src", null)?.Trim(); + var candidateSrc = !string.IsNullOrEmpty(dataSrc) ? dataSrc : src; + + if (string.IsNullOrEmpty(highResUrl)) + { + highResUrl = candidateSrc; + } + else + { + thumbUrl = candidateSrc; + } + + if (string.IsNullOrEmpty(highResUrl) || !seenUrls.Add(highResUrl)) continue; + + var alt = img.GetAttributeValue("alt", null); item.Images.Add(new UnifiedImageInfo { PageNumber = pageNum++, - ImageUrl = src, - ThumbnailUrl = src, - AltText = $"{item.Title} - {pageNum - 1}ページ" + ImageUrl = highResUrl, + ThumbnailUrl = !string.IsNullOrEmpty(thumbUrl) ? thumbUrl : highResUrl, + AltText = CleanText(alt) ?? $"{item.Title} - {pageNum - 1}ページ" }); } } } if (item.Images.Count > 0) item.ThumbnailUrl = item.Images[0].ThumbnailUrl; - return item; + return item.Images.Count > 0 ? item : null; + } + + private static (string TargetUrl, string NormalizedId, string Category) ResolveUrlAndId(string idOrUrl) + { + var trimmed = idOrUrl.Trim(); + string normalizedId = string.Empty; + string resolvedCategory = "doujinshi"; + + if (trimmed.StartsWith("http://", StringComparison.OrdinalIgnoreCase) || + trimmed.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) + { + var uri = new Uri(trimmed); + var query = System.Web.HttpUtility.ParseQueryString(uri.Query); + var pVal = query.Get("p"); + if (!string.IsNullOrWhiteSpace(pVal)) + { + normalizedId = pVal.Trim(); + return ($"{BaseUrl}/?p={normalizedId}", normalizedId, resolvedCategory); + } + + var segments = uri.AbsolutePath.Split('/', StringSplitOptions.RemoveEmptyEntries); + if (segments.Length >= 2) + { + resolvedCategory = segments[0]; + normalizedId = segments[1]; + } + else if (segments.Length == 1) + { + normalizedId = segments[0]; + } + else + { + normalizedId = trimmed; + } + + return ($"{BaseUrl}/{resolvedCategory}/{normalizedId}/", normalizedId, resolvedCategory); + } + + return ($"{BaseUrl}/{resolvedCategory}/{trimmed}/", trimmed, resolvedCategory); + } + + private static void ExtractMetadata(HtmlDocument doc, UnifiedGalleryItem item) + { + var catNodes = doc.DocumentNode.SelectNodes("//div[contains(@class, 'entry-categories')]//a[contains(@class, 'cat-link')]"); + if (catNodes != null) + { + foreach (var node in catNodes) + { + var catText = CleanText(node.InnerText); + if (!string.IsNullOrEmpty(catText)) + { + item.Category = catText; + break; + } + } + } + + var tagNodes = doc.DocumentNode.SelectNodes("//div[contains(@class, 'entry-tags')]//a[contains(@class, 'tag-link')]"); + if (tagNodes != null) + { + foreach (var node in tagNodes) + { + var tag = CleanText(node.InnerText); + if (!string.IsNullOrEmpty(tag) && !item.Tags.Contains(tag)) + { + item.Tags.Add(tag); + } + } + } + + var authorNode = doc.DocumentNode.SelectSingleNode("//div[contains(@class, 'author-info')]//*[contains(@class, 'author-name')]"); + if (authorNode != null) + { + item.Author = CleanText(authorNode.InnerText); + } + } + + private static bool IsImageExtension(string url) + { + var ext = Path.GetExtension(url.Split('?')[0]).ToLowerInvariant(); + return ext is ".jpg" or ".jpeg" or ".png" or ".webp" or ".gif" or ".bmp"; } }