fix(scraper): モモンガの画像セレクタ(#post-hentai)及び各サイトのスクレイピング抽出ロジックを修正

This commit is contained in:
armonica committed 2026-10-01 10:38:41 +09:00
1 parent fb7f4aa62b
commit a59a66a08a
2 files changed
+680 -121

No files matched your search

@@ -1,3 +1,4 @@
using System.Collections.Concurrent;
using System.Text.Json;
using System.Text.RegularExpressions;
using HtmlAgilityPack;
@@ -9,13 +10,14 @@ namespace WebImageViewer.Class.Unified.Scrapers;
/// <summary>
/// E-Hentai (e-hentai.org) 固有の統一画像スクレイパー
/// 公式JSON API (gdata) とギャラリーHTML解析を組み合わせた画像取得を実装
/// 公式JSON API (gdata) とギャラリーHTML解析、showpage API を組み合わせた画像取得を実装
/// </summary>
public class UnifiedEHentaiScraper : BaseUnifiedScraper
{
private const string BaseDomain = "e-hentai.org";
private const string BaseUrl = "https://e-hentai.org";
private const string ApiEndpoint = "https://api.e-hentai.org/api.php";
private static readonly HttpClient ApiClient = new() { Timeout = TimeSpan.FromSeconds(20) };
public override string SiteName => "E-Hentai";
public override string SiteKey => "ehentai";
@@ -28,7 +30,7 @@ public class UnifiedEHentaiScraper : BaseUnifiedScraper
public override async Task<UnifiedGalleryItem?> ScrapeAsync(string urlOrPath, CancellationToken ct = default)
{
var (gid, token) = ResolveGidAndToken(urlOrPath);
var (gid, token) = await ResolveGidAndTokenAsync(urlOrPath, ct);
if (gid <= 0 || string.IsNullOrWhiteSpace(token))
{
Logger.LogWarning("[{SiteName}] GIDとTokenを抽出できませんでした: {Input}", SiteName, urlOrPath);
@@ -40,23 +42,28 @@ public class UnifiedEHentaiScraper : BaseUnifiedScraper
// 1. 公式API (gdata) を呼び出してメタデータを取得
UnifiedGalleryItem? item = null;
int fileCount = 0;
try
{
item = await FetchMetadataViaApiAsync(gid, token, ct);
var meta = await FetchMetadataViaApiAsync(gid, token, ct);
if (meta != null)
{
item = meta.Value.Item;
fileCount = meta.Value.FileCount;
}
}
catch (Exception ex)
{
Logger.LogWarning(ex, "[{SiteName}] 公式APIからのメタデータ取得に失敗しました。HTMLからフォールバックします", SiteName);
Logger.LogWarning(ex, "[{SiteName}] 公式APIからのメタデータ取得に失敗しました", SiteName);
}
// 2. ギャラリーHTMLから画像ページリストを抽出
var doc = await LoadHtmlDocumentAsync(galleryUrl, ct: ct);
if (doc == null && item == null) return null;
if (item == null)
{
var title = doc?.DocumentNode.SelectSingleNode("//h1[@id='gn']")?.InnerText?.Trim()
?? doc?.DocumentNode.SelectSingleNode("//h1[@id='gj']")?.InnerText?.Trim()
var doc = await LoadHtmlDocumentAsync(galleryUrl, ct: ct);
if (doc == null) return null;
var title = doc.DocumentNode.SelectSingleNode("//h1[@id='gn']")?.InnerText?.Trim()
?? doc.DocumentNode.SelectSingleNode("//h1[@id='gj']")?.InnerText?.Trim()
?? $"E-Hentai ギャラリー #{gid}";
item = new UnifiedGalleryItem
{
@@ -69,73 +76,111 @@ public class UnifiedEHentaiScraper : BaseUnifiedScraper
};
}
if (doc != null)
// 2. ギャラリーページから各ページのリンクを収集
var pageEntries = await CollectPageEntriesAsync(gid, token, fileCount, ct);
if (pageEntries.Count == 0)
{
// サムネイルコンテナ (gdtm / gdtl) から各画像の詳細ページURLまたはサムネイルを抽出
var imgNodes = doc.DocumentNode.SelectNodes("//div[@id='gdt']//div[contains(@class, 'gdtm') or contains(@class, 'gdtl')]//a");
if (imgNodes != null)
{
int pageNum = 1;
foreach (var a in imgNodes)
{
var href = a.GetAttributeValue("href", "")?.Trim();
var img = a.SelectSingleNode(".//img");
var src = img?.GetAttributeValue("src", "")?.Trim();
Logger.LogWarning("[{SiteName}] ギャラリーページからページリンクを取得できませんでした: {Url}", SiteName, galleryUrl);
return item;
}
if (!string.IsNullOrEmpty(href))
// 3. 1ページ目から showkey と 1ページ目画像 URL を取得
var firstEntry = pageEntries[0];
var (showkey, firstImgUrl, firstWidth, firstHeight) = await FetchPage1DetailsAsync(firstEntry.Url, ct);
var images = new ConcurrentDictionary<int, UnifiedImageInfo>();
if (!string.IsNullOrWhiteSpace(firstImgUrl))
{
images[1] = new UnifiedImageInfo
{
PageNumber = 1,
ImageUrl = firstImgUrl,
ThumbnailUrl = firstEntry.ThumbUrl ?? item.ThumbnailUrl,
AltText = $"{item.Title} 1ページ",
Width = firstWidth,
Height = firstHeight
};
}
// 4. 2ページ目以降の画像 URL を showpage API で並列取得
var remainingEntries = pageEntries.Skip(1).ToList();
if (remainingEntries.Count > 0 && !string.IsNullOrWhiteSpace(showkey))
{
using var semaphore = new SemaphoreSlim(10);
var tasks = remainingEntries.Select(async entry =>
{
await semaphore.WaitAsync(ct);
try
{
var imgInfo = await FetchImageViaShowpageApiAsync(gid, entry.Page, entry.ImgKey, showkey, entry.ThumbUrl, item.Title, ct);
if (imgInfo != null)
{
item.Images.Add(new UnifiedImageInfo
{
PageNumber = pageNum++,
ImageUrl = href, // 閲覧ビューア側で表示またはプロキシ
ThumbnailUrl = src,
AltText = $"{item.Title} - {pageNum - 1}ページ"
});
images[entry.Page] = imgInfo;
}
}
}
catch (Exception ex)
{
Logger.LogWarning(ex, "[{SiteName}] ページ {Page} の画像URL取得に失敗しました", SiteName, entry.Page);
}
finally
{
semaphore.Release();
}
});
await Task.WhenAll(tasks);
}
return item;
item.Images = images.Values.OrderBy(img => img.PageNumber).ToList();
if (item.Images.Count > 0 && string.IsNullOrEmpty(item.ThumbnailUrl))
{
item.ThumbnailUrl = item.Images[0].ThumbnailUrl;
}
return item.Images.Count > 0 ? item : null;
}
private static (long Gid, string Token) ResolveGidAndToken(string input)
private async Task<(long Gid, string Token)> ResolveGidAndTokenAsync(string input, CancellationToken ct)
{
var trimmed = input.Trim();
var match = Regex.Match(trimmed, @"/(?:g|s)/(?<gid>\d+)/(?<token>[a-z0-9]+)", RegexOptions.IgnoreCase);
if (match.Success && long.TryParse(match.Groups["gid"].Value, out var gid))
var gMatch = Regex.Match(trimmed, @"/g/(\d+)/([a-z0-9]{10})", RegexOptions.IgnoreCase);
if (gMatch.Success) return (long.Parse(gMatch.Groups[1].Value), gMatch.Groups[2].Value.ToLowerInvariant());
var pairMatch = Regex.Match(trimmed, @"^(\d+)[/_ -]([a-z0-9]{10})$", RegexOptions.IgnoreCase);
if (pairMatch.Success) return (long.Parse(pairMatch.Groups[1].Value), pairMatch.Groups[2].Value.ToLowerInvariant());
var sMatch = Regex.Match(trimmed, @"/s/([a-z0-9]+)/(\d+)-(\d+)", RegexOptions.IgnoreCase);
if (sMatch.Success)
{
return (gid, match.Groups["token"].Value);
var pageUrl = trimmed.StartsWith("http") ? trimmed : $"{BaseUrl}/s/{sMatch.Groups[1].Value}/{sMatch.Groups[2].Value}-{sMatch.Groups[3].Value}";
try
{
var doc = await LoadHtmlDocumentAsync(pageUrl, ct: ct);
var href = doc?.DocumentNode.SelectSingleNode("//a[contains(@href, '/g/')]")?.GetAttributeValue("href", "");
if (!string.IsNullOrEmpty(href))
{
var m = Regex.Match(href, @"/g/(\d+)/([a-z0-9]{10})", RegexOptions.IgnoreCase);
if (m.Success) return (long.Parse(m.Groups[1].Value), m.Groups[2].Value.ToLowerInvariant());
}
}
catch { }
}
var matchDirect = Regex.Match(trimmed, @"^(?<gid>\d+)/(?<token>[a-z0-9]+)$", RegexOptions.IgnoreCase);
if (matchDirect.Success && long.TryParse(matchDirect.Groups["gid"].Value, out var gidDirect))
{
return (gidDirect, matchDirect.Groups["token"].Value);
}
return (0, "");
return (0, string.Empty);
}
private async Task<UnifiedGalleryItem?> FetchMetadataViaApiAsync(long gid, string token, CancellationToken ct)
private async Task<(UnifiedGalleryItem Item, int FileCount)?> FetchMetadataViaApiAsync(long gid, string token, CancellationToken ct)
{
using var client = new HttpClient();
client.Timeout = TimeSpan.FromSeconds(15);
client.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64)");
var payload = new
{
method = "gdata",
gidlist = new[] { new object[] { gid, token } },
namespace_ = 1
gidlist = new object[] { new object[] { gid, token } },
@namespace = 1
};
var content = new StringContent(
JsonSerializer.Serialize(payload).Replace("namespace_", "namespace"),
System.Text.Encoding.UTF8,
"application/json");
using var resp = await client.PostAsync(ApiEndpoint, content, ct);
var content = new StringContent(JsonSerializer.Serialize(payload), System.Text.Encoding.UTF8, "application/json");
using var resp = await ApiClient.PostAsync(ApiEndpoint, content, ct);
resp.EnsureSuccessStatusCode();
var json = await resp.Content.ReadAsStringAsync(ct);
@@ -147,9 +192,9 @@ public class UnifiedEHentaiScraper : BaseUnifiedScraper
var titleJpn = g.TryGetProperty("title_jpn", out var tj) ? tj.GetString() : null;
var category = g.TryGetProperty("category", out var c) ? c.GetString() ?? "general" : "general";
var thumb = g.TryGetProperty("thumb", out var th) ? th.GetString() : null;
int fileCount = g.TryGetProperty("filecount", out var fc) && int.TryParse(fc.GetString(), out var count) ? count : 0;
var displayTitle = !string.IsNullOrWhiteSpace(titleJpn) ? titleJpn : title;
var item = new UnifiedGalleryItem
{
Id = $"{gid}/{token}",
@@ -175,6 +220,113 @@ public class UnifiedEHentaiScraper : BaseUnifiedScraper
}
}
return item;
return (item, fileCount);
}
private async Task<List<EHPageEntry>> CollectPageEntriesAsync(long gid, string token, int totalFiles, CancellationToken ct)
{
var entries = new List<EHPageEntry>();
int pageIndex = 0;
int maxPages = totalFiles > 0 ? (int)Math.Ceiling(totalFiles / 20.0) : 5;
while (pageIndex < maxPages && pageIndex < 20)
{
var pageUrl = $"{BaseUrl}/g/{gid}/{token}/?p={pageIndex}";
var doc = await LoadHtmlDocumentAsync(pageUrl, ct: ct);
if (doc == null) break;
var linkNodes = doc.DocumentNode.SelectNodes("//div[@id='gdt']//a[contains(@href, '/s/')]");
if (linkNodes == null || linkNodes.Count == 0) break;
int addedInThisPage = 0;
foreach (var node in linkNodes)
{
var href = node.GetAttributeValue("href", "");
var match = Regex.Match(href, @"/s/([a-z0-9]+)/(\d+)-(\d+)", RegexOptions.IgnoreCase);
if (match.Success)
{
var imgKey = match.Groups[1].Value;
var pNum = int.Parse(match.Groups[3].Value);
string? thumbUrl = node.SelectSingleNode(".//img")?.GetAttributeValue("src", null);
entries.Add(new EHPageEntry(pNum, imgKey, href, thumbUrl));
addedInThisPage++;
}
}
if (addedInThisPage == 0 || (totalFiles > 0 && entries.Count >= totalFiles)) break;
pageIndex++;
}
return entries.OrderBy(e => e.Page).ToList();
}
private async Task<(string? Showkey, string? ImageUrl, int? Width, int? Height)> FetchPage1DetailsAsync(string page1Url, CancellationToken ct)
{
try
{
var doc = await LoadHtmlDocumentAsync(page1Url, ct: ct);
if (doc == null) return (null, null, null, null);
var rawHtml = doc.DocumentNode.OuterHtml;
var showkeyMatch = Regex.Match(rawHtml, @"var showkey\s*=\s*""([^""]+)"";");
var showkey = showkeyMatch.Success ? showkeyMatch.Groups[1].Value : null;
var imgNode = doc.DocumentNode.SelectSingleNode("//img[@id='img']");
if (imgNode != null)
{
var imgUrl = imgNode.GetAttributeValue("src", null);
return (showkey, imgUrl, null, null);
}
return (showkey, null, null, null);
}
catch
{
return (null, null, null, null);
}
}
private async Task<UnifiedImageInfo?> FetchImageViaShowpageApiAsync(long gid, int page, string imgKey, string showkey, string? thumbUrl, string title, CancellationToken ct)
{
var requestObj = new
{
method = "showpage",
gid = gid,
page = page,
imgkey = imgKey,
showkey = showkey
};
var content = new StringContent(JsonSerializer.Serialize(requestObj), System.Text.Encoding.UTF8, "application/json");
using var resp = await ApiClient.PostAsync(ApiEndpoint, content, ct);
resp.EnsureSuccessStatusCode();
var resJson = await resp.Content.ReadAsStringAsync(ct);
using var doc = JsonDocument.Parse(resJson);
var root = doc.RootElement;
if (root.TryGetProperty("i3", out var i3Prop))
{
var i3Html = i3Prop.GetString();
if (!string.IsNullOrWhiteSpace(i3Html))
{
var imgMatch = Regex.Match(i3Html, @"<img\s+id=""img""\s+src=""([^""]+)""", RegexOptions.IgnoreCase);
if (imgMatch.Success)
{
return new UnifiedImageInfo
{
PageNumber = page,
ImageUrl = imgMatch.Groups[1].Value,
ThumbnailUrl = thumbUrl ?? imgMatch.Groups[1].Value,
AltText = $"{title} {page}ページ"
};
}
}
}
return null;
}
private sealed record EHPageEntry(int Page, string ImgKey, string Url, string? ThumbUrl);
}
@@ -24,46 +24,183 @@ public class UnifiedMomongaScraper : BaseUnifiedScraper
public override async Task<UnifiedGalleryItem?> ScrapeAsync(string urlOrPath, CancellationToken ct = default)
{
var match = Regex.Match(urlOrPath, @"(?:momon-ga\.com/|momonga/)?(?<id>[a-z0-9_-]+)(?:\.html)?", RegexOptions.IgnoreCase);
var id = match.Success ? match.Groups["id"].Value : urlOrPath.Trim();
var (targetUrl, normalizedId, resolvedCategory) = ResolveUrlAndId(urlOrPath);
var pageUrl = urlOrPath.StartsWith("http") ? urlOrPath : $"{BaseUrl}/{id}.html";
var doc = await LoadHtmlDocumentAsync(pageUrl, ct: ct);
if (doc == null) return null;
Logger.LogInformation("[{SiteName}] スクレイピング開始 - ID: {Id}, Category: {Category}, URL: {Url}",
SiteName, normalizedId, resolvedCategory, targetUrl);
var title = doc.DocumentNode.SelectSingleNode("//h1")?.InnerText?.Trim() ?? $"モモンガ作品 ({id})";
var doc = await LoadHtmlDocumentAsync(targetUrl, ct: ct);
if (doc == null || doc.DocumentNode == null)
{
Logger.LogWarning("[{SiteName}] ドキュメントの読み込みに失敗しました: {Url}", SiteName, targetUrl);
return null;
}
// タイトルの抽出(div#post-data h1、h1、titleタグ)
var titleNode = doc.DocumentNode.SelectSingleNode("//div[@id='post-data']//h1") ??
doc.DocumentNode.SelectSingleNode("//h1") ??
doc.DocumentNode.SelectSingleNode("//title");
var title = CleanText(titleNode?.InnerText);
if (string.IsNullOrWhiteSpace(title)) title = normalizedId;
// 余分なサイト名表記を除去(例: "新妻契約 - 同人誌 - エロ漫画 momon:GA(モモンガッ!!)" -> "新妻契約")
if (title.Contains(" - "))
{
title = title.Split(" - ")[0].Trim();
}
var item = new UnifiedGalleryItem
{
Id = id,
Title = CleanText(title),
Id = normalizedId,
Title = string.IsNullOrEmpty(title) ? normalizedId : title,
Category = resolvedCategory,
SourceSite = SiteName,
SourceUrl = pageUrl,
SourceUrl = targetUrl,
FetchedAt = DateTime.UtcNow
};
var imgNodes = doc.DocumentNode.SelectNodes("//div[contains(@class, 'entry-content')]//img | //div[contains(@class, 'post-content')]//img");
// サークル、作者、タグの抽出
ExtractTagsAndMetadata(doc, item);
// 画像一覧の抽出(#post-hentai img, .entry-content img, article img)
var imgNodes = doc.DocumentNode.SelectNodes("//div[@id='post-hentai']//img") ??
doc.DocumentNode.SelectNodes("//div[contains(@class, 'entry-content')]//img") ??
doc.DocumentNode.SelectNodes("//article//img");
if (imgNodes != null)
{
int pageNum = 1;
foreach (var img in imgNodes)
{
var src = img.GetAttributeValue("src", "")?.Trim();
if (!string.IsNullOrEmpty(src))
var src = img.GetAttributeValue("src", null) ??
img.GetAttributeValue("data-src", null) ??
img.GetAttributeValue("data-lazy-src", null);
if (string.IsNullOrWhiteSpace(src)) continue;
// 広告画像やアイコン等を除外
if (src.Contains("themes", StringComparison.OrdinalIgnoreCase) ||
src.Contains("plugins", StringComparison.OrdinalIgnoreCase) ||
src.Contains("banner", StringComparison.OrdinalIgnoreCase) ||
src.Contains("ad_", StringComparison.OrdinalIgnoreCase))
{
item.Images.Add(new UnifiedImageInfo
{
PageNumber = pageNum++,
ImageUrl = src,
ThumbnailUrl = src,
AltText = $"{item.Title} - {pageNum - 1}ページ"
});
continue;
}
var alt = img.GetAttributeValue("alt", null);
int? width = int.TryParse(img.GetAttributeValue("width", null), out var w) ? w : null;
int? height = int.TryParse(img.GetAttributeValue("height", null), out var h) ? h : null;
item.Images.Add(new UnifiedImageInfo
{
PageNumber = pageNum++,
ImageUrl = src.Trim(),
ThumbnailUrl = src.Trim(),
AltText = CleanText(alt) ?? $"{item.Title} - {pageNum - 1}ページ",
Width = width,
Height = height
});
}
}
if (item.Images.Count > 0) item.ThumbnailUrl = item.Images[0].ThumbnailUrl;
return item;
// サムネイルの決定(data-thumb属性、または1枚目の画像)
var historyNode = doc.DocumentNode.SelectSingleNode("//div[contains(@class, 'history')]");
var dataThumb = historyNode?.GetAttributeValue("data-thumb", null);
if (!string.IsNullOrWhiteSpace(dataThumb))
{
item.ThumbnailUrl = dataThumb.Trim();
}
else if (item.Images.Count > 0)
{
item.ThumbnailUrl = item.Images[0].ImageUrl;
}
Logger.LogInformation("[{SiteName}] スクレイピング完了: 『{Title}』, 画像数: {Count}枚",
SiteName, item.Title, item.Images.Count);
return item.Images.Count > 0 ? item : null;
}
private static (string TargetUrl, string NormalizedId, string ResolvedCategory) ResolveUrlAndId(string idOrUrl)
{
var trimmed = idOrUrl.Trim();
string resolvedCategory = "fanzine";
string normalizedId;
if (trimmed.StartsWith("http://", StringComparison.OrdinalIgnoreCase) ||
trimmed.StartsWith("https://", StringComparison.OrdinalIgnoreCase))
{
var uri = new Uri(trimmed);
var segments = uri.AbsolutePath.Split('/', StringSplitOptions.RemoveEmptyEntries);
if (segments.Length >= 2)
{
resolvedCategory = segments[0];
normalizedId = segments[1];
}
else if (segments.Length == 1)
{
normalizedId = segments[0];
}
else
{
normalizedId = trimmed;
}
return ($"{BaseUrl}/{resolvedCategory}/{normalizedId}/", normalizedId, resolvedCategory);
}
if (trimmed.Contains('/'))
{
var parts = trimmed.Split('/', StringSplitOptions.RemoveEmptyEntries);
if (parts.Length >= 2)
{
resolvedCategory = parts[0];
normalizedId = parts[1];
}
else
{
normalizedId = parts[0];
}
}
else
{
normalizedId = trimmed;
}
return ($"{BaseUrl}/{resolvedCategory}/{normalizedId}/", normalizedId, resolvedCategory);
}
private static void ExtractTagsAndMetadata(HtmlDocument doc, UnifiedGalleryItem item)
{
var tagTables = doc.DocumentNode.SelectNodes("//div[@id='post-tag']//div[contains(@class, 'post-tag-table')]");
if (tagTables == null) return;
foreach (var table in tagTables)
{
var titleNode = table.SelectSingleNode(".//div[contains(@class, 'post-tag-title')]");
var titleText = CleanText(titleNode?.InnerText);
var linkNodes = table.SelectNodes(".//div[contains(@class, 'post-tags')]//a");
if (linkNodes == null) continue;
var tagValues = linkNodes.Select(n => CleanText(n.InnerText)).Where(s => !string.IsNullOrEmpty(s)).ToList();
if (titleText.Contains("サークル", StringComparison.OrdinalIgnoreCase) || titleText.Contains("グループ", StringComparison.OrdinalIgnoreCase))
{
item.Circle = string.Join(", ", tagValues);
}
else if (titleText.Contains("作者", StringComparison.OrdinalIgnoreCase) || titleText.Contains("漫画家", StringComparison.OrdinalIgnoreCase))
{
item.Author = string.Join(", ", tagValues);
}
else if (titleText.Contains("内容", StringComparison.OrdinalIgnoreCase) || titleText.Contains("タグ", StringComparison.OrdinalIgnoreCase) || titleText.Contains("キャラクター", StringComparison.OrdinalIgnoreCase))
{
foreach (var tag in tagValues)
{
if (!item.Tags.Contains(tag)) item.Tags.Add(tag);
}
}
}
}
}
@@ -73,6 +210,7 @@ public class UnifiedMomongaScraper : BaseUnifiedScraper
public class UnifiedAuraHentaiScraper : BaseUnifiedScraper
{
private const string BaseUrl = "https://aurahentai.com";
private const string ImageBaseUrl = "https://img.aurahentai.com";
public override string SiteName => "AuraHentai";
public override string SiteKey => "aurahentai";
@@ -85,48 +223,181 @@ public class UnifiedAuraHentaiScraper : BaseUnifiedScraper
public override async Task<UnifiedGalleryItem?> ScrapeAsync(string urlOrPath, CancellationToken ct = default)
{
var match = Regex.Match(urlOrPath, @"(?:g/|manga/|aurahentai/)?(?<id>[a-z0-9_-]+)", RegexOptions.IgnoreCase);
var id = match.Success ? match.Groups["id"].Value : urlOrPath.Trim();
var (targetUrl, normalizedId) = ResolveUrlAndId(urlOrPath);
var pageUrl = urlOrPath.StartsWith("http") ? urlOrPath : $"{BaseUrl}/g/{id}";
var doc = await LoadHtmlDocumentAsync(pageUrl, ct: ct);
if (doc == null) return null;
Logger.LogInformation("[{SiteName}] スクレイピング開始 - ID: {Id}, URL: {Url}", SiteName, normalizedId, targetUrl);
var title = doc.DocumentNode.SelectSingleNode("//h1")?.InnerText?.Trim() ?? $"AuraHentai 作品 ({id})";
var doc = await LoadHtmlDocumentAsync(targetUrl, useFlareSolverr: true, ct: ct);
if (doc == null || doc.DocumentNode == null)
{
Logger.LogWarning("[{SiteName}] ドキュメントの読み込みに失敗しました: {Url}", SiteName, targetUrl);
return null;
}
var titleNode = doc.DocumentNode.SelectSingleNode("//main//h1[not(@id='age-gate-title')]") ??
doc.DocumentNode.SelectSingleNode("//h1[not(@id='age-gate-title')]") ??
doc.DocumentNode.SelectSingleNode("//title");
var title = CleanText(titleNode?.InnerText) ?? normalizedId;
if (title.Contains(" ? "))
{
title = title.Split(" ? ")[0].Trim();
}
else if (title.Contains(" | "))
{
title = title.Split(" | ")[0].Trim();
}
var item = new UnifiedGalleryItem
{
Id = id,
Title = CleanText(title),
Id = normalizedId,
Title = string.IsNullOrEmpty(title) ? normalizedId : title,
SourceSite = SiteName,
SourceUrl = pageUrl,
SourceUrl = targetUrl,
FetchedAt = DateTime.UtcNow
};
var imgNodes = doc.DocumentNode.SelectNodes("//div[contains(@class, 'reader-area')]//img | //div[@id='reader']//img | //div[contains(@class, 'gallery-thumb')]//img");
int pageCount = ExtractMetadata(doc, item);
// 画像一覧の抽出
var imgNodes = doc.DocumentNode.SelectNodes("//div[contains(@class, 'page-grid')]//img") ??
doc.DocumentNode.SelectNodes("//main//img[contains(@src, '/g/')]");
if (imgNodes != null)
{
int pageNum = 1;
foreach (var img in imgNodes)
{
var src = img.GetAttributeValue("data-src", "")?.Trim();
if (string.IsNullOrEmpty(src)) src = img.GetAttributeValue("src", "")?.Trim();
var src = img.GetAttributeValue("src", null) ??
img.GetAttributeValue("data-src", null);
if (!string.IsNullOrEmpty(src))
if (string.IsNullOrWhiteSpace(src) || src.Contains("cover.webp") || src.Contains("logo"))
{
item.Images.Add(new UnifiedImageInfo
{
PageNumber = pageNum++,
ImageUrl = src,
ThumbnailUrl = src,
AltText = $"{item.Title} - {pageNum - 1}ページ"
});
continue;
}
var thumbUrl = src.StartsWith("http") ? src : $"{ImageBaseUrl}{src}";
var fullImageUrl = thumbUrl.Replace("/thumb/g/", "/g/");
var alt = img.GetAttributeValue("alt", null);
item.Images.Add(new UnifiedImageInfo
{
PageNumber = pageNum++,
ImageUrl = fullImageUrl,
ThumbnailUrl = thumbUrl,
AltText = CleanText(alt) ?? $"{item.Title} {pageNum - 1}ページ"
});
}
}
// HTMLから取得できなかった場合、ページ数情報があれば連番URLを補完生成
if (item.Images.Count == 0 && pageCount > 0)
{
for (int i = 1; i <= pageCount; i++)
{
var pageStr = i.ToString("D3");
item.Images.Add(new UnifiedImageInfo
{
PageNumber = i,
ImageUrl = $"{ImageBaseUrl}/g/{normalizedId}/{pageStr}.webp",
ThumbnailUrl = $"{ImageBaseUrl}/thumb/g/{normalizedId}/{pageStr}.webp",
AltText = $"{item.Title} {i}ページ"
});
}
}
if (item.Images.Count > 0) item.ThumbnailUrl = item.Images[0].ThumbnailUrl;
return item;
return item.Images.Count > 0 ? item : null;
}
private static (string TargetUrl, string NormalizedId) ResolveUrlAndId(string idOrUrl)
{
var trimmed = idOrUrl.Trim();
string normalizedId;
if (trimmed.StartsWith("http://", StringComparison.OrdinalIgnoreCase) ||
trimmed.StartsWith("https://", StringComparison.OrdinalIgnoreCase))
{
var uri = new Uri(trimmed);
var segments = uri.AbsolutePath.Split('/', StringSplitOptions.RemoveEmptyEntries);
normalizedId = segments.Length > 0 ? segments[^1] : trimmed;
}
else if (trimmed.Contains('/'))
{
var parts = trimmed.Split('/', StringSplitOptions.RemoveEmptyEntries);
normalizedId = parts.Length > 0 ? parts[^1] : trimmed;
}
else
{
normalizedId = trimmed;
}
if (int.TryParse(normalizedId, out var num) && normalizedId.Length < 6)
{
normalizedId = num.ToString("D6");
}
var fullUrl = $"{BaseUrl}/gallery/{normalizedId}";
return (fullUrl, normalizedId);
}
private static int ExtractMetadata(HtmlDocument doc, UnifiedGalleryItem item)
{
int detectedPageCount = 0;
var dlNode = doc.DocumentNode.SelectSingleNode("//main//dl");
if (dlNode == null) return detectedPageCount;
var dtNodes = dlNode.SelectNodes(".//dt");
if (dtNodes == null) return detectedPageCount;
foreach (var dt in dtNodes)
{
var label = CleanText(dt.InnerText).ToLowerInvariant();
var dd = dt.ParentNode?.SelectSingleNode(".//dd");
if (dd == null) continue;
var links = dd.SelectNodes(".//a");
var values = links != null
? links.Select(a => CleanText(a.InnerText)).Where(s => !string.IsNullOrEmpty(s)).ToList()
: new List<string> { CleanText(dd.InnerText) };
switch (label)
{
case "artists":
case "artist":
item.Author = string.Join(", ", values);
break;
case "groups":
case "group":
case "circles":
case "circle":
item.Circle = string.Join(", ", values);
break;
case "category":
if (values.Count > 0) item.Category = values[0];
break;
case "tags":
case "characters":
case "character":
case "parodies":
case "parody":
foreach (var val in values)
{
if (!item.Tags.Contains(val)) item.Tags.Add(val);
}
break;
case "pages":
var rawPages = CleanText(dd.InnerText);
var match = Regex.Match(rawPages, @"\d+");
if (match.Success && int.TryParse(match.Value, out var count))
{
detectedPageCount = count;
}
break;
}
}
return detectedPageCount;
}
}
@@ -148,47 +419,183 @@ public class UnifiedMangalearScraper : BaseUnifiedScraper
public override async Task<UnifiedGalleryItem?> ScrapeAsync(string urlOrPath, CancellationToken ct = default)
{
var match = Regex.Match(urlOrPath, @"(?:manga/|mangalear/)?(?<id>[a-z0-9_-]+)", RegexOptions.IgnoreCase);
var id = match.Success ? match.Groups["id"].Value : urlOrPath.Trim();
var (targetUrl, normalizedId, resolvedCategory) = ResolveUrlAndId(urlOrPath);
var pageUrl = urlOrPath.StartsWith("http") ? urlOrPath : $"{BaseUrl}/manga/{id}";
var doc = await LoadHtmlDocumentAsync(pageUrl, ct: ct);
if (doc == null) return null;
Logger.LogInformation("[{SiteName}] スクレイピング開始 - ID: {Id}, URL: {Url}", SiteName, normalizedId, targetUrl);
var title = doc.DocumentNode.SelectSingleNode("//h1")?.InnerText?.Trim() ?? $"Mangalear 作品 ({id})";
var doc = await LoadHtmlDocumentAsync(targetUrl, ct: ct);
if (doc == null || doc.DocumentNode == null)
{
Logger.LogWarning("[{SiteName}] ドキュメントの読み込みに失敗しました: {Url}", SiteName, targetUrl);
return null;
}
var titleNode = doc.DocumentNode.SelectSingleNode("//h1[contains(@class, 'entry-title')]") ??
doc.DocumentNode.SelectSingleNode("//h1") ??
doc.DocumentNode.SelectSingleNode("//title");
var title = CleanText(titleNode?.InnerText) ?? normalizedId;
// タイトルの整形(先頭の「▲」記号や末尾のサイト名を除去)
title = Regex.Replace(title, @"^[▲\s]+", string.Empty);
if (title.Contains(" - "))
{
title = title.Split(" - ")[0].Trim();
}
var item = new UnifiedGalleryItem
{
Id = id,
Title = CleanText(title),
Id = normalizedId,
Title = string.IsNullOrEmpty(title) ? normalizedId : title,
Category = resolvedCategory,
SourceSite = SiteName,
SourceUrl = pageUrl,
SourceUrl = targetUrl,
FetchedAt = DateTime.UtcNow
};
var imgNodes = doc.DocumentNode.SelectNodes("//div[contains(@class, 'page-break')]//img | //div[contains(@class, 'reading-content')]//img");
if (imgNodes != null)
{
int pageNum = 1;
foreach (var img in imgNodes)
{
var src = img.GetAttributeValue("data-src", "")?.Trim();
if (string.IsNullOrEmpty(src)) src = img.GetAttributeValue("src", "")?.Trim();
ExtractMetadata(doc, item);
if (!string.IsNullOrEmpty(src))
// 画像一覧の抽出
var contentNode = doc.DocumentNode.SelectSingleNode("//div[contains(@class, 'entry-content')]") ??
doc.DocumentNode.SelectSingleNode("//article");
if (contentNode != null)
{
var imgNodes = contentNode.SelectNodes(".//img");
if (imgNodes != null)
{
var seenUrls = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
int pageNum = 1;
foreach (var img in imgNodes)
{
var cssClass = img.GetAttributeValue("class", string.Empty);
if (cssClass.Contains("wp-smiley") || cssClass.Contains("avatar")) continue;
string? highResUrl = null;
string? thumbUrl = null;
var parentAnchor = img.ParentNode?.Name.Equals("a", StringComparison.OrdinalIgnoreCase) == true
? img.ParentNode
: img.Ancestors("a").FirstOrDefault();
if (parentAnchor != null)
{
var href = parentAnchor.GetAttributeValue("href", null)?.Trim();
if (!string.IsNullOrEmpty(href) && IsImageExtension(href))
{
highResUrl = href;
}
}
var src = img.GetAttributeValue("src", null)?.Trim();
var dataSrc = img.GetAttributeValue("data-src", null)?.Trim();
var candidateSrc = !string.IsNullOrEmpty(dataSrc) ? dataSrc : src;
if (string.IsNullOrEmpty(highResUrl))
{
highResUrl = candidateSrc;
}
else
{
thumbUrl = candidateSrc;
}
if (string.IsNullOrEmpty(highResUrl) || !seenUrls.Add(highResUrl)) continue;
var alt = img.GetAttributeValue("alt", null);
item.Images.Add(new UnifiedImageInfo
{
PageNumber = pageNum++,
ImageUrl = src,
ThumbnailUrl = src,
AltText = $"{item.Title} - {pageNum - 1}ページ"
ImageUrl = highResUrl,
ThumbnailUrl = !string.IsNullOrEmpty(thumbUrl) ? thumbUrl : highResUrl,
AltText = CleanText(alt) ?? $"{item.Title} - {pageNum - 1}ページ"
});
}
}
}
if (item.Images.Count > 0) item.ThumbnailUrl = item.Images[0].ThumbnailUrl;
return item;
return item.Images.Count > 0 ? item : null;
}
private static (string TargetUrl, string NormalizedId, string Category) ResolveUrlAndId(string idOrUrl)
{
var trimmed = idOrUrl.Trim();
string normalizedId = string.Empty;
string resolvedCategory = "doujinshi";
if (trimmed.StartsWith("http://", StringComparison.OrdinalIgnoreCase) ||
trimmed.StartsWith("https://", StringComparison.OrdinalIgnoreCase))
{
var uri = new Uri(trimmed);
var query = System.Web.HttpUtility.ParseQueryString(uri.Query);
var pVal = query.Get("p");
if (!string.IsNullOrWhiteSpace(pVal))
{
normalizedId = pVal.Trim();
return ($"{BaseUrl}/?p={normalizedId}", normalizedId, resolvedCategory);
}
var segments = uri.AbsolutePath.Split('/', StringSplitOptions.RemoveEmptyEntries);
if (segments.Length >= 2)
{
resolvedCategory = segments[0];
normalizedId = segments[1];
}
else if (segments.Length == 1)
{
normalizedId = segments[0];
}
else
{
normalizedId = trimmed;
}
return ($"{BaseUrl}/{resolvedCategory}/{normalizedId}/", normalizedId, resolvedCategory);
}
return ($"{BaseUrl}/{resolvedCategory}/{trimmed}/", trimmed, resolvedCategory);
}
private static void ExtractMetadata(HtmlDocument doc, UnifiedGalleryItem item)
{
var catNodes = doc.DocumentNode.SelectNodes("//div[contains(@class, 'entry-categories')]//a[contains(@class, 'cat-link')]");
if (catNodes != null)
{
foreach (var node in catNodes)
{
var catText = CleanText(node.InnerText);
if (!string.IsNullOrEmpty(catText))
{
item.Category = catText;
break;
}
}
}
var tagNodes = doc.DocumentNode.SelectNodes("//div[contains(@class, 'entry-tags')]//a[contains(@class, 'tag-link')]");
if (tagNodes != null)
{
foreach (var node in tagNodes)
{
var tag = CleanText(node.InnerText);
if (!string.IsNullOrEmpty(tag) && !item.Tags.Contains(tag))
{
item.Tags.Add(tag);
}
}
}
var authorNode = doc.DocumentNode.SelectSingleNode("//div[contains(@class, 'author-info')]//*[contains(@class, 'author-name')]");
if (authorNode != null)
{
item.Author = CleanText(authorNode.InnerText);
}
}
private static bool IsImageExtension(string url)
{
var ext = Path.GetExtension(url.Split('?')[0]).ToLowerInvariant();
return ext is ".jpg" or ".jpeg" or ".png" or ".webp" or ".gif" or ".bmp";
}
}