diff --git a/WebImageViewer/Class/Unified/Scrapers/UnifiedEHentaiScraper.cs b/WebImageViewer/Class/Unified/Scrapers/UnifiedEHentaiScraper.cs
index 8af4e6c..1c019eb 100644
--- a/WebImageViewer/Class/Unified/Scrapers/UnifiedEHentaiScraper.cs
+++ b/WebImageViewer/Class/Unified/Scrapers/UnifiedEHentaiScraper.cs
@@ -1,3 +1,4 @@
+using System.Collections.Concurrent;
using System.Text.Json;
using System.Text.RegularExpressions;
using HtmlAgilityPack;
@@ -9,13 +10,14 @@ namespace WebImageViewer.Class.Unified.Scrapers;
///
/// E-Hentai (e-hentai.org) 固有の統一画像スクレイパー
-/// 公式JSON API (gdata) とギャラリーHTML解析を組み合わせた画像取得を実装
+/// 公式JSON API (gdata) とギャラリーHTML解析、showpage API を組み合わせた画像取得を実装
///
public class UnifiedEHentaiScraper : BaseUnifiedScraper
{
private const string BaseDomain = "e-hentai.org";
private const string BaseUrl = "https://e-hentai.org";
private const string ApiEndpoint = "https://api.e-hentai.org/api.php";
+ private static readonly HttpClient ApiClient = new() { Timeout = TimeSpan.FromSeconds(20) };
public override string SiteName => "E-Hentai";
public override string SiteKey => "ehentai";
@@ -28,7 +30,7 @@ public class UnifiedEHentaiScraper : BaseUnifiedScraper
public override async Task ScrapeAsync(string urlOrPath, CancellationToken ct = default)
{
- var (gid, token) = ResolveGidAndToken(urlOrPath);
+ var (gid, token) = await ResolveGidAndTokenAsync(urlOrPath, ct);
if (gid <= 0 || string.IsNullOrWhiteSpace(token))
{
Logger.LogWarning("[{SiteName}] GIDとTokenを抽出できませんでした: {Input}", SiteName, urlOrPath);
@@ -40,23 +42,28 @@ public class UnifiedEHentaiScraper : BaseUnifiedScraper
// 1. 公式API (gdata) を呼び出してメタデータを取得
UnifiedGalleryItem? item = null;
+ int fileCount = 0;
try
{
- item = await FetchMetadataViaApiAsync(gid, token, ct);
+ var meta = await FetchMetadataViaApiAsync(gid, token, ct);
+ if (meta != null)
+ {
+ item = meta.Value.Item;
+ fileCount = meta.Value.FileCount;
+ }
}
catch (Exception ex)
{
- Logger.LogWarning(ex, "[{SiteName}] 公式APIからのメタデータ取得に失敗しました。HTMLからフォールバックします", SiteName);
+ Logger.LogWarning(ex, "[{SiteName}] 公式APIからのメタデータ取得に失敗しました", SiteName);
}
- // 2. ギャラリーHTMLから画像ページリストを抽出
- var doc = await LoadHtmlDocumentAsync(galleryUrl, ct: ct);
- if (doc == null && item == null) return null;
-
if (item == null)
{
- var title = doc?.DocumentNode.SelectSingleNode("//h1[@id='gn']")?.InnerText?.Trim()
- ?? doc?.DocumentNode.SelectSingleNode("//h1[@id='gj']")?.InnerText?.Trim()
+ var doc = await LoadHtmlDocumentAsync(galleryUrl, ct: ct);
+ if (doc == null) return null;
+
+ var title = doc.DocumentNode.SelectSingleNode("//h1[@id='gn']")?.InnerText?.Trim()
+ ?? doc.DocumentNode.SelectSingleNode("//h1[@id='gj']")?.InnerText?.Trim()
?? $"E-Hentai ギャラリー #{gid}";
item = new UnifiedGalleryItem
{
@@ -69,73 +76,111 @@ public class UnifiedEHentaiScraper : BaseUnifiedScraper
};
}
- if (doc != null)
+ // 2. ギャラリーページから各ページのリンクを収集
+ var pageEntries = await CollectPageEntriesAsync(gid, token, fileCount, ct);
+ if (pageEntries.Count == 0)
{
- // サムネイルコンテナ (gdtm / gdtl) から各画像の詳細ページURLまたはサムネイルを抽出
- var imgNodes = doc.DocumentNode.SelectNodes("//div[@id='gdt']//div[contains(@class, 'gdtm') or contains(@class, 'gdtl')]//a");
- if (imgNodes != null)
- {
- int pageNum = 1;
- foreach (var a in imgNodes)
- {
- var href = a.GetAttributeValue("href", "")?.Trim();
- var img = a.SelectSingleNode(".//img");
- var src = img?.GetAttributeValue("src", "")?.Trim();
+ Logger.LogWarning("[{SiteName}] ギャラリーページからページリンクを取得できませんでした: {Url}", SiteName, galleryUrl);
+ return item;
+ }
- if (!string.IsNullOrEmpty(href))
+ // 3. 1ページ目から showkey と 1ページ目画像 URL を取得
+ var firstEntry = pageEntries[0];
+ var (showkey, firstImgUrl, firstWidth, firstHeight) = await FetchPage1DetailsAsync(firstEntry.Url, ct);
+
+ var images = new ConcurrentDictionary();
+ if (!string.IsNullOrWhiteSpace(firstImgUrl))
+ {
+ images[1] = new UnifiedImageInfo
+ {
+ PageNumber = 1,
+ ImageUrl = firstImgUrl,
+ ThumbnailUrl = firstEntry.ThumbUrl ?? item.ThumbnailUrl,
+ AltText = $"{item.Title} 1ページ",
+ Width = firstWidth,
+ Height = firstHeight
+ };
+ }
+
+ // 4. 2ページ目以降の画像 URL を showpage API で並列取得
+ var remainingEntries = pageEntries.Skip(1).ToList();
+ if (remainingEntries.Count > 0 && !string.IsNullOrWhiteSpace(showkey))
+ {
+ using var semaphore = new SemaphoreSlim(10);
+ var tasks = remainingEntries.Select(async entry =>
+ {
+ await semaphore.WaitAsync(ct);
+ try
+ {
+ var imgInfo = await FetchImageViaShowpageApiAsync(gid, entry.Page, entry.ImgKey, showkey, entry.ThumbUrl, item.Title, ct);
+ if (imgInfo != null)
{
- item.Images.Add(new UnifiedImageInfo
- {
- PageNumber = pageNum++,
- ImageUrl = href, // 閲覧ビューア側で表示またはプロキシ
- ThumbnailUrl = src,
- AltText = $"{item.Title} - {pageNum - 1}ページ"
- });
+ images[entry.Page] = imgInfo;
}
}
- }
+ catch (Exception ex)
+ {
+ Logger.LogWarning(ex, "[{SiteName}] ページ {Page} の画像URL取得に失敗しました", SiteName, entry.Page);
+ }
+ finally
+ {
+ semaphore.Release();
+ }
+ });
+
+ await Task.WhenAll(tasks);
}
- return item;
+ item.Images = images.Values.OrderBy(img => img.PageNumber).ToList();
+ if (item.Images.Count > 0 && string.IsNullOrEmpty(item.ThumbnailUrl))
+ {
+ item.ThumbnailUrl = item.Images[0].ThumbnailUrl;
+ }
+
+ return item.Images.Count > 0 ? item : null;
}
- private static (long Gid, string Token) ResolveGidAndToken(string input)
+ private async Task<(long Gid, string Token)> ResolveGidAndTokenAsync(string input, CancellationToken ct)
{
var trimmed = input.Trim();
- var match = Regex.Match(trimmed, @"/(?:g|s)/(?\d+)/(?[a-z0-9]+)", RegexOptions.IgnoreCase);
- if (match.Success && long.TryParse(match.Groups["gid"].Value, out var gid))
+
+ var gMatch = Regex.Match(trimmed, @"/g/(\d+)/([a-z0-9]{10})", RegexOptions.IgnoreCase);
+ if (gMatch.Success) return (long.Parse(gMatch.Groups[1].Value), gMatch.Groups[2].Value.ToLowerInvariant());
+
+ var pairMatch = Regex.Match(trimmed, @"^(\d+)[/_ -]([a-z0-9]{10})$", RegexOptions.IgnoreCase);
+ if (pairMatch.Success) return (long.Parse(pairMatch.Groups[1].Value), pairMatch.Groups[2].Value.ToLowerInvariant());
+
+ var sMatch = Regex.Match(trimmed, @"/s/([a-z0-9]+)/(\d+)-(\d+)", RegexOptions.IgnoreCase);
+ if (sMatch.Success)
{
- return (gid, match.Groups["token"].Value);
+ var pageUrl = trimmed.StartsWith("http") ? trimmed : $"{BaseUrl}/s/{sMatch.Groups[1].Value}/{sMatch.Groups[2].Value}-{sMatch.Groups[3].Value}";
+ try
+ {
+ var doc = await LoadHtmlDocumentAsync(pageUrl, ct: ct);
+ var href = doc?.DocumentNode.SelectSingleNode("//a[contains(@href, '/g/')]")?.GetAttributeValue("href", "");
+ if (!string.IsNullOrEmpty(href))
+ {
+ var m = Regex.Match(href, @"/g/(\d+)/([a-z0-9]{10})", RegexOptions.IgnoreCase);
+ if (m.Success) return (long.Parse(m.Groups[1].Value), m.Groups[2].Value.ToLowerInvariant());
+ }
+ }
+ catch { }
}
- var matchDirect = Regex.Match(trimmed, @"^(?\d+)/(?[a-z0-9]+)$", RegexOptions.IgnoreCase);
- if (matchDirect.Success && long.TryParse(matchDirect.Groups["gid"].Value, out var gidDirect))
- {
- return (gidDirect, matchDirect.Groups["token"].Value);
- }
-
- return (0, "");
+ return (0, string.Empty);
}
- private async Task FetchMetadataViaApiAsync(long gid, string token, CancellationToken ct)
+ private async Task<(UnifiedGalleryItem Item, int FileCount)?> FetchMetadataViaApiAsync(long gid, string token, CancellationToken ct)
{
- using var client = new HttpClient();
- client.Timeout = TimeSpan.FromSeconds(15);
- client.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64)");
-
var payload = new
{
method = "gdata",
- gidlist = new[] { new object[] { gid, token } },
- namespace_ = 1
+ gidlist = new object[] { new object[] { gid, token } },
+ @namespace = 1
};
- var content = new StringContent(
- JsonSerializer.Serialize(payload).Replace("namespace_", "namespace"),
- System.Text.Encoding.UTF8,
- "application/json");
-
- using var resp = await client.PostAsync(ApiEndpoint, content, ct);
+ var content = new StringContent(JsonSerializer.Serialize(payload), System.Text.Encoding.UTF8, "application/json");
+ using var resp = await ApiClient.PostAsync(ApiEndpoint, content, ct);
resp.EnsureSuccessStatusCode();
var json = await resp.Content.ReadAsStringAsync(ct);
@@ -147,9 +192,9 @@ public class UnifiedEHentaiScraper : BaseUnifiedScraper
var titleJpn = g.TryGetProperty("title_jpn", out var tj) ? tj.GetString() : null;
var category = g.TryGetProperty("category", out var c) ? c.GetString() ?? "general" : "general";
var thumb = g.TryGetProperty("thumb", out var th) ? th.GetString() : null;
+ int fileCount = g.TryGetProperty("filecount", out var fc) && int.TryParse(fc.GetString(), out var count) ? count : 0;
var displayTitle = !string.IsNullOrWhiteSpace(titleJpn) ? titleJpn : title;
-
var item = new UnifiedGalleryItem
{
Id = $"{gid}/{token}",
@@ -175,6 +220,113 @@ public class UnifiedEHentaiScraper : BaseUnifiedScraper
}
}
- return item;
+ return (item, fileCount);
}
+
+ private async Task> CollectPageEntriesAsync(long gid, string token, int totalFiles, CancellationToken ct)
+ {
+ var entries = new List();
+ int pageIndex = 0;
+ int maxPages = totalFiles > 0 ? (int)Math.Ceiling(totalFiles / 20.0) : 5;
+
+ while (pageIndex < maxPages && pageIndex < 20)
+ {
+ var pageUrl = $"{BaseUrl}/g/{gid}/{token}/?p={pageIndex}";
+ var doc = await LoadHtmlDocumentAsync(pageUrl, ct: ct);
+ if (doc == null) break;
+
+ var linkNodes = doc.DocumentNode.SelectNodes("//div[@id='gdt']//a[contains(@href, '/s/')]");
+ if (linkNodes == null || linkNodes.Count == 0) break;
+
+ int addedInThisPage = 0;
+ foreach (var node in linkNodes)
+ {
+ var href = node.GetAttributeValue("href", "");
+ var match = Regex.Match(href, @"/s/([a-z0-9]+)/(\d+)-(\d+)", RegexOptions.IgnoreCase);
+ if (match.Success)
+ {
+ var imgKey = match.Groups[1].Value;
+ var pNum = int.Parse(match.Groups[3].Value);
+
+ string? thumbUrl = node.SelectSingleNode(".//img")?.GetAttributeValue("src", null);
+ entries.Add(new EHPageEntry(pNum, imgKey, href, thumbUrl));
+ addedInThisPage++;
+ }
+ }
+
+ if (addedInThisPage == 0 || (totalFiles > 0 && entries.Count >= totalFiles)) break;
+ pageIndex++;
+ }
+
+ return entries.OrderBy(e => e.Page).ToList();
+ }
+
+ private async Task<(string? Showkey, string? ImageUrl, int? Width, int? Height)> FetchPage1DetailsAsync(string page1Url, CancellationToken ct)
+ {
+ try
+ {
+ var doc = await LoadHtmlDocumentAsync(page1Url, ct: ct);
+ if (doc == null) return (null, null, null, null);
+
+ var rawHtml = doc.DocumentNode.OuterHtml;
+ var showkeyMatch = Regex.Match(rawHtml, @"var showkey\s*=\s*""([^""]+)"";");
+ var showkey = showkeyMatch.Success ? showkeyMatch.Groups[1].Value : null;
+
+ var imgNode = doc.DocumentNode.SelectSingleNode("//img[@id='img']");
+ if (imgNode != null)
+ {
+ var imgUrl = imgNode.GetAttributeValue("src", null);
+ return (showkey, imgUrl, null, null);
+ }
+
+ return (showkey, null, null, null);
+ }
+ catch
+ {
+ return (null, null, null, null);
+ }
+ }
+
+ private async Task FetchImageViaShowpageApiAsync(long gid, int page, string imgKey, string showkey, string? thumbUrl, string title, CancellationToken ct)
+ {
+ var requestObj = new
+ {
+ method = "showpage",
+ gid = gid,
+ page = page,
+ imgkey = imgKey,
+ showkey = showkey
+ };
+
+ var content = new StringContent(JsonSerializer.Serialize(requestObj), System.Text.Encoding.UTF8, "application/json");
+ using var resp = await ApiClient.PostAsync(ApiEndpoint, content, ct);
+ resp.EnsureSuccessStatusCode();
+
+ var resJson = await resp.Content.ReadAsStringAsync(ct);
+ using var doc = JsonDocument.Parse(resJson);
+ var root = doc.RootElement;
+
+ if (root.TryGetProperty("i3", out var i3Prop))
+ {
+ var i3Html = i3Prop.GetString();
+ if (!string.IsNullOrWhiteSpace(i3Html))
+ {
+ var imgMatch = Regex.Match(i3Html, @"
ScrapeAsync(string urlOrPath, CancellationToken ct = default)
{
- var match = Regex.Match(urlOrPath, @"(?:momon-ga\.com/|momonga/)?(?[a-z0-9_-]+)(?:\.html)?", RegexOptions.IgnoreCase);
- var id = match.Success ? match.Groups["id"].Value : urlOrPath.Trim();
+ var (targetUrl, normalizedId, resolvedCategory) = ResolveUrlAndId(urlOrPath);
- var pageUrl = urlOrPath.StartsWith("http") ? urlOrPath : $"{BaseUrl}/{id}.html";
- var doc = await LoadHtmlDocumentAsync(pageUrl, ct: ct);
- if (doc == null) return null;
+ Logger.LogInformation("[{SiteName}] スクレイピング開始 - ID: {Id}, Category: {Category}, URL: {Url}",
+ SiteName, normalizedId, resolvedCategory, targetUrl);
- var title = doc.DocumentNode.SelectSingleNode("//h1")?.InnerText?.Trim() ?? $"モモンガ作品 ({id})";
+ var doc = await LoadHtmlDocumentAsync(targetUrl, ct: ct);
+ if (doc == null || doc.DocumentNode == null)
+ {
+ Logger.LogWarning("[{SiteName}] ドキュメントの読み込みに失敗しました: {Url}", SiteName, targetUrl);
+ return null;
+ }
+
+ // タイトルの抽出(div#post-data h1、h1、titleタグ)
+ var titleNode = doc.DocumentNode.SelectSingleNode("//div[@id='post-data']//h1") ??
+ doc.DocumentNode.SelectSingleNode("//h1") ??
+ doc.DocumentNode.SelectSingleNode("//title");
+ var title = CleanText(titleNode?.InnerText);
+ if (string.IsNullOrWhiteSpace(title)) title = normalizedId;
+
+ // 余分なサイト名表記を除去(例: "新妻契約 - 同人誌 - エロ漫画 momon:GA(モモンガッ!!)" -> "新妻契約")
+ if (title.Contains(" - "))
+ {
+ title = title.Split(" - ")[0].Trim();
+ }
var item = new UnifiedGalleryItem
{
- Id = id,
- Title = CleanText(title),
+ Id = normalizedId,
+ Title = string.IsNullOrEmpty(title) ? normalizedId : title,
+ Category = resolvedCategory,
SourceSite = SiteName,
- SourceUrl = pageUrl,
+ SourceUrl = targetUrl,
FetchedAt = DateTime.UtcNow
};
- var imgNodes = doc.DocumentNode.SelectNodes("//div[contains(@class, 'entry-content')]//img | //div[contains(@class, 'post-content')]//img");
+ // サークル、作者、タグの抽出
+ ExtractTagsAndMetadata(doc, item);
+
+ // 画像一覧の抽出(#post-hentai img, .entry-content img, article img)
+ var imgNodes = doc.DocumentNode.SelectNodes("//div[@id='post-hentai']//img") ??
+ doc.DocumentNode.SelectNodes("//div[contains(@class, 'entry-content')]//img") ??
+ doc.DocumentNode.SelectNodes("//article//img");
+
if (imgNodes != null)
{
int pageNum = 1;
foreach (var img in imgNodes)
{
- var src = img.GetAttributeValue("src", "")?.Trim();
- if (!string.IsNullOrEmpty(src))
+ var src = img.GetAttributeValue("src", null) ??
+ img.GetAttributeValue("data-src", null) ??
+ img.GetAttributeValue("data-lazy-src", null);
+
+ if (string.IsNullOrWhiteSpace(src)) continue;
+
+ // 広告画像やアイコン等を除外
+ if (src.Contains("themes", StringComparison.OrdinalIgnoreCase) ||
+ src.Contains("plugins", StringComparison.OrdinalIgnoreCase) ||
+ src.Contains("banner", StringComparison.OrdinalIgnoreCase) ||
+ src.Contains("ad_", StringComparison.OrdinalIgnoreCase))
{
- item.Images.Add(new UnifiedImageInfo
- {
- PageNumber = pageNum++,
- ImageUrl = src,
- ThumbnailUrl = src,
- AltText = $"{item.Title} - {pageNum - 1}ページ"
- });
+ continue;
}
+
+ var alt = img.GetAttributeValue("alt", null);
+ int? width = int.TryParse(img.GetAttributeValue("width", null), out var w) ? w : null;
+ int? height = int.TryParse(img.GetAttributeValue("height", null), out var h) ? h : null;
+
+ item.Images.Add(new UnifiedImageInfo
+ {
+ PageNumber = pageNum++,
+ ImageUrl = src.Trim(),
+ ThumbnailUrl = src.Trim(),
+ AltText = CleanText(alt) ?? $"{item.Title} - {pageNum - 1}ページ",
+ Width = width,
+ Height = height
+ });
}
}
- if (item.Images.Count > 0) item.ThumbnailUrl = item.Images[0].ThumbnailUrl;
- return item;
+ // サムネイルの決定(data-thumb属性、または1枚目の画像)
+ var historyNode = doc.DocumentNode.SelectSingleNode("//div[contains(@class, 'history')]");
+ var dataThumb = historyNode?.GetAttributeValue("data-thumb", null);
+ if (!string.IsNullOrWhiteSpace(dataThumb))
+ {
+ item.ThumbnailUrl = dataThumb.Trim();
+ }
+ else if (item.Images.Count > 0)
+ {
+ item.ThumbnailUrl = item.Images[0].ImageUrl;
+ }
+
+ Logger.LogInformation("[{SiteName}] スクレイピング完了: 『{Title}』, 画像数: {Count}枚",
+ SiteName, item.Title, item.Images.Count);
+
+ return item.Images.Count > 0 ? item : null;
+ }
+
+ private static (string TargetUrl, string NormalizedId, string ResolvedCategory) ResolveUrlAndId(string idOrUrl)
+ {
+ var trimmed = idOrUrl.Trim();
+ string resolvedCategory = "fanzine";
+ string normalizedId;
+
+ if (trimmed.StartsWith("http://", StringComparison.OrdinalIgnoreCase) ||
+ trimmed.StartsWith("https://", StringComparison.OrdinalIgnoreCase))
+ {
+ var uri = new Uri(trimmed);
+ var segments = uri.AbsolutePath.Split('/', StringSplitOptions.RemoveEmptyEntries);
+ if (segments.Length >= 2)
+ {
+ resolvedCategory = segments[0];
+ normalizedId = segments[1];
+ }
+ else if (segments.Length == 1)
+ {
+ normalizedId = segments[0];
+ }
+ else
+ {
+ normalizedId = trimmed;
+ }
+
+ return ($"{BaseUrl}/{resolvedCategory}/{normalizedId}/", normalizedId, resolvedCategory);
+ }
+
+ if (trimmed.Contains('/'))
+ {
+ var parts = trimmed.Split('/', StringSplitOptions.RemoveEmptyEntries);
+ if (parts.Length >= 2)
+ {
+ resolvedCategory = parts[0];
+ normalizedId = parts[1];
+ }
+ else
+ {
+ normalizedId = parts[0];
+ }
+ }
+ else
+ {
+ normalizedId = trimmed;
+ }
+
+ return ($"{BaseUrl}/{resolvedCategory}/{normalizedId}/", normalizedId, resolvedCategory);
+ }
+
+ private static void ExtractTagsAndMetadata(HtmlDocument doc, UnifiedGalleryItem item)
+ {
+ var tagTables = doc.DocumentNode.SelectNodes("//div[@id='post-tag']//div[contains(@class, 'post-tag-table')]");
+ if (tagTables == null) return;
+
+ foreach (var table in tagTables)
+ {
+ var titleNode = table.SelectSingleNode(".//div[contains(@class, 'post-tag-title')]");
+ var titleText = CleanText(titleNode?.InnerText);
+
+ var linkNodes = table.SelectNodes(".//div[contains(@class, 'post-tags')]//a");
+ if (linkNodes == null) continue;
+
+ var tagValues = linkNodes.Select(n => CleanText(n.InnerText)).Where(s => !string.IsNullOrEmpty(s)).ToList();
+
+ if (titleText.Contains("サークル", StringComparison.OrdinalIgnoreCase) || titleText.Contains("グループ", StringComparison.OrdinalIgnoreCase))
+ {
+ item.Circle = string.Join(", ", tagValues);
+ }
+ else if (titleText.Contains("作者", StringComparison.OrdinalIgnoreCase) || titleText.Contains("漫画家", StringComparison.OrdinalIgnoreCase))
+ {
+ item.Author = string.Join(", ", tagValues);
+ }
+ else if (titleText.Contains("内容", StringComparison.OrdinalIgnoreCase) || titleText.Contains("タグ", StringComparison.OrdinalIgnoreCase) || titleText.Contains("キャラクター", StringComparison.OrdinalIgnoreCase))
+ {
+ foreach (var tag in tagValues)
+ {
+ if (!item.Tags.Contains(tag)) item.Tags.Add(tag);
+ }
+ }
+ }
}
}
@@ -73,6 +210,7 @@ public class UnifiedMomongaScraper : BaseUnifiedScraper
public class UnifiedAuraHentaiScraper : BaseUnifiedScraper
{
private const string BaseUrl = "https://aurahentai.com";
+ private const string ImageBaseUrl = "https://img.aurahentai.com";
public override string SiteName => "AuraHentai";
public override string SiteKey => "aurahentai";
@@ -85,48 +223,181 @@ public class UnifiedAuraHentaiScraper : BaseUnifiedScraper
public override async Task ScrapeAsync(string urlOrPath, CancellationToken ct = default)
{
- var match = Regex.Match(urlOrPath, @"(?:g/|manga/|aurahentai/)?(?[a-z0-9_-]+)", RegexOptions.IgnoreCase);
- var id = match.Success ? match.Groups["id"].Value : urlOrPath.Trim();
+ var (targetUrl, normalizedId) = ResolveUrlAndId(urlOrPath);
- var pageUrl = urlOrPath.StartsWith("http") ? urlOrPath : $"{BaseUrl}/g/{id}";
- var doc = await LoadHtmlDocumentAsync(pageUrl, ct: ct);
- if (doc == null) return null;
+ Logger.LogInformation("[{SiteName}] スクレイピング開始 - ID: {Id}, URL: {Url}", SiteName, normalizedId, targetUrl);
- var title = doc.DocumentNode.SelectSingleNode("//h1")?.InnerText?.Trim() ?? $"AuraHentai 作品 ({id})";
+ var doc = await LoadHtmlDocumentAsync(targetUrl, useFlareSolverr: true, ct: ct);
+ if (doc == null || doc.DocumentNode == null)
+ {
+ Logger.LogWarning("[{SiteName}] ドキュメントの読み込みに失敗しました: {Url}", SiteName, targetUrl);
+ return null;
+ }
+
+ var titleNode = doc.DocumentNode.SelectSingleNode("//main//h1[not(@id='age-gate-title')]") ??
+ doc.DocumentNode.SelectSingleNode("//h1[not(@id='age-gate-title')]") ??
+ doc.DocumentNode.SelectSingleNode("//title");
+ var title = CleanText(titleNode?.InnerText) ?? normalizedId;
+
+ if (title.Contains(" ? "))
+ {
+ title = title.Split(" ? ")[0].Trim();
+ }
+ else if (title.Contains(" | "))
+ {
+ title = title.Split(" | ")[0].Trim();
+ }
var item = new UnifiedGalleryItem
{
- Id = id,
- Title = CleanText(title),
+ Id = normalizedId,
+ Title = string.IsNullOrEmpty(title) ? normalizedId : title,
SourceSite = SiteName,
- SourceUrl = pageUrl,
+ SourceUrl = targetUrl,
FetchedAt = DateTime.UtcNow
};
- var imgNodes = doc.DocumentNode.SelectNodes("//div[contains(@class, 'reader-area')]//img | //div[@id='reader']//img | //div[contains(@class, 'gallery-thumb')]//img");
+ int pageCount = ExtractMetadata(doc, item);
+
+ // 画像一覧の抽出
+ var imgNodes = doc.DocumentNode.SelectNodes("//div[contains(@class, 'page-grid')]//img") ??
+ doc.DocumentNode.SelectNodes("//main//img[contains(@src, '/g/')]");
+
if (imgNodes != null)
{
int pageNum = 1;
foreach (var img in imgNodes)
{
- var src = img.GetAttributeValue("data-src", "")?.Trim();
- if (string.IsNullOrEmpty(src)) src = img.GetAttributeValue("src", "")?.Trim();
+ var src = img.GetAttributeValue("src", null) ??
+ img.GetAttributeValue("data-src", null);
- if (!string.IsNullOrEmpty(src))
+ if (string.IsNullOrWhiteSpace(src) || src.Contains("cover.webp") || src.Contains("logo"))
{
- item.Images.Add(new UnifiedImageInfo
- {
- PageNumber = pageNum++,
- ImageUrl = src,
- ThumbnailUrl = src,
- AltText = $"{item.Title} - {pageNum - 1}ページ"
- });
+ continue;
}
+
+ var thumbUrl = src.StartsWith("http") ? src : $"{ImageBaseUrl}{src}";
+ var fullImageUrl = thumbUrl.Replace("/thumb/g/", "/g/");
+ var alt = img.GetAttributeValue("alt", null);
+
+ item.Images.Add(new UnifiedImageInfo
+ {
+ PageNumber = pageNum++,
+ ImageUrl = fullImageUrl,
+ ThumbnailUrl = thumbUrl,
+ AltText = CleanText(alt) ?? $"{item.Title} {pageNum - 1}ページ"
+ });
+ }
+ }
+
+ // HTMLから取得できなかった場合、ページ数情報があれば連番URLを補完生成
+ if (item.Images.Count == 0 && pageCount > 0)
+ {
+ for (int i = 1; i <= pageCount; i++)
+ {
+ var pageStr = i.ToString("D3");
+ item.Images.Add(new UnifiedImageInfo
+ {
+ PageNumber = i,
+ ImageUrl = $"{ImageBaseUrl}/g/{normalizedId}/{pageStr}.webp",
+ ThumbnailUrl = $"{ImageBaseUrl}/thumb/g/{normalizedId}/{pageStr}.webp",
+ AltText = $"{item.Title} {i}ページ"
+ });
}
}
if (item.Images.Count > 0) item.ThumbnailUrl = item.Images[0].ThumbnailUrl;
- return item;
+ return item.Images.Count > 0 ? item : null;
+ }
+
+ private static (string TargetUrl, string NormalizedId) ResolveUrlAndId(string idOrUrl)
+ {
+ var trimmed = idOrUrl.Trim();
+ string normalizedId;
+
+ if (trimmed.StartsWith("http://", StringComparison.OrdinalIgnoreCase) ||
+ trimmed.StartsWith("https://", StringComparison.OrdinalIgnoreCase))
+ {
+ var uri = new Uri(trimmed);
+ var segments = uri.AbsolutePath.Split('/', StringSplitOptions.RemoveEmptyEntries);
+ normalizedId = segments.Length > 0 ? segments[^1] : trimmed;
+ }
+ else if (trimmed.Contains('/'))
+ {
+ var parts = trimmed.Split('/', StringSplitOptions.RemoveEmptyEntries);
+ normalizedId = parts.Length > 0 ? parts[^1] : trimmed;
+ }
+ else
+ {
+ normalizedId = trimmed;
+ }
+
+ if (int.TryParse(normalizedId, out var num) && normalizedId.Length < 6)
+ {
+ normalizedId = num.ToString("D6");
+ }
+
+ var fullUrl = $"{BaseUrl}/gallery/{normalizedId}";
+ return (fullUrl, normalizedId);
+ }
+
+ private static int ExtractMetadata(HtmlDocument doc, UnifiedGalleryItem item)
+ {
+ int detectedPageCount = 0;
+ var dlNode = doc.DocumentNode.SelectSingleNode("//main//dl");
+ if (dlNode == null) return detectedPageCount;
+
+ var dtNodes = dlNode.SelectNodes(".//dt");
+ if (dtNodes == null) return detectedPageCount;
+
+ foreach (var dt in dtNodes)
+ {
+ var label = CleanText(dt.InnerText).ToLowerInvariant();
+ var dd = dt.ParentNode?.SelectSingleNode(".//dd");
+ if (dd == null) continue;
+
+ var links = dd.SelectNodes(".//a");
+ var values = links != null
+ ? links.Select(a => CleanText(a.InnerText)).Where(s => !string.IsNullOrEmpty(s)).ToList()
+ : new List { CleanText(dd.InnerText) };
+
+ switch (label)
+ {
+ case "artists":
+ case "artist":
+ item.Author = string.Join(", ", values);
+ break;
+ case "groups":
+ case "group":
+ case "circles":
+ case "circle":
+ item.Circle = string.Join(", ", values);
+ break;
+ case "category":
+ if (values.Count > 0) item.Category = values[0];
+ break;
+ case "tags":
+ case "characters":
+ case "character":
+ case "parodies":
+ case "parody":
+ foreach (var val in values)
+ {
+ if (!item.Tags.Contains(val)) item.Tags.Add(val);
+ }
+ break;
+ case "pages":
+ var rawPages = CleanText(dd.InnerText);
+ var match = Regex.Match(rawPages, @"\d+");
+ if (match.Success && int.TryParse(match.Value, out var count))
+ {
+ detectedPageCount = count;
+ }
+ break;
+ }
+ }
+
+ return detectedPageCount;
}
}
@@ -148,47 +419,183 @@ public class UnifiedMangalearScraper : BaseUnifiedScraper
public override async Task ScrapeAsync(string urlOrPath, CancellationToken ct = default)
{
- var match = Regex.Match(urlOrPath, @"(?:manga/|mangalear/)?(?[a-z0-9_-]+)", RegexOptions.IgnoreCase);
- var id = match.Success ? match.Groups["id"].Value : urlOrPath.Trim();
+ var (targetUrl, normalizedId, resolvedCategory) = ResolveUrlAndId(urlOrPath);
- var pageUrl = urlOrPath.StartsWith("http") ? urlOrPath : $"{BaseUrl}/manga/{id}";
- var doc = await LoadHtmlDocumentAsync(pageUrl, ct: ct);
- if (doc == null) return null;
+ Logger.LogInformation("[{SiteName}] スクレイピング開始 - ID: {Id}, URL: {Url}", SiteName, normalizedId, targetUrl);
- var title = doc.DocumentNode.SelectSingleNode("//h1")?.InnerText?.Trim() ?? $"Mangalear 作品 ({id})";
+ var doc = await LoadHtmlDocumentAsync(targetUrl, ct: ct);
+ if (doc == null || doc.DocumentNode == null)
+ {
+ Logger.LogWarning("[{SiteName}] ドキュメントの読み込みに失敗しました: {Url}", SiteName, targetUrl);
+ return null;
+ }
+
+ var titleNode = doc.DocumentNode.SelectSingleNode("//h1[contains(@class, 'entry-title')]") ??
+ doc.DocumentNode.SelectSingleNode("//h1") ??
+ doc.DocumentNode.SelectSingleNode("//title");
+ var title = CleanText(titleNode?.InnerText) ?? normalizedId;
+
+ // タイトルの整形(先頭の「▲」記号や末尾のサイト名を除去)
+ title = Regex.Replace(title, @"^[▲\s]+", string.Empty);
+ if (title.Contains(" - "))
+ {
+ title = title.Split(" - ")[0].Trim();
+ }
var item = new UnifiedGalleryItem
{
- Id = id,
- Title = CleanText(title),
+ Id = normalizedId,
+ Title = string.IsNullOrEmpty(title) ? normalizedId : title,
+ Category = resolvedCategory,
SourceSite = SiteName,
- SourceUrl = pageUrl,
+ SourceUrl = targetUrl,
FetchedAt = DateTime.UtcNow
};
- var imgNodes = doc.DocumentNode.SelectNodes("//div[contains(@class, 'page-break')]//img | //div[contains(@class, 'reading-content')]//img");
- if (imgNodes != null)
- {
- int pageNum = 1;
- foreach (var img in imgNodes)
- {
- var src = img.GetAttributeValue("data-src", "")?.Trim();
- if (string.IsNullOrEmpty(src)) src = img.GetAttributeValue("src", "")?.Trim();
+ ExtractMetadata(doc, item);
- if (!string.IsNullOrEmpty(src))
+ // 画像一覧の抽出
+ var contentNode = doc.DocumentNode.SelectSingleNode("//div[contains(@class, 'entry-content')]") ??
+ doc.DocumentNode.SelectSingleNode("//article");
+
+ if (contentNode != null)
+ {
+ var imgNodes = contentNode.SelectNodes(".//img");
+ if (imgNodes != null)
+ {
+ var seenUrls = new HashSet(StringComparer.OrdinalIgnoreCase);
+ int pageNum = 1;
+
+ foreach (var img in imgNodes)
{
+ var cssClass = img.GetAttributeValue("class", string.Empty);
+ if (cssClass.Contains("wp-smiley") || cssClass.Contains("avatar")) continue;
+
+ string? highResUrl = null;
+ string? thumbUrl = null;
+
+ var parentAnchor = img.ParentNode?.Name.Equals("a", StringComparison.OrdinalIgnoreCase) == true
+ ? img.ParentNode
+ : img.Ancestors("a").FirstOrDefault();
+
+ if (parentAnchor != null)
+ {
+ var href = parentAnchor.GetAttributeValue("href", null)?.Trim();
+ if (!string.IsNullOrEmpty(href) && IsImageExtension(href))
+ {
+ highResUrl = href;
+ }
+ }
+
+ var src = img.GetAttributeValue("src", null)?.Trim();
+ var dataSrc = img.GetAttributeValue("data-src", null)?.Trim();
+ var candidateSrc = !string.IsNullOrEmpty(dataSrc) ? dataSrc : src;
+
+ if (string.IsNullOrEmpty(highResUrl))
+ {
+ highResUrl = candidateSrc;
+ }
+ else
+ {
+ thumbUrl = candidateSrc;
+ }
+
+ if (string.IsNullOrEmpty(highResUrl) || !seenUrls.Add(highResUrl)) continue;
+
+ var alt = img.GetAttributeValue("alt", null);
item.Images.Add(new UnifiedImageInfo
{
PageNumber = pageNum++,
- ImageUrl = src,
- ThumbnailUrl = src,
- AltText = $"{item.Title} - {pageNum - 1}ページ"
+ ImageUrl = highResUrl,
+ ThumbnailUrl = !string.IsNullOrEmpty(thumbUrl) ? thumbUrl : highResUrl,
+ AltText = CleanText(alt) ?? $"{item.Title} - {pageNum - 1}ページ"
});
}
}
}
if (item.Images.Count > 0) item.ThumbnailUrl = item.Images[0].ThumbnailUrl;
- return item;
+ return item.Images.Count > 0 ? item : null;
+ }
+
+ private static (string TargetUrl, string NormalizedId, string Category) ResolveUrlAndId(string idOrUrl)
+ {
+ var trimmed = idOrUrl.Trim();
+ string normalizedId = string.Empty;
+ string resolvedCategory = "doujinshi";
+
+ if (trimmed.StartsWith("http://", StringComparison.OrdinalIgnoreCase) ||
+ trimmed.StartsWith("https://", StringComparison.OrdinalIgnoreCase))
+ {
+ var uri = new Uri(trimmed);
+ var query = System.Web.HttpUtility.ParseQueryString(uri.Query);
+ var pVal = query.Get("p");
+ if (!string.IsNullOrWhiteSpace(pVal))
+ {
+ normalizedId = pVal.Trim();
+ return ($"{BaseUrl}/?p={normalizedId}", normalizedId, resolvedCategory);
+ }
+
+ var segments = uri.AbsolutePath.Split('/', StringSplitOptions.RemoveEmptyEntries);
+ if (segments.Length >= 2)
+ {
+ resolvedCategory = segments[0];
+ normalizedId = segments[1];
+ }
+ else if (segments.Length == 1)
+ {
+ normalizedId = segments[0];
+ }
+ else
+ {
+ normalizedId = trimmed;
+ }
+
+ return ($"{BaseUrl}/{resolvedCategory}/{normalizedId}/", normalizedId, resolvedCategory);
+ }
+
+ return ($"{BaseUrl}/{resolvedCategory}/{trimmed}/", trimmed, resolvedCategory);
+ }
+
+ private static void ExtractMetadata(HtmlDocument doc, UnifiedGalleryItem item)
+ {
+ var catNodes = doc.DocumentNode.SelectNodes("//div[contains(@class, 'entry-categories')]//a[contains(@class, 'cat-link')]");
+ if (catNodes != null)
+ {
+ foreach (var node in catNodes)
+ {
+ var catText = CleanText(node.InnerText);
+ if (!string.IsNullOrEmpty(catText))
+ {
+ item.Category = catText;
+ break;
+ }
+ }
+ }
+
+ var tagNodes = doc.DocumentNode.SelectNodes("//div[contains(@class, 'entry-tags')]//a[contains(@class, 'tag-link')]");
+ if (tagNodes != null)
+ {
+ foreach (var node in tagNodes)
+ {
+ var tag = CleanText(node.InnerText);
+ if (!string.IsNullOrEmpty(tag) && !item.Tags.Contains(tag))
+ {
+ item.Tags.Add(tag);
+ }
+ }
+ }
+
+ var authorNode = doc.DocumentNode.SelectSingleNode("//div[contains(@class, 'author-info')]//*[contains(@class, 'author-name')]");
+ if (authorNode != null)
+ {
+ item.Author = CleanText(authorNode.InnerText);
+ }
+ }
+
+ private static bool IsImageExtension(string url)
+ {
+ var ext = Path.GetExtension(url.Split('?')[0]).ToLowerInvariant();
+ return ext is ".jpg" or ".jpeg" or ".png" or ".webp" or ".gif" or ".bmp";
}
}