fix(scraper): モモンガの画像セレクタ(#post-hentai)及び各サイトのスクレイピング抽出ロジックを修正
This commit is contained in:
1 parent
fb7f4aa62b
commit
a59a66a08a
2 files changed
+672
-113
No files matched your search
@@ -1,3 +1,4 @@
|
|||||||
|
using System.Collections.Concurrent;
|
||||||
using System.Text.Json;
|
using System.Text.Json;
|
||||||
using System.Text.RegularExpressions;
|
using System.Text.RegularExpressions;
|
||||||
using HtmlAgilityPack;
|
using HtmlAgilityPack;
|
||||||
@@ -9,13 +10,14 @@ namespace WebImageViewer.Class.Unified.Scrapers;
|
|||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// E-Hentai (e-hentai.org) 固有の統一画像スクレイパー
|
/// E-Hentai (e-hentai.org) 固有の統一画像スクレイパー
|
||||||
/// 公式JSON API (gdata) とギャラリーHTML解析を組み合わせた画像取得を実装
|
/// 公式JSON API (gdata) とギャラリーHTML解析、showpage API を組み合わせた画像取得を実装
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public class UnifiedEHentaiScraper : BaseUnifiedScraper
|
public class UnifiedEHentaiScraper : BaseUnifiedScraper
|
||||||
{
|
{
|
||||||
private const string BaseDomain = "e-hentai.org";
|
private const string BaseDomain = "e-hentai.org";
|
||||||
private const string BaseUrl = "https://e-hentai.org";
|
private const string BaseUrl = "https://e-hentai.org";
|
||||||
private const string ApiEndpoint = "https://api.e-hentai.org/api.php";
|
private const string ApiEndpoint = "https://api.e-hentai.org/api.php";
|
||||||
|
private static readonly HttpClient ApiClient = new() { Timeout = TimeSpan.FromSeconds(20) };
|
||||||
|
|
||||||
public override string SiteName => "E-Hentai";
|
public override string SiteName => "E-Hentai";
|
||||||
public override string SiteKey => "ehentai";
|
public override string SiteKey => "ehentai";
|
||||||
@@ -28,7 +30,7 @@ public class UnifiedEHentaiScraper : BaseUnifiedScraper
|
|||||||
|
|
||||||
public override async Task<UnifiedGalleryItem?> ScrapeAsync(string urlOrPath, CancellationToken ct = default)
|
public override async Task<UnifiedGalleryItem?> ScrapeAsync(string urlOrPath, CancellationToken ct = default)
|
||||||
{
|
{
|
||||||
var (gid, token) = ResolveGidAndToken(urlOrPath);
|
var (gid, token) = await ResolveGidAndTokenAsync(urlOrPath, ct);
|
||||||
if (gid <= 0 || string.IsNullOrWhiteSpace(token))
|
if (gid <= 0 || string.IsNullOrWhiteSpace(token))
|
||||||
{
|
{
|
||||||
Logger.LogWarning("[{SiteName}] GIDとTokenを抽出できませんでした: {Input}", SiteName, urlOrPath);
|
Logger.LogWarning("[{SiteName}] GIDとTokenを抽出できませんでした: {Input}", SiteName, urlOrPath);
|
||||||
@@ -40,23 +42,28 @@ public class UnifiedEHentaiScraper : BaseUnifiedScraper
|
|||||||
|
|
||||||
// 1. 公式API (gdata) を呼び出してメタデータを取得
|
// 1. 公式API (gdata) を呼び出してメタデータを取得
|
||||||
UnifiedGalleryItem? item = null;
|
UnifiedGalleryItem? item = null;
|
||||||
|
int fileCount = 0;
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
item = await FetchMetadataViaApiAsync(gid, token, ct);
|
var meta = await FetchMetadataViaApiAsync(gid, token, ct);
|
||||||
|
if (meta != null)
|
||||||
|
{
|
||||||
|
item = meta.Value.Item;
|
||||||
|
fileCount = meta.Value.FileCount;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
catch (Exception ex)
|
catch (Exception ex)
|
||||||
{
|
{
|
||||||
Logger.LogWarning(ex, "[{SiteName}] 公式APIからのメタデータ取得に失敗しました。HTMLからフォールバックします", SiteName);
|
Logger.LogWarning(ex, "[{SiteName}] 公式APIからのメタデータ取得に失敗しました", SiteName);
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2. ギャラリーHTMLから画像ページリストを抽出
|
|
||||||
var doc = await LoadHtmlDocumentAsync(galleryUrl, ct: ct);
|
|
||||||
if (doc == null && item == null) return null;
|
|
||||||
|
|
||||||
if (item == null)
|
if (item == null)
|
||||||
{
|
{
|
||||||
var title = doc?.DocumentNode.SelectSingleNode("//h1[@id='gn']")?.InnerText?.Trim()
|
var doc = await LoadHtmlDocumentAsync(galleryUrl, ct: ct);
|
||||||
?? doc?.DocumentNode.SelectSingleNode("//h1[@id='gj']")?.InnerText?.Trim()
|
if (doc == null) return null;
|
||||||
|
|
||||||
|
var title = doc.DocumentNode.SelectSingleNode("//h1[@id='gn']")?.InnerText?.Trim()
|
||||||
|
?? doc.DocumentNode.SelectSingleNode("//h1[@id='gj']")?.InnerText?.Trim()
|
||||||
?? $"E-Hentai ギャラリー #{gid}";
|
?? $"E-Hentai ギャラリー #{gid}";
|
||||||
item = new UnifiedGalleryItem
|
item = new UnifiedGalleryItem
|
||||||
{
|
{
|
||||||
@@ -69,73 +76,111 @@ public class UnifiedEHentaiScraper : BaseUnifiedScraper
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
if (doc != null)
|
// 2. ギャラリーページから各ページのリンクを収集
|
||||||
|
var pageEntries = await CollectPageEntriesAsync(gid, token, fileCount, ct);
|
||||||
|
if (pageEntries.Count == 0)
|
||||||
{
|
{
|
||||||
// サムネイルコンテナ (gdtm / gdtl) から各画像の詳細ページURLまたはサムネイルを抽出
|
Logger.LogWarning("[{SiteName}] ギャラリーページからページリンクを取得できませんでした: {Url}", SiteName, galleryUrl);
|
||||||
var imgNodes = doc.DocumentNode.SelectNodes("//div[@id='gdt']//div[contains(@class, 'gdtm') or contains(@class, 'gdtl')]//a");
|
|
||||||
if (imgNodes != null)
|
|
||||||
{
|
|
||||||
int pageNum = 1;
|
|
||||||
foreach (var a in imgNodes)
|
|
||||||
{
|
|
||||||
var href = a.GetAttributeValue("href", "")?.Trim();
|
|
||||||
var img = a.SelectSingleNode(".//img");
|
|
||||||
var src = img?.GetAttributeValue("src", "")?.Trim();
|
|
||||||
|
|
||||||
if (!string.IsNullOrEmpty(href))
|
|
||||||
{
|
|
||||||
item.Images.Add(new UnifiedImageInfo
|
|
||||||
{
|
|
||||||
PageNumber = pageNum++,
|
|
||||||
ImageUrl = href, // 閲覧ビューア側で表示またはプロキシ
|
|
||||||
ThumbnailUrl = src,
|
|
||||||
AltText = $"{item.Title} - {pageNum - 1}ページ"
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return item;
|
return item;
|
||||||
}
|
}
|
||||||
|
|
||||||
private static (long Gid, string Token) ResolveGidAndToken(string input)
|
// 3. 1ページ目から showkey と 1ページ目画像 URL を取得
|
||||||
|
var firstEntry = pageEntries[0];
|
||||||
|
var (showkey, firstImgUrl, firstWidth, firstHeight) = await FetchPage1DetailsAsync(firstEntry.Url, ct);
|
||||||
|
|
||||||
|
var images = new ConcurrentDictionary<int, UnifiedImageInfo>();
|
||||||
|
if (!string.IsNullOrWhiteSpace(firstImgUrl))
|
||||||
|
{
|
||||||
|
images[1] = new UnifiedImageInfo
|
||||||
|
{
|
||||||
|
PageNumber = 1,
|
||||||
|
ImageUrl = firstImgUrl,
|
||||||
|
ThumbnailUrl = firstEntry.ThumbUrl ?? item.ThumbnailUrl,
|
||||||
|
AltText = $"{item.Title} 1ページ",
|
||||||
|
Width = firstWidth,
|
||||||
|
Height = firstHeight
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4. 2ページ目以降の画像 URL を showpage API で並列取得
|
||||||
|
var remainingEntries = pageEntries.Skip(1).ToList();
|
||||||
|
if (remainingEntries.Count > 0 && !string.IsNullOrWhiteSpace(showkey))
|
||||||
|
{
|
||||||
|
using var semaphore = new SemaphoreSlim(10);
|
||||||
|
var tasks = remainingEntries.Select(async entry =>
|
||||||
|
{
|
||||||
|
await semaphore.WaitAsync(ct);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var imgInfo = await FetchImageViaShowpageApiAsync(gid, entry.Page, entry.ImgKey, showkey, entry.ThumbUrl, item.Title, ct);
|
||||||
|
if (imgInfo != null)
|
||||||
|
{
|
||||||
|
images[entry.Page] = imgInfo;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
Logger.LogWarning(ex, "[{SiteName}] ページ {Page} の画像URL取得に失敗しました", SiteName, entry.Page);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
semaphore.Release();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
await Task.WhenAll(tasks);
|
||||||
|
}
|
||||||
|
|
||||||
|
item.Images = images.Values.OrderBy(img => img.PageNumber).ToList();
|
||||||
|
if (item.Images.Count > 0 && string.IsNullOrEmpty(item.ThumbnailUrl))
|
||||||
|
{
|
||||||
|
item.ThumbnailUrl = item.Images[0].ThumbnailUrl;
|
||||||
|
}
|
||||||
|
|
||||||
|
return item.Images.Count > 0 ? item : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task<(long Gid, string Token)> ResolveGidAndTokenAsync(string input, CancellationToken ct)
|
||||||
{
|
{
|
||||||
var trimmed = input.Trim();
|
var trimmed = input.Trim();
|
||||||
var match = Regex.Match(trimmed, @"/(?:g|s)/(?<gid>\d+)/(?<token>[a-z0-9]+)", RegexOptions.IgnoreCase);
|
|
||||||
if (match.Success && long.TryParse(match.Groups["gid"].Value, out var gid))
|
var gMatch = Regex.Match(trimmed, @"/g/(\d+)/([a-z0-9]{10})", RegexOptions.IgnoreCase);
|
||||||
|
if (gMatch.Success) return (long.Parse(gMatch.Groups[1].Value), gMatch.Groups[2].Value.ToLowerInvariant());
|
||||||
|
|
||||||
|
var pairMatch = Regex.Match(trimmed, @"^(\d+)[/_ -]([a-z0-9]{10})$", RegexOptions.IgnoreCase);
|
||||||
|
if (pairMatch.Success) return (long.Parse(pairMatch.Groups[1].Value), pairMatch.Groups[2].Value.ToLowerInvariant());
|
||||||
|
|
||||||
|
var sMatch = Regex.Match(trimmed, @"/s/([a-z0-9]+)/(\d+)-(\d+)", RegexOptions.IgnoreCase);
|
||||||
|
if (sMatch.Success)
|
||||||
{
|
{
|
||||||
return (gid, match.Groups["token"].Value);
|
var pageUrl = trimmed.StartsWith("http") ? trimmed : $"{BaseUrl}/s/{sMatch.Groups[1].Value}/{sMatch.Groups[2].Value}-{sMatch.Groups[3].Value}";
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var doc = await LoadHtmlDocumentAsync(pageUrl, ct: ct);
|
||||||
|
var href = doc?.DocumentNode.SelectSingleNode("//a[contains(@href, '/g/')]")?.GetAttributeValue("href", "");
|
||||||
|
if (!string.IsNullOrEmpty(href))
|
||||||
|
{
|
||||||
|
var m = Regex.Match(href, @"/g/(\d+)/([a-z0-9]{10})", RegexOptions.IgnoreCase);
|
||||||
|
if (m.Success) return (long.Parse(m.Groups[1].Value), m.Groups[2].Value.ToLowerInvariant());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch { }
|
||||||
}
|
}
|
||||||
|
|
||||||
var matchDirect = Regex.Match(trimmed, @"^(?<gid>\d+)/(?<token>[a-z0-9]+)$", RegexOptions.IgnoreCase);
|
return (0, string.Empty);
|
||||||
if (matchDirect.Success && long.TryParse(matchDirect.Groups["gid"].Value, out var gidDirect))
|
|
||||||
{
|
|
||||||
return (gidDirect, matchDirect.Groups["token"].Value);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return (0, "");
|
private async Task<(UnifiedGalleryItem Item, int FileCount)?> FetchMetadataViaApiAsync(long gid, string token, CancellationToken ct)
|
||||||
}
|
|
||||||
|
|
||||||
private async Task<UnifiedGalleryItem?> FetchMetadataViaApiAsync(long gid, string token, CancellationToken ct)
|
|
||||||
{
|
{
|
||||||
using var client = new HttpClient();
|
|
||||||
client.Timeout = TimeSpan.FromSeconds(15);
|
|
||||||
client.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64)");
|
|
||||||
|
|
||||||
var payload = new
|
var payload = new
|
||||||
{
|
{
|
||||||
method = "gdata",
|
method = "gdata",
|
||||||
gidlist = new[] { new object[] { gid, token } },
|
gidlist = new object[] { new object[] { gid, token } },
|
||||||
namespace_ = 1
|
@namespace = 1
|
||||||
};
|
};
|
||||||
|
|
||||||
var content = new StringContent(
|
var content = new StringContent(JsonSerializer.Serialize(payload), System.Text.Encoding.UTF8, "application/json");
|
||||||
JsonSerializer.Serialize(payload).Replace("namespace_", "namespace"),
|
using var resp = await ApiClient.PostAsync(ApiEndpoint, content, ct);
|
||||||
System.Text.Encoding.UTF8,
|
|
||||||
"application/json");
|
|
||||||
|
|
||||||
using var resp = await client.PostAsync(ApiEndpoint, content, ct);
|
|
||||||
resp.EnsureSuccessStatusCode();
|
resp.EnsureSuccessStatusCode();
|
||||||
|
|
||||||
var json = await resp.Content.ReadAsStringAsync(ct);
|
var json = await resp.Content.ReadAsStringAsync(ct);
|
||||||
@@ -147,9 +192,9 @@ public class UnifiedEHentaiScraper : BaseUnifiedScraper
|
|||||||
var titleJpn = g.TryGetProperty("title_jpn", out var tj) ? tj.GetString() : null;
|
var titleJpn = g.TryGetProperty("title_jpn", out var tj) ? tj.GetString() : null;
|
||||||
var category = g.TryGetProperty("category", out var c) ? c.GetString() ?? "general" : "general";
|
var category = g.TryGetProperty("category", out var c) ? c.GetString() ?? "general" : "general";
|
||||||
var thumb = g.TryGetProperty("thumb", out var th) ? th.GetString() : null;
|
var thumb = g.TryGetProperty("thumb", out var th) ? th.GetString() : null;
|
||||||
|
int fileCount = g.TryGetProperty("filecount", out var fc) && int.TryParse(fc.GetString(), out var count) ? count : 0;
|
||||||
|
|
||||||
var displayTitle = !string.IsNullOrWhiteSpace(titleJpn) ? titleJpn : title;
|
var displayTitle = !string.IsNullOrWhiteSpace(titleJpn) ? titleJpn : title;
|
||||||
|
|
||||||
var item = new UnifiedGalleryItem
|
var item = new UnifiedGalleryItem
|
||||||
{
|
{
|
||||||
Id = $"{gid}/{token}",
|
Id = $"{gid}/{token}",
|
||||||
@@ -175,6 +220,113 @@ public class UnifiedEHentaiScraper : BaseUnifiedScraper
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return item;
|
return (item, fileCount);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async Task<List<EHPageEntry>> CollectPageEntriesAsync(long gid, string token, int totalFiles, CancellationToken ct)
|
||||||
|
{
|
||||||
|
var entries = new List<EHPageEntry>();
|
||||||
|
int pageIndex = 0;
|
||||||
|
int maxPages = totalFiles > 0 ? (int)Math.Ceiling(totalFiles / 20.0) : 5;
|
||||||
|
|
||||||
|
while (pageIndex < maxPages && pageIndex < 20)
|
||||||
|
{
|
||||||
|
var pageUrl = $"{BaseUrl}/g/{gid}/{token}/?p={pageIndex}";
|
||||||
|
var doc = await LoadHtmlDocumentAsync(pageUrl, ct: ct);
|
||||||
|
if (doc == null) break;
|
||||||
|
|
||||||
|
var linkNodes = doc.DocumentNode.SelectNodes("//div[@id='gdt']//a[contains(@href, '/s/')]");
|
||||||
|
if (linkNodes == null || linkNodes.Count == 0) break;
|
||||||
|
|
||||||
|
int addedInThisPage = 0;
|
||||||
|
foreach (var node in linkNodes)
|
||||||
|
{
|
||||||
|
var href = node.GetAttributeValue("href", "");
|
||||||
|
var match = Regex.Match(href, @"/s/([a-z0-9]+)/(\d+)-(\d+)", RegexOptions.IgnoreCase);
|
||||||
|
if (match.Success)
|
||||||
|
{
|
||||||
|
var imgKey = match.Groups[1].Value;
|
||||||
|
var pNum = int.Parse(match.Groups[3].Value);
|
||||||
|
|
||||||
|
string? thumbUrl = node.SelectSingleNode(".//img")?.GetAttributeValue("src", null);
|
||||||
|
entries.Add(new EHPageEntry(pNum, imgKey, href, thumbUrl));
|
||||||
|
addedInThisPage++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (addedInThisPage == 0 || (totalFiles > 0 && entries.Count >= totalFiles)) break;
|
||||||
|
pageIndex++;
|
||||||
|
}
|
||||||
|
|
||||||
|
return entries.OrderBy(e => e.Page).ToList();
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task<(string? Showkey, string? ImageUrl, int? Width, int? Height)> FetchPage1DetailsAsync(string page1Url, CancellationToken ct)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var doc = await LoadHtmlDocumentAsync(page1Url, ct: ct);
|
||||||
|
if (doc == null) return (null, null, null, null);
|
||||||
|
|
||||||
|
var rawHtml = doc.DocumentNode.OuterHtml;
|
||||||
|
var showkeyMatch = Regex.Match(rawHtml, @"var showkey\s*=\s*""([^""]+)"";");
|
||||||
|
var showkey = showkeyMatch.Success ? showkeyMatch.Groups[1].Value : null;
|
||||||
|
|
||||||
|
var imgNode = doc.DocumentNode.SelectSingleNode("//img[@id='img']");
|
||||||
|
if (imgNode != null)
|
||||||
|
{
|
||||||
|
var imgUrl = imgNode.GetAttributeValue("src", null);
|
||||||
|
return (showkey, imgUrl, null, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
return (showkey, null, null, null);
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
return (null, null, null, null);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task<UnifiedImageInfo?> FetchImageViaShowpageApiAsync(long gid, int page, string imgKey, string showkey, string? thumbUrl, string title, CancellationToken ct)
|
||||||
|
{
|
||||||
|
var requestObj = new
|
||||||
|
{
|
||||||
|
method = "showpage",
|
||||||
|
gid = gid,
|
||||||
|
page = page,
|
||||||
|
imgkey = imgKey,
|
||||||
|
showkey = showkey
|
||||||
|
};
|
||||||
|
|
||||||
|
var content = new StringContent(JsonSerializer.Serialize(requestObj), System.Text.Encoding.UTF8, "application/json");
|
||||||
|
using var resp = await ApiClient.PostAsync(ApiEndpoint, content, ct);
|
||||||
|
resp.EnsureSuccessStatusCode();
|
||||||
|
|
||||||
|
var resJson = await resp.Content.ReadAsStringAsync(ct);
|
||||||
|
using var doc = JsonDocument.Parse(resJson);
|
||||||
|
var root = doc.RootElement;
|
||||||
|
|
||||||
|
if (root.TryGetProperty("i3", out var i3Prop))
|
||||||
|
{
|
||||||
|
var i3Html = i3Prop.GetString();
|
||||||
|
if (!string.IsNullOrWhiteSpace(i3Html))
|
||||||
|
{
|
||||||
|
var imgMatch = Regex.Match(i3Html, @"<img\s+id=""img""\s+src=""([^""]+)""", RegexOptions.IgnoreCase);
|
||||||
|
if (imgMatch.Success)
|
||||||
|
{
|
||||||
|
return new UnifiedImageInfo
|
||||||
|
{
|
||||||
|
PageNumber = page,
|
||||||
|
ImageUrl = imgMatch.Groups[1].Value,
|
||||||
|
ThumbnailUrl = thumbUrl ?? imgMatch.Groups[1].Value,
|
||||||
|
AltText = $"{title} {page}ページ"
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed record EHPageEntry(int Page, string ImgKey, string Url, string? ThumbUrl);
|
||||||
}
|
}
|
||||||
@@ -24,46 +24,183 @@ public class UnifiedMomongaScraper : BaseUnifiedScraper
|
|||||||
|
|
||||||
public override async Task<UnifiedGalleryItem?> ScrapeAsync(string urlOrPath, CancellationToken ct = default)
|
public override async Task<UnifiedGalleryItem?> ScrapeAsync(string urlOrPath, CancellationToken ct = default)
|
||||||
{
|
{
|
||||||
var match = Regex.Match(urlOrPath, @"(?:momon-ga\.com/|momonga/)?(?<id>[a-z0-9_-]+)(?:\.html)?", RegexOptions.IgnoreCase);
|
var (targetUrl, normalizedId, resolvedCategory) = ResolveUrlAndId(urlOrPath);
|
||||||
var id = match.Success ? match.Groups["id"].Value : urlOrPath.Trim();
|
|
||||||
|
|
||||||
var pageUrl = urlOrPath.StartsWith("http") ? urlOrPath : $"{BaseUrl}/{id}.html";
|
Logger.LogInformation("[{SiteName}] スクレイピング開始 - ID: {Id}, Category: {Category}, URL: {Url}",
|
||||||
var doc = await LoadHtmlDocumentAsync(pageUrl, ct: ct);
|
SiteName, normalizedId, resolvedCategory, targetUrl);
|
||||||
if (doc == null) return null;
|
|
||||||
|
|
||||||
var title = doc.DocumentNode.SelectSingleNode("//h1")?.InnerText?.Trim() ?? $"モモンガ作品 ({id})";
|
var doc = await LoadHtmlDocumentAsync(targetUrl, ct: ct);
|
||||||
|
if (doc == null || doc.DocumentNode == null)
|
||||||
|
{
|
||||||
|
Logger.LogWarning("[{SiteName}] ドキュメントの読み込みに失敗しました: {Url}", SiteName, targetUrl);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
// タイトルの抽出(div#post-data h1、h1、titleタグ)
|
||||||
|
var titleNode = doc.DocumentNode.SelectSingleNode("//div[@id='post-data']//h1") ??
|
||||||
|
doc.DocumentNode.SelectSingleNode("//h1") ??
|
||||||
|
doc.DocumentNode.SelectSingleNode("//title");
|
||||||
|
var title = CleanText(titleNode?.InnerText);
|
||||||
|
if (string.IsNullOrWhiteSpace(title)) title = normalizedId;
|
||||||
|
|
||||||
|
// 余分なサイト名表記を除去(例: "新妻契約 - 同人誌 - エロ漫画 momon:GA(モモンガッ!!)" -> "新妻契約")
|
||||||
|
if (title.Contains(" - "))
|
||||||
|
{
|
||||||
|
title = title.Split(" - ")[0].Trim();
|
||||||
|
}
|
||||||
|
|
||||||
var item = new UnifiedGalleryItem
|
var item = new UnifiedGalleryItem
|
||||||
{
|
{
|
||||||
Id = id,
|
Id = normalizedId,
|
||||||
Title = CleanText(title),
|
Title = string.IsNullOrEmpty(title) ? normalizedId : title,
|
||||||
|
Category = resolvedCategory,
|
||||||
SourceSite = SiteName,
|
SourceSite = SiteName,
|
||||||
SourceUrl = pageUrl,
|
SourceUrl = targetUrl,
|
||||||
FetchedAt = DateTime.UtcNow
|
FetchedAt = DateTime.UtcNow
|
||||||
};
|
};
|
||||||
|
|
||||||
var imgNodes = doc.DocumentNode.SelectNodes("//div[contains(@class, 'entry-content')]//img | //div[contains(@class, 'post-content')]//img");
|
// サークル、作者、タグの抽出
|
||||||
|
ExtractTagsAndMetadata(doc, item);
|
||||||
|
|
||||||
|
// 画像一覧の抽出(#post-hentai img, .entry-content img, article img)
|
||||||
|
var imgNodes = doc.DocumentNode.SelectNodes("//div[@id='post-hentai']//img") ??
|
||||||
|
doc.DocumentNode.SelectNodes("//div[contains(@class, 'entry-content')]//img") ??
|
||||||
|
doc.DocumentNode.SelectNodes("//article//img");
|
||||||
|
|
||||||
if (imgNodes != null)
|
if (imgNodes != null)
|
||||||
{
|
{
|
||||||
int pageNum = 1;
|
int pageNum = 1;
|
||||||
foreach (var img in imgNodes)
|
foreach (var img in imgNodes)
|
||||||
{
|
{
|
||||||
var src = img.GetAttributeValue("src", "")?.Trim();
|
var src = img.GetAttributeValue("src", null) ??
|
||||||
if (!string.IsNullOrEmpty(src))
|
img.GetAttributeValue("data-src", null) ??
|
||||||
|
img.GetAttributeValue("data-lazy-src", null);
|
||||||
|
|
||||||
|
if (string.IsNullOrWhiteSpace(src)) continue;
|
||||||
|
|
||||||
|
// 広告画像やアイコン等を除外
|
||||||
|
if (src.Contains("themes", StringComparison.OrdinalIgnoreCase) ||
|
||||||
|
src.Contains("plugins", StringComparison.OrdinalIgnoreCase) ||
|
||||||
|
src.Contains("banner", StringComparison.OrdinalIgnoreCase) ||
|
||||||
|
src.Contains("ad_", StringComparison.OrdinalIgnoreCase))
|
||||||
{
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var alt = img.GetAttributeValue("alt", null);
|
||||||
|
int? width = int.TryParse(img.GetAttributeValue("width", null), out var w) ? w : null;
|
||||||
|
int? height = int.TryParse(img.GetAttributeValue("height", null), out var h) ? h : null;
|
||||||
|
|
||||||
item.Images.Add(new UnifiedImageInfo
|
item.Images.Add(new UnifiedImageInfo
|
||||||
{
|
{
|
||||||
PageNumber = pageNum++,
|
PageNumber = pageNum++,
|
||||||
ImageUrl = src,
|
ImageUrl = src.Trim(),
|
||||||
ThumbnailUrl = src,
|
ThumbnailUrl = src.Trim(),
|
||||||
AltText = $"{item.Title} - {pageNum - 1}ページ"
|
AltText = CleanText(alt) ?? $"{item.Title} - {pageNum - 1}ページ",
|
||||||
|
Width = width,
|
||||||
|
Height = height
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// サムネイルの決定(data-thumb属性、または1枚目の画像)
|
||||||
|
var historyNode = doc.DocumentNode.SelectSingleNode("//div[contains(@class, 'history')]");
|
||||||
|
var dataThumb = historyNode?.GetAttributeValue("data-thumb", null);
|
||||||
|
if (!string.IsNullOrWhiteSpace(dataThumb))
|
||||||
|
{
|
||||||
|
item.ThumbnailUrl = dataThumb.Trim();
|
||||||
|
}
|
||||||
|
else if (item.Images.Count > 0)
|
||||||
|
{
|
||||||
|
item.ThumbnailUrl = item.Images[0].ImageUrl;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (item.Images.Count > 0) item.ThumbnailUrl = item.Images[0].ThumbnailUrl;
|
Logger.LogInformation("[{SiteName}] スクレイピング完了: 『{Title}』, 画像数: {Count}枚",
|
||||||
return item;
|
SiteName, item.Title, item.Images.Count);
|
||||||
|
|
||||||
|
return item.Images.Count > 0 ? item : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static (string TargetUrl, string NormalizedId, string ResolvedCategory) ResolveUrlAndId(string idOrUrl)
|
||||||
|
{
|
||||||
|
var trimmed = idOrUrl.Trim();
|
||||||
|
string resolvedCategory = "fanzine";
|
||||||
|
string normalizedId;
|
||||||
|
|
||||||
|
if (trimmed.StartsWith("http://", StringComparison.OrdinalIgnoreCase) ||
|
||||||
|
trimmed.StartsWith("https://", StringComparison.OrdinalIgnoreCase))
|
||||||
|
{
|
||||||
|
var uri = new Uri(trimmed);
|
||||||
|
var segments = uri.AbsolutePath.Split('/', StringSplitOptions.RemoveEmptyEntries);
|
||||||
|
if (segments.Length >= 2)
|
||||||
|
{
|
||||||
|
resolvedCategory = segments[0];
|
||||||
|
normalizedId = segments[1];
|
||||||
|
}
|
||||||
|
else if (segments.Length == 1)
|
||||||
|
{
|
||||||
|
normalizedId = segments[0];
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
normalizedId = trimmed;
|
||||||
|
}
|
||||||
|
|
||||||
|
return ($"{BaseUrl}/{resolvedCategory}/{normalizedId}/", normalizedId, resolvedCategory);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (trimmed.Contains('/'))
|
||||||
|
{
|
||||||
|
var parts = trimmed.Split('/', StringSplitOptions.RemoveEmptyEntries);
|
||||||
|
if (parts.Length >= 2)
|
||||||
|
{
|
||||||
|
resolvedCategory = parts[0];
|
||||||
|
normalizedId = parts[1];
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
normalizedId = parts[0];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
normalizedId = trimmed;
|
||||||
|
}
|
||||||
|
|
||||||
|
return ($"{BaseUrl}/{resolvedCategory}/{normalizedId}/", normalizedId, resolvedCategory);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void ExtractTagsAndMetadata(HtmlDocument doc, UnifiedGalleryItem item)
|
||||||
|
{
|
||||||
|
var tagTables = doc.DocumentNode.SelectNodes("//div[@id='post-tag']//div[contains(@class, 'post-tag-table')]");
|
||||||
|
if (tagTables == null) return;
|
||||||
|
|
||||||
|
foreach (var table in tagTables)
|
||||||
|
{
|
||||||
|
var titleNode = table.SelectSingleNode(".//div[contains(@class, 'post-tag-title')]");
|
||||||
|
var titleText = CleanText(titleNode?.InnerText);
|
||||||
|
|
||||||
|
var linkNodes = table.SelectNodes(".//div[contains(@class, 'post-tags')]//a");
|
||||||
|
if (linkNodes == null) continue;
|
||||||
|
|
||||||
|
var tagValues = linkNodes.Select(n => CleanText(n.InnerText)).Where(s => !string.IsNullOrEmpty(s)).ToList();
|
||||||
|
|
||||||
|
if (titleText.Contains("サークル", StringComparison.OrdinalIgnoreCase) || titleText.Contains("グループ", StringComparison.OrdinalIgnoreCase))
|
||||||
|
{
|
||||||
|
item.Circle = string.Join(", ", tagValues);
|
||||||
|
}
|
||||||
|
else if (titleText.Contains("作者", StringComparison.OrdinalIgnoreCase) || titleText.Contains("漫画家", StringComparison.OrdinalIgnoreCase))
|
||||||
|
{
|
||||||
|
item.Author = string.Join(", ", tagValues);
|
||||||
|
}
|
||||||
|
else if (titleText.Contains("内容", StringComparison.OrdinalIgnoreCase) || titleText.Contains("タグ", StringComparison.OrdinalIgnoreCase) || titleText.Contains("キャラクター", StringComparison.OrdinalIgnoreCase))
|
||||||
|
{
|
||||||
|
foreach (var tag in tagValues)
|
||||||
|
{
|
||||||
|
if (!item.Tags.Contains(tag)) item.Tags.Add(tag);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -73,6 +210,7 @@ public class UnifiedMomongaScraper : BaseUnifiedScraper
|
|||||||
public class UnifiedAuraHentaiScraper : BaseUnifiedScraper
|
public class UnifiedAuraHentaiScraper : BaseUnifiedScraper
|
||||||
{
|
{
|
||||||
private const string BaseUrl = "https://aurahentai.com";
|
private const string BaseUrl = "https://aurahentai.com";
|
||||||
|
private const string ImageBaseUrl = "https://img.aurahentai.com";
|
||||||
|
|
||||||
public override string SiteName => "AuraHentai";
|
public override string SiteName => "AuraHentai";
|
||||||
public override string SiteKey => "aurahentai";
|
public override string SiteKey => "aurahentai";
|
||||||
@@ -85,48 +223,181 @@ public class UnifiedAuraHentaiScraper : BaseUnifiedScraper
|
|||||||
|
|
||||||
public override async Task<UnifiedGalleryItem?> ScrapeAsync(string urlOrPath, CancellationToken ct = default)
|
public override async Task<UnifiedGalleryItem?> ScrapeAsync(string urlOrPath, CancellationToken ct = default)
|
||||||
{
|
{
|
||||||
var match = Regex.Match(urlOrPath, @"(?:g/|manga/|aurahentai/)?(?<id>[a-z0-9_-]+)", RegexOptions.IgnoreCase);
|
var (targetUrl, normalizedId) = ResolveUrlAndId(urlOrPath);
|
||||||
var id = match.Success ? match.Groups["id"].Value : urlOrPath.Trim();
|
|
||||||
|
|
||||||
var pageUrl = urlOrPath.StartsWith("http") ? urlOrPath : $"{BaseUrl}/g/{id}";
|
Logger.LogInformation("[{SiteName}] スクレイピング開始 - ID: {Id}, URL: {Url}", SiteName, normalizedId, targetUrl);
|
||||||
var doc = await LoadHtmlDocumentAsync(pageUrl, ct: ct);
|
|
||||||
if (doc == null) return null;
|
|
||||||
|
|
||||||
var title = doc.DocumentNode.SelectSingleNode("//h1")?.InnerText?.Trim() ?? $"AuraHentai 作品 ({id})";
|
var doc = await LoadHtmlDocumentAsync(targetUrl, useFlareSolverr: true, ct: ct);
|
||||||
|
if (doc == null || doc.DocumentNode == null)
|
||||||
|
{
|
||||||
|
Logger.LogWarning("[{SiteName}] ドキュメントの読み込みに失敗しました: {Url}", SiteName, targetUrl);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
var titleNode = doc.DocumentNode.SelectSingleNode("//main//h1[not(@id='age-gate-title')]") ??
|
||||||
|
doc.DocumentNode.SelectSingleNode("//h1[not(@id='age-gate-title')]") ??
|
||||||
|
doc.DocumentNode.SelectSingleNode("//title");
|
||||||
|
var title = CleanText(titleNode?.InnerText) ?? normalizedId;
|
||||||
|
|
||||||
|
if (title.Contains(" ? "))
|
||||||
|
{
|
||||||
|
title = title.Split(" ? ")[0].Trim();
|
||||||
|
}
|
||||||
|
else if (title.Contains(" | "))
|
||||||
|
{
|
||||||
|
title = title.Split(" | ")[0].Trim();
|
||||||
|
}
|
||||||
|
|
||||||
var item = new UnifiedGalleryItem
|
var item = new UnifiedGalleryItem
|
||||||
{
|
{
|
||||||
Id = id,
|
Id = normalizedId,
|
||||||
Title = CleanText(title),
|
Title = string.IsNullOrEmpty(title) ? normalizedId : title,
|
||||||
SourceSite = SiteName,
|
SourceSite = SiteName,
|
||||||
SourceUrl = pageUrl,
|
SourceUrl = targetUrl,
|
||||||
FetchedAt = DateTime.UtcNow
|
FetchedAt = DateTime.UtcNow
|
||||||
};
|
};
|
||||||
|
|
||||||
var imgNodes = doc.DocumentNode.SelectNodes("//div[contains(@class, 'reader-area')]//img | //div[@id='reader']//img | //div[contains(@class, 'gallery-thumb')]//img");
|
int pageCount = ExtractMetadata(doc, item);
|
||||||
|
|
||||||
|
// 画像一覧の抽出
|
||||||
|
var imgNodes = doc.DocumentNode.SelectNodes("//div[contains(@class, 'page-grid')]//img") ??
|
||||||
|
doc.DocumentNode.SelectNodes("//main//img[contains(@src, '/g/')]");
|
||||||
|
|
||||||
if (imgNodes != null)
|
if (imgNodes != null)
|
||||||
{
|
{
|
||||||
int pageNum = 1;
|
int pageNum = 1;
|
||||||
foreach (var img in imgNodes)
|
foreach (var img in imgNodes)
|
||||||
{
|
{
|
||||||
var src = img.GetAttributeValue("data-src", "")?.Trim();
|
var src = img.GetAttributeValue("src", null) ??
|
||||||
if (string.IsNullOrEmpty(src)) src = img.GetAttributeValue("src", "")?.Trim();
|
img.GetAttributeValue("data-src", null);
|
||||||
|
|
||||||
if (!string.IsNullOrEmpty(src))
|
if (string.IsNullOrWhiteSpace(src) || src.Contains("cover.webp") || src.Contains("logo"))
|
||||||
{
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var thumbUrl = src.StartsWith("http") ? src : $"{ImageBaseUrl}{src}";
|
||||||
|
var fullImageUrl = thumbUrl.Replace("/thumb/g/", "/g/");
|
||||||
|
var alt = img.GetAttributeValue("alt", null);
|
||||||
|
|
||||||
item.Images.Add(new UnifiedImageInfo
|
item.Images.Add(new UnifiedImageInfo
|
||||||
{
|
{
|
||||||
PageNumber = pageNum++,
|
PageNumber = pageNum++,
|
||||||
ImageUrl = src,
|
ImageUrl = fullImageUrl,
|
||||||
ThumbnailUrl = src,
|
ThumbnailUrl = thumbUrl,
|
||||||
AltText = $"{item.Title} - {pageNum - 1}ページ"
|
AltText = CleanText(alt) ?? $"{item.Title} {pageNum - 1}ページ"
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// HTMLから取得できなかった場合、ページ数情報があれば連番URLを補完生成
|
||||||
|
if (item.Images.Count == 0 && pageCount > 0)
|
||||||
|
{
|
||||||
|
for (int i = 1; i <= pageCount; i++)
|
||||||
|
{
|
||||||
|
var pageStr = i.ToString("D3");
|
||||||
|
item.Images.Add(new UnifiedImageInfo
|
||||||
|
{
|
||||||
|
PageNumber = i,
|
||||||
|
ImageUrl = $"{ImageBaseUrl}/g/{normalizedId}/{pageStr}.webp",
|
||||||
|
ThumbnailUrl = $"{ImageBaseUrl}/thumb/g/{normalizedId}/{pageStr}.webp",
|
||||||
|
AltText = $"{item.Title} {i}ページ"
|
||||||
|
});
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (item.Images.Count > 0) item.ThumbnailUrl = item.Images[0].ThumbnailUrl;
|
if (item.Images.Count > 0) item.ThumbnailUrl = item.Images[0].ThumbnailUrl;
|
||||||
return item;
|
return item.Images.Count > 0 ? item : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static (string TargetUrl, string NormalizedId) ResolveUrlAndId(string idOrUrl)
|
||||||
|
{
|
||||||
|
var trimmed = idOrUrl.Trim();
|
||||||
|
string normalizedId;
|
||||||
|
|
||||||
|
if (trimmed.StartsWith("http://", StringComparison.OrdinalIgnoreCase) ||
|
||||||
|
trimmed.StartsWith("https://", StringComparison.OrdinalIgnoreCase))
|
||||||
|
{
|
||||||
|
var uri = new Uri(trimmed);
|
||||||
|
var segments = uri.AbsolutePath.Split('/', StringSplitOptions.RemoveEmptyEntries);
|
||||||
|
normalizedId = segments.Length > 0 ? segments[^1] : trimmed;
|
||||||
|
}
|
||||||
|
else if (trimmed.Contains('/'))
|
||||||
|
{
|
||||||
|
var parts = trimmed.Split('/', StringSplitOptions.RemoveEmptyEntries);
|
||||||
|
normalizedId = parts.Length > 0 ? parts[^1] : trimmed;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
normalizedId = trimmed;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (int.TryParse(normalizedId, out var num) && normalizedId.Length < 6)
|
||||||
|
{
|
||||||
|
normalizedId = num.ToString("D6");
|
||||||
|
}
|
||||||
|
|
||||||
|
var fullUrl = $"{BaseUrl}/gallery/{normalizedId}";
|
||||||
|
return (fullUrl, normalizedId);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static int ExtractMetadata(HtmlDocument doc, UnifiedGalleryItem item)
|
||||||
|
{
|
||||||
|
int detectedPageCount = 0;
|
||||||
|
var dlNode = doc.DocumentNode.SelectSingleNode("//main//dl");
|
||||||
|
if (dlNode == null) return detectedPageCount;
|
||||||
|
|
||||||
|
var dtNodes = dlNode.SelectNodes(".//dt");
|
||||||
|
if (dtNodes == null) return detectedPageCount;
|
||||||
|
|
||||||
|
foreach (var dt in dtNodes)
|
||||||
|
{
|
||||||
|
var label = CleanText(dt.InnerText).ToLowerInvariant();
|
||||||
|
var dd = dt.ParentNode?.SelectSingleNode(".//dd");
|
||||||
|
if (dd == null) continue;
|
||||||
|
|
||||||
|
var links = dd.SelectNodes(".//a");
|
||||||
|
var values = links != null
|
||||||
|
? links.Select(a => CleanText(a.InnerText)).Where(s => !string.IsNullOrEmpty(s)).ToList()
|
||||||
|
: new List<string> { CleanText(dd.InnerText) };
|
||||||
|
|
||||||
|
switch (label)
|
||||||
|
{
|
||||||
|
case "artists":
|
||||||
|
case "artist":
|
||||||
|
item.Author = string.Join(", ", values);
|
||||||
|
break;
|
||||||
|
case "groups":
|
||||||
|
case "group":
|
||||||
|
case "circles":
|
||||||
|
case "circle":
|
||||||
|
item.Circle = string.Join(", ", values);
|
||||||
|
break;
|
||||||
|
case "category":
|
||||||
|
if (values.Count > 0) item.Category = values[0];
|
||||||
|
break;
|
||||||
|
case "tags":
|
||||||
|
case "characters":
|
||||||
|
case "character":
|
||||||
|
case "parodies":
|
||||||
|
case "parody":
|
||||||
|
foreach (var val in values)
|
||||||
|
{
|
||||||
|
if (!item.Tags.Contains(val)) item.Tags.Add(val);
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
case "pages":
|
||||||
|
var rawPages = CleanText(dd.InnerText);
|
||||||
|
var match = Regex.Match(rawPages, @"\d+");
|
||||||
|
if (match.Success && int.TryParse(match.Value, out var count))
|
||||||
|
{
|
||||||
|
detectedPageCount = count;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return detectedPageCount;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -148,47 +419,183 @@ public class UnifiedMangalearScraper : BaseUnifiedScraper
|
|||||||
|
|
||||||
public override async Task<UnifiedGalleryItem?> ScrapeAsync(string urlOrPath, CancellationToken ct = default)
|
public override async Task<UnifiedGalleryItem?> ScrapeAsync(string urlOrPath, CancellationToken ct = default)
|
||||||
{
|
{
|
||||||
var match = Regex.Match(urlOrPath, @"(?:manga/|mangalear/)?(?<id>[a-z0-9_-]+)", RegexOptions.IgnoreCase);
|
var (targetUrl, normalizedId, resolvedCategory) = ResolveUrlAndId(urlOrPath);
|
||||||
var id = match.Success ? match.Groups["id"].Value : urlOrPath.Trim();
|
|
||||||
|
|
||||||
var pageUrl = urlOrPath.StartsWith("http") ? urlOrPath : $"{BaseUrl}/manga/{id}";
|
Logger.LogInformation("[{SiteName}] スクレイピング開始 - ID: {Id}, URL: {Url}", SiteName, normalizedId, targetUrl);
|
||||||
var doc = await LoadHtmlDocumentAsync(pageUrl, ct: ct);
|
|
||||||
if (doc == null) return null;
|
|
||||||
|
|
||||||
var title = doc.DocumentNode.SelectSingleNode("//h1")?.InnerText?.Trim() ?? $"Mangalear 作品 ({id})";
|
var doc = await LoadHtmlDocumentAsync(targetUrl, ct: ct);
|
||||||
|
if (doc == null || doc.DocumentNode == null)
|
||||||
|
{
|
||||||
|
Logger.LogWarning("[{SiteName}] ドキュメントの読み込みに失敗しました: {Url}", SiteName, targetUrl);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
var titleNode = doc.DocumentNode.SelectSingleNode("//h1[contains(@class, 'entry-title')]") ??
|
||||||
|
doc.DocumentNode.SelectSingleNode("//h1") ??
|
||||||
|
doc.DocumentNode.SelectSingleNode("//title");
|
||||||
|
var title = CleanText(titleNode?.InnerText) ?? normalizedId;
|
||||||
|
|
||||||
|
// タイトルの整形(先頭の「▲」記号や末尾のサイト名を除去)
|
||||||
|
title = Regex.Replace(title, @"^[▲\s]+", string.Empty);
|
||||||
|
if (title.Contains(" - "))
|
||||||
|
{
|
||||||
|
title = title.Split(" - ")[0].Trim();
|
||||||
|
}
|
||||||
|
|
||||||
var item = new UnifiedGalleryItem
|
var item = new UnifiedGalleryItem
|
||||||
{
|
{
|
||||||
Id = id,
|
Id = normalizedId,
|
||||||
Title = CleanText(title),
|
Title = string.IsNullOrEmpty(title) ? normalizedId : title,
|
||||||
|
Category = resolvedCategory,
|
||||||
SourceSite = SiteName,
|
SourceSite = SiteName,
|
||||||
SourceUrl = pageUrl,
|
SourceUrl = targetUrl,
|
||||||
FetchedAt = DateTime.UtcNow
|
FetchedAt = DateTime.UtcNow
|
||||||
};
|
};
|
||||||
|
|
||||||
var imgNodes = doc.DocumentNode.SelectNodes("//div[contains(@class, 'page-break')]//img | //div[contains(@class, 'reading-content')]//img");
|
ExtractMetadata(doc, item);
|
||||||
|
|
||||||
|
// 画像一覧の抽出
|
||||||
|
var contentNode = doc.DocumentNode.SelectSingleNode("//div[contains(@class, 'entry-content')]") ??
|
||||||
|
doc.DocumentNode.SelectSingleNode("//article");
|
||||||
|
|
||||||
|
if (contentNode != null)
|
||||||
|
{
|
||||||
|
var imgNodes = contentNode.SelectNodes(".//img");
|
||||||
if (imgNodes != null)
|
if (imgNodes != null)
|
||||||
{
|
{
|
||||||
|
var seenUrls = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||||
int pageNum = 1;
|
int pageNum = 1;
|
||||||
|
|
||||||
foreach (var img in imgNodes)
|
foreach (var img in imgNodes)
|
||||||
{
|
{
|
||||||
var src = img.GetAttributeValue("data-src", "")?.Trim();
|
var cssClass = img.GetAttributeValue("class", string.Empty);
|
||||||
if (string.IsNullOrEmpty(src)) src = img.GetAttributeValue("src", "")?.Trim();
|
if (cssClass.Contains("wp-smiley") || cssClass.Contains("avatar")) continue;
|
||||||
|
|
||||||
if (!string.IsNullOrEmpty(src))
|
string? highResUrl = null;
|
||||||
|
string? thumbUrl = null;
|
||||||
|
|
||||||
|
var parentAnchor = img.ParentNode?.Name.Equals("a", StringComparison.OrdinalIgnoreCase) == true
|
||||||
|
? img.ParentNode
|
||||||
|
: img.Ancestors("a").FirstOrDefault();
|
||||||
|
|
||||||
|
if (parentAnchor != null)
|
||||||
{
|
{
|
||||||
|
var href = parentAnchor.GetAttributeValue("href", null)?.Trim();
|
||||||
|
if (!string.IsNullOrEmpty(href) && IsImageExtension(href))
|
||||||
|
{
|
||||||
|
highResUrl = href;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var src = img.GetAttributeValue("src", null)?.Trim();
|
||||||
|
var dataSrc = img.GetAttributeValue("data-src", null)?.Trim();
|
||||||
|
var candidateSrc = !string.IsNullOrEmpty(dataSrc) ? dataSrc : src;
|
||||||
|
|
||||||
|
if (string.IsNullOrEmpty(highResUrl))
|
||||||
|
{
|
||||||
|
highResUrl = candidateSrc;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
thumbUrl = candidateSrc;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (string.IsNullOrEmpty(highResUrl) || !seenUrls.Add(highResUrl)) continue;
|
||||||
|
|
||||||
|
var alt = img.GetAttributeValue("alt", null);
|
||||||
item.Images.Add(new UnifiedImageInfo
|
item.Images.Add(new UnifiedImageInfo
|
||||||
{
|
{
|
||||||
PageNumber = pageNum++,
|
PageNumber = pageNum++,
|
||||||
ImageUrl = src,
|
ImageUrl = highResUrl,
|
||||||
ThumbnailUrl = src,
|
ThumbnailUrl = !string.IsNullOrEmpty(thumbUrl) ? thumbUrl : highResUrl,
|
||||||
AltText = $"{item.Title} - {pageNum - 1}ページ"
|
AltText = CleanText(alt) ?? $"{item.Title} - {pageNum - 1}ページ"
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (item.Images.Count > 0) item.ThumbnailUrl = item.Images[0].ThumbnailUrl;
|
if (item.Images.Count > 0) item.ThumbnailUrl = item.Images[0].ThumbnailUrl;
|
||||||
return item;
|
return item.Images.Count > 0 ? item : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static (string TargetUrl, string NormalizedId, string Category) ResolveUrlAndId(string idOrUrl)
|
||||||
|
{
|
||||||
|
var trimmed = idOrUrl.Trim();
|
||||||
|
string normalizedId = string.Empty;
|
||||||
|
string resolvedCategory = "doujinshi";
|
||||||
|
|
||||||
|
if (trimmed.StartsWith("http://", StringComparison.OrdinalIgnoreCase) ||
|
||||||
|
trimmed.StartsWith("https://", StringComparison.OrdinalIgnoreCase))
|
||||||
|
{
|
||||||
|
var uri = new Uri(trimmed);
|
||||||
|
var query = System.Web.HttpUtility.ParseQueryString(uri.Query);
|
||||||
|
var pVal = query.Get("p");
|
||||||
|
if (!string.IsNullOrWhiteSpace(pVal))
|
||||||
|
{
|
||||||
|
normalizedId = pVal.Trim();
|
||||||
|
return ($"{BaseUrl}/?p={normalizedId}", normalizedId, resolvedCategory);
|
||||||
|
}
|
||||||
|
|
||||||
|
var segments = uri.AbsolutePath.Split('/', StringSplitOptions.RemoveEmptyEntries);
|
||||||
|
if (segments.Length >= 2)
|
||||||
|
{
|
||||||
|
resolvedCategory = segments[0];
|
||||||
|
normalizedId = segments[1];
|
||||||
|
}
|
||||||
|
else if (segments.Length == 1)
|
||||||
|
{
|
||||||
|
normalizedId = segments[0];
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
normalizedId = trimmed;
|
||||||
|
}
|
||||||
|
|
||||||
|
return ($"{BaseUrl}/{resolvedCategory}/{normalizedId}/", normalizedId, resolvedCategory);
|
||||||
|
}
|
||||||
|
|
||||||
|
return ($"{BaseUrl}/{resolvedCategory}/{trimmed}/", trimmed, resolvedCategory);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void ExtractMetadata(HtmlDocument doc, UnifiedGalleryItem item)
|
||||||
|
{
|
||||||
|
var catNodes = doc.DocumentNode.SelectNodes("//div[contains(@class, 'entry-categories')]//a[contains(@class, 'cat-link')]");
|
||||||
|
if (catNodes != null)
|
||||||
|
{
|
||||||
|
foreach (var node in catNodes)
|
||||||
|
{
|
||||||
|
var catText = CleanText(node.InnerText);
|
||||||
|
if (!string.IsNullOrEmpty(catText))
|
||||||
|
{
|
||||||
|
item.Category = catText;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var tagNodes = doc.DocumentNode.SelectNodes("//div[contains(@class, 'entry-tags')]//a[contains(@class, 'tag-link')]");
|
||||||
|
if (tagNodes != null)
|
||||||
|
{
|
||||||
|
foreach (var node in tagNodes)
|
||||||
|
{
|
||||||
|
var tag = CleanText(node.InnerText);
|
||||||
|
if (!string.IsNullOrEmpty(tag) && !item.Tags.Contains(tag))
|
||||||
|
{
|
||||||
|
item.Tags.Add(tag);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var authorNode = doc.DocumentNode.SelectSingleNode("//div[contains(@class, 'author-info')]//*[contains(@class, 'author-name')]");
|
||||||
|
if (authorNode != null)
|
||||||
|
{
|
||||||
|
item.Author = CleanText(authorNode.InnerText);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static bool IsImageExtension(string url)
|
||||||
|
{
|
||||||
|
var ext = Path.GetExtension(url.Split('?')[0]).ToLowerInvariant();
|
||||||
|
return ext is ".jpg" or ".jpeg" or ".png" or ".webp" or ".gif" or ".bmp";
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Reference in new issue
Block a user