fix(scraper): Hitomi/HentaiPaw等のスクレイピング抽出ロジックを修正

This commit is contained in:
armonica committed 2026-10-06 08:11:44 +09:00
1 parent b415c74601
commit 295d1f0731
4 files changed
+118 -35

No files matched your search

@@ -168,8 +168,8 @@ public class HitomiScraper : BaseScraper
// 高解像度 WebP 画像URLの計算
var rawImageUrl = BuildWebpImageUrl(hash, ggData);
// サムネイル画像URLの計算
var rawThumbUrl = BuildThumbnailUrl(hash, ggData);
// サムネイル画像URLの計算 (1ページ目は大サムネイル、2ページ目以降は通常サムネイル)
var rawThumbUrl = BuildThumbnailUrl(hash, ggData, isBig: pageNum == 1);
// hitomi.laの画像CDNはReferer制限があるため、プロキシURLを介して確実に配信
var referer = $"{BaseUrl}/reader/{galleryId}.html";
@@ -273,29 +273,55 @@ public class HitomiScraper : BaseScraper
return cached;
}
// フォールバック用の正時タイムスタンプ(UTC毎正時秒 + 1)
var nowSec = DateTimeOffset.UtcNow.ToUnixTimeSeconds();
var fallbackB = $"{nowSec / 3600 * 3600 + 1}/";
try
{
var ggJs = await HttpHelper.GetHtmlAsync($"{LtnDomain}/gg.js", cancellationToken);
var ggData = ParseGgJs(ggJs);
_cache.Set(cacheKey, ggData, GgCacheDuration);
var ggData = ParseGgJs(ggJs, fallbackB);
// 次の正時(0分)までの残り時間、または最大15分でキャッシュ
var nextHour = DateTimeOffset.FromUnixTimeSeconds((nowSec / 3600 + 1) * 3600);
var cacheDuration = nextHour - DateTimeOffset.UtcNow;
if (cacheDuration > GgCacheDuration) cacheDuration = GgCacheDuration;
if (cacheDuration < TimeSpan.FromSeconds(30)) cacheDuration = TimeSpan.FromSeconds(30);
_cache.Set(cacheKey, ggData, cacheDuration);
Logger.LogInformation("[{SiteName}] gg.js を更新しました (プレフィックス: {Prefix}, Case数: {Cases})",
SiteName, ggData.B, ggData.Cases.Count);
return ggData;
}
catch (Exception ex)
{
Logger.LogError(ex, "[{SiteName}] gg.js の取得に失敗しました。デフォルト値を使用します", SiteName);
return new GgData("1788710402/", new HashSet<int>());
Logger.LogError(ex, "[{SiteName}] gg.js の取得に失敗しました。動的フォールバック値を使用します", SiteName);
return new GgData(fallbackB, new HashSet<int>(), 0, 1);
}
}
/// <summary>
/// gg.js の文字列を解析して GgData を構築します。
/// </summary>
private static GgData ParseGgJs(string ggJs)
private static GgData ParseGgJs(string ggJs, string fallbackB)
{
var bMatch = Regex.Match(ggJs, @"b:\s*['""]([^'""]+)['""]");
var b = bMatch.Success ? bMatch.Groups[1].Value : "1788710402/";
var b = bMatch.Success ? bMatch.Groups[1].Value : fallbackB;
// gg.js の m(g) 関数内の初期値とcaseマッチ時の値をパース(Hitomi側の仕様変更に動的追従)
int defaultO = 0;
var defaultMatch = Regex.Match(ggJs, @"var\s+o\s*=\s*(\d+)");
if (defaultMatch.Success && int.TryParse(defaultMatch.Groups[1].Value, out var dVal))
{
defaultO = dVal;
}
int matchedO = 1;
var matchedMatch = Regex.Match(ggJs, @"o\s*=\s*(\d+);\s*break;");
if (matchedMatch.Success && int.TryParse(matchedMatch.Groups[1].Value, out var mVal))
{
matchedO = mVal;
}
var cases = new HashSet<int>();
var caseMatches = Regex.Matches(ggJs, @"case\s+(\d+):");
@@ -307,7 +333,7 @@ public class HitomiScraper : BaseScraper
}
}
return new GgData(b, cases);
return new GgData(b, cases, defaultO, matchedO);
}
/// <summary>
@@ -323,9 +349,8 @@ public class HitomiScraper : BaseScraper
var g = int.Parse(m2 + m1, NumberStyles.HexNumber);
var ggS = g.ToString(CultureInfo.InvariantCulture);
// gg.js の仕様: caseに含まれる数値は 0 (w1), 含まれないものは 1 (w2)
// (var o = 1; switch (g) { case ...: o = 0; break; } return o;)
var mVal = gg.Cases.Contains(g) ? 0 : 1;
// gg.js の m(g) ロジック: caseに一致した場合は MatchedO、不一致なら DefaultO
var mVal = gg.Cases.Contains(g) ? gg.MatchedO : gg.DefaultO;
var subNum = 1 + mVal;
return $"https://w{subNum}.{ContentDomain}/{gg.B}{ggS}/{hash}.webp";
@@ -334,7 +359,7 @@ public class HitomiScraper : BaseScraper
/// <summary>
/// ハッシュと ggData からサムネイル画像 URL を構築します。
/// </summary>
private static string BuildThumbnailUrl(string hash, GgData gg)
private static string BuildThumbnailUrl(string hash, GgData gg, bool isBig = false)
{
if (hash.Length < 3) return string.Empty;
@@ -343,14 +368,15 @@ public class HitomiScraper : BaseScraper
var m2 = hash.Substring(hash.Length - 1, 1);
var g = int.Parse(m2 + m1, NumberStyles.HexNumber);
// gg.js の仕様: caseに含まれる数値は 0 ('a' -> atn), 含まれないものは 1 ('b' -> btn)
var mVal = gg.Cases.Contains(g) ? 0 : 1;
// gg.js の m(g) ロジック: caseに一致した場合は MatchedO、不一致なら DefaultO
var mVal = gg.Cases.Contains(g) ? gg.MatchedO : gg.DefaultO;
var subLet = (char)('a' + mVal);
var tnDir = isBig ? "webpbigtn" : "webpsmalltn";
return $"https://{subLet}tn.{ContentDomain}/webpbigtn/{m2}/{m1}/{hash}.webp";
return $"https://{subLet}tn.{ContentDomain}/{tnDir}/{m2}/{m1}/{hash}.webp";
}
private record GgData(string B, HashSet<int> Cases);
private record GgData(string B, HashSet<int> Cases, int DefaultO, int MatchedO);
}
@@ -36,8 +36,23 @@ public class UnifiedHentaiPawScraper : BaseUnifiedScraper
var viewerUrl = $"{BaseUrl}/viewer?articleId={articleId}&page=1";
Logger.LogInformation("[{SiteName}] スクレイピング開始 - ID: {Id}", SiteName, articleId);
// 1. まず本編画像URLを含むビュワーページをFlareSolverr経由で取得
var viewerDoc = await LoadHtmlDocumentAsync(viewerUrl, useFlareSolverr: true, ct: ct);
var articleDoc = await LoadHtmlDocumentAsync(articleUrl, useFlareSolverr: true, ct: ct);
// 2. 記事詳細ページ(メタデータ用)の取得
// まず通常取得を試み、失敗した場合のみFlareSolverrを使用
HtmlDocument? articleDoc = null;
try
{
articleDoc = await LoadHtmlDocumentAsync(articleUrl, useFlareSolverr: false, ct: ct);
}
catch { }
// ビュワーが取得できず詳細も通常取得できなかった場合は詳細をFlareSolverrで試行
if (viewerDoc == null && articleDoc == null)
{
articleDoc = await LoadHtmlDocumentAsync(articleUrl, useFlareSolverr: true, ct: ct);
}
if (viewerDoc == null && articleDoc == null)
{
@@ -81,6 +96,13 @@ public class UnifiedHentaiPawScraper : BaseUnifiedScraper
}
ExtractImages(viewerDoc, articleDoc, galleryItem, articleId);
if (galleryItem.Images.Count == 0)
{
Logger.LogWarning("[{SiteName}] 画像URLの抽出に失敗しました: ID={Id}", SiteName, articleId);
return null;
}
return galleryItem;
}
@@ -130,7 +152,7 @@ public class UnifiedHentaiPawScraper : BaseUnifiedScraper
if (viewerDoc?.DocumentNode != null)
{
var rawHtml = viewerDoc.DocumentNode.OuterHtml;
var pattern = $@"https://cdn\.imagedeliveries\.com/{Regex.Escape(articleId)}/[a-f0-9]{{32,64}}/(?<page>\d+)\.webp";
var pattern = $@"https?://cdn\.imagedeliveries\.com/{Regex.Escape(articleId)}/[a-f0-9]{{32,64}}/(?<page>\d+)\.(?:webp|jpg|png|jpeg)";
var matches = Regex.Matches(rawHtml, pattern, RegexOptions.IgnoreCase);
var pageMap = new Dictionary<int, string>();
@@ -164,7 +186,7 @@ public class UnifiedHentaiPawScraper : BaseUnifiedScraper
if (collectedImages.Count == 0 && articleDoc?.DocumentNode != null)
{
var rawHtml = articleDoc.DocumentNode.OuterHtml;
var thumbPattern = $@"https://cdn\.imagedeliveries\.com/{Regex.Escape(articleId)}/thumbnails/(?<page>\d+)\.webp";
var thumbPattern = $@"https?://cdn\.imagedeliveries\.com/{Regex.Escape(articleId)}/thumbnails/(?<page>\d+)\.(?:webp|jpg|png|jpeg)";
var matches = Regex.Matches(rawHtml, thumbPattern, RegexOptions.IgnoreCase);
var pages = new SortedSet<int>();
@@ -115,7 +115,8 @@ public class UnifiedHitomiScraper : BaseUnifiedScraper
if (string.IsNullOrWhiteSpace(hash)) continue;
var rawImageUrl = BuildWebpImageUrl(hash, ggData);
var rawThumbUrl = BuildThumbnailUrl(hash, ggData);
// 1ページ目は大サムネイル(webpbigtn)、2ページ目以降は通常サムネイル(webpsmalltn)
var rawThumbUrl = BuildThumbnailUrl(hash, ggData, isBig: pageNum == 1);
var referer = $"{BaseUrl}/reader/{galleryId}.html";
var proxyImageUrl = $"/api/proxy-image?url={UrlEncoder.Default.Encode(rawImageUrl)}&referer={UrlEncoder.Default.Encode(referer)}";
var proxyThumbUrl = $"/api/proxy-image?url={UrlEncoder.Default.Encode(rawThumbUrl)}&referer={UrlEncoder.Default.Encode(referer)}";
@@ -180,11 +181,30 @@ public class UnifiedHitomiScraper : BaseUnifiedScraper
const string cacheKey = "unified_hitomi_gg_data";
if (_cache.TryGetValue(cacheKey, out GgData? cached) && cached != null) return cached;
// フォールバック用の正時タイムスタンプ(UTC毎正時秒 + 1)
var nowSec = DateTimeOffset.UtcNow.ToUnixTimeSeconds();
var fallbackB = $"{nowSec / 3600 * 3600 + 1}/";
try
{
var ggJs = await HttpHelper.GetHtmlAsync($"{LtnDomain}/gg.js", ct: ct);
var bMatch = Regex.Match(ggJs, @"b:\s*['""]([^'""]+)['""]");
var b = bMatch.Success ? bMatch.Groups[1].Value : "1788710402/";
var b = bMatch.Success ? bMatch.Groups[1].Value : fallbackB;
// gg.js の m(g) 関数内の初期値とcaseマッチ時の値をパース(Hitomi側の仕様変更に動的追従)
int defaultO = 0;
var defaultMatch = Regex.Match(ggJs, @"var\s+o\s*=\s*(\d+)");
if (defaultMatch.Success && int.TryParse(defaultMatch.Groups[1].Value, out var dVal))
{
defaultO = dVal;
}
int matchedO = 1;
var matchedMatch = Regex.Match(ggJs, @"o\s*=\s*(\d+);\s*break;");
if (matchedMatch.Success && int.TryParse(matchedMatch.Groups[1].Value, out var mVal))
{
matchedO = mVal;
}
var cases = new HashSet<int>();
foreach (Match m in Regex.Matches(ggJs, @"case\s+(\d+):"))
@@ -192,13 +212,21 @@ public class UnifiedHitomiScraper : BaseUnifiedScraper
if (int.TryParse(m.Groups[1].Value, out var c)) cases.Add(c);
}
var result = new GgData(b, cases);
_cache.Set(cacheKey, result, GgCacheDuration);
var result = new GgData(b, cases, defaultO, matchedO);
// 次の正時(0分)までの残り時間、または最大15分でキャッシュ
var nextHour = DateTimeOffset.FromUnixTimeSeconds((nowSec / 3600 + 1) * 3600);
var cacheDuration = nextHour - DateTimeOffset.UtcNow;
if (cacheDuration > GgCacheDuration) cacheDuration = GgCacheDuration;
if (cacheDuration < TimeSpan.FromSeconds(30)) cacheDuration = TimeSpan.FromSeconds(30);
_cache.Set(cacheKey, result, cacheDuration);
return result;
}
catch
catch (Exception ex)
{
return new GgData("1788710402/", new HashSet<int>());
Logger.LogWarning(ex, "[{SiteName}] gg.js の取得・パースに失敗したため、動的フォールバック値を使用します", SiteName);
return new GgData(fallbackB, new HashSet<int>(), 0, 1);
}
}
@@ -208,19 +236,25 @@ public class UnifiedHitomiScraper : BaseUnifiedScraper
var m1 = hash.Substring(hash.Length - 3, 2);
var m2 = hash.Substring(hash.Length - 1, 1);
var g = int.Parse(m2 + m1, NumberStyles.HexNumber);
var subNum = 1 + (gg.Cases.Contains(g) ? 0 : 1);
// gg.js の m(g) ロジック: caseに一致した場合は MatchedO、不一致なら DefaultO
var mVal = gg.Cases.Contains(g) ? gg.MatchedO : gg.DefaultO;
var subNum = 1 + mVal;
return $"https://w{subNum}.{ContentDomain}/{gg.B}{g}/{hash}.webp";
}
private static string BuildThumbnailUrl(string hash, GgData gg)
private static string BuildThumbnailUrl(string hash, GgData gg, bool isBig = false)
{
if (hash.Length < 3) return "";
var m1 = hash.Substring(hash.Length - 3, 2);
var m2 = hash.Substring(hash.Length - 1, 1);
var g = int.Parse(m2 + m1, NumberStyles.HexNumber);
var subLet = (char)('a' + (gg.Cases.Contains(g) ? 0 : 1));
return $"https://{subLet}tn.{ContentDomain}/webpbigtn/{m2}/{m1}/{hash}.webp";
// gg.js の m(g) ロジック: caseに一致した場合は MatchedO、不一致なら DefaultO
var mVal = gg.Cases.Contains(g) ? gg.MatchedO : gg.DefaultO;
var subLet = (char)('a' + mVal);
// 1ページ目(カバー)は webpbigtn、2ページ目以降のサムネイルは webpsmalltn
var tnDir = isBig ? "webpbigtn" : "webpsmalltn";
return $"https://{subLet}tn.{ContentDomain}/{tnDir}/{m2}/{m1}/{hash}.webp";
}
private record GgData(string B, HashSet<int> Cases);
private record GgData(string B, HashSet<int> Cases, int DefaultO, int MatchedO);
}
@@ -406,11 +406,11 @@ public class UnifiedAuraHentaiScraper : BaseUnifiedScraper
/// </summary>
public class UnifiedMangalearScraper : BaseUnifiedScraper
{
private const string BaseUrl = "https://mangalear.com";
private const string BaseUrl = "https://www.mangalear.blog";
public override string SiteName => "Mangalear";
public override string SiteKey => "mangalear";
public override IReadOnlyList<string> SupportedDomains => new[] { "mangalear.com", "mangalear" };
public override IReadOnlyList<string> SupportedDomains => new[] { "mangalear.blog", "www.mangalear.blog", "mangalear.com", "mangalear" };
public UnifiedMangalearScraper(UnifiedHttpHelper httpHelper, ILogger<UnifiedMangalearScraper> logger)
: base(httpHelper, logger)
@@ -528,12 +528,13 @@ public class UnifiedMangalearScraper : BaseUnifiedScraper
trimmed.StartsWith("https://", StringComparison.OrdinalIgnoreCase))
{
var uri = new Uri(trimmed);
var hostBase = uri.GetLeftPart(UriPartial.Authority);
var query = System.Web.HttpUtility.ParseQueryString(uri.Query);
var pVal = query.Get("p");
if (!string.IsNullOrWhiteSpace(pVal))
{
normalizedId = pVal.Trim();
return ($"{BaseUrl}/?p={normalizedId}", normalizedId, resolvedCategory);
return ($"{hostBase}/?p={normalizedId}", normalizedId, resolvedCategory);
}
var segments = uri.AbsolutePath.Split('/', StringSplitOptions.RemoveEmptyEntries);
@@ -551,7 +552,7 @@ public class UnifiedMangalearScraper : BaseUnifiedScraper
normalizedId = trimmed;
}
return ($"{BaseUrl}/{resolvedCategory}/{normalizedId}/", normalizedId, resolvedCategory);
return ($"{hostBase}/{resolvedCategory}/{normalizedId}/", normalizedId, resolvedCategory);
}
return ($"{BaseUrl}/{resolvedCategory}/{trimmed}/", trimmed, resolvedCategory);