修正: 大量画像ZIP化時のページ抜け防止と10秒取得試行・中断/続行ダイアログの実装
- ページ遷移直後のZIPダウンロード開始時にバックグラウンド先読みリクエストを即時中断・一時停止し接続競合を解消 - 3桁を超える大量画像でも抜けが発生しないようページ順に10秒間繰り返し取得を試行するロジックへ改修 - 10秒間試行して取得できない場合にエラーメッセージおよび「中断」「続行」ボタンをダイアログ上に表示して処理を制御 - E-Hentaiスクレイパーの大量ページ取得時の並列数制御とリトライを追加
This commit is contained in:
1 parent
bc8bce8de4
commit
cd202a1719
2 files changed
+433
-152
No files matched your search
@@ -319,12 +319,41 @@ public static class UnifiedHtmlRenderer
|
||||
border-radius: 6px;
|
||||
}
|
||||
|
||||
.zip-error-prompt {
|
||||
font-size: 0.88rem;
|
||||
line-height: 1.5;
|
||||
color: #fecaca;
|
||||
background: rgba(239, 68, 68, 0.15);
|
||||
border: 1px solid rgba(239, 68, 68, 0.4);
|
||||
padding: 12px 14px;
|
||||
border-radius: 8px;
|
||||
margin-top: 4px;
|
||||
}
|
||||
|
||||
.zip-modal-footer {
|
||||
padding: 14px 24px;
|
||||
background: rgba(0, 0, 0, 0.2);
|
||||
border-top: 1px solid var(--border-subtle);
|
||||
display: flex;
|
||||
justify-content: flex-end;
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
.zip-continue-btn {
|
||||
background: var(--accent-gradient-gallery);
|
||||
border: 1px solid transparent;
|
||||
color: #fff;
|
||||
padding: 8px 18px;
|
||||
border-radius: 8px;
|
||||
font-size: 0.85rem;
|
||||
font-weight: 700;
|
||||
cursor: pointer;
|
||||
transition: all 0.2s;
|
||||
}
|
||||
|
||||
.zip-continue-btn:hover {
|
||||
filter: brightness(1.1);
|
||||
transform: translateY(-1px);
|
||||
}
|
||||
|
||||
.zip-cancel-btn {
|
||||
@@ -1773,16 +1802,40 @@ public static class UnifiedHtmlRenderer
|
||||
this.images = images || [];
|
||||
this.loadedPages = new Set();
|
||||
this.loadingPages = new Set();
|
||||
this.activeImages = new Map();
|
||||
this.priorityQueue = [];
|
||||
this.cursor = 0;
|
||||
this.concurrency = 2; // 通信詰まりを起こさず最高効率で先読み
|
||||
this.activeCount = 0;
|
||||
this.paused = false;
|
||||
}
|
||||
|
||||
start() {
|
||||
this.paused = false;
|
||||
this.pump();
|
||||
}
|
||||
|
||||
pause() {
|
||||
this.paused = true;
|
||||
// 進行中の先読みリクエストを中断してブラウザの同時接続枠を即座に解放
|
||||
for (const [page, img] of this.activeImages.entries()) {
|
||||
img.onload = null;
|
||||
img.onerror = null;
|
||||
img.src = '';
|
||||
this.loadingPages.delete(page);
|
||||
this.priorityQueue.unshift(page);
|
||||
}
|
||||
this.activeImages.clear();
|
||||
this.activeCount = 0;
|
||||
}
|
||||
|
||||
resume() {
|
||||
if (this.paused) {
|
||||
this.paused = false;
|
||||
this.pump();
|
||||
}
|
||||
}
|
||||
|
||||
prioritize(page) {
|
||||
if (page < 1 || page > this.images.length) return;
|
||||
if (this.loadedPages.has(page) || this.loadingPages.has(page)) return;
|
||||
@@ -1792,6 +1845,7 @@ public static class UnifiedHtmlRenderer
|
||||
}
|
||||
|
||||
pump() {
|
||||
if (this.paused) return;
|
||||
while (this.activeCount < this.concurrency) {
|
||||
let targetPage = null;
|
||||
if (this.priorityQueue.length > 0) {
|
||||
@@ -1820,10 +1874,12 @@ public static class UnifiedHtmlRenderer
|
||||
|
||||
const preImg = new Image();
|
||||
preImg.decoding = 'async';
|
||||
this.activeImages.set(page, preImg);
|
||||
|
||||
const finalize = (success) => {
|
||||
this.activeImages.delete(page);
|
||||
this.loadingPages.delete(page);
|
||||
this.activeCount--;
|
||||
this.activeCount = Math.max(0, this.activeCount - 1);
|
||||
if (success) {
|
||||
this.loadedPages.add(page);
|
||||
this.applyLoadedImage(page, item.imageUrl);
|
||||
@@ -2234,9 +2290,11 @@ public static class UnifiedHtmlRenderer
|
||||
</div>
|
||||
<div class="zip-counter-text" id="zipCounterText">0 / {{totalPages}} ページ</div>
|
||||
<div class="zip-warning-text" id="zipWarningText" style="display: none;"></div>
|
||||
<div class="zip-error-prompt" id="zipErrorPrompt" style="display: none;"></div>
|
||||
</div>
|
||||
<div class="zip-modal-footer">
|
||||
<button type="button" class="zip-cancel-btn" id="zipCancelBtn">中断する</button>
|
||||
<button type="button" class="zip-continue-btn" id="zipContinueBtn" style="display: none;">続行</button>
|
||||
<button type="button" class="zip-cancel-btn" id="zipCancelBtn">中断</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -2357,11 +2415,106 @@ public static class UnifiedHtmlRenderer
|
||||
}
|
||||
|
||||
// ==========================================
|
||||
// 保存先事前指定 & 進捗モーダル付きZIPダウンロード
|
||||
// 保存先事前指定 & 10秒取得試行・中断/続行ダイアログ付きZIPダウンロード
|
||||
// ==========================================
|
||||
let isZipDownloading = false;
|
||||
let zipAbortController = null;
|
||||
|
||||
// 1ページ分の画像を最大10秒間繰り返し試行して取得する関数
|
||||
async function fetchSinglePageWithin10Seconds(imgInfo, refererUrl, total, onStatusUpdate) {
|
||||
const RETRY_LIMIT_MS = 10000; // 10秒間試行
|
||||
const startTime = Date.now();
|
||||
const pNum = imgInfo.pageNumber;
|
||||
const rawUrl = imgInfo.imageUrl;
|
||||
const isAlreadyProxy = rawUrl.startsWith('/api/proxy-image');
|
||||
const proxyUrl = isAlreadyProxy
|
||||
? rawUrl
|
||||
: `/api/proxy-image?url=${encodeURIComponent(rawUrl)}&referer=${encodeURIComponent(refererUrl || '')}`;
|
||||
|
||||
let attempt = 0;
|
||||
while (Date.now() - startTime < RETRY_LIMIT_MS) {
|
||||
if (zipAbortController && zipAbortController.signal.aborted) {
|
||||
throw new Error('Canceled');
|
||||
}
|
||||
|
||||
attempt++;
|
||||
const elapsed = Date.now() - startTime;
|
||||
const remainingMs = RETRY_LIMIT_MS - elapsed;
|
||||
if (remainingMs <= 0) break;
|
||||
|
||||
if (attempt > 1 && onStatusUpdate) {
|
||||
const remainSec = Math.ceil(remainingMs / 1000);
|
||||
onStatusUpdate(`${pNum} / ${total} ページを再試行中... (残り約 ${remainSec} 秒)`);
|
||||
}
|
||||
|
||||
// 外部ドメインのCORS失敗を避けるため、2回目以降またはプロキシURLの場合はproxyUrlを使用
|
||||
const targetFetchUrl = (!isAlreadyProxy && attempt === 1) ? rawUrl : proxyUrl;
|
||||
|
||||
const fetchCtrl = new AbortController();
|
||||
const abortListener = () => fetchCtrl.abort();
|
||||
if (zipAbortController) {
|
||||
zipAbortController.signal.addEventListener('abort', abortListener, { once: true });
|
||||
}
|
||||
const timeoutId = setTimeout(() => fetchCtrl.abort(), Math.min(remainingMs, 5000));
|
||||
|
||||
try {
|
||||
const resp = await fetch(targetFetchUrl, {
|
||||
signal: fetchCtrl.signal,
|
||||
cache: 'force-cache'
|
||||
});
|
||||
clearTimeout(timeoutId);
|
||||
if (zipAbortController) {
|
||||
zipAbortController.signal.removeEventListener('abort', abortListener);
|
||||
}
|
||||
|
||||
if (!resp.ok) {
|
||||
throw new Error(`HTTP ${resp.status}`);
|
||||
}
|
||||
|
||||
const buf = await resp.arrayBuffer();
|
||||
const data = new Uint8Array(buf);
|
||||
if (data.length === 0) {
|
||||
throw new Error('Empty image data');
|
||||
}
|
||||
|
||||
let ext = '.webp';
|
||||
const cType = (resp.headers.get('content-type') || '').toLowerCase();
|
||||
if (cType.includes('png')) ext = '.png';
|
||||
else if (cType.includes('jpeg') || cType.includes('jpg')) ext = '.jpg';
|
||||
else if (cType.includes('gif')) ext = '.gif';
|
||||
else if (cType.includes('avif')) ext = '.avif';
|
||||
else if (cType.includes('webp')) ext = '.webp';
|
||||
else {
|
||||
const urlPart = rawUrl.split('?')[0];
|
||||
const dotExt = urlPart.split('.').pop();
|
||||
if (dotExt && dotExt.length <= 4 && /^[a-z0-9]+$/i.test(dotExt)) {
|
||||
ext = '.' + dotExt.toLowerCase();
|
||||
}
|
||||
}
|
||||
|
||||
return { data, ext };
|
||||
} catch (err) {
|
||||
clearTimeout(timeoutId);
|
||||
if (zipAbortController) {
|
||||
zipAbortController.signal.removeEventListener('abort', abortListener);
|
||||
}
|
||||
if (zipAbortController && zipAbortController.signal.aborted) {
|
||||
throw new Error('Canceled');
|
||||
}
|
||||
|
||||
const afterElapsed = Date.now() - startTime;
|
||||
const waitMs = Math.min(600, RETRY_LIMIT_MS - afterElapsed);
|
||||
if (waitMs > 50) {
|
||||
await new Promise(r => setTimeout(r, waitMs));
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
async function startZipDownloadWithProgress(gallery) {
|
||||
if (isZipDownloading) return;
|
||||
if (!gallery || !gallery.images || gallery.images.length === 0) {
|
||||
@@ -2386,108 +2539,113 @@ public static class UnifiedHtmlRenderer
|
||||
});
|
||||
} catch (err) {
|
||||
if (err.name === 'AbortError') {
|
||||
// ユーザーが保存先選択をキャンセルした場合は処理を中断
|
||||
return;
|
||||
}
|
||||
console.warn('showSaveFilePicker fallback:', err);
|
||||
}
|
||||
}
|
||||
|
||||
// 2. モーダル表示と初期化
|
||||
// 2. バックグラウンド先読みダウンローダーが動いている場合は一時停止し、帯域・接続競合を防止
|
||||
if (typeof downloader !== 'undefined' && downloader && typeof downloader.pause === 'function') {
|
||||
downloader.pause();
|
||||
}
|
||||
|
||||
// 3. モーダル表示と初期化
|
||||
const modal = document.getElementById('zipProgressModal');
|
||||
const progressBar = document.getElementById('zipProgressBar');
|
||||
const statusText = document.getElementById('zipStatusText');
|
||||
const percentText = document.getElementById('zipPercentText');
|
||||
const counterText = document.getElementById('zipCounterText');
|
||||
const warningText = document.getElementById('zipWarningText');
|
||||
const errorPrompt = document.getElementById('zipErrorPrompt');
|
||||
const spinner = document.getElementById('zipModalSpinner');
|
||||
const continueBtn = document.getElementById('zipContinueBtn');
|
||||
const cancelBtn = document.getElementById('zipCancelBtn');
|
||||
const modalTitle = document.getElementById('zipModalTitle');
|
||||
|
||||
isZipDownloading = true;
|
||||
zipAbortController = new AbortController();
|
||||
|
||||
const sortedImages = [...gallery.images].sort((a, b) => a.pageNumber - b.pageNumber);
|
||||
const total = sortedImages.length;
|
||||
const padLen = Math.max(3, String(total).length);
|
||||
|
||||
modal.classList.add('active');
|
||||
progressBar.style.width = '0%';
|
||||
percentText.textContent = '0%';
|
||||
counterText.textContent = `0 / ${gallery.images.length} ページ`;
|
||||
counterText.textContent = `0 / ${total} ページ`;
|
||||
statusText.textContent = '画像のダウンロードを準備しています...';
|
||||
warningText.style.display = 'none';
|
||||
warningText.textContent = '';
|
||||
errorPrompt.style.display = 'none';
|
||||
errorPrompt.textContent = '';
|
||||
continueBtn.style.display = 'none';
|
||||
spinner.classList.remove('done');
|
||||
cancelBtn.textContent = '中断する';
|
||||
cancelBtn.textContent = '中断';
|
||||
modalTitle.textContent = '📦 ZIP一括保存中...';
|
||||
|
||||
cancelBtn.onclick = () => {
|
||||
const cleanupAndClose = () => {
|
||||
if (zipAbortController) zipAbortController.abort();
|
||||
modal.classList.remove('active');
|
||||
isZipDownloading = false;
|
||||
if (typeof downloader !== 'undefined' && downloader && typeof downloader.resume === 'function') {
|
||||
downloader.resume();
|
||||
}
|
||||
};
|
||||
|
||||
const total = gallery.images.length;
|
||||
cancelBtn.onclick = cleanupAndClose;
|
||||
|
||||
let completed = 0;
|
||||
let failedCount = 0;
|
||||
let skippedPages = [];
|
||||
const zip = new SimpleZip();
|
||||
|
||||
// 並列度3で順次確実に取得(相手サーバーへの負荷軽減とブロック防止)
|
||||
const concurrency = 3;
|
||||
let queueIndex = 0;
|
||||
|
||||
async function downloadWorker() {
|
||||
while (queueIndex < gallery.images.length) {
|
||||
try {
|
||||
// 3桁を超える大量画像でもプロキシや相手サーバーの詰まり・抜けが起きないよう1ページずつ順番に確実に取得
|
||||
for (let i = 0; i < sortedImages.length; i++) {
|
||||
if (zipAbortController.signal.aborted) throw new Error('Canceled');
|
||||
const idx = queueIndex++;
|
||||
const imgInfo = gallery.images[idx];
|
||||
const pNum = imgInfo.pageNumber;
|
||||
|
||||
const imgInfo = sortedImages[i];
|
||||
const pNum = imgInfo.pageNumber;
|
||||
statusText.textContent = `${pNum} / ${total} ページを取得中...`;
|
||||
|
||||
let data = null;
|
||||
let ext = '.webp';
|
||||
const result = await fetchSinglePageWithin10Seconds(
|
||||
imgInfo,
|
||||
gallery.sourceUrl,
|
||||
total,
|
||||
(msg) => { statusText.textContent = msg; }
|
||||
);
|
||||
|
||||
// 最大3回リトライを行い、一時的な失敗による抜けを徹底防止
|
||||
for (let attempt = 1; attempt <= 3; attempt++) {
|
||||
if (zipAbortController.signal.aborted) throw new Error('Canceled');
|
||||
try {
|
||||
const fetchCtrl = new AbortController();
|
||||
const timeoutId = setTimeout(() => fetchCtrl.abort(), 20000); // 20秒タイムアウト
|
||||
if (result && result.data && result.data.length > 0) {
|
||||
const entryName = `${String(pNum).padStart(padLen, '0')}${result.ext}`;
|
||||
zip.addFile(entryName, result.data);
|
||||
} else {
|
||||
// 10秒間試行しても取得できなかった場合:メッセージと「中断」「続行」ボタンをダイアログ上に表示して待機
|
||||
spinner.classList.add('done');
|
||||
modalTitle.textContent = '⚠️ 画像の取得に失敗しました';
|
||||
statusText.textContent = `${pNum} / ${total} ページの取得タイムアウト (10秒経過)`;
|
||||
errorPrompt.style.display = 'block';
|
||||
errorPrompt.textContent = `ページ ${pNum} の画像を10秒間取得試行しましたが取得できませんでした。処理を中断して終了しますか?それともこのページをスキップして続行しますか?`;
|
||||
continueBtn.style.display = 'inline-block';
|
||||
cancelBtn.textContent = '中断';
|
||||
|
||||
const resp = await fetch(imgInfo.imageUrl, {
|
||||
signal: fetchCtrl.signal
|
||||
const userChoice = await new Promise((resolve) => {
|
||||
continueBtn.onclick = () => resolve('continue');
|
||||
cancelBtn.onclick = () => resolve('abort');
|
||||
});
|
||||
clearTimeout(timeoutId);
|
||||
|
||||
if (!resp.ok) throw new Error(`HTTP ${resp.status}`);
|
||||
|
||||
const cType = resp.headers.get('content-type') || '';
|
||||
if (cType.includes('png')) ext = '.png';
|
||||
else if (cType.includes('jpeg') || cType.includes('jpg')) ext = '.jpg';
|
||||
else if (cType.includes('gif')) ext = '.gif';
|
||||
else if (cType.includes('webp')) ext = '.webp';
|
||||
else {
|
||||
const urlPart = imgInfo.imageUrl.split('?')[0];
|
||||
const dotExt = urlPart.split('.').pop();
|
||||
if (dotExt && dotExt.length <= 4) ext = '.' + dotExt;
|
||||
if (userChoice === 'abort') {
|
||||
// 中断が選択された場合は処理を終了
|
||||
cleanupAndClose();
|
||||
return;
|
||||
}
|
||||
|
||||
const buf = await resp.arrayBuffer();
|
||||
data = new Uint8Array(buf);
|
||||
break;
|
||||
} catch (e) {
|
||||
if (zipAbortController.signal.aborted) throw new Error('Canceled');
|
||||
if (attempt < 3) {
|
||||
await new Promise(r => setTimeout(r, attempt * 1000));
|
||||
} else {
|
||||
console.warn(`Page ${pNum} download failed after 3 attempts:`, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (data && data.length > 0) {
|
||||
const entryName = `${String(pNum).padStart(3, '0')}${ext}`;
|
||||
zip.addFile(entryName, data);
|
||||
} else {
|
||||
failedCount++;
|
||||
// 続行が選択された場合はダイアログ表示を戻して次のページへ進む
|
||||
skippedPages.push(pNum);
|
||||
errorPrompt.style.display = 'none';
|
||||
continueBtn.style.display = 'none';
|
||||
spinner.classList.remove('done');
|
||||
modalTitle.textContent = '📦 ZIP一括保存中...';
|
||||
cancelBtn.onclick = cleanupAndClose;
|
||||
}
|
||||
|
||||
completed++;
|
||||
@@ -2496,19 +2654,20 @@ public static class UnifiedHtmlRenderer
|
||||
percentText.textContent = `${pct}%`;
|
||||
counterText.textContent = `${completed} / ${total} ページ完了`;
|
||||
|
||||
if (failedCount > 0) {
|
||||
if (skippedPages.length > 0) {
|
||||
warningText.style.display = 'block';
|
||||
warningText.textContent = `※ ${failedCount} 件の画像がタイムアウト等のためスキップされました`;
|
||||
}
|
||||
warningText.textContent = `※ スキップ済み (${skippedPages.length}件): ページ ${skippedPages.join(', ')}`;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const workers = [];
|
||||
for (let i = 0; i < Math.min(concurrency, gallery.images.length); i++) {
|
||||
workers.push(downloadWorker());
|
||||
if (zip.files.length === 0) {
|
||||
spinner.classList.add('done');
|
||||
modalTitle.textContent = '⚠️ 保存できる画像がありません';
|
||||
statusText.textContent = '有効な画像を取得できなかったため処理を終了しました。';
|
||||
cancelBtn.textContent = '閉じる';
|
||||
cancelBtn.onclick = cleanupAndClose;
|
||||
return;
|
||||
}
|
||||
await Promise.all(workers);
|
||||
|
||||
statusText.textContent = 'ZIPアーカイブを作成しています...';
|
||||
const zipBytes = zip.generateUint8Array();
|
||||
@@ -2536,14 +2695,12 @@ public static class UnifiedHtmlRenderer
|
||||
spinner.classList.add('done');
|
||||
statusText.textContent = `全 ${zip.files.length} 枚の画像を正常に保存しました!`;
|
||||
cancelBtn.textContent = '閉じる';
|
||||
cancelBtn.onclick = () => {
|
||||
modal.classList.remove('active');
|
||||
isZipDownloading = false;
|
||||
};
|
||||
cancelBtn.onclick = cleanupAndClose;
|
||||
|
||||
setTimeout(() => {
|
||||
modal.classList.remove('active');
|
||||
isZipDownloading = false;
|
||||
if (isZipDownloading) {
|
||||
cleanupAndClose();
|
||||
}
|
||||
}, 2200);
|
||||
|
||||
} catch (err) {
|
||||
@@ -2551,10 +2708,16 @@ public static class UnifiedHtmlRenderer
|
||||
console.log('Download canceled by user');
|
||||
} else {
|
||||
console.error('ZIP generation error:', err);
|
||||
statusText.textContent = 'ダウンロード中にエラーが発生しました。';
|
||||
spinner.classList.add('done');
|
||||
modalTitle.textContent = '⚠️ エラーが発生しました';
|
||||
statusText.textContent = 'ダウンロード処理中にエラーが発生したため終了しました。';
|
||||
cancelBtn.textContent = '閉じる';
|
||||
cancelBtn.onclick = cleanupAndClose;
|
||||
}
|
||||
isZipDownloading = false;
|
||||
if (typeof downloader !== 'undefined' && downloader && typeof downloader.resume === 'function') {
|
||||
downloader.resume();
|
||||
}
|
||||
}
|
||||
}
|
||||
</script>
|
||||
@@ -3018,9 +3181,11 @@ public static class UnifiedHtmlRenderer
|
||||
</div>
|
||||
<div class="zip-counter-text" id="zipCounterText">0 / {{totalPages}} ページ</div>
|
||||
<div class="zip-warning-text" id="zipWarningText" style="display: none;"></div>
|
||||
<div class="zip-error-prompt" id="zipErrorPrompt" style="display: none;"></div>
|
||||
</div>
|
||||
<div class="zip-modal-footer">
|
||||
<button type="button" class="zip-cancel-btn" id="zipCancelBtn">中断する</button>
|
||||
<button type="button" class="zip-continue-btn" id="zipContinueBtn" style="display: none;">続行</button>
|
||||
<button type="button" class="zip-cancel-btn" id="zipCancelBtn">中断</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -3141,11 +3306,106 @@ public static class UnifiedHtmlRenderer
|
||||
}
|
||||
|
||||
// ==========================================
|
||||
// 保存先事前指定 & 進捗モーダル付きZIPダウンロード
|
||||
// 保存先事前指定 & 10秒取得試行・中断/続行ダイアログ付きZIPダウンロード
|
||||
// ==========================================
|
||||
let isZipDownloading = false;
|
||||
let zipAbortController = null;
|
||||
|
||||
// 1ページ分の画像を最大10秒間繰り返し試行して取得する関数
|
||||
async function fetchSinglePageWithin10Seconds(imgInfo, refererUrl, total, onStatusUpdate) {
|
||||
const RETRY_LIMIT_MS = 10000; // 10秒間試行
|
||||
const startTime = Date.now();
|
||||
const pNum = imgInfo.pageNumber;
|
||||
const rawUrl = imgInfo.imageUrl;
|
||||
const isAlreadyProxy = rawUrl.startsWith('/api/proxy-image');
|
||||
const proxyUrl = isAlreadyProxy
|
||||
? rawUrl
|
||||
: `/api/proxy-image?url=${encodeURIComponent(rawUrl)}&referer=${encodeURIComponent(refererUrl || '')}`;
|
||||
|
||||
let attempt = 0;
|
||||
while (Date.now() - startTime < RETRY_LIMIT_MS) {
|
||||
if (zipAbortController && zipAbortController.signal.aborted) {
|
||||
throw new Error('Canceled');
|
||||
}
|
||||
|
||||
attempt++;
|
||||
const elapsed = Date.now() - startTime;
|
||||
const remainingMs = RETRY_LIMIT_MS - elapsed;
|
||||
if (remainingMs <= 0) break;
|
||||
|
||||
if (attempt > 1 && onStatusUpdate) {
|
||||
const remainSec = Math.ceil(remainingMs / 1000);
|
||||
onStatusUpdate(`${pNum} / ${total} ページを再試行中... (残り約 ${remainSec} 秒)`);
|
||||
}
|
||||
|
||||
// 外部ドメインのCORS失敗を避けるため、2回目以降またはプロキシURLの場合はproxyUrlを使用
|
||||
const targetFetchUrl = (!isAlreadyProxy && attempt === 1) ? rawUrl : proxyUrl;
|
||||
|
||||
const fetchCtrl = new AbortController();
|
||||
const abortListener = () => fetchCtrl.abort();
|
||||
if (zipAbortController) {
|
||||
zipAbortController.signal.addEventListener('abort', abortListener, { once: true });
|
||||
}
|
||||
const timeoutId = setTimeout(() => fetchCtrl.abort(), Math.min(remainingMs, 5000));
|
||||
|
||||
try {
|
||||
const resp = await fetch(targetFetchUrl, {
|
||||
signal: fetchCtrl.signal,
|
||||
cache: 'force-cache'
|
||||
});
|
||||
clearTimeout(timeoutId);
|
||||
if (zipAbortController) {
|
||||
zipAbortController.signal.removeEventListener('abort', abortListener);
|
||||
}
|
||||
|
||||
if (!resp.ok) {
|
||||
throw new Error(`HTTP ${resp.status}`);
|
||||
}
|
||||
|
||||
const buf = await resp.arrayBuffer();
|
||||
const data = new Uint8Array(buf);
|
||||
if (data.length === 0) {
|
||||
throw new Error('Empty image data');
|
||||
}
|
||||
|
||||
let ext = '.webp';
|
||||
const cType = (resp.headers.get('content-type') || '').toLowerCase();
|
||||
if (cType.includes('png')) ext = '.png';
|
||||
else if (cType.includes('jpeg') || cType.includes('jpg')) ext = '.jpg';
|
||||
else if (cType.includes('gif')) ext = '.gif';
|
||||
else if (cType.includes('avif')) ext = '.avif';
|
||||
else if (cType.includes('webp')) ext = '.webp';
|
||||
else {
|
||||
const urlPart = rawUrl.split('?')[0];
|
||||
const dotExt = urlPart.split('.').pop();
|
||||
if (dotExt && dotExt.length <= 4 && /^[a-z0-9]+$/i.test(dotExt)) {
|
||||
ext = '.' + dotExt.toLowerCase();
|
||||
}
|
||||
}
|
||||
|
||||
return { data, ext };
|
||||
} catch (err) {
|
||||
clearTimeout(timeoutId);
|
||||
if (zipAbortController) {
|
||||
zipAbortController.signal.removeEventListener('abort', abortListener);
|
||||
}
|
||||
if (zipAbortController && zipAbortController.signal.aborted) {
|
||||
throw new Error('Canceled');
|
||||
}
|
||||
|
||||
const afterElapsed = Date.now() - startTime;
|
||||
const waitMs = Math.min(600, RETRY_LIMIT_MS - afterElapsed);
|
||||
if (waitMs > 50) {
|
||||
await new Promise(r => setTimeout(r, waitMs));
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
async function startZipDownloadWithProgress(gallery) {
|
||||
if (isZipDownloading) return;
|
||||
if (!gallery || !gallery.images || gallery.images.length === 0) {
|
||||
@@ -3170,108 +3430,113 @@ public static class UnifiedHtmlRenderer
|
||||
});
|
||||
} catch (err) {
|
||||
if (err.name === 'AbortError') {
|
||||
// ユーザーが保存先選択をキャンセルした場合は処理を中断
|
||||
return;
|
||||
}
|
||||
console.warn('showSaveFilePicker fallback:', err);
|
||||
}
|
||||
}
|
||||
|
||||
// 2. モーダル表示と初期化
|
||||
// 2. バックグラウンド先読みダウンローダーが動いている場合は一時停止し、帯域・接続競合を防止
|
||||
if (typeof downloader !== 'undefined' && downloader && typeof downloader.pause === 'function') {
|
||||
downloader.pause();
|
||||
}
|
||||
|
||||
// 3. モーダル表示と初期化
|
||||
const modal = document.getElementById('zipProgressModal');
|
||||
const progressBar = document.getElementById('zipProgressBar');
|
||||
const statusText = document.getElementById('zipStatusText');
|
||||
const percentText = document.getElementById('zipPercentText');
|
||||
const counterText = document.getElementById('zipCounterText');
|
||||
const warningText = document.getElementById('zipWarningText');
|
||||
const errorPrompt = document.getElementById('zipErrorPrompt');
|
||||
const spinner = document.getElementById('zipModalSpinner');
|
||||
const continueBtn = document.getElementById('zipContinueBtn');
|
||||
const cancelBtn = document.getElementById('zipCancelBtn');
|
||||
const modalTitle = document.getElementById('zipModalTitle');
|
||||
|
||||
isZipDownloading = true;
|
||||
zipAbortController = new AbortController();
|
||||
|
||||
const sortedImages = [...gallery.images].sort((a, b) => a.pageNumber - b.pageNumber);
|
||||
const total = sortedImages.length;
|
||||
const padLen = Math.max(3, String(total).length);
|
||||
|
||||
modal.classList.add('active');
|
||||
progressBar.style.width = '0%';
|
||||
percentText.textContent = '0%';
|
||||
counterText.textContent = `0 / ${gallery.images.length} ページ`;
|
||||
counterText.textContent = `0 / ${total} ページ`;
|
||||
statusText.textContent = '画像のダウンロードを準備しています...';
|
||||
warningText.style.display = 'none';
|
||||
warningText.textContent = '';
|
||||
errorPrompt.style.display = 'none';
|
||||
errorPrompt.textContent = '';
|
||||
continueBtn.style.display = 'none';
|
||||
spinner.classList.remove('done');
|
||||
cancelBtn.textContent = '中断する';
|
||||
cancelBtn.textContent = '中断';
|
||||
modalTitle.textContent = '📦 ZIP一括保存中...';
|
||||
|
||||
cancelBtn.onclick = () => {
|
||||
const cleanupAndClose = () => {
|
||||
if (zipAbortController) zipAbortController.abort();
|
||||
modal.classList.remove('active');
|
||||
isZipDownloading = false;
|
||||
if (typeof downloader !== 'undefined' && downloader && typeof downloader.resume === 'function') {
|
||||
downloader.resume();
|
||||
}
|
||||
};
|
||||
|
||||
const total = gallery.images.length;
|
||||
cancelBtn.onclick = cleanupAndClose;
|
||||
|
||||
let completed = 0;
|
||||
let failedCount = 0;
|
||||
let skippedPages = [];
|
||||
const zip = new SimpleZip();
|
||||
|
||||
// 並列度3で順次確実に取得(相手サーバーへの負荷軽減とブロック防止)
|
||||
const concurrency = 3;
|
||||
let queueIndex = 0;
|
||||
|
||||
async function downloadWorker() {
|
||||
while (queueIndex < gallery.images.length) {
|
||||
try {
|
||||
// 3桁を超える大量画像でもプロキシや相手サーバーの詰まり・抜けが起きないよう1ページずつ順番に確実に取得
|
||||
for (let i = 0; i < sortedImages.length; i++) {
|
||||
if (zipAbortController.signal.aborted) throw new Error('Canceled');
|
||||
const idx = queueIndex++;
|
||||
const imgInfo = gallery.images[idx];
|
||||
const pNum = imgInfo.pageNumber;
|
||||
|
||||
const imgInfo = sortedImages[i];
|
||||
const pNum = imgInfo.pageNumber;
|
||||
statusText.textContent = `${pNum} / ${total} ページを取得中...`;
|
||||
|
||||
let data = null;
|
||||
let ext = '.webp';
|
||||
const result = await fetchSinglePageWithin10Seconds(
|
||||
imgInfo,
|
||||
gallery.sourceUrl,
|
||||
total,
|
||||
(msg) => { statusText.textContent = msg; }
|
||||
);
|
||||
|
||||
// 最大3回リトライを行い、一時的な失敗による抜けを徹底防止
|
||||
for (let attempt = 1; attempt <= 3; attempt++) {
|
||||
if (zipAbortController.signal.aborted) throw new Error('Canceled');
|
||||
try {
|
||||
const fetchCtrl = new AbortController();
|
||||
const timeoutId = setTimeout(() => fetchCtrl.abort(), 20000); // 20秒タイムアウト
|
||||
if (result && result.data && result.data.length > 0) {
|
||||
const entryName = `${String(pNum).padStart(padLen, '0')}${result.ext}`;
|
||||
zip.addFile(entryName, result.data);
|
||||
} else {
|
||||
// 10秒間試行しても取得できなかった場合:メッセージと「中断」「続行」ボタンをダイアログ上に表示して待機
|
||||
spinner.classList.add('done');
|
||||
modalTitle.textContent = '⚠️ 画像の取得に失敗しました';
|
||||
statusText.textContent = `${pNum} / ${total} ページの取得タイムアウト (10秒経過)`;
|
||||
errorPrompt.style.display = 'block';
|
||||
errorPrompt.textContent = `ページ ${pNum} の画像を10秒間取得試行しましたが取得できませんでした。処理を中断して終了しますか?それともこのページをスキップして続行しますか?`;
|
||||
continueBtn.style.display = 'inline-block';
|
||||
cancelBtn.textContent = '中断';
|
||||
|
||||
const resp = await fetch(imgInfo.imageUrl, {
|
||||
signal: fetchCtrl.signal
|
||||
const userChoice = await new Promise((resolve) => {
|
||||
continueBtn.onclick = () => resolve('continue');
|
||||
cancelBtn.onclick = () => resolve('abort');
|
||||
});
|
||||
clearTimeout(timeoutId);
|
||||
|
||||
if (!resp.ok) throw new Error(`HTTP ${resp.status}`);
|
||||
|
||||
const cType = resp.headers.get('content-type') || '';
|
||||
if (cType.includes('png')) ext = '.png';
|
||||
else if (cType.includes('jpeg') || cType.includes('jpg')) ext = '.jpg';
|
||||
else if (cType.includes('gif')) ext = '.gif';
|
||||
else if (cType.includes('webp')) ext = '.webp';
|
||||
else {
|
||||
const urlPart = imgInfo.imageUrl.split('?')[0];
|
||||
const dotExt = urlPart.split('.').pop();
|
||||
if (dotExt && dotExt.length <= 4) ext = '.' + dotExt;
|
||||
if (userChoice === 'abort') {
|
||||
// 中断が選択された場合は処理を終了
|
||||
cleanupAndClose();
|
||||
return;
|
||||
}
|
||||
|
||||
const buf = await resp.arrayBuffer();
|
||||
data = new Uint8Array(buf);
|
||||
break;
|
||||
} catch (e) {
|
||||
if (zipAbortController.signal.aborted) throw new Error('Canceled');
|
||||
if (attempt < 3) {
|
||||
await new Promise(r => setTimeout(r, attempt * 1000));
|
||||
} else {
|
||||
console.warn(`Page ${pNum} download failed after 3 attempts:`, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (data && data.length > 0) {
|
||||
const entryName = `${String(pNum).padStart(3, '0')}${ext}`;
|
||||
zip.addFile(entryName, data);
|
||||
} else {
|
||||
failedCount++;
|
||||
// 続行が選択された場合はダイアログ表示を戻して次のページへ進む
|
||||
skippedPages.push(pNum);
|
||||
errorPrompt.style.display = 'none';
|
||||
continueBtn.style.display = 'none';
|
||||
spinner.classList.remove('done');
|
||||
modalTitle.textContent = '📦 ZIP一括保存中...';
|
||||
cancelBtn.onclick = cleanupAndClose;
|
||||
}
|
||||
|
||||
completed++;
|
||||
@@ -3280,19 +3545,20 @@ public static class UnifiedHtmlRenderer
|
||||
percentText.textContent = `${pct}%`;
|
||||
counterText.textContent = `${completed} / ${total} ページ完了`;
|
||||
|
||||
if (failedCount > 0) {
|
||||
if (skippedPages.length > 0) {
|
||||
warningText.style.display = 'block';
|
||||
warningText.textContent = `※ ${failedCount} 件の画像がタイムアウト等のためスキップされました`;
|
||||
}
|
||||
warningText.textContent = `※ スキップ済み (${skippedPages.length}件): ページ ${skippedPages.join(', ')}`;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const workers = [];
|
||||
for (let i = 0; i < Math.min(concurrency, gallery.images.length); i++) {
|
||||
workers.push(downloadWorker());
|
||||
if (zip.files.length === 0) {
|
||||
spinner.classList.add('done');
|
||||
modalTitle.textContent = '⚠️ 保存できる画像がありません';
|
||||
statusText.textContent = '有効な画像を取得できなかったため処理を終了しました。';
|
||||
cancelBtn.textContent = '閉じる';
|
||||
cancelBtn.onclick = cleanupAndClose;
|
||||
return;
|
||||
}
|
||||
await Promise.all(workers);
|
||||
|
||||
statusText.textContent = 'ZIPアーカイブを作成しています...';
|
||||
const zipBytes = zip.generateUint8Array();
|
||||
@@ -3320,14 +3586,12 @@ public static class UnifiedHtmlRenderer
|
||||
spinner.classList.add('done');
|
||||
statusText.textContent = `全 ${zip.files.length} 枚の画像を正常に保存しました!`;
|
||||
cancelBtn.textContent = '閉じる';
|
||||
cancelBtn.onclick = () => {
|
||||
modal.classList.remove('active');
|
||||
isZipDownloading = false;
|
||||
};
|
||||
cancelBtn.onclick = cleanupAndClose;
|
||||
|
||||
setTimeout(() => {
|
||||
modal.classList.remove('active');
|
||||
isZipDownloading = false;
|
||||
if (isZipDownloading) {
|
||||
cleanupAndClose();
|
||||
}
|
||||
}, 2200);
|
||||
|
||||
} catch (err) {
|
||||
@@ -3335,10 +3599,16 @@ public static class UnifiedHtmlRenderer
|
||||
console.log('Download canceled by user');
|
||||
} else {
|
||||
console.error('ZIP generation error:', err);
|
||||
statusText.textContent = 'ダウンロード中にエラーが発生しました。';
|
||||
spinner.classList.add('done');
|
||||
modalTitle.textContent = '⚠️ エラーが発生しました';
|
||||
statusText.textContent = 'ダウンロード処理中にエラーが発生したため終了しました。';
|
||||
cancelBtn.textContent = '閉じる';
|
||||
cancelBtn.onclick = cleanupAndClose;
|
||||
}
|
||||
isZipDownloading = false;
|
||||
if (typeof downloader !== 'undefined' && downloader && typeof downloader.resume === 'function') {
|
||||
downloader.resume();
|
||||
}
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Text.Json;
|
||||
using System.Text.RegularExpressions;
|
||||
using HtmlAgilityPack;
|
||||
@@ -102,26 +102,37 @@ public class UnifiedEHentaiScraper : BaseUnifiedScraper
|
||||
};
|
||||
}
|
||||
|
||||
// 4. 2ページ目以降の画像 URL を showpage API で並列取得
|
||||
// 4. 2ページ目以降の画像 URL を showpage API で並列取得(3桁ページでもレート制限で抜けないよう並列数制御+リトライ)
|
||||
var remainingEntries = pageEntries.Skip(1).ToList();
|
||||
if (remainingEntries.Count > 0 && !string.IsNullOrWhiteSpace(showkey))
|
||||
{
|
||||
using var semaphore = new SemaphoreSlim(10);
|
||||
using var semaphore = new SemaphoreSlim(4);
|
||||
var tasks = remainingEntries.Select(async entry =>
|
||||
{
|
||||
await semaphore.WaitAsync(ct);
|
||||
try
|
||||
{
|
||||
for (int attempt = 1; attempt <= 3; attempt++)
|
||||
{
|
||||
try
|
||||
{
|
||||
var imgInfo = await FetchImageViaShowpageApiAsync(gid, entry.Page, entry.ImgKey, showkey, entry.ThumbUrl, item.Title, ct);
|
||||
if (imgInfo != null)
|
||||
{
|
||||
images[entry.Page] = imgInfo;
|
||||
break;
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
catch (Exception ex) when (attempt == 3)
|
||||
{
|
||||
Logger.LogWarning(ex, "[{SiteName}] ページ {Page} の画像URL取得に失敗しました", SiteName, entry.Page);
|
||||
}
|
||||
catch
|
||||
{
|
||||
await Task.Delay(attempt * 500, ct);
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
semaphore.Release();
|
||||
@@ -229,7 +240,7 @@ public class UnifiedEHentaiScraper : BaseUnifiedScraper
|
||||
int pageIndex = 0;
|
||||
int maxPages = totalFiles > 0 ? (int)Math.Ceiling(totalFiles / 20.0) : 5;
|
||||
|
||||
while (pageIndex < maxPages && pageIndex < 20)
|
||||
while (pageIndex < maxPages && pageIndex < 100)
|
||||
{
|
||||
var pageUrl = $"{BaseUrl}/g/{gid}/{token}/?p={pageIndex}";
|
||||
var doc = await LoadHtmlDocumentAsync(pageUrl, ct: ct);
|
||||
|
||||
Reference in new issue
Block a user