接続方法変更

This commit is contained in:
armonica committed 2026-08-24 20:10:15 +09:00
1 parent 518f22d36e
commit 4a93843d3f
1 file changed
+20 -5
+20 -5
View File
@@ -6,6 +6,7 @@ using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics; using System.Diagnostics;
using System.Net; using System.Net;
using System.Net.Http.Headers;
using System.Security.Cryptography; using System.Security.Cryptography;
using System.ServiceModel.Syndication; using System.ServiceModel.Syndication;
using System.Text; using System.Text;
@@ -130,25 +131,39 @@ internal class CmnUtil
{ {
Console.WriteLine($"Fetching: {pageURL}"); Console.WriteLine($"Fetching: {pageURL}");
// 1. User-Agent が未設定なら付与(Bot遮断対策) // 1. User-Agent が未設定なら付与
if (!client.DefaultRequestHeaders.UserAgent.Any()) if (!client.DefaultRequestHeaders.UserAgent.Any())
{ {
client.DefaultRequestHeaders.UserAgent.ParseAdd("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"); client.DefaultRequestHeaders.UserAgent.ParseAdd("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36");
} }
// 2. HTMLのダウンロード // 2. キャッシュ無効化ヘッダーを付与したリクエストを生成
string htmlContent = await client.GetStringAsync(pageURL); string separator = pageURL.Contains('?') ? "&" : "?";
string cacheBusterUrl = $"{pageURL}{separator}_tutc={DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()}";
using var request = new HttpRequestMessage(HttpMethod.Get, cacheBusterUrl);
request.Headers.CacheControl = new CacheControlHeaderValue
{
NoCache = true,
NoStore = true,
MustRevalidate = true
};
request.Headers.Pragma.ParseAdd("no-cache");
// 3. HTMLのダウンロード
using var response = await client.SendAsync(request);
response.EnsureSuccessStatusCode();
string htmlContent = await response.Content.ReadAsStringAsync();
if (string.IsNullOrWhiteSpace(htmlContent)) if (string.IsNullOrWhiteSpace(htmlContent))
{ {
Console.WriteLine("Warning: 取得したHTMLが空です。"); Console.WriteLine("Warning: 取得したHTMLが空です。");
} }
// 3. AngleSharpのコンテキスト生成(通信なし・最速設定) // 4. AngleSharpのコンテキスト生成(メモリ内パース)
var config = Configuration.Default; var config = Configuration.Default;
var context = BrowsingContext.New(config); var context = BrowsingContext.New(config);
// 4. アドレス(pageURL)を紐付けてパースすることで about:blank を防ぐ // 5. アドレス(pageURL)を紐付けてパース
IDocument document = await context.OpenAsync(req => req.Content(htmlContent).Address(pageURL)); IDocument document = await context.OpenAsync(req => req.Content(htmlContent).Address(pageURL));
return document; return document;