接続方法変更
This commit is contained in:
1 parent
518f22d36e
commit
4a93843d3f
1 file changed
+20
-5
@@ -6,6 +6,7 @@ using System;
|
|||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Diagnostics;
|
using System.Diagnostics;
|
||||||
using System.Net;
|
using System.Net;
|
||||||
|
using System.Net.Http.Headers;
|
||||||
using System.Security.Cryptography;
|
using System.Security.Cryptography;
|
||||||
using System.ServiceModel.Syndication;
|
using System.ServiceModel.Syndication;
|
||||||
using System.Text;
|
using System.Text;
|
||||||
@@ -130,25 +131,39 @@ internal class CmnUtil
|
|||||||
{
|
{
|
||||||
Console.WriteLine($"Fetching: {pageURL}");
|
Console.WriteLine($"Fetching: {pageURL}");
|
||||||
|
|
||||||
// 1. User-Agent が未設定なら付与(Bot遮断対策)
|
// 1. User-Agent が未設定なら付与
|
||||||
if (!client.DefaultRequestHeaders.UserAgent.Any())
|
if (!client.DefaultRequestHeaders.UserAgent.Any())
|
||||||
{
|
{
|
||||||
client.DefaultRequestHeaders.UserAgent.ParseAdd("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36");
|
client.DefaultRequestHeaders.UserAgent.ParseAdd("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36");
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2. HTMLのダウンロード
|
// 2. キャッシュ無効化ヘッダーを付与したリクエストを生成
|
||||||
string htmlContent = await client.GetStringAsync(pageURL);
|
string separator = pageURL.Contains('?') ? "&" : "?";
|
||||||
|
string cacheBusterUrl = $"{pageURL}{separator}_tutc={DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()}";
|
||||||
|
using var request = new HttpRequestMessage(HttpMethod.Get, cacheBusterUrl);
|
||||||
|
request.Headers.CacheControl = new CacheControlHeaderValue
|
||||||
|
{
|
||||||
|
NoCache = true,
|
||||||
|
NoStore = true,
|
||||||
|
MustRevalidate = true
|
||||||
|
};
|
||||||
|
request.Headers.Pragma.ParseAdd("no-cache");
|
||||||
|
|
||||||
|
// 3. HTMLのダウンロード
|
||||||
|
using var response = await client.SendAsync(request);
|
||||||
|
response.EnsureSuccessStatusCode();
|
||||||
|
string htmlContent = await response.Content.ReadAsStringAsync();
|
||||||
|
|
||||||
if (string.IsNullOrWhiteSpace(htmlContent))
|
if (string.IsNullOrWhiteSpace(htmlContent))
|
||||||
{
|
{
|
||||||
Console.WriteLine("Warning: 取得したHTMLが空です。");
|
Console.WriteLine("Warning: 取得したHTMLが空です。");
|
||||||
}
|
}
|
||||||
|
|
||||||
// 3. AngleSharpのコンテキスト生成(通信なし・最速設定)
|
// 4. AngleSharpのコンテキスト生成(メモリ内パース)
|
||||||
var config = Configuration.Default;
|
var config = Configuration.Default;
|
||||||
var context = BrowsingContext.New(config);
|
var context = BrowsingContext.New(config);
|
||||||
|
|
||||||
// 4. アドレス(pageURL)を紐付けてパースすることで about:blank を防ぐ
|
// 5. アドレス(pageURL)を紐付けてパース
|
||||||
IDocument document = await context.OpenAsync(req => req.Content(htmlContent).Address(pageURL));
|
IDocument document = await context.OpenAsync(req => req.Content(htmlContent).Address(pageURL));
|
||||||
|
|
||||||
return document;
|
return document;
|
||||||
|
|||||||
Reference in new issue
Block a user