init public

This commit is contained in:
GAJ3T 2026-10-23 16:43:34 -04:00
parent 9e99855d54
commit 47f37d3220
80 changed files with 11104 additions and 1 deletions

15
.dockerignore Normal file
View File

@ -0,0 +1,15 @@
.git
.gitattributes
.gitignore
.github
.vs
**/.vs
**/bin
**/obj
**/TestResults
**/*.log
**/*.suo
**/*.user
docs
src/WebAccess.MCP.TestRig
clean.ps1

63
.gitattributes vendored Normal file
View File

@ -0,0 +1,63 @@
###############################################################################
# Set default behavior to automatically normalize line endings.
###############################################################################
* text=auto
###############################################################################
# Set default behavior for command prompt diff.
#
# This is need for earlier builds of msysgit that does not have it on by
# default for csharp files.
# Note: This is only used by command line
###############################################################################
#*.cs diff=csharp
###############################################################################
# Set the merge driver for project and solution files
#
# Merging from the command prompt will add diff markers to the files if there
# are conflicts (Merging from VS is not affected by the settings below, in VS
# the diff markers are never inserted). Diff markers may cause the following
# file extensions to fail to load in VS. An alternative would be to treat
# these files as binary and thus will always conflict and require user
# intervention with every merge. To do so, just uncomment the entries below
###############################################################################
#*.sln merge=binary
#*.csproj merge=binary
#*.vbproj merge=binary
#*.vcxproj merge=binary
#*.vcproj merge=binary
#*.dbproj merge=binary
#*.fsproj merge=binary
#*.lsproj merge=binary
#*.wixproj merge=binary
#*.modelproj merge=binary
#*.sqlproj merge=binary
#*.wwaproj merge=binary
###############################################################################
# behavior for image files
#
# image files are treated as binary by default.
###############################################################################
#*.jpg binary
#*.png binary
#*.gif binary
###############################################################################
# diff behavior for common document formats
#
# Convert binary document formats to text before diffing them. This feature
# is only available from the command line. Turn it on by uncommenting the
# entries below.
###############################################################################
#*.doc diff=astextplain
#*.DOC diff=astextplain
#*.docx diff=astextplain
#*.DOCX diff=astextplain
#*.dot diff=astextplain
#*.DOT diff=astextplain
#*.pdf diff=astextplain
#*.PDF diff=astextplain
#*.rtf diff=astextplain
#*.RTF diff=astextplain

27
Containerfile Normal file
View File

@ -0,0 +1,27 @@
FROM mcr.microsoft.com/dotnet/sdk:10.0-noble AS build
WORKDIR /src
COPY src/WebAccess.MCP/WebAccess.MCP.csproj src/WebAccess.MCP/
RUN dotnet restore src/WebAccess.MCP/WebAccess.MCP.csproj
COPY src/WebAccess.MCP/ src/WebAccess.MCP/
RUN dotnet publish src/WebAccess.MCP/WebAccess.MCP.csproj \
--configuration Release \
--framework net10.0 \
--no-restore \
--self-contained false \
-p:UseAppHost=false \
--output /app/publish
FROM mcr.microsoft.com/playwright/dotnet:v1.62.0-noble AS final
WORKDIR /app
COPY --from=build --chown=pwuser:pwuser /app/publish/ ./
EXPOSE 8080
USER pwuser
ENTRYPOINT ["dotnet", "WebAccess.MCP.dll"]

View File

@ -0,0 +1,156 @@
[CmdletBinding()]
param(
[string] $DestinationPath = (
Join-Path $PSScriptRoot (
"artifacts\WebAccess.MCP-deployment-{0}\WebAccess.MCP" -f
(Get-Date -Format "yyyyMMdd-HHmmss"))),
[switch] $SkipBuild,
[switch] $NoArchive
)
$ErrorActionPreference = "Stop"
$repositoryRoot = $PSScriptRoot
$destinationRoot = [System.IO.Path]::GetFullPath($DestinationPath)
$archivePath = "$destinationRoot.zip"
function Copy-DirectoryFiles {
param(
[Parameter(Mandatory)]
[string] $Source,
[Parameter(Mandatory)]
[string] $Destination,
[string[]] $ExcludedDirectoryNames = @(),
[string[]] $ExcludedFileNames = @()
)
$sourceRoot = [System.IO.Path]::GetFullPath($Source).TrimEnd("\")
Get-ChildItem -LiteralPath $sourceRoot -File -Recurse |
Where-Object {
$relativePath = $_.FullName.Substring($sourceRoot.Length).TrimStart("\")
$directoryNames = $relativePath.Split("\")
$_.Name -notin $ExcludedFileNames -and
-not ($directoryNames | Where-Object { $_ -in $ExcludedDirectoryNames })
} |
ForEach-Object {
$relativePath = $_.FullName.Substring($sourceRoot.Length).TrimStart("\")
$targetPath = Join-Path $Destination $relativePath
$targetDirectory = Split-Path -Parent $targetPath
if (-not (Test-Path -LiteralPath $targetDirectory -PathType Container)) {
New-Item -ItemType Directory -Path $targetDirectory -Force | Out-Null
}
Copy-Item -LiteralPath $_.FullName -Destination $targetPath
}
}
if (Test-Path -LiteralPath $destinationRoot) {
throw "Destination already exists: $destinationRoot"
}
if (-not $NoArchive -and (Test-Path -LiteralPath $archivePath)) {
throw "Archive already exists: $archivePath"
}
$requiredPaths = @(
(Join-Path $repositoryRoot ".dockerignore"),
(Join-Path $repositoryRoot "Containerfile"),
(Join-Path $repositoryRoot "QUICKSTART.md"),
(Join-Path $repositoryRoot "Readme.md"),
(Join-Path $repositoryRoot "Start-WebAccessMcp.cmd"),
(Join-Path $repositoryRoot "deploy\compose.yml"),
(Join-Path $repositoryRoot "deploy\Start-WebAccessStack.ps1"),
(Join-Path $repositoryRoot "deploy\searxng\settings.yml"),
(Join-Path $repositoryRoot "src\WebAccess.MCP\WebAccess.MCP.csproj")
)
foreach ($requiredPath in $requiredPaths) {
if (-not (Test-Path -LiteralPath $requiredPath -PathType Leaf)) {
throw "Required deployment file is missing: $requiredPath"
}
}
if (-not $SkipBuild) {
if (-not (Get-Command "podman" -ErrorAction SilentlyContinue)) {
throw "Podman is not installed or is not available in PATH. Install or start Podman, or use -SkipBuild to create the bundle without validating the image build."
}
& podman info *> $null
if ($LASTEXITCODE -ne 0) {
throw "Podman is not running. Start the Podman machine, or use -SkipBuild to create the bundle without validating the image build."
}
Write-Host "Building the WebAccess.MCP container image..."
& podman build `
--file (Join-Path $repositoryRoot "Containerfile") `
--tag "localhost/webaccess-mcp:compose" `
$repositoryRoot
if ($LASTEXITCODE -ne 0) {
throw "The WebAccess.MCP container image build failed with exit code $LASTEXITCODE."
}
}
Write-Host "Creating deployment bundle at $destinationRoot..."
New-Item -ItemType Directory -Path $destinationRoot -Force | Out-Null
Copy-Item `
-LiteralPath (Join-Path $repositoryRoot ".dockerignore") `
-Destination (Join-Path $destinationRoot ".dockerignore")
Copy-Item `
-LiteralPath (Join-Path $repositoryRoot "Containerfile") `
-Destination (Join-Path $destinationRoot "Containerfile")
Copy-Item `
-LiteralPath (Join-Path $repositoryRoot "QUICKSTART.md") `
-Destination (Join-Path $destinationRoot "QUICKSTART.md")
Copy-Item `
-LiteralPath (Join-Path $repositoryRoot "Readme.md") `
-Destination (Join-Path $destinationRoot "Readme.md")
Copy-Item `
-LiteralPath (Join-Path $repositoryRoot "Start-WebAccessMcp.cmd") `
-Destination (Join-Path $destinationRoot "Start-WebAccessMcp.cmd")
Copy-DirectoryFiles `
-Source (Join-Path $repositoryRoot "deploy") `
-Destination (Join-Path $destinationRoot "deploy") `
-ExcludedDirectoryNames @("bin", "obj") `
-ExcludedFileNames @(".env")
Copy-DirectoryFiles `
-Source (Join-Path $repositoryRoot "src\WebAccess.MCP") `
-Destination (Join-Path $destinationRoot "src\WebAccess.MCP") `
-ExcludedDirectoryNames @("bin", "obj")
Copy-DirectoryFiles `
-Source (Join-Path $repositoryRoot "docs") `
-Destination (Join-Path $destinationRoot "docs")
$environmentPath = Join-Path $destinationRoot "deploy\.env"
if (Test-Path -LiteralPath $environmentPath) {
throw "The deployment bundle unexpectedly contains deploy\.env. Remove the bundle and investigate before distributing it."
}
if (-not $NoArchive) {
Write-Host "Creating archive at $archivePath..."
Compress-Archive -LiteralPath $destinationRoot -DestinationPath $archivePath
}
Write-Host ""
Write-Host "Deployment bundle created:"
Write-Host " Folder: $destinationRoot"
if (-not $NoArchive) {
Write-Host " Archive: $archivePath"
}
Write-Host ""
Write-Host "The recipient can extract/copy the WebAccess.MCP folder, open it, and run:"
Write-Host " .\Start-WebAccessMcp.cmd"

191
QUICKSTART.md Normal file
View File

@ -0,0 +1,191 @@
# WebAccess.MCP Quick Start
Use this guide when the computer already has:
- Podman Desktop with a running Podman machine.
- Podman Compose.
- LM Studio 0.3.17 or later.
- A tool-capable model installed in LM Studio.
- Git.
All commands in this quick-start use **PowerShell (Standard user)**. Do not run
PowerShell as an administrator for this workflow.
## 1. Clone the Repository
**PowerShell (Standard user)**
```powershell
$McpRoot = Join-Path $env:SystemDrive "ai\mcp"
New-Item -ItemType Directory -Force -Path $McpRoot | Out-Null
Set-Location $McpRoot
git clone <repository-url> WebAccess.MCP
Set-Location ".\WebAccess.MCP"
```
Replace `<repository-url>` with the repository URL supplied by the project
owner.
`$env:SystemDrive` uses the Windows system drive, which is normally `C:`. The
resulting default repository location is:
```text
C:\ai\mcp\WebAccess.MCP
```
To use another local drive, set `$McpRoot` explicitly before cloning:
**PowerShell (Standard user)**
```powershell
$McpRoot = "E:\ai\mcp"
```
## 2. Start the Entire Stack
From the cloned repository root, run one command:
**PowerShell (Standard user)**
```powershell
.\Start-WebAccessMcp.cmd
```
The command:
1. Verifies Podman and Podman Compose.
2. Creates local installation secrets if they do not exist.
3. Builds the WebAccess.MCP container image.
4. Starts PostgreSQL, SearXNG, and WebAccess.MCP.
5. Displays container status and the local endpoint.
The first build can take several minutes.
If an earlier startup created failed or incomplete containers, running the same
command again replaces them with the corrected stack.
When startup succeeds, the MCP endpoint is:
```text
http://127.0.0.1:28081/mcp
```
The startup command does not report success until this health endpoint responds:
```text
http://127.0.0.1:28081/health
```
## 3. Add WebAccess.MCP to LM Studio
1. Open LM Studio.
2. Open the **Program** tab in the right sidebar.
3. Select **Install > Edit mcp.json**.
4. Add `webaccess` inside the existing `mcpServers` object.
For an empty configuration, use:
```json
{
"mcpServers": {
"webaccess": {
"url": "http://127.0.0.1:28081/mcp"
}
}
}
```
If other MCP servers already exist, preserve them and add:
```json
"webaccess": {
"url": "http://127.0.0.1:28081/mcp"
}
```
Include a comma between server entries. Do not add `command`, `args`, headers,
or a Podman path to the `webaccess` entry.
5. Save `mcp.json`.
6. Click **Refresh** in LM Studio.
7. Confirm that `webaccess` is connected and exposes 15 tools.
## 4. Test It
Load the tool-capable model and ask:
> Use `web_search` to find the official LM Studio MCP documentation and return
> the source URL.
LM Studio should display a `web_search` tool call.
## Normal Startup After a Restart
Start Podman Desktop and its Podman machine, then run from the repository root:
**PowerShell (Standard user)**
```powershell
.\Start-WebAccessMcp.cmd
```
The command is safe to run again. It preserves the existing secrets and
PostgreSQL page cache.
## Stop the Stack
From the repository root:
**PowerShell (Standard user)**
```powershell
Set-Location ".\deploy"
podman compose --project-name webaccess-mcp stop
```
## If Startup Fails
Check Podman:
**PowerShell (Standard user)**
```powershell
podman info
podman compose version
```
Check the stack:
**PowerShell (Standard user)**
```powershell
Set-Location ".\deploy"
podman compose --project-name webaccess-mcp ps --all
podman compose --project-name webaccess-mcp logs --tail 200
```
Test the HTTP server itself:
**PowerShell (Standard user)**
```powershell
Invoke-RestMethod "http://127.0.0.1:28081/health"
```
Expected result:
```text
status
------
healthy
```
Do not use a plain browser request or `curl` GET to `/mcp` as the health test.
`/mcp` expects MCP protocol requests and can return a protocol or method error
even when the server is reachable. Use `/health` to test connectivity and LM
Studio to test the MCP handshake.
For detailed instructions, see:
- `docs\Standing-Up-the-Podman-Compose-Stack.md`
- `docs\Adding-WebAccess-MCP-to-LM-Studio.md`

153
README.md
View File

@ -1,2 +1,153 @@
# webaccess-mcp # WebAccess.MCP
WebAccess.MCP is a Streamable HTTP MCP server that provides web search and
Playwright-backed browser tools to LM Studio.
## Start After Copying or Cloning the Repository
These instructions assume the computer already has:
- Podman Desktop.
- A running Podman machine.
- Podman Compose.
- LM Studio 0.3.17 or later.
- A tool-capable model installed in LM Studio.
Confirm that `Containerfile` exists directly in the copied repository root.
Open **PowerShell (Standard user)** in the repository root. Do not run
PowerShell as administrator.
Run the PowerShell startup script:
```powershell
powershell.exe -NoProfile -ExecutionPolicy Bypass -File ".\deploy\Start-WebAccessStack.ps1"
```
Alternatively, run the shorter wrapper:
```powershell
.\Start-WebAccessMcp.cmd
```
Both commands:
1. Verify that Podman is installed and its machine is running.
2. Verify that Podman Compose is installed.
3. Create `deploy\.env` with machine-specific secrets if it is missing.
4. Build the WebAccess.MCP container image.
5. Start PostgreSQL, SearXNG, and WebAccess.MCP.
6. Wait for the WebAccess.MCP health endpoint.
7. Print container status, endpoint URLs, and the LM Studio configuration.
The first build can take several minutes.
## Verify WebAccess.MCP
Use **PowerShell (Standard user)**:
```powershell
curl.exe http://127.0.0.1:28081/health
```
Expected response:
```json
{"status":"healthy"}
```
The MCP endpoint is:
```text
http://127.0.0.1:28081/mcp
```
A plain GET request to `/mcp` can return HTTP 405 because it is an MCP protocol
endpoint, not a normal web page. Use `/health` for connectivity checks and LM
Studio for the MCP handshake.
## Configure LM Studio
No `mcp.json` change is needed if it already contains this exact server entry:
```json
{
"mcpServers": {
"webaccess": {
"url": "http://127.0.0.1:28081/mcp"
}
}
}
```
If `mcp.json` already contains other MCP servers:
1. Open LM Studio.
2. Open the **Program** tab.
3. Select **Install > Edit mcp.json**.
4. Keep the existing server entries.
5. Add this property inside the existing `mcpServers` object:
```json
"webaccess": {
"url": "http://127.0.0.1:28081/mcp"
}
```
6. Include a comma between server properties.
7. Save `mcp.json`.
8. Click **Refresh** in LM Studio.
9. Confirm that `webaccess` connects and exposes 15 tools.
Do not add `command`, `args`, environment variables, authentication headers, a
Podman path, or a container name to the `webaccess` entry.
When LM Studio and Podman run on the same computer, `127.0.0.1` is the correct
address.
## Test in LM Studio
Load the tool-capable model and ask:
> Use `web_search` to find the official LM Studio MCP documentation and return
> the source URL.
LM Studio should display a `web_search` tool call.
## Start Again Later
Start Podman Desktop and its Podman machine. Then open **PowerShell (Standard
user)** in the repository root and run:
```powershell
.\Start-WebAccessMcp.cmd
```
The command preserves the existing secrets and PostgreSQL page cache.
## Stop the Stack
Use **PowerShell (Standard user)** from the repository root:
```powershell
Set-Location ".\deploy"
podman compose --project-name webaccess-mcp stop
```
## Troubleshooting
If startup fails, run from the repository root using **PowerShell (Standard
user)**:
```powershell
Set-Location ".\deploy"
podman compose --project-name webaccess-mcp ps
podman compose --project-name webaccess-mcp logs --tail 200
```
Detailed guides:
- `QUICKSTART.md`
- `docs\Standing-Up-the-Podman-Compose-Stack.md`
- `docs\Adding-WebAccess-MCP-to-LM-Studio.md`
- `docs\Running-in-Podman-Desktop-and-LM-Studio.md`

10
Start-WebAccessMcp.cmd Normal file
View File

@ -0,0 +1,10 @@
@echo off
setlocal
"%SystemRoot%\System32\WindowsPowerShell\v1.0\powershell.exe" ^
-NoLogo ^
-NoProfile ^
-ExecutionPolicy Bypass ^
-File "%~dp0deploy\Start-WebAccessStack.ps1"
exit /b %ERRORLEVEL%

7
clean.ps1 Normal file
View File

@ -0,0 +1,7 @@
# Delete all "bin" and "obj" folders recursively from the current directory
Get-ChildItem -Path . -Directory -Recurse -Force |
Where-Object { $_.Name -in @('bin','obj') } |
ForEach-Object {
Write-Host "Deleting: $($_.FullName)"
Remove-Item -LiteralPath $_.FullName -Recurse -Force -ErrorAction SilentlyContinue
}

2
deploy/.env Normal file
View File

@ -0,0 +1,2 @@
WEBACCESS_POSTGRES_PASSWORD=d0eadcf239bed2d046b23dc7597b4a4f9bc243f23a0e9bf998bca7f8f91fb0d3
SEARXNG_SECRET=28ff5159aaa18974a5717df2b988a2b1ec1dceb3dde38b510290c8ba73a0ac50

View File

@ -0,0 +1,152 @@
[CmdletBinding()]
param()
$ErrorActionPreference = "Stop"
$projectName = "webaccess-mcp"
$environmentPath = Join-Path $PSScriptRoot ".env"
$healthUrl = "http://127.0.0.1:28081/health"
function Invoke-Podman {
param(
[Parameter(Mandatory)]
[string[]] $Arguments,
[switch] $DiscardOutput
)
$previousErrorActionPreference = $ErrorActionPreference
try {
$ErrorActionPreference = "Continue"
if ($DiscardOutput) {
& podman @Arguments *> $null
}
else {
& podman @Arguments
}
$script:PodmanExitCode = $LASTEXITCODE
}
finally {
$ErrorActionPreference = $previousErrorActionPreference
}
}
function New-HexSecret {
$bytes = New-Object byte[] 32
$generator = [System.Security.Cryptography.RandomNumberGenerator]::Create()
try {
$generator.GetBytes($bytes)
}
finally {
$generator.Dispose()
}
return -join ($bytes | ForEach-Object { $_.ToString("x2") })
}
if (-not (Get-Command "podman" -ErrorAction SilentlyContinue)) {
throw "Podman is not installed or is not available in PATH."
}
Invoke-Podman -Arguments @("info") -DiscardOutput
if ($script:PodmanExitCode -ne 0) {
throw "Podman is not running. Start the Podman machine in Podman Desktop and try again."
}
Invoke-Podman -Arguments @("compose", "version") -DiscardOutput
if ($script:PodmanExitCode -ne 0) {
throw "Podman Compose is not installed. Install it from Podman Desktop Settings > Resources."
}
if (-not (Test-Path -LiteralPath $environmentPath -PathType Leaf)) {
@"
WEBACCESS_POSTGRES_PASSWORD=$(New-HexSecret)
SEARXNG_SECRET=$(New-HexSecret)
"@ | Set-Content -LiteralPath $environmentPath -Encoding Ascii
Write-Host "Created deploy\.env with new installation secrets."
}
Push-Location $PSScriptRoot
try {
Write-Host "Building and starting PostgreSQL, SearXNG, and WebAccess.MCP..."
Invoke-Podman -Arguments @(
"compose",
"--project-name",
$projectName,
"up",
"--detach",
"--build")
if ($script:PodmanExitCode -ne 0) {
throw "Podman Compose failed with exit code $script:PodmanExitCode."
}
Write-Host ""
Invoke-Podman -Arguments @("compose", "--project-name", $projectName, "ps")
if ($script:PodmanExitCode -ne 0) {
throw "The stack started, but its status could not be displayed."
}
Write-Host ""
Write-Host "Waiting for WebAccess.MCP to become ready..."
$isReady = $false
for ($attempt = 1; $attempt -le 30; $attempt++) {
try {
$response = Invoke-RestMethod `
-Uri $healthUrl `
-Method Get `
-TimeoutSec 3
if ($response.status -eq "healthy") {
$isReady = $true
break
}
}
catch {
if ($attempt -lt 30) {
Start-Sleep -Seconds 2
}
}
}
if (-not $isReady) {
Write-Host ""
Write-Host "WebAccess.MCP did not become reachable. Current status:"
Invoke-Podman -Arguments @("compose", "--project-name", $projectName, "ps")
Write-Host ""
Write-Host "WebAccess.MCP logs:"
Invoke-Podman -Arguments @(
"compose",
"--project-name",
$projectName,
"logs",
"--tail",
"200",
"webaccess")
throw "WebAccess.MCP did not pass its readiness check at $healthUrl."
}
Write-Host ""
Write-Host "WebAccess.MCP health check passed at $healthUrl"
Write-Host "WebAccess.MCP is available at http://127.0.0.1:28081/mcp"
Write-Host "SearXNG diagnostics are available at http://127.0.0.1:28080/"
Write-Host ""
Write-Host "Add this entry inside the mcpServers object in LM Studio's mcp.json:"
Write-Host ""
Write-Host ' "webaccess": {'
Write-Host ' "url": "http://127.0.0.1:28081/mcp"'
Write-Host ' }'
Write-Host ""
Write-Host "Save mcp.json, then click Refresh in LM Studio."
}
finally {
Pop-Location
}

77
deploy/compose.yml Normal file
View File

@ -0,0 +1,77 @@
services:
postgres:
image: docker.io/library/postgres:16
environment:
POSTGRES_DB: webaccess
POSTGRES_USER: webaccess
POSTGRES_PASSWORD: ${WEBACCESS_POSTGRES_PASSWORD:?WEBACCESS_POSTGRES_PASSWORD must be provided}
volumes:
- "postgres-data:/var/lib/postgresql/data"
healthcheck:
test:
- CMD-SHELL
- pg_isready -U webaccess -d webaccess
interval: 5s
timeout: 3s
retries: 12
start_period: 10s
networks:
- webaccess
searxng:
image: docker.io/searxng/searxng:2025.10.21-a8f3644cd@sha256:4c5024bccf5474d24afc8a85e2e7ebcb6a6c3b148654b5bc13eabfcf55414830
environment:
SEARXNG_SECRET: ${SEARXNG_SECRET:?SEARXNG_SECRET must be provided}
ports:
- "127.0.0.1:28080:8080"
volumes:
- "./searxng/settings.yml:/etc/searxng/settings.yml:ro"
healthcheck:
test:
- CMD
- /usr/bin/wget
- --quiet
- --output-document=/dev/null
- http://127.0.0.1:8080/config
interval: 5s
timeout: 3s
retries: 12
start_period: 10s
networks:
- webaccess
webaccess:
build:
context: ..
image: localhost/webaccess-mcp:compose
environment:
ConnectionStrings__WebCache: Host=postgres;Port=5432;Database=webaccess;Username=webaccess;Password=${WEBACCESS_POSTGRES_PASSWORD:?WEBACCESS_POSTGRES_PASSWORD must be provided}
Search__SearXNG__BaseUrl: http://searxng:8080/
Search__SearXNG__Timeout: "00:00:15"
Search__SearXNG__DefaultEngines__0: google
Search__SearXNG__DefaultEngines__1: brave
Search__SearXNG__DefaultEngines__2: duckduckgo
ports:
- "127.0.0.1:28081:8080"
healthcheck:
test:
- CMD-SHELL
- timeout 3 bash -c '</dev/tcp/127.0.0.1/8080'
interval: 5s
timeout: 3s
retries: 12
start_period: 20s
depends_on:
postgres:
condition: service_healthy
searxng:
condition: service_healthy
networks:
- webaccess
networks:
webaccess:
driver: bridge
volumes:
postgres-data:

View File

@ -0,0 +1,5 @@
Repository: docker.io/searxng/searxng
Tag: 2025.10.21-a8f3644cd
Digest: sha256:4c5024bccf5474d24afc8a85e2e7ebcb6a6c3b148654b5bc13eabfcf55414830
PinnedReference: docker.io/searxng/searxng:2025.10.21-a8f3644cd@sha256:4c5024bccf5474d24afc8a85e2e7ebcb6a6c3b148654b5bc13eabfcf55414830
Architecture: linux/amd64

View File

@ -0,0 +1,29 @@
use_default_settings:
engines:
keep_only:
- google
- brave
- duckduckgo
general:
debug: false
instance_name: "WebAccess.MCP SearXNG"
search:
safe_search: 0
autocomplete: ""
formats:
- html
- json
server:
limiter: false
image_proxy: false
engines:
- name: google
disabled: false
- name: brave
disabled: false
- name: duckduckgo
disabled: false

View File

@ -0,0 +1,453 @@
# Add WebAccess.MCP to LM Studio
This guide explains, step by step, how to add a running WebAccess.MCP server to
LM Studio's MCP configuration.
All PowerShell commands in this guide use **PowerShell (Standard user)**. No
step requires **PowerShell (Admin)**.
WebAccess.MCP uses a remote Streamable HTTP connection. LM Studio connects to:
```text
http://127.0.0.1:28081/mcp
```
It does not launch WebAccess.MCP as a command.
## Fast Path After Cloning
If Podman Desktop, LM Studio, and a tool-capable model are already installed:
1. Open PowerShell in the cloned WebAccess.MCP repository.
2. Run:
```powershell
.\Start-WebAccessMcp.cmd
```
3. Wait for the script to report that WebAccess.MCP is available.
4. In LM Studio, open **Program > Install > Edit mcp.json**.
5. Add this entry inside `mcpServers`:
```json
"webaccess": {
"url": "http://127.0.0.1:28081/mcp"
}
```
6. Save `mcp.json`.
7. Click **Refresh** in LM Studio.
The startup command creates the local secrets, builds the image, and starts the
complete PostgreSQL, SearXNG, and WebAccess.MCP stack.
## Before You Begin
Confirm:
- LM Studio 0.3.17 or later is installed.
- The WebAccess.MCP Podman Compose stack is running.
- The `webaccess` container is running.
- Port 28081 is reachable on the local computer.
- A tool-capable instruct/chat model is installed in LM Studio.
Verify WebAccess.MCP from PowerShell:
```powershell
Test-NetConnection 127.0.0.1 -Port 28081
```
The result must include:
```text
TcpTestSucceeded : True
```
If it returns `False`, start or troubleshoot the Podman Compose stack before
changing LM Studio.
## 1. Open LM Studio's MCP Configuration
1. Start LM Studio.
2. Open the **Program** tab in the right sidebar.
3. Select **Install**.
4. Select **Edit mcp.json**.
Use LM Studio's editor rather than searching for the file in Windows. LM Studio
opens the correct configuration for the current installation.
The top-level JSON structure is:
```json
{
"mcpServers": {
}
}
```
All MCP server definitions belong inside `mcpServers`.
## 2. Back Up an Existing Configuration
If the editor already contains MCP servers, copy the complete JSON into a
temporary text file before changing it.
Do not share that backup without reviewing it. Other MCP entries can contain
tokens, API keys, local paths, or environment variables.
If the file is empty or contains only an empty `mcpServers` object, continue to
the next section.
## 3. Add WebAccess.MCP to an Empty Configuration
Replace the empty configuration with:
```json
{
"mcpServers": {
"webaccess": {
"url": "http://127.0.0.1:28081/mcp"
}
}
}
```
The server name is `webaccess`. The name identifies this connection inside LM
Studio and does not change the server endpoint.
## 4. Add WebAccess.MCP Alongside Existing Servers
If `mcp.json` already contains servers, do not replace them. Add `webaccess`
inside the existing `mcpServers` object.
For example, starting with:
```json
{
"mcpServers": {
"example-server": {
"url": "https://example.com/mcp"
}
}
}
```
Add a comma after the existing server and insert the WebAccess.MCP entry:
```json
{
"mcpServers": {
"example-server": {
"url": "https://example.com/mcp"
},
"webaccess": {
"url": "http://127.0.0.1:28081/mcp"
}
}
}
```
If `webaccess` is not the final entry, add a comma after its closing brace:
```json
{
"mcpServers": {
"webaccess": {
"url": "http://127.0.0.1:28081/mcp"
},
"another-server": {
"url": "https://example.com/another-mcp"
}
}
}
```
JSON does not allow a trailing comma after the final property.
## 5. Check the WebAccess.MCP Entry
The entry must contain only the HTTP URL:
```json
"webaccess": {
"url": "http://127.0.0.1:28081/mcp"
}
```
Do not add:
- `command`
- `args`
- `env`
- authentication headers
- a Podman executable path
- a .NET executable path
Do not use these incorrect URLs:
```text
http://127.0.0.1:28080/
http://127.0.0.1:28081/
http://searxng:8080/
http://webaccess:8080/mcp
```
Reasons:
- Port 28080 is the SearXNG diagnostic endpoint, not MCP.
- The required MCP route is `/mcp`.
- Compose service names such as `searxng` and `webaccess` resolve only inside the
private container network.
Use exactly:
```text
http://127.0.0.1:28081/mcp
```
## 6. Save the Configuration
1. Review the complete JSON.
2. Confirm all object names and string values use double quotes.
3. Confirm properties are separated by commas.
4. Confirm there is no trailing comma after the final property.
5. Save `mcp.json` in LM Studio.
LM Studio should parse the file and show `webaccess` in the Program tab. If it
reports a JSON error, correct the syntax before continuing.
## 7. Enable WebAccess.MCP
1. Return to the **Program** tab.
2. Locate `webaccess`.
3. Enable the server if it is disabled.
4. Allow LM Studio to connect and discover its tools.
When the connection succeeds, LM Studio should discover 15 WebAccess.MCP tools.
If LM Studio continues to show a connection error after the JSON is corrected:
1. Disable `webaccess`.
2. Confirm the Podman stack is running.
3. Enable `webaccess` again.
4. Restart LM Studio if it does not retry the connection.
## 8. Load a Tool-Capable Model
WebAccess.MCP makes tools available, but the loaded model decides when and how
to call them.
1. Open **Chat** in LM Studio.
2. Load an instruct/chat model that supports tool use or function calling.
3. Use a model with enough context for tool definitions and returned web
content. At least 16K context is a useful starting point when supported by
the model and hardware.
4. Start a new chat after loading the model.
5. Enable the `webaccess` tools for the chat if LM Studio presents a tool or
integrations selector.
A base completion model without tool support is not suitable.
## 9. Test the MCP Connection
Ask the model:
> Use `web_search` to find the official LM Studio MCP documentation. Return the
> page title and source URL.
Expected behavior:
1. LM Studio displays a `web_search` tool call.
2. WebAccess.MCP sends the search through its dedicated SearXNG service.
3. The model summarizes the returned results and includes the source URL.
Then test the browser-backed tools:
> Open the official LM Studio MCP documentation URL, report its page title, and
> list the first five links.
LM Studio should show browser tool calls from WebAccess.MCP.
The browser context persists across calls. A later click, typing, or page-read
operation uses the same application-level browser context unless navigation
changes it.
## 10. Confirm the Tool Count
WebAccess.MCP exposes 15 tools. LM Studio's tool view should show 15 tools for
the `webaccess` server.
If the connection succeeds but fewer tools appear:
1. Restart the WebAccess.MCP Compose service.
2. Disable and re-enable `webaccess` in LM Studio.
3. Restart LM Studio.
4. Confirm the MCP URL has not been redirected to another service.
## 11. Example Configurations
### WebAccess.MCP only
```json
{
"mcpServers": {
"webaccess": {
"url": "http://127.0.0.1:28081/mcp"
}
}
}
```
### WebAccess.MCP plus two other remote servers
```json
{
"mcpServers": {
"webaccess": {
"url": "http://127.0.0.1:28081/mcp"
},
"documentation": {
"url": "https://example.com/documentation/mcp"
},
"issue-tracker": {
"url": "https://example.com/issues/mcp",
"headers": {
"Authorization": "Bearer <token>"
}
}
}
}
```
The example token is only a placeholder for the unrelated example server.
WebAccess.MCP does not require a token or headers.
### WebAccess.MCP plus a local command-based server
```json
{
"mcpServers": {
"webaccess": {
"url": "http://127.0.0.1:28081/mcp"
},
"local-example": {
"command": "example-command",
"args": [
"--serve"
]
}
}
}
```
Only the command-based server uses `command` and `args`. Do not copy those
properties into the `webaccess` entry.
## 12. Troubleshooting
### LM Studio says `mcp.json` is invalid
Common JSON errors include:
- A missing comma between server entries.
- A trailing comma after the final entry.
- Single quotes instead of double quotes.
- A missing closing brace.
- Adding `webaccess` outside `mcpServers`.
- Two properties with the same `webaccess` name.
Restore the backup if necessary, then add the entry again carefully.
### `webaccess` does not appear in the Program tab
Confirm the entry is inside the top-level `mcpServers` object and that the file
saved successfully:
```json
{
"mcpServers": {
"webaccess": {
"url": "http://127.0.0.1:28081/mcp"
}
}
}
```
Restart LM Studio after saving if the Program tab does not refresh.
### LM Studio reports that WebAccess.MCP is disconnected
Verify the stack in PowerShell:
```powershell
Set-Location "D:\ai\mcp\WebAccess.MCP\deploy"
podman compose --project-name webaccess-mcp ps
Test-NetConnection 127.0.0.1 -Port 28081
podman compose --project-name webaccess-mcp logs --tail 200 webaccess
```
The `webaccess` service must be running and the port test must return `True`.
### The model answers without calling a tool
1. Confirm the `webaccess` server and its tools are enabled for the chat.
2. Use an explicit instruction such as "Use the `web_search` tool."
3. Confirm the model supports tool/function calling.
4. Start a new chat after changing models or tool settings.
5. Try a stronger instruct model if the connection works but tool selection is
unreliable.
### `web_search` fails but browser tools work
The LM Studio MCP connection is probably healthy. Check the separate SearXNG
service:
```powershell
Test-NetConnection 127.0.0.1 -Port 28080
Invoke-RestMethod "http://127.0.0.1:28080/config" | Out-Null
podman compose --project-name webaccess-mcp logs --tail 200 searxng webaccess
```
External search providers can rate-limit requests or present anti-bot
challenges. WebAccess.MCP does not bypass those restrictions.
### The MCP entry works until the computer restarts
The JSON configuration persists, but the Podman machine and Compose services
must be running.
Start them before using WebAccess.MCP:
```powershell
Set-Location "D:\ai\mcp\WebAccess.MCP\deploy"
podman compose --project-name webaccess-mcp up --detach
```
Then enable or reconnect `webaccess` in LM Studio if needed.
## Quick Reference
WebAccess.MCP's complete LM Studio entry is:
```json
{
"mcpServers": {
"webaccess": {
"url": "http://127.0.0.1:28081/mcp"
}
}
}
```
Connection checklist:
1. Start the Podman machine.
2. Start the `webaccess-mcp` Compose project.
3. Verify port 28081.
4. Start LM Studio 0.3.17 or later.
5. Open **Program > Install > Edit mcp.json**.
6. Add the `webaccess` URL entry.
7. Save valid JSON.
8. Enable `webaccess`.
9. Load a tool-capable model.
10. Test with an explicit `web_search` request.

View File

@ -0,0 +1,658 @@
# Install WebAccess.MCP, Podman Desktop, and LM Studio on a New Windows PC
This guide is for a person starting with a Windows computer that does not yet
have Podman Desktop, LM Studio, or WebAccess.MCP installed.
PowerShell privilege labels in this guide mean:
- **PowerShell (Standard user)**: open PowerShell normally.
- **PowerShell (Admin)**: right-click PowerShell and select **Run as
administrator**.
Use standard-user PowerShell unless a step is explicitly labeled
**PowerShell (Admin)**.
For a shorter operations-only guide that assumes Podman and the repository are
already installed, see `docs\Standing-Up-the-Podman-Compose-Stack.md`.
For a focused, step-by-step LM Studio MCP configuration guide, see
`docs\Adding-WebAccess-MCP-to-LM-Studio.md`.
It creates one predictable AI workspace on a drive of your choice:
```text
<drive>:\ai\
llm\
models\ LM Studio model files
mcp\
WebAccess.MCP\ This MCP server and its deployment files
```
For example, if `D:` has enough free space:
```text
D:\ai\llm\models
D:\ai\mcp\WebAccess.MCP
```
The examples below use `D:`. Substitute another local drive everywhere if
needed. Use the same drive consistently.
> Podman Desktop and LM Studio are normal Windows applications, so their
> executables and application settings are installed in Windows-managed
> locations. Podman also stores container images and volumes inside its managed
> Linux virtual machine. The model files, MCP source, and deployment
> configuration are organized beneath `<drive>:\ai`.
## What This Installation Provides
Podman Compose runs three containers:
- `webaccess`: the .NET 10 MCP server, Playwright, and headless Chromium.
- `searxng`: the dedicated search service used by `web_search`.
- `postgres`: durable page-cache storage.
LM Studio connects to WebAccess.MCP through Streamable HTTP at:
```text
http://127.0.0.1:28081/mcp
```
SearXNG is available for local diagnostics at:
```text
http://127.0.0.1:28080/
```
Both ports bind only to the local computer, not to the LAN.
## Before You Begin
You need:
- 64-bit Windows 10 build 19043 or later, or Windows 11.
- Administrator access for enabling WSL 2.
- Hardware virtualization enabled in the computer's BIOS or UEFI.
- At least 6 GB of RAM available to the Podman machine. More RAM is strongly
recommended because LM Studio must run a model at the same time.
- Enough free disk space for Podman images and one or more LLMs. Models commonly
consume several to tens of gigabytes each.
- Internet access during installation, image building, and model download.
- A trusted copy of the WebAccess.MCP repository.
The optional integration TestRig also requires the .NET 10 SDK. It is not
required for normal use because WebAccess.MCP builds inside a container.
## 1. Choose the AI Drive and Create the Folder Layout
Choose a local drive with enough free space. Avoid a network share, removable
drive, OneDrive, or another synchronized folder.
**PowerShell (Standard user)**
```powershell
$AiDrive = "D:"
$AiRoot = Join-Path $AiDrive "ai"
New-Item -ItemType Directory -Force -Path `
(Join-Path $AiRoot "llm\models"), `
(Join-Path $AiRoot "mcp")
```
Verify the layout:
```powershell
Get-ChildItem $AiRoot -Recurse
```
Keep `$AiRoot` set in this PowerShell window while completing the installation.
In a new PowerShell window, set it again:
```powershell
$AiRoot = "D:\ai"
```
## 2. Install WSL 2
Podman uses a small Linux virtual machine on Windows. WSL 2 is the recommended
provider.
**PowerShell (Admin)**
```powershell
wsl --install --no-distribution
```
Restart Windows if prompted. After restarting, open PowerShell and verify WSL:
```powershell
wsl --status
```
If WSL installation fails, confirm that virtualization is enabled in BIOS/UEFI
and that Windows is fully updated before continuing.
## 3. Install and Configure Podman Desktop
### Install the application
Use either method below.
**Option A: Windows Package Manager**
Open PowerShell and run:
```powershell
winget install --exact --id RedHat.Podman-Desktop
```
**Option B: Windows installer**
1. Download Podman Desktop from
[podman-desktop.io/downloads](https://podman-desktop.io/downloads).
2. Run the downloaded installer.
3. Choose whether to install it for the current user or all users.
4. Complete the installer and start Podman Desktop.
### Complete Podman Desktop onboarding
In Podman Desktop:
1. Select **Start Onboarding**.
2. Allow Podman Desktop to install the Podman engine.
3. Keep **WSL 2** selected as the machine provider.
4. Create the Podman machine when prompted.
5. Install the Compose CLI when prompted.
6. Complete onboarding and open the dashboard.
If onboarding was previously skipped:
1. Open **Settings > Resources**.
2. In the **Podman** tile, select **Setup Podman**.
3. Create and start a WSL-based Podman machine.
4. In the **Compose** tile, select **Setup**.
The official Podman Desktop minimum is 6 GB of RAM for its machine. If the
computer has sufficient memory, assign more resources in Podman Desktop so
Chromium, PostgreSQL, and SearXNG can run comfortably alongside LM Studio.
Open a new PowerShell window and verify both components:
```powershell
podman info
podman compose version
```
Do not continue until both commands succeed. If `podman info` cannot connect,
open Podman Desktop and start its Podman machine.
## 4. Install LM Studio and Set Its Model Folder
1. Download LM Studio for Windows from
[lmstudio.ai/download](https://lmstudio.ai/download).
2. Run the installer and start LM Studio.
3. Complete the initial setup.
4. Open **My Models** in LM Studio.
5. Use the model-directory control to change the models directory to:
```text
D:\ai\llm\models
```
LM Studio may create organization and model subdirectories beneath this folder.
Do not manually flatten those directories.
WebAccess.MCP requires LM Studio 0.3.17 or later. Check the installed version in
LM Studio and install any offered update before configuring MCP.
## 5. Put WebAccess.MCP Under the AI Folder
The final repository location must be:
```text
D:\ai\mcp\WebAccess.MCP
```
Use one of the following methods.
### Option A: Copy a supplied folder
If WebAccess.MCP was supplied as a folder, copy the entire folder into:
```text
D:\ai\mcp\
```
Do not copy only the `deploy` directory. The container build also needs the
production source beneath `src`.
### Option B: Extract a supplied ZIP file
Extract the ZIP and move or rename the extracted repository folder so that this
file exists:
```text
D:\ai\mcp\WebAccess.MCP\deploy\compose.yml
```
Avoid an accidental double folder such as:
```text
D:\ai\mcp\WebAccess.MCP\WebAccess.MCP\deploy\compose.yml
```
### Option C: Clone with Git
If the person distributing WebAccess.MCP provides a Git repository URL, install
Git and clone it:
```powershell
winget install --exact --id Git.Git
```
Close and reopen PowerShell after Git installation, then run:
```powershell
$AiRoot = "D:\ai"
Set-Location (Join-Path $AiRoot "mcp")
git clone <repository-url> WebAccess.MCP
```
Replace `<repository-url>` with the trusted URL supplied by the project owner.
### Verify the repository
```powershell
$AiRoot = "D:\ai"
$WebAccessRoot = Join-Path $AiRoot "mcp\WebAccess.MCP"
Test-Path (Join-Path $WebAccessRoot "deploy\compose.yml")
Test-Path (Join-Path $WebAccessRoot "Containerfile")
Test-Path (Join-Path $WebAccessRoot "src\WebAccess.MCP\WebAccess.MCP.csproj")
```
All three commands must return `True`.
## 6. Create Installation Secrets
WebAccess.MCP needs a local PostgreSQL password and SearXNG secret. Generate
unique values rather than copying secrets from another computer.
Open PowerShell and run:
```powershell
$AiRoot = "D:\ai"
$DeployRoot = Join-Path $AiRoot "mcp\WebAccess.MCP\deploy"
Set-Location $DeployRoot
function New-HexSecret {
$bytes = New-Object byte[] 32
$generator = [System.Security.Cryptography.RandomNumberGenerator]::Create()
try {
$generator.GetBytes($bytes)
}
finally {
$generator.Dispose()
}
-join ($bytes | ForEach-Object { $_.ToString("x2") })
}
@"
WEBACCESS_POSTGRES_PASSWORD=$(New-HexSecret)
SEARXNG_SECRET=$(New-HexSecret)
"@ | Set-Content -Encoding Ascii .env
```
Verify that the file exists without printing its secret values:
```powershell
Test-Path .env
```
The repository ignores `deploy\.env`. Keep it private, do not send it to anyone,
and do not commit it to Git.
## 7. Build and Start WebAccess.MCP
From `D:\ai\mcp\WebAccess.MCP\deploy`, run:
```powershell
.\Start-WebAccessStack.ps1
```
The script creates `deploy\.env` with random installation secrets if needed,
verifies Podman and Compose, and runs the entire Compose stack. The first run
downloads the base images and builds WebAccess.MCP. It can take several minutes
because the Playwright image includes Chromium and its Linux dependencies.
Check the services:
```powershell
podman compose --project-name webaccess-mcp ps
```
Expected result:
- `postgres` is running and healthy.
- `searxng` is running and healthy.
- `webaccess` is running.
Compose waits for PostgreSQL and SearXNG to become healthy before starting
WebAccess.MCP.
In Podman Desktop, open **Containers**. The three services should appear in a
`webaccess-mcp (compose)` group. Expand the group to inspect status, ports, and
logs.
If a service does not start:
```powershell
podman compose --project-name webaccess-mcp logs --tail 200
```
## 8. Verify the Running Services
Verify both local ports:
```powershell
Test-NetConnection 127.0.0.1 -Port 28080
Test-NetConnection 127.0.0.1 -Port 28081
```
Both results should include:
```text
TcpTestSucceeded : True
```
Verify the dedicated SearXNG instance:
```powershell
(Invoke-RestMethod "http://127.0.0.1:28080/config").instance_name
```
The expected name is:
```text
WebAccess.MCP SearXNG
```
The `/mcp` URL is a protocol endpoint, not a normal website. Opening it in a
browser is not a complete MCP test. LM Studio will perform the MCP handshake in
a later step.
## 9. Download a Tool-Capable Model in LM Studio
WebAccess.MCP supplies tools, but the selected language model decides when and
how to call them. Choose an **instruct/chat model with tool-use or function-call
support**. A plain base model is not suitable.
In LM Studio:
1. Open **Discover** (`Ctrl+2`).
2. Search for a tool-capable instruct model appropriate for the computer.
3. Select a GGUF quantization. A 4-bit option is a practical starting point for
many computers; larger quantizations need more RAM or VRAM.
4. Download the model.
5. Confirm in **My Models** that it was stored beneath:
```text
D:\ai\llm\models
```
6. Open **Chat** and load the model.
Prefer a model with enough context for tool schemas and returned web content.
At least 16K context is a useful starting point when the model and hardware
support it. Model availability changes frequently, so this guide does not lock
the installation to one model family.
If LM Studio cannot use MCP tools reliably with the chosen model, first confirm
that the model card advertises tool/function calling. Then try a stronger
instruct model or a less aggressive quantization if the hardware permits.
## 10. Introduce WebAccess.MCP to LM Studio
LM Studio 0.3.17 and later supports remote MCP servers by URL.
1. Keep the Podman stack running.
2. In LM Studio, open the **Program** tab in the right sidebar.
3. Select **Install > Edit mcp.json**.
4. If the file is empty, enter:
```json
{
"mcpServers": {
"webaccess": {
"url": "http://127.0.0.1:28081/mcp"
}
}
}
```
If the file already contains MCP servers, add only this property inside its
existing `mcpServers` object:
```json
"webaccess": {
"url": "http://127.0.0.1:28081/mcp"
}
```
Preserve the existing entries and include commas between JSON properties.
This is a remote Streamable HTTP MCP server:
- Do not add `command` or `args`.
- Do not add authentication headers.
- Do not use `http://127.0.0.1:28080`; that is SearXNG, not MCP.
- Do not use `localhost` if local security software resolves it unexpectedly;
use the exact `127.0.0.1` URL shown above.
5. Save `mcp.json`.
6. Confirm that `webaccess` appears and is enabled in the Program tab.
7. Start or return to a chat with the tool-capable model loaded.
8. Enable the WebAccess.MCP tools for that chat if LM Studio asks.
## 11. Test LM Studio and WebAccess.MCP Together
Ask the model:
> Use `web_search` to find the official LM Studio MCP documentation. Return the
> page title and source URL.
LM Studio should display a `web_search` tool call and its result.
Then test the browser tools:
> Open the official LM Studio MCP documentation URL, report the page title, and
> list the first five links.
WebAccess.MCP has 15 tools. LM Studio should show those tools as available when
the MCP connection is healthy.
The browser context is intentionally long-lived. Navigation and interaction
state persist across tool calls even though the MCP HTTP transport is stateless.
## 12. Start and Stop the System
### Start a normal session
1. Start Podman Desktop.
2. Make sure the Podman machine is running.
3. Open PowerShell and run:
```powershell
Set-Location "D:\ai\mcp\WebAccess.MCP\deploy"
podman compose --project-name webaccess-mcp up --detach
```
4. Start LM Studio.
5. Load the desired tool-capable model.
6. Confirm that `webaccess` is enabled in the Program tab.
### Stop a normal session
Close LM Studio if desired, then run:
```powershell
Set-Location "D:\ai\mcp\WebAccess.MCP\deploy"
podman compose --project-name webaccess-mcp stop
```
This preserves the containers and PostgreSQL page cache.
### Start the stopped containers again
```powershell
Set-Location "D:\ai\mcp\WebAccess.MCP\deploy"
podman compose --project-name webaccess-mcp start
```
### Remove containers but preserve data
```powershell
Set-Location "D:\ai\mcp\WebAccess.MCP\deploy"
podman compose --project-name webaccess-mcp down
```
This removes the containers and private network but preserves the PostgreSQL
volume.
## 13. Update WebAccess.MCP
Before updating, stop the stack:
```powershell
Set-Location "D:\ai\mcp\WebAccess.MCP\deploy"
podman compose --project-name webaccess-mcp down
```
Replace the supplied repository files with the newer version, or use Git:
```powershell
Set-Location "D:\ai\mcp\WebAccess.MCP"
git pull
```
Do not overwrite or delete `deploy\.env`.
Rebuild and start:
```powershell
Set-Location "D:\ai\mcp\WebAccess.MCP\deploy"
podman compose --project-name webaccess-mcp up --detach --build
```
The LM Studio `mcp.json` entry does not need to change as long as the endpoint
remains `http://127.0.0.1:28081/mcp`.
## 14. Optional Full Integration Test
This section is for maintainers or advanced troubleshooting. Install the .NET
10 SDK, keep the stack running, and execute from the repository root:
```powershell
Set-Location "D:\ai\mcp\WebAccess.MCP"
dotnet run --project ".\src\WebAccess.MCP.TestRig\WebAccess.MCP.TestRig.csproj" -- `
--mcp-url "http://127.0.0.1:28081/mcp" `
--real-searxng `
--real-searxng-engine google `
--real-searxng-engine brave `
--real-searxng-engine duckduckgo
```
The TestRig validates the HTTP MCP transport, the 15-tool contract, browser
state and interactions, page caching, and the externally managed SearXNG
service. External providers can still rate-limit or challenge automated search
requests.
## 15. Troubleshooting
### Podman commands are not recognized
Close and reopen PowerShell after installing Podman Desktop. If the commands
remain unavailable, return to Podman Desktop onboarding or **Settings >
Resources** and install Podman and Compose.
### Podman cannot connect
Open Podman Desktop and start the Podman machine. Then retry:
```powershell
podman info
```
If the machine cannot start, verify WSL 2:
```powershell
wsl --status
```
### LM Studio cannot connect to WebAccess.MCP
1. Confirm LM Studio is version 0.3.17 or later.
2. Confirm the URL is exactly `http://127.0.0.1:28081/mcp`.
3. Confirm `webaccess` is enabled in LM Studio's Program tab.
4. Run:
```powershell
Set-Location "D:\ai\mcp\WebAccess.MCP\deploy"
podman compose --project-name webaccess-mcp ps
Test-NetConnection 127.0.0.1 -Port 28081
podman compose --project-name webaccess-mcp logs --tail 200 webaccess
```
5. Check that `mcp.json` is valid JSON.
### A host port is already in use
Identify the owning process:
```powershell
Get-NetTCPConnection -State Listen -LocalPort 28080,28081 |
Select-Object -Property LocalAddress, LocalPort, OwningProcess
```
Stop or reconfigure the conflicting application. Do not substitute port 18080;
that port belongs to an unrelated deployment.
### PostgreSQL rejects the password after `.env` was changed
PostgreSQL initializes its password only when its data volume is first created.
Restore the original `WEBACCESS_POSTGRES_PASSWORD`.
If the password is lost and deleting the WebAccess.MCP page cache is acceptable,
perform an intentional reset:
```powershell
Set-Location "D:\ai\mcp\WebAccess.MCP\deploy"
podman compose --project-name webaccess-mcp down --volumes
podman compose --project-name webaccess-mcp up --detach --build
```
`down --volumes` permanently deletes this stack's PostgreSQL data. Do not use it
for normal shutdown.
### Search fails while browser tools work
1. Open `http://127.0.0.1:28080/config`.
2. Inspect the `searxng` and `webaccess` logs in Podman Desktop.
3. Confirm the dedicated SearXNG service is healthy.
External search engines can temporarily fail, rate-limit requests, or present
anti-bot challenges. WebAccess.MCP does not bypass those restrictions.
### The Compose group is missing in Podman Desktop
Confirm Podman Desktop is connected to the same Podman machine used by
PowerShell. Starting with the exact `--project-name webaccess-mcp` command adds
the Compose labels Podman Desktop uses to display the services as one group.
### LM Studio saves models somewhere else
Return to **My Models**, change the model directory to
`D:\ai\llm\models`, and use LM Studio's own directory controls to move or
redownload models. Do not manually reorganize a model while LM Studio is
running.

View File

@ -0,0 +1,624 @@
# Stand Up the WebAccess.MCP Podman Compose Stack
This runbook covers only the Podman Compose stack: configure it, build it, start
it, verify it, inspect it, stop it, and rebuild it.
All commands in this runbook use **PowerShell (Standard user)**. No command in
this runbook requires **PowerShell (Admin)** because Podman and its machine are
prerequisites.
For the short clone, one-command startup, and LM Studio connection path, see
`QUICKSTART.md` in the repository root.
For complete first-time Windows, Podman Desktop, LM Studio, folder-layout, and
model-installation instructions, see
`docs\Running-in-Podman-Desktop-and-LM-Studio.md`.
To build and create a clean deployment folder and ZIP for another computer, run
`Export-WebAccessDeployment.ps1` from the repository root. See **Create a
Deployment Bundle** below.
## The Short Answer
After cloning or extracting the repository, open PowerShell in the repository
root and run one command:
```powershell
.\Start-WebAccessMcp.cmd
```
That command does the whole job:
1. Checks that Podman is installed and running.
2. Checks that Podman Compose is installed.
3. Creates `deploy\.env` with random secrets if it does not exist.
4. Runs Compose with `up --detach --build`.
5. Starts PostgreSQL, SearXNG, and WebAccess.MCP in dependency order.
6. Displays the status, local URLs, LM Studio JSON, and refresh instruction.
It does not replace or print an existing `.env` file.
The root command launches `deploy\Start-WebAccessStack.ps1` with a temporary
PowerShell execution-policy bypass. It does not change the computer's permanent
execution policy.
The direct Compose command used underneath is:
```powershell
Set-Location "D:\ai\mcp\WebAccess.MCP\deploy"
podman compose --project-name webaccess-mcp up --detach --build
```
You do not start the containers separately.
### Complete clone-to-LM-Studio flow
```powershell
git clone <repository-url> WebAccess.MCP
Set-Location ".\WebAccess.MCP"
.\Start-WebAccessMcp.cmd
```
Replace `<repository-url>` with the repository URL supplied by the project
owner.
After the command succeeds:
1. In LM Studio, open **Program > Install > Edit mcp.json**.
2. Add this entry inside the existing `mcpServers` object:
```json
"webaccess": {
"url": "http://127.0.0.1:28081/mcp"
}
```
3. Save `mcp.json`.
4. Click **Refresh** in LM Studio.
That is the complete normal installation path for someone who already has
Podman Desktop, LM Studio, and a tool-capable model.
## Stack Overview
The Compose project starts three services:
| Service | Purpose | Host access |
| --- | --- | --- |
| `webaccess` | WebAccess.MCP, Playwright, and Chromium | `http://127.0.0.1:28081/mcp` |
| `searxng` | Dedicated search service | `http://127.0.0.1:28080/` |
| `postgres` | Durable WebAccess.MCP page cache | Not published |
All services share a private Compose network. WebAccess.MCP reaches SearXNG at
`http://searxng:8080/` and PostgreSQL at `postgres:5432`.
The host bindings use `127.0.0.1`, so WebAccess.MCP and SearXNG are available
only on the computer running the stack. They are not exposed to the LAN.
## Requirements
Before using this runbook:
- Podman Desktop must be installed.
- A Podman machine must be running.
- The Compose provider must be installed.
- The complete WebAccess.MCP repository must be present locally.
The examples assume:
```text
D:\ai\mcp\WebAccess.MCP
```
Replace `D:` consistently if the AI folder is on another local drive.
Verify Podman and Compose:
```powershell
podman info
podman compose version
```
Both commands must succeed.
## 1. Open the Deployment Directory
Open PowerShell and run:
```powershell
$WebAccessRoot = "D:\ai\mcp\WebAccess.MCP"
$DeployRoot = Join-Path $WebAccessRoot "deploy"
Set-Location $DeployRoot
```
Verify the required deployment files:
```powershell
Test-Path ".\compose.yml"
Test-Path "..\Containerfile"
Test-Path ".\searxng\settings.yml"
```
All three commands must return `True`.
## 2. Environment File
`Start-WebAccessStack.ps1` creates this file automatically on its first run.
The stack requires:
- `WEBACCESS_POSTGRES_PASSWORD`
- `SEARXNG_SECRET`
The following manual procedure is needed only if the script cannot be used.
Generate unique values for this installation:
```powershell
function New-HexSecret {
$bytes = New-Object byte[] 32
$generator = [System.Security.Cryptography.RandomNumberGenerator]::Create()
try {
$generator.GetBytes($bytes)
}
finally {
$generator.Dispose()
}
-join ($bytes | ForEach-Object { $_.ToString("x2") })
}
@"
WEBACCESS_POSTGRES_PASSWORD=$(New-HexSecret)
SEARXNG_SECRET=$(New-HexSecret)
"@ | Set-Content -Encoding Ascii ".env"
```
Verify that the file exists without displaying its contents:
```powershell
Test-Path ".env"
```
Keep `.env` private. Do not commit, publish, or send it to another person. The
repository's `.gitignore` excludes `deploy\.env`.
If `.env` already exists for a working installation, keep it. Do not generate a
new password over an existing PostgreSQL volume. Skip this section on every
normal start.
## 3. Review the Effective Compose Model
Validate the Compose file:
```powershell
podman compose --project-name webaccess-mcp config --quiet
```
A successful command produces no errors.
Do not run `podman compose config` without `--quiet` in logs or support messages:
its expanded output can contain the PostgreSQL password.
## 4. Build and Start the Stack
The recommended command is:
```powershell
Set-Location ..
.\Start-WebAccessMcp.cmd
```
Alternatively, from the `deploy` directory run the PowerShell launcher directly:
```powershell
.\Start-WebAccessStack.ps1
```
Or run Compose directly:
```powershell
podman compose --project-name webaccess-mcp up --detach --build
```
This is the single command that stands up the whole stack. Do not run separate
`podman run` commands for PostgreSQL, SearXNG, or WebAccess.MCP.
This command:
1. Pulls the PostgreSQL, SearXNG, .NET, and Playwright base images as needed.
2. Builds the `webaccess` image from the repository source.
3. Creates the private network and PostgreSQL volume.
4. Starts PostgreSQL and waits for it to become healthy.
5. Starts SearXNG and waits for it to become healthy.
6. Starts WebAccess.MCP.
The first build can take several minutes. Later builds generally reuse cached
image layers.
## 5. Check Container Status
Run:
```powershell
podman compose --project-name webaccess-mcp ps
```
Expected state:
- `postgres`: running and healthy.
- `searxng`: running and healthy.
- `webaccess`: running.
In Podman Desktop, open **Containers** and expand the
`webaccess-mcp (compose)` group. It should contain the same three services.
If `webaccess` is absent, first confirm that PostgreSQL and SearXNG reached
healthy status. Compose does not start WebAccess.MCP until both dependencies are
healthy.
## 6. Verify Host Connectivity
Verify that SearXNG and WebAccess.MCP are listening:
```powershell
Test-NetConnection 127.0.0.1 -Port 28080
Test-NetConnection 127.0.0.1 -Port 28081
```
Both results must include:
```text
TcpTestSucceeded : True
```
Verify that the correct SearXNG instance is running:
```powershell
(Invoke-RestMethod "http://127.0.0.1:28080/config").instance_name
```
Expected output:
```text
WebAccess.MCP SearXNG
```
The MCP endpoint is:
```text
http://127.0.0.1:28081/mcp
```
It is a Streamable HTTP MCP protocol endpoint, not a normal website. A browser
GET is not a complete health check or MCP handshake.
## 7. Inspect Logs
Show the most recent logs from every service:
```powershell
podman compose --project-name webaccess-mcp logs --tail 200
```
Show one service:
```powershell
podman compose --project-name webaccess-mcp logs --tail 200 webaccess
podman compose --project-name webaccess-mcp logs --tail 200 searxng
podman compose --project-name webaccess-mcp logs --tail 200 postgres
```
Follow WebAccess.MCP logs:
```powershell
podman compose --project-name webaccess-mcp logs --follow webaccess
```
Press `Ctrl+C` to stop following logs. This does not stop the service.
## 8. Normal Start and Stop Commands
Run all commands from:
```text
D:\ai\mcp\WebAccess.MCP\deploy
```
### Stop without removing containers
```powershell
podman compose --project-name webaccess-mcp stop
```
### Start stopped containers
```powershell
podman compose --project-name webaccess-mcp start
```
### Restart all services
```powershell
podman compose --project-name webaccess-mcp restart
```
### Restart one service
```powershell
podman compose --project-name webaccess-mcp restart webaccess
```
### Remove containers and the private network
```powershell
podman compose --project-name webaccess-mcp down
```
`down` preserves the named PostgreSQL volume. The page cache remains available
the next time the stack starts.
### Start after `down`
```powershell
podman compose --project-name webaccess-mcp up --detach
```
## 9. Rebuild After a Source Update
After replacing the source or running `git pull`, execute:
```powershell
Set-Location "D:\ai\mcp\WebAccess.MCP\deploy"
podman compose --project-name webaccess-mcp up --detach --build
```
Compose rebuilds the WebAccess.MCP image and recreates services when necessary.
The PostgreSQL volume and its cached page data are preserved.
To force a completely fresh WebAccess.MCP image build without deleting
PostgreSQL data:
```powershell
podman compose --project-name webaccess-mcp build --no-cache webaccess
podman compose --project-name webaccess-mcp up --detach
```
Use the no-cache build only for troubleshooting stale build layers. It is
slower than a normal rebuild.
## 10. Create a Deployment Bundle
Run this from the development repository root to build WebAccess.MCP and create
a clean bundle that can be copied or uploaded:
```powershell
Set-Location "D:\path\to\the\WebAccess.MCP-repository"
.\Export-WebAccessDeployment.ps1
```
By default, the script creates a timestamped folder and ZIP beneath
`artifacts`, for example:
```text
artifacts\
WebAccess.MCP-deployment-20260821-193000\
WebAccess.MCP\
.dockerignore
deploy\
docs\
src\
WebAccess.MCP.zip
```
The script:
1. Builds the production container image with Podman.
2. Copies only the production source and required Compose/container files.
3. Excludes `bin`, `obj`, tests, TestRig files, IDE files, Git metadata, and
`deploy\.env`.
4. Includes the startup script and operator documentation.
5. Creates a ZIP suitable for uploading or transferring.
To choose the output folder:
```powershell
.\Export-WebAccessDeployment.ps1 `
-DestinationPath "D:\release\WebAccess.MCP"
```
This also creates:
```text
D:\release\WebAccess.MCP.zip
```
The destination and ZIP must not already exist. The script refuses to overwrite
an existing release.
To create the bundle without ZIP compression:
```powershell
.\Export-WebAccessDeployment.ps1 `
-DestinationPath "D:\release\WebAccess.MCP" `
-NoArchive
```
To package without building the image first:
```powershell
.\Export-WebAccessDeployment.ps1 -SkipBuild
```
Use `-SkipBuild` only when Podman is unavailable or the image has already been
validated separately. The default behavior proves that the bundled source can
build before creating the release.
The bundle never contains `.env` or installation secrets. On the destination
computer, extract or copy the `WebAccess.MCP` folder to:
```text
<drive>:\ai\mcp\WebAccess.MCP
```
Then run:
```powershell
Set-Location "<drive>:\ai\mcp\WebAccess.MCP"
.\Start-WebAccessMcp.cmd
```
The startup script creates new machine-specific secrets and starts the complete
Compose stack.
## 11. Intentional Data Reset
The following operation removes the containers, network, and PostgreSQL volume:
```powershell
podman compose --project-name webaccess-mcp down --volumes
```
This permanently deletes WebAccess.MCP's page-cache database. It does not delete
the source repository or `.env`.
Use it only when:
- A completely clean database is explicitly required.
- The PostgreSQL password was lost or intentionally changed.
- The PostgreSQL volume is known to be unusable.
Start a new stack afterward:
```powershell
podman compose --project-name webaccess-mcp up --detach --build
```
Do not use `down --volumes` for normal shutdown or routine updates.
## 12. Troubleshooting
### Podman cannot connect
Open Podman Desktop and start its Podman machine, then run:
```powershell
podman info
```
### The Compose provider is unavailable
In Podman Desktop, open **Settings > Resources**. In the **Compose** tile,
select **Setup**, then verify:
```powershell
podman compose version
```
### Compose says a required variable is missing
Confirm that PowerShell is in the `deploy` directory and that `.env` exists:
```powershell
Get-Location
Test-Path ".env"
```
The file must define both `WEBACCESS_POSTGRES_PASSWORD` and `SEARXNG_SECRET`.
Do not print secret values into shared terminal logs.
### A service exits during startup
Inspect status and logs:
```powershell
podman compose --project-name webaccess-mcp ps --all
podman compose --project-name webaccess-mcp logs --tail 200
```
Resolve the first unhealthy or failed dependency before troubleshooting
downstream services.
### Port 28080 or 28081 is already in use
Identify the owning process:
```powershell
Get-NetTCPConnection -State Listen -LocalPort 28080,28081 |
Select-Object -Property LocalAddress, LocalPort, OwningProcess
```
Stop or reconfigure the conflicting application. Do not switch SearXNG to port
18080; that port belongs to an unrelated deployment.
### PostgreSQL rejects the password
If the PostgreSQL volume already exists, it retains the password used when it
was created. Restore the original value in `.env`.
If the original password is unavailable and losing cached pages is acceptable,
use the intentional `down --volumes` reset described above.
### SearXNG is unhealthy
Inspect its logs:
```powershell
podman compose --project-name webaccess-mcp logs --tail 200 searxng
```
Confirm that `deploy\searxng\settings.yml` exists and that port 28080 is not
occupied.
### WebAccess.MCP is not listening
Inspect its logs and dependencies:
```powershell
podman compose --project-name webaccess-mcp ps
podman compose --project-name webaccess-mcp logs --tail 200 webaccess
```
PostgreSQL and SearXNG must be healthy. The host port test must succeed:
```powershell
Test-NetConnection 127.0.0.1 -Port 28081
```
### Search providers return errors
First verify the local SearXNG service and review logs:
```powershell
Invoke-RestMethod "http://127.0.0.1:28080/config" | Out-Null
podman compose --project-name webaccess-mcp logs --tail 200 searxng webaccess
```
Google, Brave, or DuckDuckGo can temporarily rate-limit requests or present
anti-bot challenges. The stack does not bypass provider restrictions.
## Command Summary
Run from `D:\ai\mcp\WebAccess.MCP\deploy`:
```powershell
# Validate
podman compose --project-name webaccess-mcp config --quiet
# First build or rebuild
podman compose --project-name webaccess-mcp up --detach --build
# Status
podman compose --project-name webaccess-mcp ps
# Logs
podman compose --project-name webaccess-mcp logs --tail 200
# Stop and start without removing containers
podman compose --project-name webaccess-mcp stop
podman compose --project-name webaccess-mcp start
# Remove containers while preserving PostgreSQL data
podman compose --project-name webaccess-mcp down
# Destructive reset, including PostgreSQL data
podman compose --project-name webaccess-mcp down --volumes
```

View File

@ -0,0 +1,226 @@
# WebAccess.MCP Architecture
## Purpose
WebAccess.MCP is a .NET 10 web-access service for LM Studio and compatible Model Context Protocol (MCP) clients. The target production host is an ASP.NET Core application exposing MCP through Streamable HTTP. Browser interaction remains Playwright-backed, while web search is delegated over HTTP to a dedicated SearXNG service.
The repository contains three sibling projects:
- `WebAccess.MCP`, the production server.
- `WebAccess.MCP.TestRig`, an independent integration client with no project reference to production.
- `WebAccess.MCP.Tests`, focused unit tests for production infrastructure.
## Migration State
The source is being migrated in small, independently validated steps. Production now uses ASP.NET Core Streamable HTTP MCP, the TestRig connects to a separately started HTTP server, and the application-owned SearXNG lifecycle subsystem has been removed. Container, Compose, deployment, and LM Studio migration remain future work.
Future changes must move toward the target described below without combining unrelated transport, testing, lifecycle, container, deployment, or LM Studio changes into one step.
## Target Runtime Topology
The deployed topology is:
LM Studio
|
| Streamable HTTP MCP
v
WebAccess.MCP container
|
+--> Playwright / headless Chromium
|
+--> HTTP --> dedicated SearXNG container
| |
| +--> Google
| +--> Brave
| +--> DuckDuckGo
|
+--> PostgreSQL --> durable page-cache persistence
Podman Compose owns the WebAccess.MCP, dedicated SearXNG, and PostgreSQL containers, private network, dependency and readiness orchestration, host port mappings, and service startup and shutdown. PostgreSQL belongs to the `webaccess-mcp` project and is unrelated to the separate `postgres-dev` stack.
WebAccess.MCP does not invoke Podman, Docker, or Compose; does not know where a container runtime is installed; and does not start, stop, or otherwise own SearXNG. From the application's perspective, SearXNG is only an HTTP service identified by configuration.
The existing `ai-searxng` deployment is unrelated and off limits. Its container, configuration, data, identity, and host port 18080 must not be reused or treated as a template or dependency.
## MCP Transport and Endpoints
The target production host uses:
- ASP.NET Core on .NET 10.
- `ModelContextProtocol.AspNetCore` 2.2.0.
- Streamable HTTP MCP.
- A stateless MCP HTTP transport initially.
- The MCP endpoint `/mcp`.
The container listens on port 8080. Compose publishes that endpoint only on host loopback:
http://127.0.0.1:28081/mcp
LM Studio will ultimately connect to this URL instead of launching a native WebAccess.MCP command.
Stateless MCP transport does not make browser navigation stateless. Browser state is owned by application singletons independently of MCP transport sessions. A stateful transport should be introduced only if a later MCP requirement needs transport-level session state.
Neither the WebAccess.MCP endpoint nor SearXNG should bind to LAN interfaces by default.
## Browser Runtime and Tools
Browser tools use Microsoft.Playwright 1.62.0 and headless Chromium. The existing browser design is preserved during the transport and container migration:
- One long-lived Playwright runtime.
- One long-lived Chromium browser.
- One persistent browser context and page, created lazily on the first browser operation.
- Serialized access to the persistent page so navigation and interaction state survive safely across tool calls.
Chromium itself currently starts eagerly as part of hosted-service startup. Converting Chromium startup to lazy initialization is explicitly deferred until the HTTP and container migration is stable.
The expected MCP tool surface is 16 tools:
- `health_check`
- `open_url`
- `get_page_title`
- `get_page_url`
- `get_page_info`
- `get_page_text`
- `find_page_text`
- `get_links`
- `get_interactive_elements`
- `click`
- `type`
- `press_key`
- `go_back`
- `go_forward`
- `reload`
- `web_search`
Tool names and contracts should remain stable throughout the transport migration.
## Search Architecture
`web_search` does not use Playwright and does not directly scrape Google, Brave, DuckDuckGo, Bing, or SearXNG pages. Its application flow is:
SearchTools
-> IWebSearchService
-> SearXngWebSearchService
-> HTTP GET /search?format=json
-> dedicated SearXNG
-> normalized WebSearchResponse
The dedicated SearXNG instance performs searches using the intended selectors:
- `google`
- `brave`
- `duckduckgo`
Bing is not required. The standard selected engines require no downstream search-engine API keys, and the separate `braveapi` engine is not used.
The search service retains query and result-limit validation, configured default engines, per-call selector overrides, selector normalization, JSON response handling, HTTP/HTTPS URL validation, result normalization, deduplication, timeout handling, and safe MCP error translation.
Each successful `web_search` response includes one short deterministic `nextStepHint` encouraging targeted research through `find_page_text(url,query)` before `get_page_text`. This is guidance only: `web_search` does not call either tool, and search providers, ranking, filtering, ordering, and result records are unaffected. The hint is intentionally small and isolated so the experiment is easy to revert. Adjacent-page behavior and loop protection remain separate concerns.
## Dedicated SearXNG Service
The Compose service name is `searxng`, and WebAccess.MCP reaches it through the private Compose network at:
http://searxng:8080/
SearXNG listens on container port 8080. Compose publishes diagnostic and host access only on loopback:
http://127.0.0.1:28080/
The dedicated instance must enable JSON output and use independent configuration and data beneath the WebAccess.MCP deployment boundary. Its lifecycle and readiness are Compose responsibilities.
## Application Configuration
Application configuration uses standard .NET configuration providers. Application-level SearXNG configuration contains only HTTP client concerns:
- `Search:SearXNG:BaseUrl`
- `Search:SearXNG:Timeout`
- `Search:SearXNG:DefaultEngines`
Compose supplies the production base URL as `http://searxng:8080/`.
Page-cache configuration uses:
- `ConnectionStrings:WebCache` for the internal PostgreSQL connection.
- `WebCache:DefaultExpiration`, which defaults to one hour and must be greater than zero.
- `WebCache:ChunkSize`, which defaults to 4000 characters and supports values from 1 through 50000.
PostgreSQL is the durable page-cache persistence layer. `WebCacheService` owns requested-URL validation, safe fragment-free URL identity, expiry decisions, and non-overlapping text chunk creation. Cache keys are lowercase SHA-256 hashes of normalized requested URLs, and one current snapshot is retained per unique cache key. `PostgresWebCacheRepository` owns factory-scoped EF Core persistence and transactional replacement of page metadata and ordered chunks.
Cached page snapshots use contiguous, 1-based chunks whose configurable logical size defaults to 4000 characters; optional raw HTML remains on the parent snapshot. URL-based page reads request individual chunks instead of repeatedly returning the beginning of a page. This persistence layer will also support later repeated-work suppression and search enrichment without duplicating normalized text on the parent row. Redis is not part of this design at this time.
PostgreSQL also provides deterministic lexical relevance search over cached chunk text. Each chunk has an automatically maintained `tsvector` generated with PostgreSQL's `simple` text-search configuration and indexed with GIN. Full-text search identifies candidate locations within one selected cached page; deterministic in-process scoring then weights the exact query more strongly than caller-supplied related terms and rewards multiple nearby matches. It does not use an LLM, embeddings, pgvector, or additional PostgreSQL extensions.
`find_page_text(url,query,relatedTerms?)` exposes that retrieval through MCP. `url` and `query` are required; optional related terms are trimmed, bounded, deduplicated hints used only for deterministic matching inside the already selected cached document. They never alter `web_search`, select another URL, invoke an LLM, or become cached page content. A fresh snapshot is searched entirely through PostgreSQL-backed data; a missing or expired snapshot is populated with one serialized browser navigation/extraction and then searched.
Storage chunks are not treated as semantic boundaries. After candidate discovery, the service reconstructs the complete bounded normalized snapshot from persisted `StartOffset`, `Length`, and `Text`, then returns one hard-bounded context window of approximately the configured 4000-character page size around the strongest match. That window may span multiple storage chunks without gaps, overlap, or duplicated boundary text and reports the corresponding 1-based `get_page_text` page or page range. If targeted retrieval is weak or finds nothing, the response does not claim the information is absent; it directs the caller to sequential `get_page_text` reading as the correctness fallback. Repeat/loop protection remains separate.
The preferred targeted-research flow is `web_search` -> `find_page_text` -> `get_page_text` only for selected returned pages when snippets are insufficient. Neither `open_url` nor `get_page_text` page 1 is required to seed the cache. Sequential page reading remains valid for broad summaries, comprehensive document reading, and explicit continuation when a selected page is incomplete. Automatic adjacent-page expansion is not implemented and remains a separate concern.
`get_page_text` returns one configured-size text page per call. Its public `page` parameter is 1-based and defaults to 1; the former public `maxChars` parameter is removed. URL-based reads return a fresh PostgreSQL snapshot when available. A miss or expired snapshot causes one serialized browser navigation and extraction, followed by transactional snapshot replacement, while the requested page is returned directly from the normalized text already in memory. Reads of the current interactive browser page bypass durable URL caching because click, type, and key operations may have changed its state.
Later pages contain only their new, non-overlapping text. PostgreSQL failures fail open to the existing browser path, and a failed cache write does not suppress an otherwise successful browser result. The existing 50000-character normalized extraction ceiling remains an internal safety limit; a final available page indicates when the stored snapshot was truncated. Repeat/loop protection remains a separate future step.
The removed application-owned lifecycle subsystem and its fields must not be reintroduced:
- `AutoStart`
- `StopOnShutdown`
- `StartupTimeout`
- `ProbeInterval`
- `StartExecutable`
- `StartArguments`
- `StopExecutable`
- `StopArguments`
- `WorkingDirectory`
No application configuration should contain a Podman executable path, Windows container-runtime path, or generic SearXNG launcher command.
Use `Microsoft.Extensions.Logging` for application diagnostics. Normal structured console and container logging is allowed for the HTTP host. Prefer `ILogger` over ad hoc `Console.WriteLine` diagnostics.
## TestRig Direction
`WebAccess.MCP.TestRig` remains a separate sibling .NET 10 console project with no direct production project reference.
`WebAccess.MCP.TestRig` connects to a separately started production server through the real Streamable HTTP MCP endpoint. It does not own production as a child process. Deterministic browser loopback fixtures and fake-SearXNG validation remain useful; obsolete application lifecycle tests have been removed.
Integration validation should continue to verify the 15 expected tools, persistent browser navigation state, deterministic browser interactions, normalized search behavior, error handling, and an explicitly selected real SearXNG environment.
## Container Strategy
The initial WebAccess.MCP container prioritizes compatibility over image-size optimization:
- Build and publish with .NET 10 on Ubuntu Noble.
- Use `mcr.microsoft.com/playwright/dotnet:v1.62.0-noble` as the compatible final-image baseline.
- Keep the application package at Microsoft.Playwright 1.62.0.
- Use the matching image's Chromium binaries and Linux browser dependencies.
Image-size optimization and a thinner runtime image are later work. They must not be combined with the architecture reset.
## Deployment Boundary
The complete deployed stack will eventually live beneath:
D:\ai\mcp\WebAccess.MCP
The conceptual runtime layout is:
D:\ai\mcp\WebAccess.MCP\
compose.yml
.env
webaccess\
searxng\
settings.yml
data\
searxng\
logs\
The existing native publish in that directory remains untouched until the container stack has been deployed and validated and its removal is explicitly approved.
## Security and External-Service Behavior
Host port mappings use loopback by default:
- WebAccess.MCP: `127.0.0.1:28081` to container port 8080.
- SearXNG: `127.0.0.1:28080` to container port 8080.
LAN exposure requires a separate explicit decision. Browser and search operations must not attempt to bypass CAPTCHAs, anti-bot protections, access controls, or provider restrictions. External-engine failures should be surfaced clearly without introducing application-owned process or container recovery behavior.

View File

@ -0,0 +1,341 @@
using System.Net;
using System.Net.Sockets;
using System.Text;
using System.Text.Json;
internal sealed class FakeSearXngServer : IAsyncDisposable
{
public const string SearchPath = "/searxng/search";
private readonly TcpListener listener;
private readonly CancellationTokenSource shutdown;
private readonly Task serverTask;
private readonly Lock requestsLock = new();
private readonly List<FakeSearXngRequest> requests = [];
private int disposed;
private FakeSearXngServer(TcpListener listener, CancellationToken cancellationToken)
{
this.listener = listener;
shutdown = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
var endpoint = (IPEndPoint)listener.LocalEndpoint;
BaseUrl = new Uri($"http://127.0.0.1:{endpoint.Port}/searxng/");
serverTask = RunAsync(shutdown.Token);
}
public Uri BaseUrl { get; }
public int RequestCount
{
get
{
lock (requestsLock)
{
return requests.Count;
}
}
}
public IReadOnlyList<FakeSearXngRequest> Requests
{
get
{
lock (requestsLock)
{
return requests.ToArray();
}
}
}
public static FakeSearXngServer Start(int port, CancellationToken cancellationToken)
{
ArgumentOutOfRangeException.ThrowIfLessThan(port, 1);
ArgumentOutOfRangeException.ThrowIfGreaterThan(port, 65535);
var listener = new TcpListener(IPAddress.Loopback, port);
try
{
listener.Start();
return new FakeSearXngServer(listener, cancellationToken);
}
catch
{
listener.Stop();
throw;
}
}
public async ValueTask DisposeAsync()
{
if (Interlocked.Exchange(ref disposed, 1) != 0)
{
return;
}
shutdown.Cancel();
listener.Stop();
try
{
await serverTask;
}
catch (OperationCanceledException) when (shutdown.IsCancellationRequested)
{
}
catch (SocketException) when (shutdown.IsCancellationRequested)
{
}
finally
{
shutdown.Dispose();
}
}
private async Task RunAsync(CancellationToken cancellationToken)
{
while (!cancellationToken.IsCancellationRequested)
{
using var client = await listener.AcceptTcpClientAsync(cancellationToken);
await HandleClientAsync(client, cancellationToken);
}
}
private async Task HandleClientAsync(
TcpClient client,
CancellationToken cancellationToken)
{
await using var stream = client.GetStream();
using var reader = new StreamReader(
stream,
Encoding.ASCII,
detectEncodingFromByteOrderMarks: false,
leaveOpen: true);
var requestLine = await reader.ReadLineAsync(cancellationToken)
?? throw new InvalidDataException("Fake SearXNG received an empty HTTP request.");
string? headerLine;
do
{
headerLine = await reader.ReadLineAsync(cancellationToken);
}
while (!string.IsNullOrEmpty(headerLine));
var requestParts = requestLine.Split(' ', StringSplitOptions.RemoveEmptyEntries);
if (requestParts.Length != 3)
{
await WriteResponseAsync(
stream,
"400 Bad Request",
"""{"error":"Invalid HTTP request line."}""",
cancellationToken);
return;
}
var method = requestParts[0];
if (!Uri.TryCreate($"http://127.0.0.1{requestParts[1]}", UriKind.Absolute, out var requestUri))
{
await WriteResponseAsync(
stream,
"400 Bad Request",
"""{"error":"Invalid HTTP request target."}""",
cancellationToken);
return;
}
var query = ParseQuery(requestUri.Query);
query.TryGetValue("q", out var searchQuery);
query.TryGetValue("format", out var format);
var request = new FakeSearXngRequest(
method,
requestUri.AbsolutePath,
searchQuery,
format);
lock (requestsLock)
{
requests.Add(request);
}
if (!string.Equals(method, "GET", StringComparison.Ordinal))
{
await WriteResponseAsync(
stream,
"405 Method Not Allowed",
"""{"error":"Only GET is supported."}""",
cancellationToken);
return;
}
if (!string.Equals(request.Path, SearchPath, StringComparison.Ordinal))
{
await WriteResponseAsync(
stream,
"404 Not Found",
"""{"error":"Unexpected fake SearXNG route."}""",
cancellationToken);
return;
}
var response = CreateResponse(searchQuery);
await WriteResponseAsync(
stream,
response.Status,
response.Body,
cancellationToken);
}
private static Dictionary<string, string> ParseQuery(string query)
{
var values = new Dictionary<string, string>(StringComparer.Ordinal);
foreach (var component in query.TrimStart('?').Split(
'&',
StringSplitOptions.RemoveEmptyEntries))
{
var separatorIndex = component.IndexOf('=');
var encodedName = separatorIndex >= 0
? component[..separatorIndex]
: component;
var encodedValue = separatorIndex >= 0
? component[(separatorIndex + 1)..]
: string.Empty;
var name = DecodeQueryComponent(encodedName);
var value = DecodeQueryComponent(encodedValue);
values[name] = value;
}
return values;
}
private static string DecodeQueryComponent(string value) =>
Uri.UnescapeDataString(value.Replace("+", " ", StringComparison.Ordinal));
private static FakeSearXngResponse CreateResponse(string? query) =>
query switch
{
"!google !bing default engine fixture" => new(
"200 OK",
JsonSerializer.Serialize(new
{
results = new object[]
{
new
{
title = " Alpha Result ",
url = "https://example.com/alpha",
content = " Alpha snippet text ",
engine = "google"
},
new
{
title = "Duplicate Alpha",
url = "https://example.com/alpha",
content = "duplicate",
engine = "bing"
},
new
{
title = "Invalid Mail",
url = "mailto:test@example.com",
content = "invalid",
engine = "google"
},
new
{
title = "Beta Result",
url = "https://example.com/beta",
content = "Beta snippet",
engines = new[] { "google", "bing", "GOOGLE", "" }
},
new
{
title = "Gamma Result",
url = "https://example.com/gamma",
content = "Gamma snippet",
engine = "google"
}
}
})),
"!brave !duckduckgo override engine fixture" => SuccessfulResult(
"Override Result",
"https://example.com/override",
"Override snippet",
"brave"),
"!google !bing empty engines fixture" => SuccessfulResult(
"Empty Engines Result",
"https://example.com/empty-engines",
"Empty engines snippet",
"google"),
"!google !bing zero results fixture" => new(
"200 OK",
"""{"results":[]}"""),
"!google !bing forbidden fixture" => new(
"403 Forbidden",
"""{"error":"JSON output disabled."}"""),
"!google !bing malformed json fixture" => new(
"200 OK",
"""{"results":["""),
"!google !bing C# & .NET search fixture" => SuccessfulResult(
"Encoded Query Result",
"https://example.com/encoded",
"Encoded query snippet",
"google"),
_ => new(
"400 Bad Request",
"""{"error":"Unexpected fake SearXNG query."}""")
};
private static FakeSearXngResponse SuccessfulResult(
string title,
string url,
string content,
string engine) =>
new(
"200 OK",
JsonSerializer.Serialize(new
{
results = new[]
{
new
{
title,
url,
content,
engine
}
}
}));
private static async Task WriteResponseAsync(
NetworkStream stream,
string status,
string body,
CancellationToken cancellationToken)
{
var entity = Encoding.UTF8.GetBytes(body);
var headers = Encoding.ASCII.GetBytes(
$"HTTP/1.1 {status}\r\n"
+ "Content-Type: application/json; charset=utf-8\r\n"
+ $"Content-Length: {entity.Length}\r\n"
+ "Cache-Control: no-store\r\n"
+ "Connection: close\r\n"
+ "\r\n");
await stream.WriteAsync(headers, cancellationToken);
await stream.WriteAsync(entity, cancellationToken);
}
private sealed record FakeSearXngResponse(
string Status,
string Body);
}
internal sealed record FakeSearXngRequest(
string Method,
string Path,
string? Query,
string? Format);

View File

@ -0,0 +1,362 @@
using System.Net;
using System.Net.Sockets;
using System.Text;
internal sealed class InteractionTestServer : IAsyncDisposable
{
private const string ExampleDomainPage =
"""
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="description" content="Deterministic Example Domain page for browser validation.">
<meta property="og:site_name" content="WebAccess.MCP TestRig">
<title>Example Domain</title>
</head>
<body>
<main>
<h1>Example Domain</h1>
<p>This page is for use in documentation examples without needing permission.</p>
<a href="/example-target">More information</a>
</main>
</body>
</html>
""";
private const string ExampleTargetPage =
"""
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>Example Target</title>
</head>
<body>
<main>
<h1>Example Target</h1>
</main>
</body>
</html>
""";
private const string InteractionPage =
"""
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>WebAccess MCP Interaction Test</title>
</head>
<body>
<main>
<h1>Interaction Test</h1>
<label for="name">Name</label>
<input id="name" name="name" placeholder="Type your name">
<button type="button" id="apply">Apply</button>
<p id="status">Ready</p>
</main>
<script>
const nameInput = document.getElementById('name');
const status = document.getElementById('status');
nameInput.addEventListener('keydown', event => {
if (event.key === 'Enter') {
status.textContent = `Submitted: ${nameInput.value}`;
}
});
document.getElementById('apply').addEventListener('click', () => {
status.textContent = `Clicked: ${nameInput.value}`;
});
</script>
</body>
</html>
""";
private const string HistoryAPage =
"""
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>WebAccess MCP History A</title>
</head>
<body>
<main>
<h1>History A</h1>
</main>
</body>
</html>
""";
private readonly TcpListener listener;
private readonly CancellationTokenSource shutdown;
private readonly Task serverTask;
private int disposed;
private int historyBLoadCount;
private int searchablePageLoadCount;
private InteractionTestServer(TcpListener listener, CancellationToken cancellationToken)
{
this.listener = listener;
shutdown = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
var endpoint = (IPEndPoint)listener.LocalEndpoint;
var baseUrl = $"http://127.0.0.1:{endpoint.Port}";
InteractionUrl = new Uri($"{baseUrl}/");
ExampleDomainUrl = new Uri($"{baseUrl}/example-domain");
RedirectToExampleDomainUrl = new Uri($"{baseUrl}/redirect-example-domain");
ExampleTargetUrl = new Uri($"{baseUrl}/example-target");
LongTextUrl = new Uri($"{baseUrl}/long-text");
SearchablePageUrl = new Uri($"{baseUrl}/searchable-page");
HistoryAUrl = new Uri($"{baseUrl}/history-a");
HistoryBUrl = new Uri($"{baseUrl}/history-b");
serverTask = RunAsync(shutdown.Token);
}
public Uri InteractionUrl { get; }
public Uri ExampleDomainUrl { get; }
public Uri RedirectToExampleDomainUrl { get; }
public Uri ExampleTargetUrl { get; }
public Uri LongTextUrl { get; }
public Uri SearchablePageUrl { get; }
public int SearchablePageLoadCount => Volatile.Read(ref searchablePageLoadCount);
public Uri HistoryAUrl { get; }
public Uri HistoryBUrl { get; }
public static InteractionTestServer Start(CancellationToken cancellationToken)
{
var listener = new TcpListener(IPAddress.Loopback, 0);
try
{
listener.Start();
return new InteractionTestServer(listener, cancellationToken);
}
catch
{
listener.Stop();
throw;
}
}
public async ValueTask DisposeAsync()
{
if (Interlocked.Exchange(ref disposed, 1) != 0)
{
return;
}
shutdown.Cancel();
listener.Stop();
try
{
await serverTask;
}
catch (OperationCanceledException) when (shutdown.IsCancellationRequested)
{
}
catch (SocketException) when (shutdown.IsCancellationRequested)
{
}
finally
{
shutdown.Dispose();
}
}
private async Task RunAsync(CancellationToken cancellationToken)
{
var clientTasks = new List<Task>();
try
{
while (!cancellationToken.IsCancellationRequested)
{
var client = await listener.AcceptTcpClientAsync(cancellationToken);
clientTasks.Add(HandleClientSafelyAsync(client, cancellationToken));
}
}
finally
{
await Task.WhenAll(clientTasks);
}
}
private async Task HandleClientSafelyAsync(
TcpClient client,
CancellationToken cancellationToken)
{
using (client)
{
try
{
await HandleClientAsync(client, cancellationToken);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
}
catch (IOException exception)
{
if (!cancellationToken.IsCancellationRequested)
{
Console.Error.WriteLine(
$"[Interaction fixture] Client connection ended before the response completed: {exception.Message}");
}
}
catch (SocketException exception)
{
if (!cancellationToken.IsCancellationRequested)
{
Console.Error.WriteLine(
$"[Interaction fixture] Client connection failed: {exception.Message}");
}
}
}
}
private async Task HandleClientAsync(
TcpClient client,
CancellationToken cancellationToken)
{
await using var stream = client.GetStream();
using var reader = new StreamReader(
stream,
Encoding.ASCII,
detectEncodingFromByteOrderMarks: false,
leaveOpen: true);
var requestLine = await reader.ReadLineAsync(cancellationToken);
if (requestLine is null)
{
return;
}
string? headerLine;
do
{
headerLine = await reader.ReadLineAsync(cancellationToken);
}
while (!string.IsNullOrEmpty(headerLine));
var requestParts = requestLine.Split(' ', StringSplitOptions.RemoveEmptyEntries);
var method = requestParts.ElementAtOrDefault(0);
var path = requestParts.ElementAtOrDefault(1);
var supportedMethod = method == "GET" || method == "HEAD";
if (supportedMethod && path == "/redirect-example-domain")
{
var redirectHeaders = Encoding.ASCII.GetBytes(
"HTTP/1.1 302 Found\r\n"
+ "Location: /example-domain\r\n"
+ "Content-Length: 0\r\n"
+ "Cache-Control: no-store\r\n"
+ "Connection: close\r\n"
+ "\r\n");
await stream.WriteAsync(redirectHeaders, cancellationToken);
return;
}
string? page = supportedMethod
? path switch
{
"/" => InteractionPage,
"/example-domain" => ExampleDomainPage,
"/example-target" => ExampleTargetPage,
"/long-text" => CreateLongTextPage(),
"/searchable-page" => CreateSearchablePage(
method == "GET"
? Interlocked.Increment(ref searchablePageLoadCount)
: Volatile.Read(ref searchablePageLoadCount)),
"/history-a" => HistoryAPage,
"/history-b" => CreateHistoryBPage(
method == "GET"
? Interlocked.Increment(ref historyBLoadCount)
: Volatile.Read(ref historyBLoadCount)),
_ => null
}
: null;
var status = page is not null ? "200 OK" : "404 Not Found";
var entity = page is not null ? Encoding.UTF8.GetBytes(page) : [];
var body = method == "HEAD" ? [] : entity;
var headers = Encoding.ASCII.GetBytes(
$"HTTP/1.1 {status}\r\n"
+ "Content-Type: text/html; charset=utf-8\r\n"
+ $"Content-Length: {entity.Length}\r\n"
+ "Cache-Control: no-store\r\n"
+ "Connection: close\r\n"
+ "\r\n");
await stream.WriteAsync(headers, cancellationToken);
if (body.Length > 0)
{
await stream.WriteAsync(body, cancellationToken);
}
}
private static string CreateHistoryBPage(int loadCount) =>
$$"""
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>WebAccess MCP History B</title>
</head>
<body>
<main>
<h1>History B</h1>
<p>History B Load: {{loadCount}}</p>
</main>
</body>
</html>
""";
private static string CreateLongTextPage() =>
$"""
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>WebAccess MCP Long Text</title>
</head>
<body>{new string('A', 60_000)}</body>
</html>
""";
private static string CreateSearchablePage(int loadCount) =>
$"""
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>WebAccess MCP Searchable Page</title>
</head>
<body>
<span>{CreateSearchableText(loadCount)}</span>
</body>
</html>
""";
private static string CreateSearchableText(int loadCount)
{
const string firstHalf = "request throttling ";
var prefix = new string('x', 4_000 - firstHalf.Length);
var usefulText =
$"behavior continues across the storage boundary. "
+ "Rate limiting enforces a request quota and a concurrency limit. "
+ $"Searchable Page Load: {loadCount}.";
var suffix = new string('y', 4_000 - usefulText.Length);
return prefix + firstHalf + usefulText + suffix;
}
}

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,14 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="ModelContextProtocol" Version="2.2.0" />
</ItemGroup>
</Project>

View File

@ -0,0 +1,465 @@
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using WebAccess.MCP.Browser;
using WebAccess.MCP.WebCache;
namespace WebAccess.MCP.Tests;
public sealed class BrowserPageTextCoordinatorTests
{
private static readonly DateTimeOffset TestNow =
new(2026, 8, 20, 4, 0, 0, TimeSpan.Zero);
[Fact]
public async Task PageZeroIsRejectedBeforeCacheOrBrowserAccess()
{
var cache = new FakeWebCacheService();
var reader = new FakeBrowserPageTextReader();
var coordinator = CreateCoordinator(reader, cache);
await Assert.ThrowsAsync<ArgumentOutOfRangeException>(
() => coordinator.GetPageTextAsync(
"https://example.com/page",
0,
CancellationToken.None));
Assert.Equal(0, cache.LookupCalls);
Assert.Equal(0, reader.TotalReads);
}
[Fact]
public async Task UrlCacheHitReturnsPageOneWithoutBrowser()
{
var cache = new FakeWebCacheService
{
LookupResult = CreateHit(1, 2, 0, "cached page one", hasMore: true)
};
var reader = new FakeBrowserPageTextReader();
var coordinator = CreateCoordinator(reader, cache);
var result = await coordinator.GetPageTextAsync(
"https://example.com/page",
1,
CancellationToken.None);
Assert.Contains("Page 1 of 2", result, StringComparison.Ordinal);
Assert.Contains("cached page one", result, StringComparison.Ordinal);
Assert.Contains("Request page 2 only if needed", result, StringComparison.Ordinal);
Assert.Equal(0, reader.TotalReads);
Assert.Equal(0, cache.StoreCalls);
}
[Fact]
public async Task UrlCacheHitPageTwoContainsNoPageOneText()
{
var cache = new FakeWebCacheService
{
LookupResult = CreateHit(2, 2, 4_000, "SECOND-CHUNK", hasMore: false)
};
var reader = new FakeBrowserPageTextReader();
var coordinator = CreateCoordinator(reader, cache);
var result = await coordinator.GetPageTextAsync(
"https://example.com/page",
2,
CancellationToken.None);
Assert.Contains("SECOND-CHUNK", result, StringComparison.Ordinal);
Assert.DoesNotContain("FIRST-CHUNK", result, StringComparison.Ordinal);
Assert.DoesNotContain("More content available", result, StringComparison.Ordinal);
Assert.Equal(0, reader.TotalReads);
}
[Fact]
public async Task FreshMissingChunkDoesNotInvokeBrowser()
{
var cache = new FakeWebCacheService
{
LookupResult = new WebCacheLookupResult(
WebCacheLookupStatus.ChunkNotFound,
CreateReadModel(null, 4, 0, null, hasMore: false))
};
var reader = new FakeBrowserPageTextReader();
var coordinator = CreateCoordinator(reader, cache);
var result = await coordinator.GetPageTextAsync(
"https://example.com/page",
7,
CancellationToken.None);
Assert.Equal(
"Page 7 does not exist. This snapshot has 4 pages.",
result);
Assert.Equal(0, reader.TotalReads);
}
[Fact]
public async Task CacheMissNavigatesExtractsAndStoresOnceThenReturnsRequestedPage()
{
var cache = new FakeWebCacheService
{
LookupResult = new WebCacheLookupResult(
WebCacheLookupStatus.PageNotFound,
null)
};
var reader = new FakeBrowserPageTextReader
{
NavigationSnapshot = CreateSnapshot(
new string('a', 4_000) + "b",
finalUrl: "https://example.com/final",
statusCode: 404,
title: "Not Found")
};
var coordinator = CreateCoordinator(reader, cache);
var result = await coordinator.GetPageTextAsync(
"https://example.com/requested",
2,
CancellationToken.None);
Assert.Equal("b", GetTextLine(result));
Assert.Equal(1, reader.NavigationReads);
Assert.Equal(0, reader.CurrentReads);
Assert.Equal(1, cache.StoreCalls);
Assert.NotNull(cache.LastWrite);
Assert.Equal("https://example.com/requested", cache.LastWrite.RequestedUrl);
Assert.Equal("https://example.com/final", cache.LastWrite.FinalUrl);
Assert.Equal(404, cache.LastWrite.HttpStatusCode);
Assert.Equal("Not Found", cache.LastWrite.Title);
Assert.Equal("text/html", cache.LastWrite.ContentType);
}
[Fact]
public async Task ExpiredEntryRefreshesAndStoresSnapshot()
{
var cache = new FakeWebCacheService
{
LookupResult = new WebCacheLookupResult(WebCacheLookupStatus.Expired, null)
};
var reader = new FakeBrowserPageTextReader
{
NavigationSnapshot = CreateSnapshot("refreshed")
};
var coordinator = CreateCoordinator(reader, cache);
var result = await coordinator.GetPageTextAsync(
"https://example.com/page",
1,
CancellationToken.None);
Assert.Contains("refreshed", result, StringComparison.Ordinal);
Assert.Equal(1, reader.NavigationReads);
Assert.Equal(1, cache.StoreCalls);
}
[Fact]
public async Task CacheReadFailureFallsBackToBrowser()
{
var cache = new FakeWebCacheService
{
LookupException = CreateCacheException()
};
var reader = new FakeBrowserPageTextReader
{
NavigationSnapshot = CreateSnapshot("browser fallback")
};
var coordinator = CreateCoordinator(reader, cache);
var result = await coordinator.GetPageTextAsync(
"https://example.com/page",
1,
CancellationToken.None);
Assert.Contains("browser fallback", result, StringComparison.Ordinal);
Assert.Equal(1, reader.NavigationReads);
Assert.Equal(1, cache.StoreCalls);
}
[Fact]
public async Task CacheWriteFailureStillReturnsBrowserPage()
{
var cache = new FakeWebCacheService
{
LookupResult = new WebCacheLookupResult(
WebCacheLookupStatus.PageNotFound,
null),
StoreException = CreateCacheException()
};
var reader = new FakeBrowserPageTextReader
{
NavigationSnapshot = CreateSnapshot("available despite write failure")
};
var coordinator = CreateCoordinator(reader, cache);
var result = await coordinator.GetPageTextAsync(
"https://example.com/page",
1,
CancellationToken.None);
Assert.Contains(
"available despite write failure",
result,
StringComparison.Ordinal);
Assert.Equal(1, reader.NavigationReads);
Assert.Equal(1, cache.StoreCalls);
}
[Fact]
public async Task UrlOmittedBypassesCacheAndReadsCurrentPage()
{
var cache = new FakeWebCacheService();
var reader = new FakeBrowserPageTextReader
{
CurrentSnapshot = CreateSnapshot(
new string('a', 4_000) + "CURRENT-PAGE-TWO")
};
var coordinator = CreateCoordinator(reader, cache);
var result = await coordinator.GetPageTextAsync(
null,
2,
CancellationToken.None);
Assert.Equal("CURRENT-PAGE-TWO", GetTextLine(result));
Assert.Equal(0, cache.LookupCalls);
Assert.Equal(0, cache.StoreCalls);
Assert.Equal(1, reader.CurrentReads);
Assert.Equal(0, reader.NavigationReads);
}
[Fact]
public async Task FourThousandCharactersProduceOneFinalPage()
{
var coordinator = CreateCurrentPageCoordinator(new string('x', 4_000));
var result = await coordinator.GetPageTextAsync(null, 1, CancellationToken.None);
Assert.StartsWith("Page 1 of 1 | chars 0-3999 of 4000", result);
Assert.Equal(4_000, GetTextLine(result).Length);
Assert.DoesNotContain("More content available", result, StringComparison.Ordinal);
}
[Fact]
public async Task FourThousandOneCharactersProduceTwoNonOverlappingPages()
{
var coordinator = CreateCurrentPageCoordinator(new string('a', 4_000) + "b");
var first = await coordinator.GetPageTextAsync(null, 1, CancellationToken.None);
var second = await coordinator.GetPageTextAsync(null, 2, CancellationToken.None);
Assert.Equal(4_000, GetTextLine(first).Length);
Assert.Equal("b", GetTextLine(second));
Assert.StartsWith("Page 2 of 2 | chars 4000-4000 of 4001", second);
Assert.Contains("Request page 2 only if needed", first, StringComparison.Ordinal);
Assert.DoesNotContain("More content available", second, StringComparison.Ordinal);
}
[Fact]
public async Task EightThousandOneCharactersProduceThreeNonOverlappingPages()
{
var text = new string('a', 4_000) + new string('b', 4_000) + "c";
var coordinator = CreateCurrentPageCoordinator(text);
var second = await coordinator.GetPageTextAsync(null, 2, CancellationToken.None);
var third = await coordinator.GetPageTextAsync(null, 3, CancellationToken.None);
Assert.Equal(new string('b', 4_000), GetTextLine(second));
Assert.DoesNotContain(new string('a', 100), GetTextLine(second), StringComparison.Ordinal);
Assert.Equal("c", GetTextLine(third));
Assert.StartsWith("Page 3 of 3 | chars 8000-8000 of 8001", third);
}
[Fact]
public async Task ZeroTextReturnsConciseStatusMetadata()
{
var coordinator = CreateCurrentPageCoordinator(
string.Empty,
statusCode: 429);
var result = await coordinator.GetPageTextAsync(null, 1, CancellationToken.None);
Assert.StartsWith("Page 0 of 0 | 0 characters | HTTP 429", result);
Assert.Contains("No readable page text", result, StringComparison.Ordinal);
}
[Fact]
public async Task InternallyTruncatedSnapshotReportsLimitOnFinalPage()
{
var source = new string('x', BrowserPageTextReader.MaximumSnapshotCharacters + 1);
var (boundedText, isComplete) = BrowserPageTextReader.ApplySnapshotLimit(source);
var coordinator = CreateCurrentPageCoordinator(boundedText, isComplete: isComplete);
var result = await coordinator.GetPageTextAsync(null, 13, CancellationToken.None);
Assert.Equal(BrowserPageTextReader.MaximumSnapshotCharacters, boundedText.Length);
Assert.False(isComplete);
Assert.StartsWith("Page 13 of 13 | chars 48000-49999 of 50000", result);
Assert.Contains(
"Snapshot truncated at the internal 50000-character extraction limit",
result,
StringComparison.Ordinal);
}
private static BrowserPageTextCoordinator CreateCurrentPageCoordinator(
string text,
int? statusCode = 200,
bool isComplete = true)
{
var reader = new FakeBrowserPageTextReader
{
CurrentSnapshot = CreateSnapshot(
text,
statusCode: statusCode,
isComplete: isComplete)
};
return CreateCoordinator(reader, new FakeWebCacheService());
}
private static BrowserPageTextCoordinator CreateCoordinator(
FakeBrowserPageTextReader reader,
FakeWebCacheService cache) =>
new(
reader,
cache,
Options.Create(
new WebCacheOptions
{
DefaultExpiration = TimeSpan.FromHours(1),
ChunkSize = 4_000
}),
NullLogger<BrowserPageTextCoordinator>.Instance);
private static WebCacheLookupResult CreateHit(
int page,
int pageCount,
int startOffset,
string text,
bool hasMore) =>
new(
WebCacheLookupStatus.Hit,
CreateReadModel(text, pageCount, startOffset, page, hasMore));
private static WebCachePageChunkRead CreateReadModel(
string? text,
int pageCount,
int startOffset,
int? page,
bool hasMore) =>
new(
"https://example.com/page",
"https://example.com/final",
200,
"Example",
"text/html",
TestNow,
TestNow.AddHours(1),
true,
pageCount == 0 ? 0 : startOffset + (text?.Length ?? 0),
page,
pageCount,
page is null ? null : startOffset,
text?.Length,
text,
hasMore);
private static BrowserPageTextSnapshot CreateSnapshot(
string text,
string finalUrl = "https://example.com/page",
int? statusCode = 200,
string title = "Example",
bool isComplete = true) =>
new(
finalUrl,
statusCode,
title,
"text/html",
text,
isComplete);
private static string GetTextLine(string result) =>
result.Split(Environment.NewLine, StringSplitOptions.None)[1];
private static WebCacheUnavailableException CreateCacheException() =>
new("Cache unavailable.", new InvalidOperationException("Synthetic failure."));
private sealed class FakeBrowserPageTextReader : IBrowserPageTextReader
{
public BrowserPageTextSnapshot CurrentSnapshot { get; set; } =
CreateSnapshot("current");
public BrowserPageTextSnapshot NavigationSnapshot { get; set; } =
CreateSnapshot("navigated");
public int CurrentReads { get; private set; }
public int NavigationReads { get; private set; }
public int TotalReads => CurrentReads + NavigationReads;
public Task<BrowserPageTextSnapshot> ReadCurrentAsync(
CancellationToken cancellationToken)
{
CurrentReads++;
cancellationToken.ThrowIfCancellationRequested();
return Task.FromResult(CurrentSnapshot);
}
public Task<BrowserPageTextSnapshot> NavigateAndReadAsync(
Uri uri,
CancellationToken cancellationToken)
{
NavigationReads++;
cancellationToken.ThrowIfCancellationRequested();
return Task.FromResult(NavigationSnapshot);
}
}
private sealed class FakeWebCacheService : IWebCacheService
{
public WebCacheLookupResult LookupResult { get; set; } =
new(WebCacheLookupStatus.PageNotFound, null);
public WebCacheUnavailableException? LookupException { get; set; }
public WebCacheUnavailableException? StoreException { get; set; }
public int LookupCalls { get; private set; }
public int StoreCalls { get; private set; }
public WebCachePageWriteRequest? LastWrite { get; private set; }
public Task<WebCacheLookupResult> TryGetFreshChunkAsync(
string requestedUrl,
int chunkNumber,
CancellationToken cancellationToken)
{
LookupCalls++;
cancellationToken.ThrowIfCancellationRequested();
return LookupException is null
? Task.FromResult(LookupResult)
: Task.FromException<WebCacheLookupResult>(LookupException);
}
public Task StoreAsync(
WebCachePageWriteRequest request,
CancellationToken cancellationToken)
{
StoreCalls++;
LastWrite = request;
cancellationToken.ThrowIfCancellationRequested();
return StoreException is null
? Task.CompletedTask
: Task.FromException(StoreException);
}
public Task<WebCacheSearchResult> SearchFreshPageAsync(
string requestedUrl,
string query,
IReadOnlyList<string>? relatedTerms,
CancellationToken cancellationToken) =>
throw new NotSupportedException();
}
}

View File

@ -0,0 +1,337 @@
using Microsoft.Extensions.Logging.Abstractions;
using WebAccess.MCP.Browser;
using WebAccess.MCP.WebCache;
namespace WebAccess.MCP.Tests;
public sealed class BrowserPageTextSearchCoordinatorTests
{
[Theory]
[InlineData("not a URL")]
[InlineData("ftp://example.com/page")]
public async Task InvalidUrlFailsBeforeCacheOrBrowser(string url)
{
var cache = new FakeCacheService();
var reader = new FakePageTextReader();
await Assert.ThrowsAsync<ArgumentException>(
() => CreateCoordinator(reader, cache).FindPageTextAsync(
url,
"request throttling",
null,
CancellationToken.None));
Assert.Equal(0, cache.SearchCalls);
Assert.Equal(0, reader.NavigationReads);
}
[Theory]
[InlineData("")]
[InlineData(" ")]
public async Task InvalidQueryFailsBeforeCacheOrBrowser(string query)
{
var cache = new FakeCacheService();
var reader = new FakePageTextReader();
await Assert.ThrowsAsync<ArgumentException>(
() => CreateCoordinator(reader, cache).FindPageTextAsync(
"https://example.com/page",
query,
null,
CancellationToken.None));
Assert.Equal(0, cache.SearchCalls);
Assert.Equal(0, reader.NavigationReads);
}
[Fact]
public void RelatedTermLimitsAreValidatedBeforeWork()
{
var tooMany = Enumerable.Range(1, 9).Select(index => $"term {index}").ToArray();
Assert.Throws<ArgumentException>(
() => BrowserPageTextSearchCoordinator.ValidateInputs(
"https://example.com/page",
"request throttling",
tooMany));
Assert.Throws<ArgumentException>(
() => BrowserPageTextSearchCoordinator.ValidateInputs(
"https://example.com/page",
"request throttling",
[new string('x', 101)]));
}
[Fact]
public async Task FreshSearchUsesPostgreSqlWithoutBrowserOrStore()
{
var cache = new FakeCacheService();
cache.Results.Enqueue(CreateHit("exact cached context", 2, 3));
var reader = new FakePageTextReader();
var output = await CreateCoordinator(reader, cache).FindPageTextAsync(
"https://example.com/page",
"request throttling",
["rate limiting"],
CancellationToken.None);
Assert.Equal(1, cache.SearchCalls);
Assert.Equal(0, cache.StoreCalls);
Assert.Equal(0, reader.NavigationReads);
Assert.Contains("Pages 2-3", output);
Assert.Contains("exact cached context", output);
Assert.Equal(["rate limiting"], cache.LastRelatedTerms);
}
[Fact]
public async Task SecondDifferentQueryAndSynonymSetRemainCacheOnly()
{
var cache = new FakeCacheService();
cache.Results.Enqueue(CreateHit("first context", 1, 1));
cache.Results.Enqueue(CreateHit("second context", 2, 2));
var reader = new FakePageTextReader();
var coordinator = CreateCoordinator(reader, cache);
await coordinator.FindPageTextAsync(
"https://example.com/page",
"request throttling",
["rate limiting"],
CancellationToken.None);
await coordinator.FindPageTextAsync(
"https://example.com/page",
"request quota",
["concurrency limit"],
CancellationToken.None);
Assert.Equal(2, cache.SearchCalls);
Assert.Equal(0, cache.StoreCalls);
Assert.Equal(0, reader.NavigationReads);
Assert.Equal(["request throttling", "request quota"], cache.Queries);
}
[Theory]
[InlineData((int)WebCacheSearchStatus.PageNotFound)]
[InlineData((int)WebCacheSearchStatus.Expired)]
public async Task ColdOrExpiredSnapshotPopulatesOnceThenRetrieves(
int initialStatusValue)
{
var cache = new FakeCacheService();
cache.Results.Enqueue(CreateStatus((WebCacheSearchStatus)initialStatusValue));
cache.Results.Enqueue(CreateHit("retrieved context", 2, 2));
var reader = new FakePageTextReader
{
Snapshot = CreateSnapshot("complete normalized snapshot")
};
var output = await CreateCoordinator(reader, cache).FindPageTextAsync(
"https://example.com/page",
"request throttling",
["rate limiting"],
CancellationToken.None);
Assert.Equal(1, reader.NavigationReads);
Assert.Equal(1, cache.StoreCalls);
Assert.Equal(2, cache.SearchCalls);
Assert.Contains("retrieved context", output);
Assert.DoesNotContain("complete normalized snapshot", output);
}
[Fact]
public async Task NoMatchPreservesSequentialCorrectnessFallback()
{
var cache = new FakeCacheService();
cache.Results.Enqueue(CreateStatus(WebCacheSearchStatus.NoMatches));
var output = await CreateCoordinator(
new FakePageTextReader(),
cache)
.FindPageTextAsync(
"https://example.com/page",
"unmatched",
null,
CancellationToken.None);
Assert.Equal(
"Targeted cached search did not identify a confident location. "
+ "Use get_page_text sequentially if broader reading is needed.",
output);
Assert.DoesNotContain("absent", output, StringComparison.OrdinalIgnoreCase);
Assert.DoesNotContain("does not exist", output, StringComparison.OrdinalIgnoreCase);
}
[Fact]
public void FormattingReturnsPageRangeAndBoundedContextWithoutRanks()
{
var context = new string('x', 4_000);
var output = BrowserPageTextSearchCoordinator.FormatResult(
CreateHit(context, 4, 5));
Assert.StartsWith("Found targeted context at Pages 4-5 of 8.", output);
Assert.Contains(context, output);
Assert.DoesNotContain("Score", output, StringComparison.OrdinalIgnoreCase);
Assert.DoesNotContain("Rank", output, StringComparison.OrdinalIgnoreCase);
Assert.Contains("get_page_text", output);
}
[Fact]
public async Task DatabaseAndStoreFailuresDoNotLeakPageText()
{
var databaseCache = new FakeCacheService
{
SearchException = CreateCacheException()
};
var databaseException = await Assert.ThrowsAsync<PageTextSearchUnavailableException>(
() => CreateCoordinator(new FakePageTextReader(), databaseCache)
.FindPageTextAsync(
"https://example.com/page",
"request throttling",
null,
CancellationToken.None));
var storeCache = new FakeCacheService
{
StoreException = CreateCacheException()
};
storeCache.Results.Enqueue(CreateStatus(WebCacheSearchStatus.PageNotFound));
var storeException = await Assert.ThrowsAsync<PageTextSearchUnavailableException>(
() => CreateCoordinator(
new FakePageTextReader
{
Snapshot = CreateSnapshot(new string('z', 50_000))
},
storeCache)
.FindPageTextAsync(
"https://example.com/page",
"request throttling",
null,
CancellationToken.None));
Assert.Contains("unavailable", databaseException.Message);
Assert.Contains("could not be stored", storeException.Message);
Assert.DoesNotContain(new string('z', 100), storeException.Message);
}
private static BrowserPageTextSearchCoordinator CreateCoordinator(
FakePageTextReader reader,
FakeCacheService cache) =>
new(
reader,
cache,
NullLogger<BrowserPageTextSearchCoordinator>.Instance);
private static WebCacheSearchResult CreateHit(
string context,
int startPage,
int endPage) =>
new(
WebCacheSearchStatus.Hit,
"https://example.com/page",
"https://example.com/final",
DateTimeOffset.UtcNow,
DateTimeOffset.UtcNow.AddHours(1),
true,
8_001,
8,
[
new WebCacheSearchHit(
startPage,
endPage,
(startPage - 1) * 4_000,
context.Length,
10_000,
context)
]);
private static WebCacheSearchResult CreateStatus(WebCacheSearchStatus status) =>
new(
status,
"https://example.com/page",
"https://example.com/final",
DateTimeOffset.UtcNow,
DateTimeOffset.UtcNow.AddHours(1),
true,
8_001,
8,
[]);
private static BrowserPageTextSnapshot CreateSnapshot(string text) =>
new(
"https://example.com/final",
200,
"Example",
"text/html",
text,
true);
private static WebCacheUnavailableException CreateCacheException() =>
new("Unavailable.", new InvalidOperationException("Synthetic."));
private sealed class FakePageTextReader : IBrowserPageTextReader
{
public BrowserPageTextSnapshot Snapshot { get; set; } =
CreateSnapshot("snapshot");
public int NavigationReads { get; private set; }
public Task<BrowserPageTextSnapshot> NavigateAndReadAsync(
Uri uri,
CancellationToken cancellationToken)
{
NavigationReads++;
cancellationToken.ThrowIfCancellationRequested();
return Task.FromResult(Snapshot);
}
public Task<BrowserPageTextSnapshot> ReadCurrentAsync(
CancellationToken cancellationToken) =>
throw new NotSupportedException();
}
private sealed class FakeCacheService : IWebCacheService
{
public Queue<WebCacheSearchResult> Results { get; } = new();
public WebCacheUnavailableException? SearchException { get; set; }
public WebCacheUnavailableException? StoreException { get; set; }
public int SearchCalls { get; private set; }
public int StoreCalls { get; private set; }
public List<string> Queries { get; } = [];
public IReadOnlyList<string>? LastRelatedTerms { get; private set; }
public Task<WebCacheSearchResult> SearchFreshPageAsync(
string requestedUrl,
string query,
IReadOnlyList<string>? relatedTerms,
CancellationToken cancellationToken)
{
SearchCalls++;
Queries.Add(query);
LastRelatedTerms = relatedTerms;
cancellationToken.ThrowIfCancellationRequested();
return SearchException is null
? Task.FromResult(Results.Dequeue())
: Task.FromException<WebCacheSearchResult>(SearchException);
}
public Task StoreAsync(
WebCachePageWriteRequest request,
CancellationToken cancellationToken)
{
StoreCalls++;
cancellationToken.ThrowIfCancellationRequested();
return StoreException is null
? Task.CompletedTask
: Task.FromException(StoreException);
}
public Task<WebCacheLookupResult> TryGetFreshChunkAsync(
string requestedUrl,
int chunkNumber,
CancellationToken cancellationToken) =>
throw new NotSupportedException();
}
}

View File

@ -0,0 +1,24 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<IsPackable>false</IsPackable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.1" />
<PackageReference Include="xunit" Version="2.9.3" />
<PackageReference Include="xunit.runner.visualstudio" Version="3.1.4" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\WebAccess.MCP\WebAccess.MCP.csproj" />
</ItemGroup>
<ItemGroup>
<Using Include="Xunit" />
</ItemGroup>
</Project>

View File

@ -0,0 +1,128 @@
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using WebAccess.MCP.WebCache;
namespace WebAccess.MCP.Tests;
public sealed class WebCacheFoundationTests
{
[Fact]
public void DefaultExpirationIsExactlyOneHour()
{
var options = new WebCacheOptions();
Assert.Equal(TimeSpan.FromHours(1), options.DefaultExpiration);
}
[Fact]
public void ChunkSizeDefaultsToExactlyFourThousand()
{
var options = new WebCacheOptions();
Assert.Equal(4_000, options.ChunkSize);
}
[Theory]
[InlineData(0)]
[InlineData(-1)]
public void NonPositiveExpirationIsRejected(int seconds)
{
var options = new WebCacheOptions
{
DefaultExpiration = TimeSpan.FromSeconds(seconds)
};
var result = new WebCacheOptionsValidator().Validate(null, options);
Assert.True(result.Failed);
Assert.Contains(
"must be greater than zero",
result.FailureMessage,
StringComparison.Ordinal);
}
[Theory]
[InlineData(0)]
[InlineData(50_001)]
public void OutOfRangeChunkSizeIsRejected(int chunkSize)
{
var options = new WebCacheOptions
{
ChunkSize = chunkSize
};
var result = new WebCacheOptionsValidator().Validate(null, options);
Assert.True(result.Failed);
Assert.Contains("1 through 50000", result.FailureMessage, StringComparison.Ordinal);
}
[Fact]
public void ModelContainsRequiredCacheStructure()
{
using var context = CreateContext();
var model = context.Model;
var page = model.FindEntityType(typeof(CachedWebPage));
var chunk = model.FindEntityType(typeof(CachedWebPageChunk));
Assert.NotNull(page);
Assert.NotNull(chunk);
var foreignKey = Assert.Single(chunk.GetForeignKeys());
Assert.Same(page, foreignKey.PrincipalEntityType);
Assert.Equal(DeleteBehavior.Cascade, foreignKey.DeleteBehavior);
Assert.Contains(
page.GetIndexes(),
index => index.IsUnique
&& index.Properties.Select(property => property.Name)
.SequenceEqual([nameof(CachedWebPage.CacheKey)]));
Assert.Contains(
page.GetIndexes(),
index => index.Properties.Select(property => property.Name)
.SequenceEqual([nameof(CachedWebPage.ExpiresAtUtc)]));
Assert.Contains(
chunk.GetIndexes(),
index => index.IsUnique
&& index.Properties.Select(property => property.Name)
.SequenceEqual(
[
nameof(CachedWebPageChunk.CachedWebPageId),
nameof(CachedWebPageChunk.ChunkNumber)
]));
}
[Fact]
public void RegistrationAndModelCreationDoNotConnectToPostgres()
{
var configuration = new ConfigurationBuilder()
.AddInMemoryCollection(
new Dictionary<string, string?>
{
[$"ConnectionStrings:{WebCacheServiceCollectionExtensions.ConnectionStringName}"] =
"Host=127.0.0.1;Port=1;Database=webaccess;Username=webaccess;Timeout=1",
[$"{WebCacheOptions.SectionName}:DefaultExpiration"] = "01:00:00",
[$"{WebCacheOptions.SectionName}:ChunkSize"] = "4000"
})
.Build();
var services = new ServiceCollection();
services.AddWebCachePersistence(configuration);
using var serviceProvider = services.BuildServiceProvider();
var factory = serviceProvider.GetRequiredService<IDbContextFactory<WebCacheDbContext>>();
_ = serviceProvider.GetRequiredService<IWebCacheService>();
using var context = factory.CreateDbContext();
Assert.NotNull(context.Model.FindEntityType(typeof(CachedWebPage)));
}
private static WebCacheDbContext CreateContext()
{
var options = new DbContextOptionsBuilder<WebCacheDbContext>()
.UseNpgsql("Host=127.0.0.1;Port=1;Database=webaccess;Username=webaccess;Timeout=1")
.Options;
return new WebCacheDbContext(options);
}
}

View File

@ -0,0 +1,412 @@
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Metadata;
using WebAccess.MCP.WebCache;
namespace WebAccess.MCP.Tests;
public sealed class WebCacheRelevanceSearchTests
{
private static readonly DateTimeOffset TestNow =
new(2026, 8, 20, 6, 0, 0, TimeSpan.Zero);
[Theory]
[InlineData("not a URL")]
[InlineData("ftp://example.com/page")]
public async Task InvalidUrlIsRejectedBeforeRepositoryAccess(string url)
{
var repository = new FakeRepository();
var service = CreateService(repository);
await Assert.ThrowsAsync<ArgumentException>(
() => service.SearchFreshPageAsync(
url,
"request throttling",
null,
CancellationToken.None));
Assert.Equal(0, repository.SearchCalls);
}
[Theory]
[InlineData("")]
[InlineData(" ")]
public async Task EmptyQueryIsRejectedBeforeRepositoryAccess(string query)
{
var repository = new FakeRepository();
var service = CreateService(repository);
await Assert.ThrowsAsync<ArgumentException>(
() => service.SearchFreshPageAsync(
"https://example.com/page",
query,
null,
CancellationToken.None));
Assert.Equal(0, repository.SearchCalls);
}
[Fact]
public async Task RelatedTermsMayBeOmittedNullOrEmpty()
{
foreach (IReadOnlyList<string>? relatedTerms in new IReadOnlyList<string>?[]
{
null,
[]
})
{
var repository = new FakeRepository();
var service = CreateService(repository);
await service.SearchFreshPageAsync(
"https://example.com/page",
"request throttling",
relatedTerms,
CancellationToken.None);
Assert.Equal(1, repository.SearchCalls);
}
}
[Fact]
public void RelatedTermsAreTrimmedBlankFilteredAndCaseInsensitiveDeduplicated()
{
var result = WebCacheService.NormalizeRelatedTerms(
[" rate limiting ", "", " ", "RATE LIMITING", "request quota"]);
Assert.Equal(["rate limiting", "request quota"], result);
}
[Fact]
public void MoreThanEightRelatedTermsIsRejected()
{
var terms = Enumerable.Range(1, 9).Select(index => $"term {index}").ToArray();
Assert.Throws<ArgumentException>(
() => WebCacheService.NormalizeRelatedTerms(terms));
}
[Fact]
public void RelatedTermLongerThanOneHundredCharactersIsRejected()
{
Assert.Throws<ArgumentException>(
() => WebCacheService.NormalizeRelatedTerms([new string('x', 101)]));
}
[Fact]
public void CandidateQueryContainsExactAndRelatedTokensWithoutSqlSyntax()
{
var candidate = WebCacheService.CreateCandidateQuery(
"request throttling behavior",
["rate limiting", "request quota"]);
Assert.Contains("\"request\"", candidate);
Assert.Contains("\"throttling\"", candidate);
Assert.Contains("\"behavior\"", candidate);
Assert.Contains("\"rate\"", candidate);
Assert.Contains("\"limiting\"", candidate);
Assert.Contains("\"quota\"", candidate);
Assert.DoesNotContain("SELECT", candidate, StringComparison.OrdinalIgnoreCase);
}
[Fact]
public void ExactPhraseOutranksRepeatedRelatedOnlyRegion()
{
var exactRegion = "The request throttling behavior is deterministic and safe.";
var relatedRegion = string.Join(
' ',
Enumerable.Repeat("rate limiting request quota", 8));
var document = relatedRegion.PadRight(4_000, 'x') + exactRegion.PadRight(4_000, 'y');
var chunks = WebCacheService.CreateChunks(document, 4_000);
var hit = WebCacheService.CreateStrongestContext(
chunks,
document.Length,
"request throttling behavior",
["rate limiting", "request quota"],
4_000);
Assert.NotNull(hit);
Assert.Contains(exactRegion, hit.Context);
Assert.Equal(2, hit.EndPage);
}
[Fact]
public void StrongRelatedPhraseCanRetrieveWhenExactWordingIsAbsent()
{
var document = "A generic introduction. "
+ "Rate limiting controls a request quota for each client. "
+ new string('x', 5_000);
var chunks = WebCacheService.CreateChunks(document, 4_000);
var hit = WebCacheService.CreateStrongestContext(
chunks,
document.Length,
"request throttling behavior",
["rate limiting", "request quota"],
4_000);
Assert.NotNull(hit);
Assert.Contains("Rate limiting controls a request quota", hit.Context);
}
[Fact]
public void MultipleNearbyRelatedTermsAreConfidentButOneIsolatedTokenIsNot()
{
var strongDocument = "throttling and quota are discussed together" + new string('x', 4_500);
var weakDocument = "quota" + new string('x', 4_500);
var strong = WebCacheService.CreateStrongestContext(
WebCacheService.CreateChunks(strongDocument, 4_000),
strongDocument.Length,
"!!!",
["throttling", "quota"],
4_000);
var weak = WebCacheService.CreateStrongestContext(
WebCacheService.CreateChunks(weakDocument, 4_000),
weakDocument.Length,
"!!!",
["quota"],
4_000);
Assert.NotNull(strong);
Assert.Null(weak);
}
[Fact]
public void CrossChunkSentenceIsReturnedInOrderWithoutDuplication()
{
const string firstHalf = "request throttling ";
const string secondHalf = "behavior continues across the storage boundary.";
var firstChunk = new string('a', 4_000 - firstHalf.Length) + firstHalf;
var secondChunk = secondHalf + new string('b', 4_000 - secondHalf.Length);
var document = firstChunk + secondChunk;
var chunks = WebCacheService.CreateChunks(document, 4_000);
var hit = WebCacheService.CreateStrongestContext(
chunks,
document.Length,
"request throttling behavior",
["rate limiting"],
4_000);
Assert.NotNull(hit);
Assert.Equal(1, hit.StartPage);
Assert.Equal(2, hit.EndPage);
Assert.Contains(firstHalf + secondHalf, hit.Context);
Assert.Equal(
1,
CountOccurrences(hit.Context, firstHalf + secondHalf));
Assert.InRange(hit.Context.Length, 1, 4_000);
}
[Fact]
public void ContextClampsToDocumentStartAndEnd()
{
var startDocument = "request throttling behavior" + new string('s', 5_000);
var endDocument = new string('e', 5_000) + "request throttling behavior";
var startHit = WebCacheService.CreateStrongestContext(
WebCacheService.CreateChunks(startDocument, 4_000),
startDocument.Length,
"request throttling behavior",
[],
4_000);
var endHit = WebCacheService.CreateStrongestContext(
WebCacheService.CreateChunks(endDocument, 4_000),
endDocument.Length,
"request throttling behavior",
[],
4_000);
Assert.NotNull(startHit);
Assert.NotNull(endHit);
Assert.Equal(0, startHit.StartOffset);
Assert.Equal(endDocument.Length, endHit.StartOffset + endHit.Length);
}
[Fact]
public async Task PageNotFoundExpiredAndNoMatchMapCorrectly()
{
var notFoundRepository = new FakeRepository { SearchPage = null };
var notFound = await CreateService(notFoundRepository).SearchFreshPageAsync(
"https://example.com/missing",
"request throttling",
null,
CancellationToken.None);
var expiredRepository = new FakeRepository
{
SearchPage = CreatePage(expiresAtUtc: TestNow)
};
var expired = await CreateService(expiredRepository).SearchFreshPageAsync(
"https://example.com/page",
"request throttling",
null,
CancellationToken.None);
var noMatchRepository = new FakeRepository
{
SearchPage = CreatePage(hits: [])
};
var noMatch = await CreateService(noMatchRepository).SearchFreshPageAsync(
"https://example.com/page",
"punctuation !!!",
null,
CancellationToken.None);
Assert.Equal(WebCacheSearchStatus.PageNotFound, notFound.Status);
Assert.Equal(WebCacheSearchStatus.Expired, expired.Status);
Assert.Equal(WebCacheSearchStatus.NoMatches, noMatch.Status);
}
[Fact]
public async Task PostgreSqlFailureRemainsUnavailable()
{
var expected = new WebCacheUnavailableException(
"Unavailable.",
new InvalidOperationException("Synthetic."));
var repository = new FakeRepository { SearchException = expected };
var actual = await Assert.ThrowsAsync<WebCacheUnavailableException>(
() => CreateService(repository).SearchFreshPageAsync(
"https://example.com/page",
"request throttling",
null,
CancellationToken.None));
Assert.Same(expected, actual);
}
[Fact]
public void ProviderQueryIsPageScopedGinBackedAndParameterized()
{
using var context = CreateContext();
var sql = PostgresWebCacheRepository.BuildSearchQuery(
context,
42,
"\"request\" OR \"throttling\"")
.ToQueryString();
Assert.Contains("websearch_to_tsquery('simple'", sql, StringComparison.OrdinalIgnoreCase);
Assert.Contains("@candidateQuery", sql, StringComparison.Ordinal);
Assert.Contains("@cachedWebPageId", sql, StringComparison.Ordinal);
Assert.Contains("@@", sql, StringComparison.Ordinal);
Assert.Contains("ts_rank", sql, StringComparison.OrdinalIgnoreCase);
Assert.DoesNotContain("LIKE", sql, StringComparison.OrdinalIgnoreCase);
Assert.DoesNotContain("ILIKE", sql, StringComparison.OrdinalIgnoreCase);
}
[Fact]
public void ExistingSearchVectorAndIndexesRemainUnchanged()
{
using var context = CreateContext();
var model = context.GetService<IDesignTimeModel>().Model;
var chunk = model.FindEntityType(typeof(CachedWebPageChunk));
var page = model.FindEntityType(typeof(CachedWebPage));
Assert.NotNull(chunk);
Assert.NotNull(page);
var vector = chunk.FindProperty(nameof(CachedWebPageChunk.SearchVector));
Assert.NotNull(vector);
Assert.Equal("simple", vector.GetTsVectorConfig());
Assert.Contains(
chunk.GetIndexes(),
index => index.Properties.Select(property => property.Name)
.SequenceEqual([nameof(CachedWebPageChunk.SearchVector)])
&& string.Equals(index.GetMethod(), "GIN", StringComparison.Ordinal));
Assert.Contains(
chunk.GetIndexes(),
index => index.IsUnique
&& index.Properties.Select(property => property.Name)
.SequenceEqual(
[
nameof(CachedWebPageChunk.CachedWebPageId),
nameof(CachedWebPageChunk.ChunkNumber)
]));
Assert.Contains(
page.GetIndexes(),
index => index.IsUnique
&& index.Properties.Select(property => property.Name)
.SequenceEqual([nameof(CachedWebPage.CacheKey)]));
}
private static WebCacheService CreateService(FakeRepository repository) =>
new(
repository,
Microsoft.Extensions.Options.Options.Create(new WebCacheOptions()),
new FakeTimeProvider(TestNow));
private static WebCacheRepositorySearchPage CreatePage(
DateTimeOffset? expiresAtUtc = null,
IReadOnlyList<WebCacheRepositorySearchHit>? hits = null)
{
var document = "request throttling behavior" + new string('x', 4_500);
var chunks = WebCacheService.CreateChunks(document, 4_000);
return new WebCacheRepositorySearchPage(
1,
"https://example.com/page",
"https://example.com/final",
TestNow.AddMinutes(-5),
expiresAtUtc ?? TestNow.AddHours(1),
true,
document.Length,
chunks.Count,
chunks,
hits ?? [new WebCacheRepositorySearchHit(1, 1.0f)]);
}
private static WebCacheDbContext CreateContext()
{
var options = new DbContextOptionsBuilder<WebCacheDbContext>()
.UseNpgsql("Host=127.0.0.1;Port=1;Database=webaccess;Username=webaccess;Timeout=1")
.Options;
return new WebCacheDbContext(options);
}
private static int CountOccurrences(string value, string fragment) =>
(value.Length - value.Replace(fragment, string.Empty, StringComparison.Ordinal).Length)
/ fragment.Length;
private sealed class FakeTimeProvider(DateTimeOffset utcNow) : TimeProvider
{
public override DateTimeOffset GetUtcNow() => utcNow;
}
private sealed class FakeRepository : IWebCacheRepository
{
public WebCacheRepositorySearchPage? SearchPage { get; set; } = CreatePage();
public WebCacheUnavailableException? SearchException { get; set; }
public int SearchCalls { get; private set; }
public string? LastCandidateQuery { get; private set; }
public Task<WebCacheRepositorySearchPage?> SearchPageAsync(
string cacheKey,
string candidateQuery,
DateTimeOffset currentUtc,
CancellationToken cancellationToken)
{
SearchCalls++;
LastCandidateQuery = candidateQuery;
cancellationToken.ThrowIfCancellationRequested();
return SearchException is null
? Task.FromResult(SearchPage)
: Task.FromException<WebCacheRepositorySearchPage?>(SearchException);
}
public Task<WebCacheRepositoryPage?> TryGetPageChunkAsync(
string cacheKey,
int chunkNumber,
CancellationToken cancellationToken) =>
throw new NotSupportedException();
public Task StoreOrReplaceAsync(
WebCacheStoredPage page,
IReadOnlyList<WebCacheStoredChunk> chunks,
CancellationToken cancellationToken) =>
throw new NotSupportedException();
}
}

View File

@ -0,0 +1,360 @@
using Microsoft.Extensions.Options;
using WebAccess.MCP.WebCache;
namespace WebAccess.MCP.Tests;
public sealed class WebCacheServiceTests
{
private static readonly DateTimeOffset TestNow =
new(2026, 8, 20, 4, 0, 0, TimeSpan.Zero);
[Fact]
public void CacheKeyIsDeterministic()
{
var first = WebCacheService.CreateCacheIdentity(
"https://Example.com/path?q=value#first");
var second = WebCacheService.CreateCacheIdentity(
"https://Example.com/path?q=value#first");
Assert.Equal(first.NormalizedUrl, second.NormalizedUrl);
Assert.Equal(first.CacheKey, second.CacheKey);
Assert.Matches("^[0-9a-f]{64}$", first.CacheKey);
}
[Fact]
public void UrlFragmentsDoNotChangeCacheIdentity()
{
var first = WebCacheService.CreateCacheIdentity(
"https://example.com/path?q=value#first");
var second = WebCacheService.CreateCacheIdentity(
"https://example.com/path?q=value#second");
Assert.Equal(first.NormalizedUrl, second.NormalizedUrl);
Assert.Equal(first.CacheKey, second.CacheKey);
}
[Fact]
public void MeaningfulQueryStringsChangeCacheIdentity()
{
var first = WebCacheService.CreateCacheIdentity(
"https://example.com/path?q=first");
var second = WebCacheService.CreateCacheIdentity(
"https://example.com/path?q=second");
Assert.NotEqual(first.CacheKey, second.CacheKey);
}
[Theory]
[InlineData("not a URL")]
[InlineData("ftp://example.com/resource")]
public async Task InvalidUrlsAreRejectedBeforeRepositoryReads(string requestedUrl)
{
var repository = new FakeWebCacheRepository();
var service = CreateService(repository);
await Assert.ThrowsAsync<ArgumentException>(
() => service.TryGetFreshChunkAsync(requestedUrl, 1, CancellationToken.None));
Assert.Equal(0, repository.ReadCallCount);
}
[Fact]
public async Task InvalidUrlIsRejectedBeforeRepositoryWrites()
{
var repository = new FakeWebCacheRepository();
var service = CreateService(repository);
await Assert.ThrowsAsync<ArgumentException>(
() => service.StoreAsync(
CreateRequest("content", "file:///temporary/page"),
CancellationToken.None));
Assert.Equal(0, repository.WriteCallCount);
}
[Fact]
public async Task ChunkNumberMustBeOneOrGreaterBeforeRepositoryRead()
{
var repository = new FakeWebCacheRepository();
var service = CreateService(repository);
await Assert.ThrowsAsync<ArgumentOutOfRangeException>(
() => service.TryGetFreshChunkAsync(
"https://example.com/page",
0,
CancellationToken.None));
Assert.Equal(0, repository.ReadCallCount);
}
[Theory]
[InlineData(0, 0)]
[InlineData(4_000, 1)]
[InlineData(4_001, 2)]
[InlineData(8_000, 2)]
[InlineData(8_001, 3)]
public void ChunkCountsMatchBoundaries(int textLength, int expectedChunkCount)
{
var chunks = WebCacheService.CreateChunks(new string('x', textLength), 4_000);
Assert.Equal(expectedChunkCount, chunks.Count);
}
[Fact]
public void ChunksAreContiguousAndReproduceOriginalText()
{
var text = string.Concat(
Enumerable.Range(0, 8_001).Select(index => (char)('a' + index % 26)));
var chunks = WebCacheService.CreateChunks(text, 4_000);
Assert.Equal([1, 2, 3], chunks.Select(chunk => chunk.ChunkNumber));
Assert.Equal([0, 4_000, 8_000], chunks.Select(chunk => chunk.StartOffset));
Assert.Equal([4_000, 4_000, 1], chunks.Select(chunk => chunk.Length));
Assert.Equal(text, string.Concat(chunks.Select(chunk => chunk.Text)));
}
[Fact]
public async Task StoreUsesInjectedTimeAndConfiguredExpiration()
{
var repository = new FakeWebCacheRepository();
var timeProvider = new FakeTimeProvider(TestNow);
var service = CreateService(repository, timeProvider);
await service.StoreAsync(
CreateRequest("content"),
CancellationToken.None);
Assert.NotNull(repository.LastPage);
Assert.Equal(TestNow, repository.LastPage.FetchedAtUtc);
Assert.Equal(TestNow.AddHours(1), repository.LastPage.ExpiresAtUtc);
Assert.Equal("content".Length, repository.LastPage.TotalTextLength);
}
[Fact]
public async Task FreshRequestedChunkIsReturnedWithHasMore()
{
var repository = new FakeWebCacheRepository();
var service = CreateService(repository);
await service.StoreAsync(
CreateRequest(new string('x', 4_001)),
CancellationToken.None);
var result = await service.TryGetFreshChunkAsync(
"https://example.com/page",
1,
CancellationToken.None);
Assert.Equal(WebCacheLookupStatus.Hit, result.Status);
Assert.NotNull(result.Page);
Assert.Equal(1, result.Page.ChunkNumber);
Assert.Equal(2, result.Page.ChunkCount);
Assert.Equal(0, result.Page.StartOffset);
Assert.Equal(4_000, result.Page.Length);
Assert.True(result.Page.HasMore);
}
[Fact]
public async Task EntryExpiringAtCurrentTimeIsExpired()
{
var repository = new FakeWebCacheRepository();
var timeProvider = new FakeTimeProvider(TestNow);
var service = CreateService(repository, timeProvider);
await service.StoreAsync(CreateRequest("content"), CancellationToken.None);
timeProvider.UtcNow = TestNow.AddHours(1);
var result = await service.TryGetFreshChunkAsync(
"https://example.com/page",
1,
CancellationToken.None);
Assert.Equal(WebCacheLookupStatus.Expired, result.Status);
Assert.Null(result.Page);
}
[Fact]
public async Task ExpiredEntryIsAMiss()
{
var repository = new FakeWebCacheRepository();
var timeProvider = new FakeTimeProvider(TestNow);
var service = CreateService(repository, timeProvider);
await service.StoreAsync(CreateRequest("content"), CancellationToken.None);
timeProvider.UtcNow = TestNow.AddHours(2);
var result = await service.TryGetFreshChunkAsync(
"https://example.com/page",
1,
CancellationToken.None);
Assert.Equal(WebCacheLookupStatus.Expired, result.Status);
}
[Fact]
public async Task MissingChunkIsDistinctFromMissingPage()
{
var repository = new FakeWebCacheRepository();
var service = CreateService(repository);
await service.StoreAsync(CreateRequest("content"), CancellationToken.None);
var missingChunk = await service.TryGetFreshChunkAsync(
"https://example.com/page",
2,
CancellationToken.None);
var missingPage = await service.TryGetFreshChunkAsync(
"https://example.com/other",
1,
CancellationToken.None);
Assert.Equal(WebCacheLookupStatus.ChunkNotFound, missingChunk.Status);
Assert.NotNull(missingChunk.Page);
Assert.Equal(1, missingChunk.Page.ChunkCount);
Assert.Equal(WebCacheLookupStatus.PageNotFound, missingPage.Status);
Assert.Null(missingPage.Page);
}
[Fact]
public async Task ZeroTextSnapshotReturnsFreshMetadataWithoutChunk()
{
var repository = new FakeWebCacheRepository();
var service = CreateService(repository);
await service.StoreAsync(CreateRequest(string.Empty), CancellationToken.None);
var result = await service.TryGetFreshChunkAsync(
"https://example.com/page",
1,
CancellationToken.None);
Assert.Equal(WebCacheLookupStatus.Hit, result.Status);
Assert.NotNull(result.Page);
Assert.Equal(0, result.Page.TotalTextLength);
Assert.Equal(0, result.Page.ChunkCount);
Assert.Null(result.Page.ChunkNumber);
Assert.Null(result.Page.Text);
Assert.False(result.Page.HasMore);
}
[Fact]
public async Task ReplacementKeepsOneCurrentSnapshotPerCacheKey()
{
var repository = new FakeWebCacheRepository();
var service = CreateService(repository);
await service.StoreAsync(
CreateRequest("old", title: "Old"),
CancellationToken.None);
await service.StoreAsync(
CreateRequest("replacement", title: "New"),
CancellationToken.None);
var result = await service.TryGetFreshChunkAsync(
"https://example.com/page",
1,
CancellationToken.None);
Assert.Equal(1, repository.EntryCount);
Assert.Equal(2, repository.WriteCallCount);
Assert.Equal(WebCacheLookupStatus.Hit, result.Status);
Assert.Equal("New", result.Page?.Title);
Assert.Equal("replacement", result.Page?.Text);
}
private static WebCacheService CreateService(
FakeWebCacheRepository repository,
FakeTimeProvider? timeProvider = null) =>
new(
repository,
Options.Create(
new WebCacheOptions
{
DefaultExpiration = TimeSpan.FromHours(1),
ChunkSize = 4_000
}),
timeProvider ?? new FakeTimeProvider(TestNow));
private static WebCachePageWriteRequest CreateRequest(
string normalizedText,
string requestedUrl = "https://example.com/page",
string? title = "Example") =>
new(
requestedUrl,
requestedUrl,
200,
title,
"text/html",
null,
normalizedText,
true);
private sealed class FakeTimeProvider(DateTimeOffset utcNow) : TimeProvider
{
public DateTimeOffset UtcNow { get; set; } = utcNow;
public override DateTimeOffset GetUtcNow() => UtcNow;
}
private sealed class FakeWebCacheRepository : IWebCacheRepository
{
private readonly Dictionary<string, StoredEntry> entries = new(StringComparer.Ordinal);
public int ReadCallCount { get; private set; }
public int WriteCallCount { get; private set; }
public int EntryCount => entries.Count;
public WebCacheStoredPage? LastPage { get; private set; }
public Task<WebCacheRepositoryPage?> TryGetPageChunkAsync(
string cacheKey,
int chunkNumber,
CancellationToken cancellationToken)
{
ReadCallCount++;
cancellationToken.ThrowIfCancellationRequested();
if (!entries.TryGetValue(cacheKey, out var entry))
{
return Task.FromResult<WebCacheRepositoryPage?>(null);
}
var chunk = entry.Chunks.SingleOrDefault(
candidate => candidate.ChunkNumber == chunkNumber);
var page = entry.Page;
return Task.FromResult<WebCacheRepositoryPage?>(
new WebCacheRepositoryPage(
page.RequestedUrl,
page.FinalUrl,
page.HttpStatusCode,
page.Title,
page.ContentType,
page.FetchedAtUtc,
page.ExpiresAtUtc,
page.TotalTextLength,
page.IsComplete,
entry.Chunks.Count,
chunk));
}
public Task StoreOrReplaceAsync(
WebCacheStoredPage page,
IReadOnlyList<WebCacheStoredChunk> chunks,
CancellationToken cancellationToken)
{
WriteCallCount++;
cancellationToken.ThrowIfCancellationRequested();
LastPage = page;
entries[page.CacheKey] = new StoredEntry(page, chunks.ToArray());
return Task.CompletedTask;
}
public Task<WebCacheRepositorySearchPage?> SearchPageAsync(
string cacheKey,
string candidateQuery,
DateTimeOffset currentUtc,
CancellationToken cancellationToken) =>
throw new NotSupportedException();
private sealed record StoredEntry(
WebCacheStoredPage Page,
IReadOnlyList<WebCacheStoredChunk> Chunks);
}
}

View File

@ -0,0 +1,81 @@
using System.Text.Json;
using WebAccess.MCP.Models;
namespace WebAccess.MCP.Tests;
public sealed class WebSearchResponseTests
{
[Fact]
public void SerializationAddsOneTopLevelHintWithoutChangingResults()
{
var results = new[]
{
new WebSearchResult(
"First",
"https://example.com/first",
"First snippet",
["google"]),
new WebSearchResult(
"Second",
"https://example.com/second",
"Second snippet",
["bing"])
};
var response = new WebSearchResponse(
"rate limiting",
["google", "bing"],
results);
var json = JsonSerializer.Serialize(response);
using var document = JsonDocument.Parse(json);
var root = document.RootElement;
Assert.Equal(
WebSearchResponse.NextStepHintValue,
root.GetProperty("nextStepHint").GetString());
Assert.Equal(1, CountOccurrences(json, "\"nextStepHint\""));
Assert.Equal("rate limiting", root.GetProperty("query").GetString());
Assert.Equal(
["google", "bing"],
root.GetProperty("requestedEngines")
.EnumerateArray()
.Select(item => item.GetString()));
var serializedResults = root.GetProperty("results").EnumerateArray().ToArray();
Assert.Equal(2, serializedResults.Length);
Assert.Equal("First", serializedResults[0].GetProperty("title").GetString());
Assert.Equal("https://example.com/first", serializedResults[0].GetProperty("url").GetString());
Assert.Equal("First snippet", serializedResults[0].GetProperty("snippet").GetString());
Assert.Equal(["google"], serializedResults[0].GetProperty("engines")
.EnumerateArray()
.Select(item => item.GetString()));
Assert.False(serializedResults[0].TryGetProperty("nextStepHint", out _));
Assert.False(serializedResults[1].TryGetProperty("nextStepHint", out _));
Assert.Equal(
["query", "requestedEngines", "nextStepHint", "results"],
root.EnumerateObject().Select(property => property.Name));
}
[Fact]
public void EmptyResultsStillContainExactHint()
{
var response = new WebSearchResponse(
"no results",
[],
[]);
var json = JsonSerializer.Serialize(response);
using var document = JsonDocument.Parse(json);
Assert.Equal(
WebSearchResponse.NextStepHintValue,
document.RootElement.GetProperty("nextStepHint").GetString());
Assert.Equal(1, CountOccurrences(json, "\"nextStepHint\""));
Assert.Empty(document.RootElement.GetProperty("results").EnumerateArray());
}
private static int CountOccurrences(string value, string fragment) =>
(value.Length - value.Replace(fragment, string.Empty, StringComparison.Ordinal).Length)
/ fragment.Length;
}

5
src/WebAccess.MCP.slnx Normal file
View File

@ -0,0 +1,5 @@
<Solution>
<Project Path="WebAccess.MCP.Tests/WebAccess.MCP.Tests.csproj" />
<Project Path="WebAccess.MCP.TestRig/WebAccess.MCP.TestRig.csproj" />
<Project Path="WebAccess.MCP/WebAccess.MCP.csproj" />
</Solution>

View File

@ -0,0 +1,22 @@
using Microsoft.Playwright;
namespace WebAccess.MCP.Browser;
internal static class BrowserNavigation
{
public static async Task<IResponse?> NavigateAsync(
IPage page,
Uri uri,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
var response = await page.GotoAsync(
uri.AbsoluteUri,
new PageGotoOptions
{
WaitUntil = WaitUntilState.DOMContentLoaded
});
cancellationToken.ThrowIfCancellationRequested();
return response;
}
}

View File

@ -0,0 +1,205 @@
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using WebAccess.MCP.WebCache;
namespace WebAccess.MCP.Browser;
internal sealed class BrowserPageTextCoordinator(
IBrowserPageTextReader pageTextReader,
IWebCacheService webCacheService,
IOptions<WebCacheOptions> options,
ILogger<BrowserPageTextCoordinator> logger)
{
private readonly int chunkSize = options.Value.ChunkSize;
public async Task<string> GetPageTextAsync(
string? url,
int page,
CancellationToken cancellationToken)
{
ValidatePageNumber(page);
if (url is null)
{
var currentSnapshot = await pageTextReader.ReadCurrentAsync(cancellationToken);
return FormatSnapshot(currentSnapshot, page);
}
var uri = ValidateUrl(url);
WebCacheLookupResult? cacheResult = null;
try
{
cacheResult = await webCacheService.TryGetFreshChunkAsync(
url,
page,
cancellationToken);
}
catch (WebCacheUnavailableException exception)
{
logger.LogWarning(
"WebCache read failed ({ExceptionType}); using browser fallback.",
exception.GetType().Name);
}
if (cacheResult?.Status == WebCacheLookupStatus.Hit)
{
return FormatCachedPage(cacheResult.Page!);
}
if (cacheResult?.Status == WebCacheLookupStatus.ChunkNotFound)
{
return FormatMissingPage(page, cacheResult.Page!.ChunkCount);
}
var browserSnapshot = await pageTextReader.NavigateAndReadAsync(
uri,
cancellationToken);
var request = new WebCachePageWriteRequest(
url,
browserSnapshot.FinalUrl,
browserSnapshot.HttpStatusCode,
browserSnapshot.Title,
browserSnapshot.ContentType,
null,
browserSnapshot.NormalizedText,
browserSnapshot.IsComplete);
try
{
await webCacheService.StoreAsync(request, cancellationToken);
}
catch (WebCacheUnavailableException exception)
{
logger.LogWarning(
"WebCache write failed ({ExceptionType}); returning browser result without persistence.",
exception.GetType().Name);
}
return FormatSnapshot(browserSnapshot, page);
}
internal static void ValidatePageNumber(int page)
{
if (page < 1)
{
throw new ArgumentOutOfRangeException(
nameof(page),
page,
"The page number must be one or greater.");
}
}
private string FormatSnapshot(BrowserPageTextSnapshot snapshot, int requestedPage)
{
var chunks = WebCacheService.CreateChunks(snapshot.NormalizedText, chunkSize);
if (chunks.Count == 0)
{
return FormatZeroText(snapshot.HttpStatusCode, snapshot.IsComplete);
}
if (requestedPage > chunks.Count)
{
return FormatMissingPage(requestedPage, chunks.Count);
}
var chunk = chunks[requestedPage - 1];
return FormatPage(
chunk.ChunkNumber,
chunks.Count,
chunk.StartOffset,
chunk.Length,
chunk.Text,
snapshot.NormalizedText.Length,
chunk.ChunkNumber < chunks.Count,
snapshot.IsComplete);
}
private static string FormatCachedPage(WebCachePageChunkRead page)
{
if (page.ChunkCount == 0)
{
return FormatZeroText(page.HttpStatusCode, page.IsComplete);
}
return FormatPage(
page.ChunkNumber!.Value,
page.ChunkCount,
page.StartOffset!.Value,
page.Length!.Value,
page.Text!,
page.TotalTextLength,
page.HasMore,
page.IsComplete);
}
private static string FormatPage(
int page,
int pageCount,
int startOffset,
int length,
string text,
int totalTextLength,
bool hasMore,
bool isComplete)
{
var result =
$"Page {page} of {pageCount} | chars {startOffset}-{startOffset + length - 1} "
+ $"of {totalTextLength}"
+ Environment.NewLine
+ text;
if (hasMore)
{
result += Environment.NewLine
+ Environment.NewLine
+ $"More content available. Request page {page + 1} only if needed.";
}
else if (!isComplete)
{
result += Environment.NewLine
+ Environment.NewLine
+ "[Snapshot truncated at the internal 50000-character extraction limit.]";
}
return result;
}
private static string FormatZeroText(int? httpStatusCode, bool isComplete)
{
var status = httpStatusCode is null ? string.Empty : $" | HTTP {httpStatusCode}";
var result = $"Page 0 of 0 | 0 characters{status}"
+ Environment.NewLine
+ "No readable page text was extracted.";
if (!isComplete)
{
result += Environment.NewLine
+ Environment.NewLine
+ "[Snapshot truncated at the internal 50000-character extraction limit.]";
}
return result;
}
private static string FormatMissingPage(int requestedPage, int pageCount) =>
$"Page {requestedPage} does not exist. This snapshot has {pageCount} pages.";
private static Uri ValidateUrl(string url)
{
if (string.IsNullOrWhiteSpace(url))
{
throw new ArgumentException("A non-empty URL is required.", nameof(url));
}
if (!Uri.TryCreate(url, UriKind.Absolute, out var uri)
|| (uri.Scheme != Uri.UriSchemeHttp && uri.Scheme != Uri.UriSchemeHttps))
{
throw new ArgumentException(
"The URL must be an absolute HTTP or HTTPS URL.",
nameof(url));
}
return uri;
}
}

View File

@ -0,0 +1,95 @@
using Microsoft.Playwright;
namespace WebAccess.MCP.Browser;
internal sealed class BrowserPageTextReader(IBrowserSessionManager sessionManager)
: IBrowserPageTextReader
{
internal const int MaximumSnapshotCharacters = 50_000;
public Task<BrowserPageTextSnapshot> ReadCurrentAsync(
CancellationToken cancellationToken) =>
sessionManager.WithDefaultPageAsync(
page => ExtractSnapshotAsync(page, null, cancellationToken),
cancellationToken);
public Task<BrowserPageTextSnapshot> NavigateAndReadAsync(
Uri uri,
CancellationToken cancellationToken) =>
sessionManager.WithDefaultPageAsync(
async page =>
{
var response = await BrowserNavigation.NavigateAsync(
page,
uri,
cancellationToken);
return await ExtractSnapshotAsync(page, response, cancellationToken);
},
cancellationToken);
internal static string NormalizePageText(string text)
{
var lines = text
.Replace("\r\n", "\n", StringComparison.Ordinal)
.Replace('\r', '\n')
.Split('\n');
var normalizedLines = new List<string>(lines.Length);
foreach (var line in lines)
{
var normalizedLine = string.Join(
" ",
line.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries));
if (normalizedLine.Length > 0)
{
normalizedLines.Add(normalizedLine);
}
else if (normalizedLines.Count > 0 && normalizedLines[^1].Length > 0)
{
normalizedLines.Add(string.Empty);
}
}
if (normalizedLines.Count > 0 && normalizedLines[^1].Length == 0)
{
normalizedLines.RemoveAt(normalizedLines.Count - 1);
}
return string.Join(Environment.NewLine, normalizedLines);
}
private static async Task<BrowserPageTextSnapshot> ExtractSnapshotAsync(
IPage page,
IResponse? response,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
var bodyText = await page.Locator("body").InnerTextAsync();
var title = await page.TitleAsync();
var contentType = await page.EvaluateAsync<string?>(
"() => document.contentType || null");
cancellationToken.ThrowIfCancellationRequested();
var (normalizedText, isComplete) = ApplySnapshotLimit(
NormalizePageText(bodyText));
return new BrowserPageTextSnapshot(
page.Url,
response?.Status,
title,
string.IsNullOrWhiteSpace(contentType) ? null : contentType.Trim(),
normalizedText,
isComplete);
}
internal static (string Text, bool IsComplete) ApplySnapshotLimit(
string normalizedText)
{
ArgumentNullException.ThrowIfNull(normalizedText);
return normalizedText.Length <= MaximumSnapshotCharacters
? (normalizedText, true)
: (normalizedText[..MaximumSnapshotCharacters], false);
}
}

View File

@ -0,0 +1,133 @@
using Microsoft.Extensions.Logging;
using WebAccess.MCP.WebCache;
namespace WebAccess.MCP.Browser;
internal sealed class BrowserPageTextSearchCoordinator(
IBrowserPageTextReader pageTextReader,
IWebCacheService webCacheService,
ILogger<BrowserPageTextSearchCoordinator> logger)
{
public async Task<string> FindPageTextAsync(
string url,
string query,
IReadOnlyList<string>? relatedTerms,
CancellationToken cancellationToken)
{
var input = ValidateInputs(url, query, relatedTerms);
var searchResult = await SearchAsync(
url,
input.Query,
input.RelatedTerms,
cancellationToken);
if (searchResult.Status is WebCacheSearchStatus.Hit
or WebCacheSearchStatus.NoMatches)
{
return FormatResult(searchResult);
}
var uri = WebCacheService.CreateCacheIdentity(url).NormalizedUrl;
var snapshot = await pageTextReader.NavigateAndReadAsync(
new Uri(uri),
cancellationToken);
var request = new WebCachePageWriteRequest(
url,
snapshot.FinalUrl,
snapshot.HttpStatusCode,
snapshot.Title,
snapshot.ContentType,
null,
snapshot.NormalizedText,
snapshot.IsComplete);
try
{
await webCacheService.StoreAsync(request, cancellationToken);
}
catch (WebCacheUnavailableException exception)
{
logger.LogWarning(
"WebCache store failed during relevance lookup ({FailureType}).",
exception.GetType().Name);
throw new PageTextSearchUnavailableException(
"The page was fetched, but its searchable snapshot could not be stored.");
}
searchResult = await SearchAsync(
url,
input.Query,
input.RelatedTerms,
cancellationToken);
if (searchResult.Status is WebCacheSearchStatus.PageNotFound
or WebCacheSearchStatus.Expired)
{
throw new PageTextSearchUnavailableException(
"The searchable page snapshot was unavailable after refresh.");
}
return FormatResult(searchResult);
}
internal static WebCacheSearchInput ValidateInputs(
string url,
string query,
IReadOnlyList<string>? relatedTerms)
{
_ = WebCacheService.CreateCacheIdentity(url);
return new WebCacheSearchInput(
WebCacheService.ValidateSearchQuery(query),
WebCacheService.NormalizeRelatedTerms(relatedTerms));
}
internal static string FormatResult(WebCacheSearchResult result)
{
if (result.Status == WebCacheSearchStatus.NoMatches || result.Hits.Count == 0)
{
return "Targeted cached search did not identify a confident location. "
+ "Use get_page_text sequentially if broader reading is needed.";
}
var hit = result.Hits[0];
var pageLocation = hit.StartPage == hit.EndPage
? $"Page {hit.StartPage}"
: $"Pages {hit.StartPage}-{hit.EndPage}";
return $"Found targeted context at {pageLocation} of {result.ChunkCount}."
+ Environment.NewLine
+ Environment.NewLine
+ hit.Context
+ Environment.NewLine
+ Environment.NewLine
+ "Use get_page_text for the identified page or page range if more context is needed.";
}
private async Task<WebCacheSearchResult> SearchAsync(
string url,
string query,
IReadOnlyList<string> relatedTerms,
CancellationToken cancellationToken)
{
try
{
return await webCacheService.SearchFreshPageAsync(
url,
query,
relatedTerms,
cancellationToken);
}
catch (WebCacheUnavailableException exception)
{
logger.LogWarning(
"WebCache relevance lookup failed ({FailureType}).",
exception.GetType().Name);
throw new PageTextSearchUnavailableException(
"Cached page relevance search is unavailable.");
}
}
}
internal sealed record WebCacheSearchInput(
string Query,
IReadOnlyList<string> RelatedTerms);

View File

@ -0,0 +1,9 @@
namespace WebAccess.MCP.Browser;
internal sealed record BrowserPageTextSnapshot(
string FinalUrl,
int? HttpStatusCode,
string Title,
string? ContentType,
string NormalizedText,
bool IsComplete);

View File

@ -0,0 +1,137 @@
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Playwright;
namespace WebAccess.MCP.Browser;
internal sealed class BrowserSessionManager(
IBrowserRuntime browserRuntime,
ILogger<BrowserSessionManager> logger) : IBrowserSessionManager, IHostedService
{
private readonly SemaphoreSlim sessionLock = new(1, 1);
private IBrowserContext? defaultContext;
private IPage? defaultPage;
private bool stopping;
public Task StartAsync(CancellationToken cancellationToken) => Task.CompletedTask;
public async Task<T> WithDefaultPageAsync<T>(
Func<IPage, Task<T>> operation,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(operation);
await sessionLock.WaitAsync(cancellationToken);
try
{
ObjectDisposedException.ThrowIf(stopping, this);
var page = await GetOrCreateDefaultPageAsync(cancellationToken);
cancellationToken.ThrowIfCancellationRequested();
return await operation(page);
}
finally
{
sessionLock.Release();
}
}
public async Task WithDefaultPageAsync(
Func<IPage, Task> operation,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(operation);
await WithDefaultPageAsync(
async page =>
{
await operation(page);
return true;
},
cancellationToken);
}
public async Task StopAsync(CancellationToken cancellationToken)
{
await sessionLock.WaitAsync(CancellationToken.None);
try
{
stopping = true;
var contextToClose = defaultContext;
defaultContext = null;
defaultPage = null;
if (contextToClose is null)
{
return;
}
logger.LogInformation("Disposing default browser session.");
try
{
await contextToClose.CloseAsync();
}
catch (Exception exception)
{
logger.LogWarning(exception, "Default browser session did not close cleanly during shutdown.");
}
}
finally
{
sessionLock.Release();
}
}
private async Task<IPage> GetOrCreateDefaultPageAsync(CancellationToken cancellationToken)
{
if (defaultPage is not null)
{
return defaultPage;
}
IBrowserContext? context = null;
try
{
cancellationToken.ThrowIfCancellationRequested();
logger.LogInformation("Creating default browser context.");
context = await browserRuntime.Browser.NewContextAsync();
cancellationToken.ThrowIfCancellationRequested();
logger.LogInformation("Creating default browser page.");
var page = await context.NewPageAsync();
cancellationToken.ThrowIfCancellationRequested();
defaultContext = context;
defaultPage = page;
logger.LogInformation("Default browser session is ready.");
return page;
}
catch (Exception exception)
{
logger.LogError(exception, "Default browser session initialization failed.");
if (context is not null)
{
try
{
await context.CloseAsync();
}
catch (Exception cleanupException)
{
logger.LogWarning(
cleanupException,
"Partially initialized default browser context did not close cleanly.");
}
}
throw;
}
}
}

View File

@ -0,0 +1,10 @@
namespace WebAccess.MCP.Browser;
internal interface IBrowserPageTextReader
{
Task<BrowserPageTextSnapshot> ReadCurrentAsync(CancellationToken cancellationToken);
Task<BrowserPageTextSnapshot> NavigateAndReadAsync(
Uri uri,
CancellationToken cancellationToken);
}

View File

@ -0,0 +1,8 @@
using Microsoft.Playwright;
namespace WebAccess.MCP.Browser;
public interface IBrowserRuntime
{
IBrowser Browser { get; }
}

View File

@ -0,0 +1,14 @@
using Microsoft.Playwright;
namespace WebAccess.MCP.Browser;
public interface IBrowserSessionManager
{
Task<T> WithDefaultPageAsync<T>(
Func<IPage, Task<T>> operation,
CancellationToken cancellationToken = default);
Task WithDefaultPageAsync(
Func<IPage, Task> operation,
CancellationToken cancellationToken = default);
}

View File

@ -0,0 +1,4 @@
namespace WebAccess.MCP.Browser;
internal sealed class PageTextSearchUnavailableException(string message)
: Exception(message);

View File

@ -0,0 +1,124 @@
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Playwright;
namespace WebAccess.MCP.Browser;
internal sealed class PlaywrightBrowserRuntime(
ILogger<PlaywrightBrowserRuntime> logger) : IBrowserRuntime, IHostedService
{
private readonly SemaphoreSlim lifecycleLock = new(1, 1);
private IPlaywright? playwright;
private IBrowser? browser;
public IBrowser Browser => browser
?? throw new InvalidOperationException("The Chromium browser runtime has not been initialized.");
public async Task StartAsync(CancellationToken cancellationToken)
{
await lifecycleLock.WaitAsync(cancellationToken);
try
{
if (browser is not null)
{
return;
}
logger.LogInformation("Initializing Playwright.");
IPlaywright? initializedPlaywright = null;
IBrowser? launchedBrowser = null;
try
{
initializedPlaywright = await Playwright.CreateAsync();
logger.LogInformation("Launching headless Chromium.");
launchedBrowser = await initializedPlaywright.Chromium.LaunchAsync(
new BrowserTypeLaunchOptions
{
Headless = true
});
cancellationToken.ThrowIfCancellationRequested();
playwright = initializedPlaywright;
browser = launchedBrowser;
logger.LogInformation("Chromium is ready.");
}
catch (Exception exception)
{
logger.LogError(exception, "Playwright browser initialization failed.");
if (launchedBrowser is not null)
{
await CloseBrowserAfterFailedStartupAsync(launchedBrowser);
}
initializedPlaywright?.Dispose();
throw;
}
}
finally
{
lifecycleLock.Release();
}
}
public async Task StopAsync(CancellationToken cancellationToken)
{
await lifecycleLock.WaitAsync(CancellationToken.None);
try
{
var browserToClose = browser;
var playwrightToDispose = playwright;
browser = null;
playwright = null;
if (browserToClose is null && playwrightToDispose is null)
{
return;
}
logger.LogInformation("Shutting Chromium down.");
try
{
if (browserToClose is not null)
{
await browserToClose.CloseAsync();
}
}
catch (Exception exception)
{
logger.LogWarning(exception, "Chromium did not close cleanly during shutdown.");
}
finally
{
playwrightToDispose?.Dispose();
}
logger.LogInformation("Playwright browser runtime has stopped.");
}
finally
{
lifecycleLock.Release();
}
}
private async Task CloseBrowserAfterFailedStartupAsync(IBrowser browserToClose)
{
try
{
await browserToClose.CloseAsync();
}
catch (Exception exception)
{
logger.LogWarning(exception, "Chromium did not close cleanly after startup failed.");
}
}
}

View File

@ -0,0 +1,617 @@
using System.ComponentModel;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Playwright;
using ModelContextProtocol;
using ModelContextProtocol.Server;
using WebAccess.MCP.Browser;
using WebAccess.MCP.Models;
namespace WebAccess.MCP.McpTools;
[McpServerToolType]
public sealed class BrowserTools(
IBrowserSessionManager sessionManager,
IServiceProvider serviceProvider)
{
private const int MaximumLinks = 200;
private const int MaximumInteractiveElements = 200;
[McpServerTool(Name = "open_url", UseStructuredContent = true)]
[Description(
WebToolGuidance.MinimalBrowsing +
" Open a specific URL in the persistent browser and return its final URL and title.")]
public Task<OpenUrlResult> OpenUrlAsync(
[Description("The absolute HTTP or HTTPS URL to open.")] string url,
CancellationToken cancellationToken)
{
var uri = ValidateUrl(url);
return sessionManager.WithDefaultPageAsync(
async page =>
{
await BrowserNavigation.NavigateAsync(page, uri, cancellationToken);
var title = await page.TitleAsync();
return new OpenUrlResult(page.Url, title);
},
cancellationToken);
}
[McpServerTool(Name = "get_page_title")]
[Description("Use when only the title of the currently open page is needed.")]
public Task<string> GetPageTitleAsync(CancellationToken cancellationToken) =>
sessionManager.WithDefaultPageAsync(
page => page.TitleAsync(),
cancellationToken);
[McpServerTool(Name = "get_page_url")]
[Description("Use when only the current URL of the persistent browser page is needed.")]
public Task<string> GetPageUrlAsync(CancellationToken cancellationToken) =>
sessionManager.WithDefaultPageAsync(
page => Task.FromResult(page.Url),
cancellationToken);
[McpServerTool(Name = "get_page_info")]
[Description("Use when lightweight metadata about the currently open page is needed, such as title, URL, description, site name, or language.")]
public Task<PageInfo> GetPageInfoAsync(CancellationToken cancellationToken) =>
sessionManager.WithDefaultPageAsync(
async page =>
{
cancellationToken.ThrowIfCancellationRequested();
var title = await page.TitleAsync();
var metadata = await page.EvaluateAsync<string?[]>(
"""
() => [
document.querySelector('meta[name="description"]')?.getAttribute('content') ?? null,
document.querySelector('meta[property="og:site_name"]')?.getAttribute('content') ?? null,
document.documentElement.getAttribute('lang')
]
""");
cancellationToken.ThrowIfCancellationRequested();
return new PageInfo(
title,
page.Url,
NormalizeOptionalValue(metadata.ElementAtOrDefault(0)),
NormalizeOptionalValue(metadata.ElementAtOrDefault(1)),
NormalizeOptionalValue(metadata.ElementAtOrDefault(2)));
},
cancellationToken);
[McpServerTool(Name = "get_page_text")]
[Description(
WebToolGuidance.MinimalBrowsing +
" Read one bounded page of normalized readable text. When a URL is supplied, use a fresh cached snapshot when available; otherwise navigate and cache it. "
+ "Start with page 1 and request later pages only when the information needed is not present. "
+ "For specific facts in a selected long URL, do not start at page 1; use find_page_text first. "
+ "Use this tool for returned pages or when sequential reading is actually needed.")]
public Task<string> GetPageTextAsync(
[Description("Optional absolute HTTP or HTTPS URL; omit to read the current interactive browser page.")] string? url = null,
[Description("One-based text page to return. Defaults to 1; request later pages only if needed.")] int page = 1,
CancellationToken cancellationToken = default)
{
BrowserPageTextCoordinator.ValidatePageNumber(page);
return serviceProvider
.GetRequiredService<BrowserPageTextCoordinator>()
.GetPageTextAsync(url, page, cancellationToken);
}
[McpServerTool(Name = "find_page_text")]
[Description(
"Use this first after selecting a URL when looking for specific facts. Searches the cached snapshot using the exact query plus optional related terms and returns one bounded context window with a 1-based page or page range. "
+ "Missing or expired snapshots are fetched and cached automatically. "
+ "Use get_page_text sequentially if targeted retrieval is weak or more context is needed.")]
public async Task<string> FindPageTextAsync(
[Description("The absolute HTTP or HTTPS URL whose text should be searched.")] string url,
[Description("The information to locate within the selected page.")] string query,
[Description("Optional related terms or synonyms used only for deterministic matching in the cached page. Maximum 8 items, 100 characters each.")] string[]? relatedTerms = null,
CancellationToken cancellationToken = default)
{
BrowserPageTextSearchCoordinator.ValidateInputs(url, query, relatedTerms);
try
{
return await serviceProvider
.GetRequiredService<BrowserPageTextSearchCoordinator>()
.FindPageTextAsync(url, query, relatedTerms, cancellationToken);
}
catch (PageTextSearchUnavailableException exception)
{
throw new McpException(exception.Message);
}
}
[McpServerTool(Name = "get_links")]
[Description("Use when links available on the currently open page need to be inspected.")]
public Task<PageLink[]> GetLinksAsync(CancellationToken cancellationToken) =>
sessionManager.WithDefaultPageAsync(
async page =>
{
cancellationToken.ThrowIfCancellationRequested();
var candidates = await page.EvaluateAsync<string?[][]>(
"""
() => Array.from(document.querySelectorAll('a[href]'), anchor => [
anchor.innerText,
anchor.getAttribute('aria-label'),
anchor.getAttribute('title'),
anchor.href
])
""");
cancellationToken.ThrowIfCancellationRequested();
var links = new List<PageLink>(Math.Min(candidates.Length, MaximumLinks));
var seenUrls = new HashSet<string>(StringComparer.Ordinal);
foreach (var candidate in candidates)
{
cancellationToken.ThrowIfCancellationRequested();
if (candidate.Length < 4)
{
continue;
}
var text = FirstNonEmptyValue(candidate[0], candidate[1], candidate[2]);
if (text is null
|| !Uri.TryCreate(candidate[3], UriKind.Absolute, out var uri)
|| (uri.Scheme != Uri.UriSchemeHttp && uri.Scheme != Uri.UriSchemeHttps))
{
continue;
}
var normalizedUrl = uri.AbsoluteUri;
if (!seenUrls.Add(normalizedUrl))
{
continue;
}
links.Add(new PageLink(text, normalizedUrl));
if (links.Count == MaximumLinks)
{
break;
}
}
return links.ToArray();
},
cancellationToken);
[McpServerTool(Name = "get_interactive_elements")]
[Description("Use before click or type to discover visible controls on the currently open page.")]
public Task<InteractiveElement[]> GetInteractiveElementsAsync(CancellationToken cancellationToken) =>
sessionManager.WithDefaultPageAsync(
async page =>
{
cancellationToken.ThrowIfCancellationRequested();
var candidates = await page.EvaluateAsync<string?[][]>(
$$"""
() => {
const maximumElements = {{MaximumInteractiveElements}};
const interactiveRoles = new Set([
'button', 'checkbox', 'combobox', 'link', 'listbox', 'menuitem',
'menuitemcheckbox', 'menuitemradio', 'option', 'radio', 'searchbox',
'slider', 'spinbutton', 'switch', 'tab', 'textbox', 'treeitem'
]);
const selector = 'a[href], button, input, textarea, select, summary, [role], [contenteditable]';
const results = [];
const normalize = value => {
const normalized = value?.replace(/\s+/g, ' ').trim();
return normalized || null;
};
const inferRole = element => {
const explicitRole = normalize(element.getAttribute('role'))?.toLowerCase();
if (explicitRole) {
return explicitRole;
}
const tag = element.tagName.toLowerCase();
if (tag === 'a' && element.hasAttribute('href')) return 'link';
if (tag === 'button' || tag === 'summary') return 'button';
if (tag === 'textarea' || element.isContentEditable) return 'textbox';
if (tag === 'select') return element.multiple || element.size > 1 ? 'listbox' : 'combobox';
if (tag === 'input') {
switch (element.type) {
case 'button':
case 'reset':
case 'submit':
case 'image':
return 'button';
case 'checkbox':
return 'checkbox';
case 'radio':
return 'radio';
case 'range':
return 'slider';
case 'number':
return 'spinbutton';
case 'search':
return 'searchbox';
default:
return 'textbox';
}
}
return null;
};
for (const element of document.querySelectorAll(selector)) {
const tag = element.tagName.toLowerCase();
const explicitRole = normalize(element.getAttribute('role'))?.toLowerCase();
const nativeInteractive = ['a', 'button', 'input', 'textarea', 'select', 'summary'].includes(tag);
const editable = element.isContentEditable;
if (!nativeInteractive && !editable && !interactiveRoles.has(explicitRole)) {
continue;
}
const style = window.getComputedStyle(element);
if (style.display === 'none'
|| style.visibility === 'hidden'
|| style.visibility === 'collapse'
|| element.getClientRects().length === 0) {
continue;
}
const inputType = tag === 'input' ? normalize(element.type) : null;
const buttonText = tag === 'input'
&& ['button', 'reset', 'submit', 'image'].includes(element.type)
? normalize(element.value)
: null;
const text = buttonText ?? (tag === 'input' ? null : normalize(element.innerText));
const label = 'labels' in element
? normalize(Array.from(element.labels ?? [], item => item.innerText).join(' '))
: null;
const placeholder = normalize(element.getAttribute('placeholder'));
const name = normalize(element.getAttribute('name'));
const id = normalize(element.id);
const ariaLabel = normalize(element.getAttribute('aria-label'));
const href = tag === 'a' ? normalize(element.href) : null;
const role = inferRole(element);
const disabled = Boolean(element.disabled)
|| element.getAttribute('aria-disabled')?.toLowerCase() === 'true';
if (!role && !text && !label && !placeholder && !name && !id && !ariaLabel && !href) {
continue;
}
results.push([
tag,
role,
text,
label,
placeholder,
name,
id,
inputType,
ariaLabel,
href,
disabled ? 'true' : 'false'
]);
if (results.length === maximumElements) {
break;
}
}
return results;
}
""");
cancellationToken.ThrowIfCancellationRequested();
return candidates
.Select(CreateInteractiveElement)
.ToArray();
},
cancellationToken);
[McpServerTool(Name = "click")]
[Description("Clicks an element on the persistent page. Prefer role or text targeting; use CSS only as a fallback.")]
public Task<string> ClickAsync(
string? text = null,
string? role = null,
string? name = null,
string? css = null,
bool exact = true,
int index = 0,
CancellationToken cancellationToken = default)
{
ValidateIndex(index);
ValidateTargetSelection(
[IsSpecified(text), IsSpecified(role), IsSpecified(css)],
IsSpecified(name),
IsSpecified(role),
"click");
return sessionManager.WithDefaultPageAsync(
async page =>
{
cancellationToken.ThrowIfCancellationRequested();
ILocator locator;
if (IsSpecified(text))
{
locator = page.GetByText(
text!,
new PageGetByTextOptions
{
Exact = exact
});
}
else if (IsSpecified(role))
{
locator = page.GetByRole(
ParseAriaRole(role!),
new PageGetByRoleOptions
{
Name = NormalizeOptionalValue(name),
Exact = exact
});
}
else
{
locator = page.Locator(css!);
}
await locator.Nth(index).ClickAsync();
cancellationToken.ThrowIfCancellationRequested();
return page.Url;
},
cancellationToken);
}
[McpServerTool(Name = "type")]
[Description("Enters text into an element on the persistent page. Prefer label, placeholder, or role targeting; use CSS only as a fallback.")]
public Task<string> TypeAsync(
string value,
string? label = null,
string? placeholder = null,
string? role = null,
string? name = null,
string? css = null,
bool exact = true,
int index = 0,
bool clear = true,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(value);
ValidateIndex(index);
ValidateTargetSelection(
[IsSpecified(label), IsSpecified(placeholder), IsSpecified(role), IsSpecified(css)],
IsSpecified(name),
IsSpecified(role),
"type");
return sessionManager.WithDefaultPageAsync(
async page =>
{
cancellationToken.ThrowIfCancellationRequested();
ILocator locator;
if (IsSpecified(label))
{
locator = page.GetByLabel(
label!,
new PageGetByLabelOptions
{
Exact = exact
});
}
else if (IsSpecified(placeholder))
{
locator = page.GetByPlaceholder(
placeholder!,
new PageGetByPlaceholderOptions
{
Exact = exact
});
}
else if (IsSpecified(role))
{
locator = page.GetByRole(
ParseAriaRole(role!),
new PageGetByRoleOptions
{
Name = NormalizeOptionalValue(name),
Exact = exact
});
}
else
{
locator = page.Locator(css!);
}
var indexedLocator = locator.Nth(index);
if (clear)
{
await indexedLocator.FillAsync(value);
}
else
{
await indexedLocator.PressSequentiallyAsync(value);
}
cancellationToken.ThrowIfCancellationRequested();
return page.Url;
},
cancellationToken);
}
[McpServerTool(Name = "press_key")]
[Description("Presses a Playwright-supported key or key combination on the persistent browser page.")]
public Task<string> PressKeyAsync(string key, CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(key))
{
throw new ArgumentException("A non-empty key or key combination is required.", nameof(key));
}
return sessionManager.WithDefaultPageAsync(
async page =>
{
cancellationToken.ThrowIfCancellationRequested();
await page.Keyboard.PressAsync(key);
cancellationToken.ThrowIfCancellationRequested();
return page.Url;
},
cancellationToken);
}
[McpServerTool(Name = "go_back")]
[Description("Use to navigate the persistent browser page to the previous entry in its browser history.")]
public Task<NavigationResult> GoBackAsync(CancellationToken cancellationToken) =>
sessionManager.WithDefaultPageAsync(
async page =>
{
cancellationToken.ThrowIfCancellationRequested();
var response = await page.GoBackAsync(
new PageGoBackOptions
{
WaitUntil = WaitUntilState.DOMContentLoaded
});
cancellationToken.ThrowIfCancellationRequested();
return new NavigationResult(response is not null, page.Url);
},
cancellationToken);
[McpServerTool(Name = "go_forward")]
[Description("Use to navigate the persistent browser page to the next entry in its browser history after navigating backward.")]
public Task<NavigationResult> GoForwardAsync(CancellationToken cancellationToken) =>
sessionManager.WithDefaultPageAsync(
async page =>
{
cancellationToken.ThrowIfCancellationRequested();
var response = await page.GoForwardAsync(
new PageGoForwardOptions
{
WaitUntil = WaitUntilState.DOMContentLoaded
});
cancellationToken.ThrowIfCancellationRequested();
return new NavigationResult(response is not null, page.Url);
},
cancellationToken);
[McpServerTool(Name = "reload")]
[Description("Use to reload the currently open page in the persistent browser session.")]
public Task<NavigationResult> ReloadAsync(CancellationToken cancellationToken) =>
sessionManager.WithDefaultPageAsync(
async page =>
{
cancellationToken.ThrowIfCancellationRequested();
await page.ReloadAsync(
new PageReloadOptions
{
WaitUntil = WaitUntilState.DOMContentLoaded
});
cancellationToken.ThrowIfCancellationRequested();
return new NavigationResult(true, page.Url);
},
cancellationToken);
private static InteractiveElement CreateInteractiveElement(string?[] candidate)
{
if (candidate.Length < 11
|| NormalizeOptionalValue(candidate[0]) is not { } kind
|| !bool.TryParse(candidate[10], out var disabled))
{
throw new InvalidOperationException("Interactive element data had an invalid shape.");
}
return new InteractiveElement(
kind,
NormalizeOptionalValue(candidate[1]),
NormalizeOptionalValue(candidate[2]),
NormalizeOptionalValue(candidate[3]),
NormalizeOptionalValue(candidate[4]),
NormalizeOptionalValue(candidate[5]),
NormalizeOptionalValue(candidate[6]),
NormalizeOptionalValue(candidate[7]),
NormalizeOptionalValue(candidate[8]),
NormalizeOptionalValue(candidate[9]),
disabled);
}
private static bool IsSpecified(string? value) => !string.IsNullOrWhiteSpace(value);
private static void ValidateTargetSelection(
bool[] targetModes,
bool hasName,
bool hasRole,
string toolName)
{
var selectedModeCount = targetModes.Count(selected => selected);
if (selectedModeCount != 1)
{
throw new ArgumentException(
$"The {toolName} tool requires exactly one targeting mode.");
}
if (hasName && !hasRole)
{
throw new ArgumentException(
$"The name parameter can only be used with role targeting for the {toolName} tool.");
}
}
private static void ValidateIndex(int index)
{
if (index < 0)
{
throw new ArgumentOutOfRangeException(nameof(index), index, "The index must be zero or greater.");
}
}
private static AriaRole ParseAriaRole(string role)
{
var roleName = role.Trim();
if (!Enum.GetNames<AriaRole>().Contains(roleName, StringComparer.OrdinalIgnoreCase)
|| !Enum.TryParse(roleName, ignoreCase: true, out AriaRole parsedRole))
{
throw new ArgumentException($"'{role}' is not a supported Playwright ARIA role.", nameof(role));
}
return parsedRole;
}
private static string? NormalizeOptionalValue(string? value) =>
string.IsNullOrWhiteSpace(value) ? null : value.Trim();
private static Uri ValidateUrl(string url)
{
if (string.IsNullOrWhiteSpace(url))
{
throw new ArgumentException("A non-empty URL is required.", nameof(url));
}
if (!Uri.TryCreate(url, UriKind.Absolute, out var uri)
|| (uri.Scheme != Uri.UriSchemeHttp && uri.Scheme != Uri.UriSchemeHttps))
{
throw new ArgumentException("The URL must be an absolute HTTP or HTTPS URL.", nameof(url));
}
return uri;
}
private static string? FirstNonEmptyValue(params string?[] values)
{
foreach (var value in values)
{
var normalizedValue = NormalizeOptionalValue(value);
if (normalizedValue is not null)
{
return normalizedValue;
}
}
return null;
}
}

View File

@ -0,0 +1,12 @@
using System.ComponentModel;
using ModelContextProtocol.Server;
namespace WebAccess.MCP.McpTools;
[McpServerToolType]
public static class DiagnosticTools
{
[McpServerTool(Name = "health_check")]
[Description("Checks whether the WebAccess.MCP server is running and able to respond.")]
public static string HealthCheck() => "WebAccess.MCP is running.";
}

View File

@ -0,0 +1,48 @@
using System.ComponentModel;
using ModelContextProtocol;
using ModelContextProtocol.Server;
using WebAccess.MCP.Models;
using WebAccess.MCP.Search;
namespace WebAccess.MCP.McpTools;
[McpServerToolType]
public sealed class SearchTools(IWebSearchService searchService)
{
[McpServerTool(Name = "web_search")]
[Description(
WebToolGuidance.MinimalBrowsing +
" Search the web for current and relevant sources. For general web requests without a specified source or URL, use this tool first. "
+ "If the user specifies a source or URL, use it directly instead. After selecting a result URL for specific facts, prefer find_page_text on that URL instead of opening it or starting at get_page_text page 1.")]
public async Task<WebSearchResponse> WebSearchAsync(
[Description("The web search query.")] string query,
[Description("The maximum number of accepted results to return, from 1 through 100.")] int maxResults = 10,
[Description("Optional SearXNG engine selector names without the leading !.")] string[]? engines = null,
CancellationToken cancellationToken = default)
{
try
{
return await searchService.SearchAsync(query, maxResults, engines, cancellationToken);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
throw;
}
catch (OperationCanceledException exception)
{
throw new McpException("SearXNG search timed out.", exception);
}
catch (ArgumentException exception)
{
throw new McpException(exception.Message, exception);
}
catch (HttpRequestException exception)
{
throw new McpException(exception.Message, exception);
}
catch (InvalidDataException exception)
{
throw new McpException(exception.Message, exception);
}
}
}

View File

@ -0,0 +1,10 @@
namespace WebAccess.MCP.McpTools;
internal static class WebToolGuidance
{
public const string MinimalBrowsing =
"Prefer the minimum browsing needed. " +
"If the user names a source or URL, use it directly. " +
"Stop once the requested information is found. " +
"Use additional sources only when requested or when the current source is unavailable, incomplete, ambiguous, or conflicting.";
}

View File

@ -0,0 +1,16 @@
using System.Text.Json.Serialization;
namespace WebAccess.MCP.Models;
public sealed record InteractiveElement(
[property: JsonPropertyName("kind")] string Kind,
[property: JsonPropertyName("role")] string? Role,
[property: JsonPropertyName("text")] string? Text,
[property: JsonPropertyName("label")] string? Label,
[property: JsonPropertyName("placeholder")] string? Placeholder,
[property: JsonPropertyName("name")] string? Name,
[property: JsonPropertyName("id")] string? Id,
[property: JsonPropertyName("inputType")] string? InputType,
[property: JsonPropertyName("ariaLabel")] string? AriaLabel,
[property: JsonPropertyName("href")] string? Href,
[property: JsonPropertyName("disabled")] bool Disabled);

View File

@ -0,0 +1,7 @@
using System.Text.Json.Serialization;
namespace WebAccess.MCP.Models;
public sealed record NavigationResult(
[property: JsonPropertyName("navigated")] bool Navigated,
[property: JsonPropertyName("url")] string Url);

View File

@ -0,0 +1,7 @@
using System.Text.Json.Serialization;
namespace WebAccess.MCP.Models;
public sealed record OpenUrlResult(
[property: JsonPropertyName("url")] string Url,
[property: JsonPropertyName("title")] string Title);

View File

@ -0,0 +1,10 @@
using System.Text.Json.Serialization;
namespace WebAccess.MCP.Models;
public sealed record PageInfo(
[property: JsonPropertyName("title")] string Title,
[property: JsonPropertyName("url")] string Url,
[property: JsonPropertyName("description")] string? Description,
[property: JsonPropertyName("siteName")] string? SiteName,
[property: JsonPropertyName("language")] string? Language);

View File

@ -0,0 +1,7 @@
using System.Text.Json.Serialization;
namespace WebAccess.MCP.Models;
public sealed record PageLink(
[property: JsonPropertyName("text")] string Text,
[property: JsonPropertyName("url")] string Url);

View File

@ -0,0 +1,16 @@
using System.Text.Json.Serialization;
namespace WebAccess.MCP.Models;
public sealed record WebSearchResponse(
[property: JsonPropertyName("query"), JsonPropertyOrder(0)] string Query,
[property: JsonPropertyName("requestedEngines"), JsonPropertyOrder(1)] IReadOnlyList<string> RequestedEngines,
[property: JsonPropertyName("results"), JsonPropertyOrder(3)] IReadOnlyList<WebSearchResult> Results)
{
public const string NextStepHintValue =
"For specific facts from a selected result, use find_page_text(url, query) before get_page_text.";
[JsonPropertyName("nextStepHint")]
[JsonPropertyOrder(2)]
public string NextStepHint => NextStepHintValue;
}

View File

@ -0,0 +1,9 @@
using System.Text.Json.Serialization;
namespace WebAccess.MCP.Models;
public sealed record WebSearchResult(
[property: JsonPropertyName("title")] string Title,
[property: JsonPropertyName("url")] string Url,
[property: JsonPropertyName("snippet")] string? Snippet,
[property: JsonPropertyName("engines")] IReadOnlyList<string> Engines);

View File

@ -0,0 +1,52 @@
using Microsoft.AspNetCore.Hosting;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Options;
using WebAccess.MCP.Browser;
using WebAccess.MCP.Search;
using WebAccess.MCP.WebCache;
var builder = WebApplication.CreateBuilder(args);
if (string.IsNullOrWhiteSpace(
builder.Configuration[WebHostDefaults.ServerUrlsKey]))
{
builder.WebHost.UseUrls("http://0.0.0.0:8080");
}
builder.Services.AddSingleton<PlaywrightBrowserRuntime>();
builder.Services.AddSingleton<IBrowserRuntime>(services =>
services.GetRequiredService<PlaywrightBrowserRuntime>());
builder.Services.AddHostedService(services =>
services.GetRequiredService<PlaywrightBrowserRuntime>());
builder.Services.AddSingleton<BrowserSessionManager>();
builder.Services.AddSingleton<IBrowserSessionManager>(services =>
services.GetRequiredService<BrowserSessionManager>());
builder.Services.AddSingleton<IBrowserPageTextReader, BrowserPageTextReader>();
builder.Services.AddSingleton<BrowserPageTextCoordinator>();
builder.Services.AddSingleton<BrowserPageTextSearchCoordinator>();
// Hosted services stop in reverse registration order, so the session closes before Chromium.
builder.Services.AddHostedService(services =>
services.GetRequiredService<BrowserSessionManager>());
builder.Services
.AddOptions<SearXngOptions>()
.BindConfiguration(SearXngOptions.SectionName);
builder.Services.AddSingleton<IValidateOptions<SearXngOptions>, SearXngOptionsValidator>();
builder.Services.AddHttpClient<IWebSearchService, SearXngWebSearchService>();
builder.Services.AddWebCachePersistence(builder.Configuration);
builder.Services
.AddMcpServer()
.WithHttpTransport(options =>
{
options.Stateless = true;
})
.WithToolsFromAssembly();
var app = builder.Build();
app.MapGet("/health", () => Results.Ok(new { status = "healthy" }));
app.MapMcp("/mcp");
await app.RunAsync();

View File

@ -0,0 +1,12 @@
using WebAccess.MCP.Models;
namespace WebAccess.MCP.Search;
public interface IWebSearchService
{
Task<WebSearchResponse> SearchAsync(
string query,
int maxResults = 10,
IReadOnlyCollection<string>? engines = null,
CancellationToken cancellationToken = default);
}

View File

@ -0,0 +1,52 @@
using System.Text.RegularExpressions;
namespace WebAccess.MCP.Search;
internal static partial class SearXngEngineSelectors
{
public static IReadOnlyList<string> Normalize(IEnumerable<string?> selectors, string parameterName)
{
ArgumentNullException.ThrowIfNull(selectors);
var normalized = new List<string>();
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var selectorValue in selectors)
{
if (selectorValue is null)
{
throw new ArgumentException("Engine selectors cannot be null.", parameterName);
}
var selector = selectorValue.Trim();
if (selector.Length == 0)
{
throw new ArgumentException("Engine selectors cannot be empty or whitespace.", parameterName);
}
if (selector.StartsWith('!'))
{
throw new ArgumentException(
$"Engine selector '{selector}' must not include the leading '!'.",
parameterName);
}
if (!ValidSelectorPattern().IsMatch(selector))
{
throw new ArgumentException(
$"Engine selector '{selector}' may contain only letters, digits, underscores, hyphens, and periods.",
parameterName);
}
if (seen.Add(selector))
{
normalized.Add(selector);
}
}
return normalized;
}
[GeneratedRegex(@"\A[A-Za-z0-9_.-]+\z", RegexOptions.CultureInvariant)]
private static partial Regex ValidSelectorPattern();
}

View File

@ -0,0 +1,29 @@
namespace WebAccess.MCP.Search;
public sealed class SearXngOptions
{
public const string SectionName = "Search:SearXNG";
/// <summary>
/// Gets or sets the root URL of the SearXNG instance. The default is <see langword="null"/>,
/// which disables search until a URL is configured. Change this to the HTTP or HTTPS root
/// URL of the target instance; any path prefix is preserved, and other URI schemes are not supported.
/// </summary>
public string? BaseUrl { get; set; }
/// <summary>
/// Gets or sets the maximum duration of an HTTP search request. The default is 15 seconds.
/// Change this when the target SearXNG instance needs a different response-time allowance;
/// the value must be greater than zero and does not prevent earlier caller cancellation.
/// </summary>
public TimeSpan Timeout { get; set; } = TimeSpan.FromSeconds(15);
/// <summary>
/// Gets or sets the SearXNG engine selector identifiers used when a call supplies no overrides.
/// The default is an empty collection, which uses the SearXNG instance's configured default engines.
/// Change this to selectors such as <c>google</c>, <c>bing</c>, <c>brave</c>, or
/// <c>duckduckgo</c>, without the leading <c>!</c>. Configured selectors must exist and be
/// enabled on the target SearXNG instance; WebAccess.MCP does not maintain an engine allowlist.
/// </summary>
public string[] DefaultEngines { get; set; } = [];
}

View File

@ -0,0 +1,46 @@
using Microsoft.Extensions.Options;
namespace WebAccess.MCP.Search;
internal sealed class SearXngOptionsValidator : IValidateOptions<SearXngOptions>
{
public ValidateOptionsResult Validate(string? name, SearXngOptions options)
{
if (string.IsNullOrWhiteSpace(options.BaseUrl))
{
return ValidateOptionsResult.Fail(
$"{SearXngOptions.SectionName}:BaseUrl is required when search is used.");
}
if (!Uri.TryCreate(options.BaseUrl, UriKind.Absolute, out var baseUri))
{
return ValidateOptionsResult.Fail(
$"{SearXngOptions.SectionName}:BaseUrl must be an absolute URI.");
}
if (baseUri.Scheme != Uri.UriSchemeHttp && baseUri.Scheme != Uri.UriSchemeHttps)
{
return ValidateOptionsResult.Fail(
$"{SearXngOptions.SectionName}:BaseUrl must use the HTTP or HTTPS scheme; '{baseUri.Scheme}' is unsupported.");
}
if (options.Timeout <= TimeSpan.Zero)
{
return ValidateOptionsResult.Fail(
$"{SearXngOptions.SectionName}:Timeout must be greater than zero.");
}
try
{
SearXngEngineSelectors.Normalize(
options.DefaultEngines ?? [],
$"{SearXngOptions.SectionName}:DefaultEngines");
}
catch (ArgumentException exception)
{
return ValidateOptionsResult.Fail(exception.Message);
}
return ValidateOptionsResult.Success;
}
}

View File

@ -0,0 +1,239 @@
using System.Net;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Text.RegularExpressions;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using WebAccess.MCP.Models;
namespace WebAccess.MCP.Search;
internal sealed partial class SearXngWebSearchService : IWebSearchService
{
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web);
private readonly HttpClient _httpClient;
private readonly SearXngOptions _options;
private readonly ILogger<SearXngWebSearchService> _logger;
private readonly Uri _searchEndpoint;
private readonly IReadOnlyList<string> _defaultEngines;
public SearXngWebSearchService(
HttpClient httpClient,
IOptions<SearXngOptions> options,
ILogger<SearXngWebSearchService> logger)
{
_httpClient = httpClient;
_options = options.Value;
_logger = logger;
_searchEndpoint = CreateSearchEndpoint(_options.BaseUrl!);
_defaultEngines = SearXngEngineSelectors.Normalize(
_options.DefaultEngines ?? [],
$"{SearXngOptions.SectionName}:DefaultEngines");
_httpClient.Timeout = _options.Timeout;
}
public async Task<WebSearchResponse> SearchAsync(
string query,
int maxResults = 10,
IReadOnlyCollection<string>? engines = null,
CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(query);
if (maxResults is < 1 or > 100)
{
throw new ArgumentOutOfRangeException(
nameof(maxResults),
maxResults,
"Maximum results must be between 1 and 100.");
}
var effectiveEngines = ResolveEffectiveEngines(engines);
var effectiveQuery = effectiveEngines.Count == 0
? query
: $"{string.Join(' ', effectiveEngines.Select(static engine => $"!{engine}"))} {query}";
var requestUri = BuildRequestUri(effectiveQuery);
_logger.LogDebug(
"Sending SearXNG search request with {EngineCount} explicit engine selectors and a maximum of {MaxResults} results.",
effectiveEngines.Count,
maxResults);
using var request = new HttpRequestMessage(HttpMethod.Get, requestUri);
using var response = await _httpClient.SendAsync(
request,
HttpCompletionOption.ResponseHeadersRead,
cancellationToken);
if (!response.IsSuccessStatusCode)
{
var message = response.StatusCode == HttpStatusCode.Forbidden
? "SearXNG returned HTTP 403 (Forbidden). JSON output may not be enabled for the configured instance."
: $"SearXNG search request failed with HTTP {(int)response.StatusCode} ({response.StatusCode}).";
_logger.LogWarning(
"SearXNG search request failed with HTTP {StatusCode} ({ReasonPhrase}).",
(int)response.StatusCode,
response.ReasonPhrase);
throw new HttpRequestException(message, inner: null, response.StatusCode);
}
SearXngSearchResponse? payload;
try
{
await using var contentStream = await response.Content.ReadAsStreamAsync(cancellationToken);
payload = await JsonSerializer.DeserializeAsync<SearXngSearchResponse>(
contentStream,
JsonOptions,
cancellationToken);
}
catch (JsonException exception)
{
throw new InvalidDataException("SearXNG returned malformed or incompatible JSON.", exception);
}
if (payload?.Results is null)
{
throw new InvalidDataException(
"SearXNG returned structurally unusable JSON: the results array is missing or null.");
}
var results = NormalizeResults(payload.Results, maxResults);
_logger.LogDebug(
"SearXNG search request produced {ResultCount} accepted results.",
results.Count);
return new WebSearchResponse(query, effectiveEngines, results);
}
private IReadOnlyList<string> ResolveEffectiveEngines(IReadOnlyCollection<string>? engines)
{
if (engines is not null)
{
var perCallEngines = SearXngEngineSelectors.Normalize(engines, nameof(engines));
if (perCallEngines.Count > 0)
{
return perCallEngines;
}
}
return _defaultEngines;
}
private Uri BuildRequestUri(string effectiveQuery)
{
var builder = new UriBuilder(_searchEndpoint)
{
Query = $"q={Uri.EscapeDataString(effectiveQuery)}&format=json"
};
return builder.Uri;
}
private static Uri CreateSearchEndpoint(string baseUrl)
{
var normalizedBaseUrl = baseUrl.EndsWith("/", StringComparison.Ordinal)
? baseUrl
: $"{baseUrl}/";
return new Uri(new Uri(normalizedBaseUrl, UriKind.Absolute), "search");
}
private static IReadOnlyList<WebSearchResult> NormalizeResults(
IReadOnlyList<SearXngSearchResult?> source,
int maxResults)
{
var results = new List<WebSearchResult>(Math.Min(source.Count, maxResults));
var seenUrls = new HashSet<Uri>();
foreach (var sourceResult in source)
{
if (sourceResult is null)
{
continue;
}
var title = NormalizeText(sourceResult.Title);
if (title is null ||
!Uri.TryCreate(sourceResult.Url, UriKind.Absolute, out var uri) ||
(uri.Scheme != Uri.UriSchemeHttp && uri.Scheme != Uri.UriSchemeHttps) ||
!seenUrls.Add(uri))
{
continue;
}
var engines = sourceResult.Engines is { Count: > 0 }
? NormalizeReportedEngines(sourceResult.Engines)
: NormalizeReportedEngines(
sourceResult.Engine is null ? [] : [sourceResult.Engine]);
results.Add(new WebSearchResult(
title,
uri.AbsoluteUri,
NormalizeText(sourceResult.Content),
engines));
if (results.Count == maxResults)
{
break;
}
}
return results;
}
private static IReadOnlyList<string> NormalizeReportedEngines(IEnumerable<string?> engines)
{
var normalized = new List<string>();
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var engineValue in engines)
{
var engine = engineValue?.Trim();
if (!string.IsNullOrEmpty(engine) && seen.Add(engine))
{
normalized.Add(engine);
}
}
return normalized;
}
private static string? NormalizeText(string? value)
{
if (string.IsNullOrWhiteSpace(value))
{
return null;
}
return ExcessiveWhitespacePattern().Replace(value.Trim(), " ");
}
[GeneratedRegex(@"\s+", RegexOptions.CultureInvariant)]
private static partial Regex ExcessiveWhitespacePattern();
private sealed class SearXngSearchResponse
{
[JsonPropertyName("results")]
public List<SearXngSearchResult?>? Results { get; init; }
}
private sealed class SearXngSearchResult
{
[JsonPropertyName("title")]
public string? Title { get; init; }
[JsonPropertyName("url")]
public string? Url { get; init; }
[JsonPropertyName("content")]
public string? Content { get; init; }
[JsonPropertyName("engine")]
public string? Engine { get; init; }
[JsonPropertyName("engines")]
public List<string?>? Engines { get; init; }
}
}

View File

@ -0,0 +1,25 @@
<Project Sdk="Microsoft.NET.Sdk.Web">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<UserSecretsId>0639bfa9-c879-4cc4-8bd0-528bc7f97e85</UserSecretsId>
</PropertyGroup>
<ItemGroup>
<InternalsVisibleTo Include="WebAccess.MCP.Tests" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="10.0.4">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
<PackageReference Include="Microsoft.Playwright" Version="1.62.0" />
<PackageReference Include="ModelContextProtocol.AspNetCore" Version="2.2.0" />
<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="10.0.3" />
</ItemGroup>
</Project>

View File

@ -0,0 +1,32 @@
namespace WebAccess.MCP.WebCache;
internal sealed class CachedWebPage
{
public long Id { get; set; }
public string CacheKey { get; set; } = string.Empty;
public string RequestedUrl { get; set; } = string.Empty;
public string? FinalUrl { get; set; }
public int? HttpStatusCode { get; set; }
public string? Title { get; set; }
public string? ContentType { get; set; }
public string? RawHtml { get; set; }
public string? ContentHash { get; set; }
public DateTimeOffset FetchedAtUtc { get; set; }
public DateTimeOffset ExpiresAtUtc { get; set; }
public int TotalTextLength { get; set; }
public bool IsComplete { get; set; }
public List<CachedWebPageChunk> Chunks { get; } = [];
}

View File

@ -0,0 +1,22 @@
using NpgsqlTypes;
namespace WebAccess.MCP.WebCache;
internal sealed class CachedWebPageChunk
{
public long Id { get; set; }
public long CachedWebPageId { get; set; }
public int ChunkNumber { get; set; }
public int StartOffset { get; set; }
public int Length { get; set; }
public string Text { get; set; } = string.Empty;
public NpgsqlTsVector SearchVector { get; set; } = null!;
public CachedWebPage CachedWebPage { get; set; } = null!;
}

View File

@ -0,0 +1,20 @@
namespace WebAccess.MCP.WebCache;
internal interface IWebCacheRepository
{
Task<WebCacheRepositoryPage?> TryGetPageChunkAsync(
string cacheKey,
int chunkNumber,
CancellationToken cancellationToken);
Task StoreOrReplaceAsync(
WebCacheStoredPage page,
IReadOnlyList<WebCacheStoredChunk> chunks,
CancellationToken cancellationToken);
Task<WebCacheRepositorySearchPage?> SearchPageAsync(
string cacheKey,
string candidateQuery,
DateTimeOffset currentUtc,
CancellationToken cancellationToken);
}

View File

@ -0,0 +1,19 @@
namespace WebAccess.MCP.WebCache;
internal interface IWebCacheService
{
Task<WebCacheLookupResult> TryGetFreshChunkAsync(
string requestedUrl,
int chunkNumber,
CancellationToken cancellationToken);
Task StoreAsync(
WebCachePageWriteRequest request,
CancellationToken cancellationToken);
Task<WebCacheSearchResult> SearchFreshPageAsync(
string requestedUrl,
string query,
IReadOnlyList<string>? relatedTerms,
CancellationToken cancellationToken);
}

View File

@ -0,0 +1,138 @@
// <auto-generated />
using System;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
using WebAccess.MCP.WebCache;
#nullable disable
namespace WebAccess.MCP.WebCache.Migrations
{
[DbContext(typeof(WebCacheDbContext))]
[Migration("20260820035916_InitialWebCache")]
partial class InitialWebCache
{
/// <inheritdoc />
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "10.0.4")
.HasAnnotation("Relational:MaxIdentifierLength", 63);
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
modelBuilder.Entity("WebAccess.MCP.WebCache.CachedWebPage", b =>
{
b.Property<long>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("bigint");
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<long>("Id"));
b.Property<string>("CacheKey")
.IsRequired()
.HasMaxLength(64)
.HasColumnType("character(64)")
.IsFixedLength();
b.Property<string>("ContentHash")
.HasMaxLength(64)
.HasColumnType("character(64)")
.IsFixedLength();
b.Property<string>("ContentType")
.HasMaxLength(255)
.HasColumnType("character varying(255)");
b.Property<DateTimeOffset>("ExpiresAtUtc")
.HasColumnType("timestamp with time zone");
b.Property<DateTimeOffset>("FetchedAtUtc")
.HasColumnType("timestamp with time zone");
b.Property<string>("FinalUrl")
.HasColumnType("text");
b.Property<int?>("HttpStatusCode")
.HasColumnType("integer");
b.Property<bool>("IsComplete")
.HasColumnType("boolean");
b.Property<string>("RawHtml")
.HasColumnType("text");
b.Property<string>("RequestedUrl")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Title")
.HasColumnType("text");
b.Property<int>("TotalTextLength")
.HasColumnType("integer");
b.HasKey("Id");
b.HasIndex("CacheKey");
b.HasIndex("ExpiresAtUtc");
b.ToTable("cached_web_pages", (string)null);
});
modelBuilder.Entity("WebAccess.MCP.WebCache.CachedWebPageChunk", b =>
{
b.Property<long>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("bigint");
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<long>("Id"));
b.Property<long>("CachedWebPageId")
.HasColumnType("bigint");
b.Property<int>("ChunkNumber")
.HasColumnType("integer");
b.Property<int>("Length")
.HasColumnType("integer");
b.Property<int>("StartOffset")
.HasColumnType("integer");
b.Property<string>("Text")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.HasIndex("CachedWebPageId", "ChunkNumber")
.IsUnique();
b.ToTable("cached_web_page_chunks", (string)null);
});
modelBuilder.Entity("WebAccess.MCP.WebCache.CachedWebPageChunk", b =>
{
b.HasOne("WebAccess.MCP.WebCache.CachedWebPage", "CachedWebPage")
.WithMany("Chunks")
.HasForeignKey("CachedWebPageId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("CachedWebPage");
});
modelBuilder.Entity("WebAccess.MCP.WebCache.CachedWebPage", b =>
{
b.Navigation("Chunks");
});
#pragma warning restore 612, 618
}
}
}

View File

@ -0,0 +1,89 @@
using System;
using Microsoft.EntityFrameworkCore.Migrations;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
#nullable disable
namespace WebAccess.MCP.WebCache.Migrations
{
/// <inheritdoc />
public partial class InitialWebCache : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.CreateTable(
name: "cached_web_pages",
columns: table => new
{
Id = table.Column<long>(type: "bigint", nullable: false)
.Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn),
CacheKey = table.Column<string>(type: "character(64)", fixedLength: true, maxLength: 64, nullable: false),
RequestedUrl = table.Column<string>(type: "text", nullable: false),
FinalUrl = table.Column<string>(type: "text", nullable: true),
HttpStatusCode = table.Column<int>(type: "integer", nullable: true),
Title = table.Column<string>(type: "text", nullable: true),
ContentType = table.Column<string>(type: "character varying(255)", maxLength: 255, nullable: true),
RawHtml = table.Column<string>(type: "text", nullable: true),
ContentHash = table.Column<string>(type: "character(64)", fixedLength: true, maxLength: 64, nullable: true),
FetchedAtUtc = table.Column<DateTimeOffset>(type: "timestamp with time zone", nullable: false),
ExpiresAtUtc = table.Column<DateTimeOffset>(type: "timestamp with time zone", nullable: false),
TotalTextLength = table.Column<int>(type: "integer", nullable: false),
IsComplete = table.Column<bool>(type: "boolean", nullable: false)
},
constraints: table =>
{
table.PrimaryKey("PK_cached_web_pages", x => x.Id);
});
migrationBuilder.CreateTable(
name: "cached_web_page_chunks",
columns: table => new
{
Id = table.Column<long>(type: "bigint", nullable: false)
.Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn),
CachedWebPageId = table.Column<long>(type: "bigint", nullable: false),
ChunkNumber = table.Column<int>(type: "integer", nullable: false),
StartOffset = table.Column<int>(type: "integer", nullable: false),
Length = table.Column<int>(type: "integer", nullable: false),
Text = table.Column<string>(type: "text", nullable: false)
},
constraints: table =>
{
table.PrimaryKey("PK_cached_web_page_chunks", x => x.Id);
table.ForeignKey(
name: "FK_cached_web_page_chunks_cached_web_pages_CachedWebPageId",
column: x => x.CachedWebPageId,
principalTable: "cached_web_pages",
principalColumn: "Id",
onDelete: ReferentialAction.Cascade);
});
migrationBuilder.CreateIndex(
name: "IX_cached_web_page_chunks_CachedWebPageId_ChunkNumber",
table: "cached_web_page_chunks",
columns: new[] { "CachedWebPageId", "ChunkNumber" },
unique: true);
migrationBuilder.CreateIndex(
name: "IX_cached_web_pages_CacheKey",
table: "cached_web_pages",
column: "CacheKey");
migrationBuilder.CreateIndex(
name: "IX_cached_web_pages_ExpiresAtUtc",
table: "cached_web_pages",
column: "ExpiresAtUtc");
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(
name: "cached_web_page_chunks");
migrationBuilder.DropTable(
name: "cached_web_pages");
}
}
}

View File

@ -0,0 +1,139 @@
// <auto-generated />
using System;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
using WebAccess.MCP.WebCache;
#nullable disable
namespace WebAccess.MCP.WebCache.Migrations
{
[DbContext(typeof(WebCacheDbContext))]
[Migration("20260820041329_MakeCacheKeyUnique")]
partial class MakeCacheKeyUnique
{
/// <inheritdoc />
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "10.0.4")
.HasAnnotation("Relational:MaxIdentifierLength", 63);
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
modelBuilder.Entity("WebAccess.MCP.WebCache.CachedWebPage", b =>
{
b.Property<long>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("bigint");
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<long>("Id"));
b.Property<string>("CacheKey")
.IsRequired()
.HasMaxLength(64)
.HasColumnType("character(64)")
.IsFixedLength();
b.Property<string>("ContentHash")
.HasMaxLength(64)
.HasColumnType("character(64)")
.IsFixedLength();
b.Property<string>("ContentType")
.HasMaxLength(255)
.HasColumnType("character varying(255)");
b.Property<DateTimeOffset>("ExpiresAtUtc")
.HasColumnType("timestamp with time zone");
b.Property<DateTimeOffset>("FetchedAtUtc")
.HasColumnType("timestamp with time zone");
b.Property<string>("FinalUrl")
.HasColumnType("text");
b.Property<int?>("HttpStatusCode")
.HasColumnType("integer");
b.Property<bool>("IsComplete")
.HasColumnType("boolean");
b.Property<string>("RawHtml")
.HasColumnType("text");
b.Property<string>("RequestedUrl")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Title")
.HasColumnType("text");
b.Property<int>("TotalTextLength")
.HasColumnType("integer");
b.HasKey("Id");
b.HasIndex("CacheKey")
.IsUnique();
b.HasIndex("ExpiresAtUtc");
b.ToTable("cached_web_pages", (string)null);
});
modelBuilder.Entity("WebAccess.MCP.WebCache.CachedWebPageChunk", b =>
{
b.Property<long>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("bigint");
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<long>("Id"));
b.Property<long>("CachedWebPageId")
.HasColumnType("bigint");
b.Property<int>("ChunkNumber")
.HasColumnType("integer");
b.Property<int>("Length")
.HasColumnType("integer");
b.Property<int>("StartOffset")
.HasColumnType("integer");
b.Property<string>("Text")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.HasIndex("CachedWebPageId", "ChunkNumber")
.IsUnique();
b.ToTable("cached_web_page_chunks", (string)null);
});
modelBuilder.Entity("WebAccess.MCP.WebCache.CachedWebPageChunk", b =>
{
b.HasOne("WebAccess.MCP.WebCache.CachedWebPage", "CachedWebPage")
.WithMany("Chunks")
.HasForeignKey("CachedWebPageId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("CachedWebPage");
});
modelBuilder.Entity("WebAccess.MCP.WebCache.CachedWebPage", b =>
{
b.Navigation("Chunks");
});
#pragma warning restore 612, 618
}
}
}

View File

@ -0,0 +1,37 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace WebAccess.MCP.WebCache.Migrations
{
/// <inheritdoc />
public partial class MakeCacheKeyUnique : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropIndex(
name: "IX_cached_web_pages_CacheKey",
table: "cached_web_pages");
migrationBuilder.CreateIndex(
name: "IX_cached_web_pages_CacheKey",
table: "cached_web_pages",
column: "CacheKey",
unique: true);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropIndex(
name: "IX_cached_web_pages_CacheKey",
table: "cached_web_pages");
migrationBuilder.CreateIndex(
name: "IX_cached_web_pages_CacheKey",
table: "cached_web_pages",
column: "CacheKey");
}
}
}

View File

@ -0,0 +1,151 @@
// <auto-generated />
using System;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
using NpgsqlTypes;
using WebAccess.MCP.WebCache;
#nullable disable
namespace WebAccess.MCP.WebCache.Migrations
{
[DbContext(typeof(WebCacheDbContext))]
[Migration("20260820061730_AddChunkFullTextSearch")]
partial class AddChunkFullTextSearch
{
/// <inheritdoc />
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "10.0.4")
.HasAnnotation("Relational:MaxIdentifierLength", 63);
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
modelBuilder.Entity("WebAccess.MCP.WebCache.CachedWebPage", b =>
{
b.Property<long>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("bigint");
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<long>("Id"));
b.Property<string>("CacheKey")
.IsRequired()
.HasMaxLength(64)
.HasColumnType("character(64)")
.IsFixedLength();
b.Property<string>("ContentHash")
.HasMaxLength(64)
.HasColumnType("character(64)")
.IsFixedLength();
b.Property<string>("ContentType")
.HasMaxLength(255)
.HasColumnType("character varying(255)");
b.Property<DateTimeOffset>("ExpiresAtUtc")
.HasColumnType("timestamp with time zone");
b.Property<DateTimeOffset>("FetchedAtUtc")
.HasColumnType("timestamp with time zone");
b.Property<string>("FinalUrl")
.HasColumnType("text");
b.Property<int?>("HttpStatusCode")
.HasColumnType("integer");
b.Property<bool>("IsComplete")
.HasColumnType("boolean");
b.Property<string>("RawHtml")
.HasColumnType("text");
b.Property<string>("RequestedUrl")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Title")
.HasColumnType("text");
b.Property<int>("TotalTextLength")
.HasColumnType("integer");
b.HasKey("Id");
b.HasIndex("CacheKey")
.IsUnique();
b.HasIndex("ExpiresAtUtc");
b.ToTable("cached_web_pages", (string)null);
});
modelBuilder.Entity("WebAccess.MCP.WebCache.CachedWebPageChunk", b =>
{
b.Property<long>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("bigint");
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<long>("Id"));
b.Property<long>("CachedWebPageId")
.HasColumnType("bigint");
b.Property<int>("ChunkNumber")
.HasColumnType("integer");
b.Property<int>("Length")
.HasColumnType("integer");
b.Property<NpgsqlTsVector>("SearchVector")
.IsRequired()
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("tsvector")
.HasAnnotation("Npgsql:TsVectorConfig", "simple")
.HasAnnotation("Npgsql:TsVectorProperties", new[] { "Text" });
b.Property<int>("StartOffset")
.HasColumnType("integer");
b.Property<string>("Text")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.HasIndex("SearchVector");
NpgsqlIndexBuilderExtensions.HasMethod(b.HasIndex("SearchVector"), "GIN");
b.HasIndex("CachedWebPageId", "ChunkNumber")
.IsUnique();
b.ToTable("cached_web_page_chunks", (string)null);
});
modelBuilder.Entity("WebAccess.MCP.WebCache.CachedWebPageChunk", b =>
{
b.HasOne("WebAccess.MCP.WebCache.CachedWebPage", "CachedWebPage")
.WithMany("Chunks")
.HasForeignKey("CachedWebPageId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("CachedWebPage");
});
modelBuilder.Entity("WebAccess.MCP.WebCache.CachedWebPage", b =>
{
b.Navigation("Chunks");
});
#pragma warning restore 612, 618
}
}
}

View File

@ -0,0 +1,41 @@
using Microsoft.EntityFrameworkCore.Migrations;
using NpgsqlTypes;
#nullable disable
namespace WebAccess.MCP.WebCache.Migrations
{
/// <inheritdoc />
public partial class AddChunkFullTextSearch : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<NpgsqlTsVector>(
name: "SearchVector",
table: "cached_web_page_chunks",
type: "tsvector",
nullable: false)
.Annotation("Npgsql:TsVectorConfig", "simple")
.Annotation("Npgsql:TsVectorProperties", new[] { "Text" });
migrationBuilder.CreateIndex(
name: "IX_cached_web_page_chunks_SearchVector",
table: "cached_web_page_chunks",
column: "SearchVector")
.Annotation("Npgsql:IndexMethod", "GIN");
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropIndex(
name: "IX_cached_web_page_chunks_SearchVector",
table: "cached_web_page_chunks");
migrationBuilder.DropColumn(
name: "SearchVector",
table: "cached_web_page_chunks");
}
}
}

View File

@ -0,0 +1,148 @@
// <auto-generated />
using System;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
using NpgsqlTypes;
using WebAccess.MCP.WebCache;
#nullable disable
namespace WebAccess.MCP.WebCache.Migrations
{
[DbContext(typeof(WebCacheDbContext))]
partial class WebCacheDbContextModelSnapshot : ModelSnapshot
{
protected override void BuildModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "10.0.4")
.HasAnnotation("Relational:MaxIdentifierLength", 63);
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
modelBuilder.Entity("WebAccess.MCP.WebCache.CachedWebPage", b =>
{
b.Property<long>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("bigint");
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<long>("Id"));
b.Property<string>("CacheKey")
.IsRequired()
.HasMaxLength(64)
.HasColumnType("character(64)")
.IsFixedLength();
b.Property<string>("ContentHash")
.HasMaxLength(64)
.HasColumnType("character(64)")
.IsFixedLength();
b.Property<string>("ContentType")
.HasMaxLength(255)
.HasColumnType("character varying(255)");
b.Property<DateTimeOffset>("ExpiresAtUtc")
.HasColumnType("timestamp with time zone");
b.Property<DateTimeOffset>("FetchedAtUtc")
.HasColumnType("timestamp with time zone");
b.Property<string>("FinalUrl")
.HasColumnType("text");
b.Property<int?>("HttpStatusCode")
.HasColumnType("integer");
b.Property<bool>("IsComplete")
.HasColumnType("boolean");
b.Property<string>("RawHtml")
.HasColumnType("text");
b.Property<string>("RequestedUrl")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Title")
.HasColumnType("text");
b.Property<int>("TotalTextLength")
.HasColumnType("integer");
b.HasKey("Id");
b.HasIndex("CacheKey")
.IsUnique();
b.HasIndex("ExpiresAtUtc");
b.ToTable("cached_web_pages", (string)null);
});
modelBuilder.Entity("WebAccess.MCP.WebCache.CachedWebPageChunk", b =>
{
b.Property<long>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("bigint");
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<long>("Id"));
b.Property<long>("CachedWebPageId")
.HasColumnType("bigint");
b.Property<int>("ChunkNumber")
.HasColumnType("integer");
b.Property<int>("Length")
.HasColumnType("integer");
b.Property<NpgsqlTsVector>("SearchVector")
.IsRequired()
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("tsvector")
.HasAnnotation("Npgsql:TsVectorConfig", "simple")
.HasAnnotation("Npgsql:TsVectorProperties", new[] { "Text" });
b.Property<int>("StartOffset")
.HasColumnType("integer");
b.Property<string>("Text")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.HasIndex("SearchVector");
NpgsqlIndexBuilderExtensions.HasMethod(b.HasIndex("SearchVector"), "GIN");
b.HasIndex("CachedWebPageId", "ChunkNumber")
.IsUnique();
b.ToTable("cached_web_page_chunks", (string)null);
});
modelBuilder.Entity("WebAccess.MCP.WebCache.CachedWebPageChunk", b =>
{
b.HasOne("WebAccess.MCP.WebCache.CachedWebPage", "CachedWebPage")
.WithMany("Chunks")
.HasForeignKey("CachedWebPageId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("CachedWebPage");
});
modelBuilder.Entity("WebAccess.MCP.WebCache.CachedWebPage", b =>
{
b.Navigation("Chunks");
});
#pragma warning restore 612, 618
}
}
}

View File

@ -0,0 +1,248 @@
using System.Data;
using Microsoft.EntityFrameworkCore;
namespace WebAccess.MCP.WebCache;
internal sealed class PostgresWebCacheRepository(
Lazy<IDbContextFactory<WebCacheDbContext>> contextFactory) : IWebCacheRepository
{
public async Task<WebCacheRepositoryPage?> TryGetPageChunkAsync(
string cacheKey,
int chunkNumber,
CancellationToken cancellationToken)
{
await using var context = await GetContextFactory()
.CreateDbContextAsync(cancellationToken);
await using var transaction = await context.Database.BeginTransactionAsync(
IsolationLevel.RepeatableRead,
cancellationToken);
var page = await context.CachedWebPages
.AsNoTracking()
.Where(cachedPage => cachedPage.CacheKey == cacheKey)
.Select(cachedPage => new
{
cachedPage.Id,
cachedPage.RequestedUrl,
cachedPage.FinalUrl,
cachedPage.HttpStatusCode,
cachedPage.Title,
cachedPage.ContentType,
cachedPage.FetchedAtUtc,
cachedPage.ExpiresAtUtc,
cachedPage.TotalTextLength,
cachedPage.IsComplete,
ChunkCount = cachedPage.Chunks.Count
})
.SingleOrDefaultAsync(cancellationToken);
if (page is null)
{
await transaction.CommitAsync(cancellationToken);
return null;
}
var chunk = await context.CachedWebPageChunks
.AsNoTracking()
.Where(pageChunk =>
pageChunk.CachedWebPageId == page.Id
&& pageChunk.ChunkNumber == chunkNumber)
.Select(pageChunk => new WebCacheStoredChunk(
pageChunk.ChunkNumber,
pageChunk.StartOffset,
pageChunk.Length,
pageChunk.Text))
.SingleOrDefaultAsync(cancellationToken);
var result = new WebCacheRepositoryPage(
page.RequestedUrl,
page.FinalUrl,
page.HttpStatusCode,
page.Title,
page.ContentType,
page.FetchedAtUtc,
page.ExpiresAtUtc,
page.TotalTextLength,
page.IsComplete,
page.ChunkCount,
chunk);
await transaction.CommitAsync(cancellationToken);
return result;
}
public async Task StoreOrReplaceAsync(
WebCacheStoredPage page,
IReadOnlyList<WebCacheStoredChunk> chunks,
CancellationToken cancellationToken)
{
await using var context = await GetContextFactory()
.CreateDbContextAsync(cancellationToken);
await using var transaction =
await context.Database.BeginTransactionAsync(cancellationToken);
await context.Database.ExecuteSqlInterpolatedAsync(
$"SELECT pg_advisory_xact_lock(hashtextextended({page.CacheKey}, 0))",
cancellationToken);
var entity = await context.CachedWebPages
.Include(cachedPage => cachedPage.Chunks)
.SingleOrDefaultAsync(
cachedPage => cachedPage.CacheKey == page.CacheKey,
cancellationToken);
if (entity is null)
{
entity = new CachedWebPage
{
CacheKey = page.CacheKey
};
context.CachedWebPages.Add(entity);
}
else
{
context.CachedWebPageChunks.RemoveRange(entity.Chunks);
await context.SaveChangesAsync(cancellationToken);
entity.Chunks.Clear();
}
ApplySnapshot(entity, page);
foreach (var chunk in chunks)
{
entity.Chunks.Add(
new CachedWebPageChunk
{
ChunkNumber = chunk.ChunkNumber,
StartOffset = chunk.StartOffset,
Length = chunk.Length,
Text = chunk.Text
});
}
await context.SaveChangesAsync(cancellationToken);
await transaction.CommitAsync(cancellationToken);
}
public async Task<WebCacheRepositorySearchPage?> SearchPageAsync(
string cacheKey,
string candidateQuery,
DateTimeOffset currentUtc,
CancellationToken cancellationToken)
{
await using var context = await GetContextFactory()
.CreateDbContextAsync(cancellationToken);
await using var transaction = await context.Database.BeginTransactionAsync(
IsolationLevel.RepeatableRead,
cancellationToken);
var page = await context.CachedWebPages
.AsNoTracking()
.Where(cachedPage => cachedPage.CacheKey == cacheKey)
.Select(cachedPage => new
{
cachedPage.Id,
cachedPage.RequestedUrl,
cachedPage.FinalUrl,
cachedPage.FetchedAtUtc,
cachedPage.ExpiresAtUtc,
cachedPage.IsComplete,
cachedPage.TotalTextLength,
ChunkCount = cachedPage.Chunks.Count
})
.SingleOrDefaultAsync(cancellationToken);
if (page is null)
{
await transaction.CommitAsync(cancellationToken);
return null;
}
IReadOnlyList<WebCacheRepositorySearchHit> hits = [];
IReadOnlyList<WebCacheStoredChunk> chunks = [];
if (page.ExpiresAtUtc > currentUtc
&& !string.IsNullOrEmpty(candidateQuery))
{
var candidates = await BuildSearchQuery(context, page.Id, candidateQuery)
.ToListAsync(cancellationToken);
hits = candidates
.Select(candidate => new WebCacheRepositorySearchHit(
candidate.PageNumber,
candidate.Rank))
.ToArray();
if (hits.Count > 0)
{
chunks = await context.CachedWebPageChunks
.AsNoTracking()
.Where(pageChunk => pageChunk.CachedWebPageId == page.Id)
.OrderBy(pageChunk => pageChunk.ChunkNumber)
.Select(pageChunk => new WebCacheStoredChunk(
pageChunk.ChunkNumber,
pageChunk.StartOffset,
pageChunk.Length,
pageChunk.Text))
.ToArrayAsync(cancellationToken);
}
}
var result = new WebCacheRepositorySearchPage(
page.Id,
page.RequestedUrl,
page.FinalUrl,
page.FetchedAtUtc,
page.ExpiresAtUtc,
page.IsComplete,
page.TotalTextLength,
page.ChunkCount,
chunks,
hits);
await transaction.CommitAsync(cancellationToken);
return result;
}
internal static IQueryable<WebCacheSearchCandidate> BuildSearchQuery(
WebCacheDbContext context,
long cachedWebPageId,
string candidateQuery) =>
context.CachedWebPageChunks
.AsNoTracking()
.Where(pageChunk =>
pageChunk.CachedWebPageId == cachedWebPageId
&& pageChunk.SearchVector.Matches(
EF.Functions.WebSearchToTsQuery("simple", candidateQuery)))
.OrderByDescending(pageChunk => pageChunk.SearchVector.Rank(
EF.Functions.WebSearchToTsQuery("simple", candidateQuery)))
.ThenBy(pageChunk => pageChunk.ChunkNumber)
.Select(pageChunk => new WebCacheSearchCandidate(
pageChunk.ChunkNumber,
pageChunk.SearchVector.Rank(
EF.Functions.WebSearchToTsQuery("simple", candidateQuery))))
.Take(WebCacheSearchConstants.MaximumCandidateChunks);
private static void ApplySnapshot(CachedWebPage entity, WebCacheStoredPage page)
{
entity.RequestedUrl = page.RequestedUrl;
entity.FinalUrl = page.FinalUrl;
entity.HttpStatusCode = page.HttpStatusCode;
entity.Title = page.Title;
entity.ContentType = page.ContentType;
entity.RawHtml = page.RawHtml;
entity.ContentHash = page.ContentHash;
entity.FetchedAtUtc = page.FetchedAtUtc;
entity.ExpiresAtUtc = page.ExpiresAtUtc;
entity.TotalTextLength = page.TotalTextLength;
entity.IsComplete = page.IsComplete;
}
private IDbContextFactory<WebCacheDbContext> GetContextFactory()
{
try
{
return contextFactory.Value;
}
catch (InvalidOperationException exception)
{
throw new WebCacheUnavailableException(
"WebCache persistence is not configured.",
exception);
}
}
}

View File

@ -0,0 +1,50 @@
using Microsoft.EntityFrameworkCore;
namespace WebAccess.MCP.WebCache;
internal sealed class WebCacheDbContext(DbContextOptions<WebCacheDbContext> options)
: DbContext(options)
{
internal DbSet<CachedWebPage> CachedWebPages => Set<CachedWebPage>();
internal DbSet<CachedWebPageChunk> CachedWebPageChunks => Set<CachedWebPageChunk>();
protected override void OnModelCreating(ModelBuilder modelBuilder)
{
var page = modelBuilder.Entity<CachedWebPage>();
page.ToTable("cached_web_pages");
page.HasKey(cachedPage => cachedPage.Id);
page.Property(cachedPage => cachedPage.CacheKey)
.HasMaxLength(64)
.IsFixedLength()
.IsRequired();
page.Property(cachedPage => cachedPage.RequestedUrl).IsRequired();
page.Property(cachedPage => cachedPage.ContentType).HasMaxLength(255);
page.Property(cachedPage => cachedPage.ContentHash)
.HasMaxLength(64)
.IsFixedLength();
page.HasIndex(cachedPage => cachedPage.CacheKey).IsUnique();
page.HasIndex(cachedPage => cachedPage.ExpiresAtUtc);
var chunk = modelBuilder.Entity<CachedWebPageChunk>();
chunk.ToTable("cached_web_page_chunks");
chunk.HasKey(pageChunk => pageChunk.Id);
chunk.Property(pageChunk => pageChunk.Text).IsRequired();
chunk.HasGeneratedTsVectorColumn(
pageChunk => pageChunk.SearchVector,
"simple",
pageChunk => new { pageChunk.Text })
.HasIndex(pageChunk => pageChunk.SearchVector)
.HasMethod("GIN");
chunk.HasIndex(pageChunk => new
{
pageChunk.CachedWebPageId,
pageChunk.ChunkNumber
})
.IsUnique();
chunk.HasOne(pageChunk => pageChunk.CachedWebPage)
.WithMany(cachedPage => cachedPage.Chunks)
.HasForeignKey(pageChunk => pageChunk.CachedWebPageId)
.OnDelete(DeleteBehavior.Cascade);
}
}

View File

@ -0,0 +1,71 @@
namespace WebAccess.MCP.WebCache;
internal sealed record WebCachePageWriteRequest(
string RequestedUrl,
string? FinalUrl,
int? HttpStatusCode,
string? Title,
string? ContentType,
string? RawHtml,
string NormalizedText,
bool IsComplete);
internal enum WebCacheLookupStatus
{
Hit,
PageNotFound,
Expired,
ChunkNotFound
}
internal sealed record WebCacheLookupResult(
WebCacheLookupStatus Status,
WebCachePageChunkRead? Page);
internal sealed record WebCachePageChunkRead(
string RequestedUrl,
string? FinalUrl,
int? HttpStatusCode,
string? Title,
string? ContentType,
DateTimeOffset FetchedAtUtc,
DateTimeOffset ExpiresAtUtc,
bool IsComplete,
int TotalTextLength,
int? ChunkNumber,
int ChunkCount,
int? StartOffset,
int? Length,
string? Text,
bool HasMore);
internal sealed record WebCacheIdentity(
string NormalizedUrl,
string CacheKey);
internal enum WebCacheSearchStatus
{
Hit,
NoMatches,
PageNotFound,
Expired
}
internal sealed record WebCacheSearchResult(
WebCacheSearchStatus Status,
string RequestedUrl,
string? FinalUrl,
DateTimeOffset? FetchedAtUtc,
DateTimeOffset? ExpiresAtUtc,
bool? IsComplete,
int? TotalTextLength,
int? ChunkCount,
IReadOnlyList<WebCacheSearchHit> Hits);
internal sealed record WebCacheSearchHit(
int StartPage,
int EndPage,
int StartOffset,
int Length,
double Score,
string Context);

View File

@ -0,0 +1,21 @@
namespace WebAccess.MCP.WebCache;
internal sealed class WebCacheOptions
{
public const string SectionName = "WebCache";
public const int MaximumChunkSize = 50_000;
/// <summary>
/// Gets or sets how long a fetched page remains fresh by default.
/// The default is exactly one hour.
/// </summary>
public TimeSpan DefaultExpiration { get; set; } = TimeSpan.FromHours(1);
/// <summary>
/// Gets or sets the logical normalized-text chunk size. The default is 4000
/// characters. Smaller values reduce per-read model context usage but may
/// require more reads; larger values reduce paging calls but consume more
/// model context.
/// </summary>
public int ChunkSize { get; set; } = 4_000;
}

View File

@ -0,0 +1,28 @@
using Microsoft.Extensions.Options;
namespace WebAccess.MCP.WebCache;
internal sealed class WebCacheOptionsValidator : IValidateOptions<WebCacheOptions>
{
public ValidateOptionsResult Validate(string? name, WebCacheOptions options)
{
var failures = new List<string>();
if (options.DefaultExpiration <= TimeSpan.Zero)
{
failures.Add(
$"{WebCacheOptions.SectionName}:DefaultExpiration must be greater than zero.");
}
if (options.ChunkSize is < 1 or > WebCacheOptions.MaximumChunkSize)
{
failures.Add(
$"{WebCacheOptions.SectionName}:ChunkSize must be from 1 through "
+ $"{WebCacheOptions.MaximumChunkSize}.");
}
return failures.Count == 0
? ValidateOptionsResult.Success
: ValidateOptionsResult.Fail(failures);
}
}

View File

@ -0,0 +1,54 @@
namespace WebAccess.MCP.WebCache;
internal sealed record WebCacheStoredPage(
string CacheKey,
string RequestedUrl,
string? FinalUrl,
int? HttpStatusCode,
string? Title,
string? ContentType,
string? RawHtml,
string ContentHash,
DateTimeOffset FetchedAtUtc,
DateTimeOffset ExpiresAtUtc,
int TotalTextLength,
bool IsComplete);
internal sealed record WebCacheStoredChunk(
int ChunkNumber,
int StartOffset,
int Length,
string Text);
internal sealed record WebCacheRepositoryPage(
string RequestedUrl,
string? FinalUrl,
int? HttpStatusCode,
string? Title,
string? ContentType,
DateTimeOffset FetchedAtUtc,
DateTimeOffset ExpiresAtUtc,
int TotalTextLength,
bool IsComplete,
int ChunkCount,
WebCacheStoredChunk? Chunk);
internal sealed record WebCacheRepositorySearchPage(
long Id,
string RequestedUrl,
string? FinalUrl,
DateTimeOffset FetchedAtUtc,
DateTimeOffset ExpiresAtUtc,
bool IsComplete,
int TotalTextLength,
int ChunkCount,
IReadOnlyList<WebCacheStoredChunk> Chunks,
IReadOnlyList<WebCacheRepositorySearchHit> Hits);
internal sealed record WebCacheRepositorySearchHit(
int PageNumber,
float Rank);
internal sealed record WebCacheSearchCandidate(
int PageNumber,
float Rank);

View File

@ -0,0 +1,10 @@
namespace WebAccess.MCP.WebCache;
internal static class WebCacheSearchConstants
{
public const int MaximumHits = 5;
public const int MaximumQueryLength = 1_000;
public const int MaximumRelatedTerms = 8;
public const int MaximumRelatedTermLength = 100;
public const int MaximumCandidateChunks = 64;
}

View File

@ -0,0 +1,697 @@
using System.Data.Common;
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Options;
namespace WebAccess.MCP.WebCache;
internal sealed class WebCacheService(
IWebCacheRepository repository,
IOptions<WebCacheOptions> options,
TimeProvider timeProvider) : IWebCacheService
{
private const double ExactPhraseWeight = 10_000;
private const double QueryTokenWeight = 1_000;
private const double RelatedPhraseWeight = 300;
private const double RelatedTokenWeight = 75;
private static readonly Regex SearchTokenPattern = new(
@"[\p{L}\p{N}_][\p{L}\p{N}_.-]*",
RegexOptions.Compiled | RegexOptions.CultureInvariant);
private readonly WebCacheOptions options = options.Value;
public async Task<WebCacheLookupResult> TryGetFreshChunkAsync(
string requestedUrl,
int chunkNumber,
CancellationToken cancellationToken)
{
if (chunkNumber < 1)
{
throw new ArgumentOutOfRangeException(
nameof(chunkNumber),
chunkNumber,
"The chunk number must be one or greater.");
}
var identity = CreateCacheIdentity(requestedUrl);
WebCacheRepositoryPage? persisted;
try
{
persisted = await repository.TryGetPageChunkAsync(
identity.CacheKey,
chunkNumber,
cancellationToken);
}
catch (DbException exception)
{
throw new WebCacheUnavailableException(
"WebCache persistence is unavailable.",
exception);
}
catch (TimeoutException exception)
{
throw new WebCacheUnavailableException(
"WebCache persistence timed out.",
exception);
}
if (persisted is null)
{
return new WebCacheLookupResult(WebCacheLookupStatus.PageNotFound, null);
}
if (persisted.ExpiresAtUtc <= timeProvider.GetUtcNow())
{
return new WebCacheLookupResult(WebCacheLookupStatus.Expired, null);
}
if (persisted.ChunkCount == 0)
{
return new WebCacheLookupResult(
WebCacheLookupStatus.Hit,
CreateReadModel(persisted, null));
}
if (persisted.Chunk is null)
{
return new WebCacheLookupResult(
WebCacheLookupStatus.ChunkNotFound,
CreateReadModel(persisted, null));
}
return new WebCacheLookupResult(
WebCacheLookupStatus.Hit,
CreateReadModel(persisted, persisted.Chunk));
}
public async Task StoreAsync(
WebCachePageWriteRequest request,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(request);
ArgumentNullException.ThrowIfNull(request.NormalizedText);
var identity = CreateCacheIdentity(request.RequestedUrl);
var fetchedAtUtc = timeProvider.GetUtcNow();
var chunks = CreateChunks(request.NormalizedText, options.ChunkSize);
var page = new WebCacheStoredPage(
identity.CacheKey,
request.RequestedUrl,
request.FinalUrl,
request.HttpStatusCode,
request.Title,
request.ContentType,
request.RawHtml,
CreateHash(request.NormalizedText),
fetchedAtUtc,
fetchedAtUtc + options.DefaultExpiration,
request.NormalizedText.Length,
request.IsComplete);
try
{
await repository.StoreOrReplaceAsync(page, chunks, cancellationToken);
}
catch (DbUpdateException exception)
{
throw new WebCacheUnavailableException(
"WebCache persistence update failed.",
exception);
}
catch (DbException exception)
{
throw new WebCacheUnavailableException(
"WebCache persistence is unavailable.",
exception);
}
catch (TimeoutException exception)
{
throw new WebCacheUnavailableException(
"WebCache persistence timed out.",
exception);
}
}
public async Task<WebCacheSearchResult> SearchFreshPageAsync(
string requestedUrl,
string query,
IReadOnlyList<string>? relatedTerms,
CancellationToken cancellationToken)
{
var identity = CreateCacheIdentity(requestedUrl);
var normalizedQuery = ValidateSearchQuery(query);
var normalizedRelatedTerms = NormalizeRelatedTerms(relatedTerms);
var candidateQuery = CreateCandidateQuery(
normalizedQuery,
normalizedRelatedTerms);
WebCacheRepositorySearchPage? persisted;
try
{
persisted = await repository.SearchPageAsync(
identity.CacheKey,
candidateQuery,
timeProvider.GetUtcNow(),
cancellationToken);
}
catch (DbException exception)
{
throw new WebCacheUnavailableException(
"WebCache persistence is unavailable.",
exception);
}
catch (TimeoutException exception)
{
throw new WebCacheUnavailableException(
"WebCache persistence timed out.",
exception);
}
if (persisted is null)
{
return new WebCacheSearchResult(
WebCacheSearchStatus.PageNotFound,
requestedUrl,
null,
null,
null,
null,
null,
null,
[]);
}
if (persisted.ExpiresAtUtc <= timeProvider.GetUtcNow())
{
return CreateSearchResult(
WebCacheSearchStatus.Expired,
persisted,
[]);
}
WebCacheSearchHit? hit;
try
{
hit = persisted.Hits.Count == 0
? null
: CreateStrongestContext(
persisted.Chunks,
persisted.TotalTextLength,
normalizedQuery,
normalizedRelatedTerms,
options.ChunkSize);
}
catch (InvalidDataException exception)
{
throw new WebCacheUnavailableException(
"The cached page snapshot is invalid.",
exception);
}
var hits = hit is null ? [] : new[] { hit };
return CreateSearchResult(
hits.Length == 0
? WebCacheSearchStatus.NoMatches
: WebCacheSearchStatus.Hit,
persisted,
hits);
}
internal static WebCacheIdentity CreateCacheIdentity(string requestedUrl)
{
if (string.IsNullOrWhiteSpace(requestedUrl))
{
throw new ArgumentException("A non-empty URL is required.", nameof(requestedUrl));
}
if (!Uri.TryCreate(requestedUrl, UriKind.Absolute, out var uri)
|| (uri.Scheme != Uri.UriSchemeHttp && uri.Scheme != Uri.UriSchemeHttps))
{
throw new ArgumentException(
"The URL must be an absolute HTTP or HTTPS URL.",
nameof(requestedUrl));
}
var normalizedUri = new UriBuilder(uri)
{
Fragment = string.Empty
}.Uri.AbsoluteUri;
return new WebCacheIdentity(normalizedUri, CreateHash(normalizedUri));
}
internal static IReadOnlyList<WebCacheStoredChunk> CreateChunks(
string normalizedText,
int chunkSize)
{
ArgumentNullException.ThrowIfNull(normalizedText);
if (chunkSize is < 1 or > WebCacheOptions.MaximumChunkSize)
{
throw new ArgumentOutOfRangeException(
nameof(chunkSize),
chunkSize,
$"The chunk size must be from 1 through {WebCacheOptions.MaximumChunkSize}.");
}
if (normalizedText.Length == 0)
{
return [];
}
var chunks = new List<WebCacheStoredChunk>(
(normalizedText.Length + chunkSize - 1) / chunkSize);
for (var startOffset = 0; startOffset < normalizedText.Length; startOffset += chunkSize)
{
var length = Math.Min(chunkSize, normalizedText.Length - startOffset);
chunks.Add(
new WebCacheStoredChunk(
chunks.Count + 1,
startOffset,
length,
normalizedText.Substring(startOffset, length)));
}
return chunks;
}
internal static string ValidateSearchQuery(string query)
{
if (string.IsNullOrWhiteSpace(query))
{
throw new ArgumentException(
"A non-empty cache search query is required.",
nameof(query));
}
var normalizedQuery = query.Trim();
if (normalizedQuery.Length > WebCacheSearchConstants.MaximumQueryLength)
{
throw new ArgumentException(
$"The cache search query must not exceed "
+ $"{WebCacheSearchConstants.MaximumQueryLength} characters.",
nameof(query));
}
return normalizedQuery;
}
internal static IReadOnlyList<string> NormalizeRelatedTerms(
IReadOnlyList<string>? relatedTerms)
{
if (relatedTerms is null || relatedTerms.Count == 0)
{
return [];
}
var normalized = new List<string>(
Math.Min(relatedTerms.Count, WebCacheSearchConstants.MaximumRelatedTerms));
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var term in relatedTerms)
{
if (string.IsNullOrWhiteSpace(term))
{
continue;
}
var value = term.Trim();
if (value.Length > WebCacheSearchConstants.MaximumRelatedTermLength)
{
throw new ArgumentException(
$"Each related term must not exceed "
+ $"{WebCacheSearchConstants.MaximumRelatedTermLength} characters.",
nameof(relatedTerms));
}
if (seen.Add(value))
{
normalized.Add(value);
}
}
if (normalized.Count > WebCacheSearchConstants.MaximumRelatedTerms)
{
throw new ArgumentException(
$"No more than {WebCacheSearchConstants.MaximumRelatedTerms} "
+ "related terms may be supplied.",
nameof(relatedTerms));
}
return normalized;
}
internal static string CreateCandidateQuery(
string query,
IReadOnlyList<string> relatedTerms)
{
var tokens = ExtractTokens(query)
.Concat(relatedTerms.SelectMany(ExtractTokens))
.Distinct(StringComparer.OrdinalIgnoreCase)
.Take(WebCacheSearchConstants.MaximumCandidateChunks)
.Select(token => $"\"{token}\"")
.ToArray();
return string.Join(" OR ", tokens);
}
internal static WebCacheSearchHit? CreateStrongestContext(
IReadOnlyList<WebCacheStoredChunk> chunks,
int totalTextLength,
string query,
IReadOnlyList<string> relatedTerms,
int contextLength)
{
if (chunks.Count == 0 || totalTextLength == 0)
{
return null;
}
var document = ReconstructDocument(chunks, totalTextLength);
var matches = FindWeightedMatches(document, query, relatedTerms);
if (matches.Count == 0)
{
return null;
}
ScoredWindow? best = null;
foreach (var anchor in matches)
{
var length = Math.Min(contextLength, document.Length);
var start = Math.Clamp(
anchor.Start - (length - Math.Min(anchor.Length, length)) / 2,
0,
document.Length - length);
var end = start + length;
var windowMatches = matches
.Where(match => match.Start < end && match.Start + match.Length > start)
.OrderBy(match => match.Start)
.ToArray();
if (!IsConfident(windowMatches))
{
continue;
}
var score = ScoreMatches(windowMatches);
var candidate = new ScoredWindow(start, length, score);
if (best is null
|| candidate.Score > best.Score
|| (candidate.Score == best.Score && candidate.Start < best.Start))
{
best = candidate;
}
}
if (best is null)
{
return null;
}
var overlappingChunks = chunks
.Where(chunk =>
chunk.StartOffset < best.Start + best.Length
&& chunk.StartOffset + chunk.Length > best.Start)
.OrderBy(chunk => chunk.ChunkNumber)
.ToArray();
if (overlappingChunks.Length == 0)
{
return null;
}
return new WebCacheSearchHit(
overlappingChunks[0].ChunkNumber,
overlappingChunks[^1].ChunkNumber,
best.Start,
best.Length,
best.Score,
document.Substring(best.Start, best.Length));
}
private static string CreateHash(string value)
{
var hash = SHA256.HashData(Encoding.UTF8.GetBytes(value));
return Convert.ToHexString(hash).ToLowerInvariant();
}
private static WebCachePageChunkRead CreateReadModel(
WebCacheRepositoryPage page,
WebCacheStoredChunk? chunk) =>
new(
page.RequestedUrl,
page.FinalUrl,
page.HttpStatusCode,
page.Title,
page.ContentType,
page.FetchedAtUtc,
page.ExpiresAtUtc,
page.IsComplete,
page.TotalTextLength,
chunk?.ChunkNumber,
page.ChunkCount,
chunk?.StartOffset,
chunk?.Length,
chunk?.Text,
chunk is not null && chunk.ChunkNumber < page.ChunkCount);
private static WebCacheSearchResult CreateSearchResult(
WebCacheSearchStatus status,
WebCacheRepositorySearchPage page,
IReadOnlyList<WebCacheSearchHit> hits) =>
new(
status,
page.RequestedUrl,
page.FinalUrl,
page.FetchedAtUtc,
page.ExpiresAtUtc,
page.IsComplete,
page.TotalTextLength,
page.ChunkCount,
hits);
private static IReadOnlyList<string> ExtractTokens(string value) =>
SearchTokenPattern.Matches(value)
.Select(match => match.Value)
.ToArray();
private static string ReconstructDocument(
IReadOnlyList<WebCacheStoredChunk> chunks,
int totalTextLength)
{
var buffer = new char[totalTextLength];
var expectedOffset = 0;
foreach (var chunk in chunks.OrderBy(chunk => chunk.ChunkNumber))
{
if (chunk.StartOffset != expectedOffset
|| chunk.Length != chunk.Text.Length
|| chunk.StartOffset + chunk.Length > totalTextLength)
{
throw new InvalidDataException(
"Cached page chunks are not contiguous and valid.");
}
chunk.Text.CopyTo(0, buffer, chunk.StartOffset, chunk.Length);
expectedOffset += chunk.Length;
}
if (expectedOffset != totalTextLength)
{
throw new InvalidDataException(
"Cached page chunks do not cover the persisted text length.");
}
return new string(buffer);
}
private static IReadOnlyList<WeightedMatch> FindWeightedMatches(
string document,
string query,
IReadOnlyList<string> relatedTerms)
{
var matches = new List<WeightedMatch>();
AddPhraseMatches(matches, document, query, ExactPhraseWeight, MatchKind.ExactPhrase, "query");
foreach (var token in ExtractTokens(query).Distinct(StringComparer.OrdinalIgnoreCase))
{
AddTokenMatches(
matches,
document,
token,
QueryTokenWeight,
MatchKind.QueryToken,
$"query:{token}");
}
foreach (var term in relatedTerms)
{
var termTokens = ExtractTokens(term)
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToArray();
if (termTokens.Length > 1)
{
AddPhraseMatches(
matches,
document,
term,
RelatedPhraseWeight,
MatchKind.RelatedPhrase,
$"related:{term}");
}
foreach (var token in termTokens)
{
AddTokenMatches(
matches,
document,
token,
RelatedTokenWeight,
MatchKind.RelatedToken,
$"related-token:{term}:{token}");
}
}
return matches;
}
private static void AddPhraseMatches(
List<WeightedMatch> matches,
string document,
string phrase,
double weight,
MatchKind kind,
string group)
{
if (!ExtractTokens(phrase).Any())
{
return;
}
for (var start = 0; start < document.Length;)
{
var index = document.IndexOf(phrase, start, StringComparison.OrdinalIgnoreCase);
if (index < 0)
{
break;
}
matches.Add(new WeightedMatch(index, phrase.Length, weight, kind, group));
start = index + Math.Max(phrase.Length, 1);
}
}
private static void AddTokenMatches(
List<WeightedMatch> matches,
string document,
string token,
double weight,
MatchKind kind,
string group)
{
for (var start = 0; start < document.Length;)
{
var index = document.IndexOf(token, start, StringComparison.OrdinalIgnoreCase);
if (index < 0)
{
break;
}
var beforeBoundary = index == 0 || !IsTokenCharacter(document[index - 1]);
var after = index + token.Length;
var afterBoundary = after == document.Length || !IsTokenCharacter(document[after]);
if (beforeBoundary && afterBoundary)
{
matches.Add(new WeightedMatch(index, token.Length, weight, kind, group));
}
start = index + Math.Max(token.Length, 1);
}
}
private static bool IsTokenCharacter(char value) =>
char.IsLetterOrDigit(value) || value == '_';
private static bool IsConfident(IReadOnlyList<WeightedMatch> matches)
{
if (matches.Any(match => match.Kind is MatchKind.ExactPhrase or MatchKind.QueryToken)
|| matches.Any(match => match.Kind == MatchKind.RelatedPhrase))
{
return true;
}
return matches
.Where(match => match.Kind == MatchKind.RelatedToken)
.Select(match => match.Group)
.Distinct(StringComparer.OrdinalIgnoreCase)
.Take(2)
.Count() >= 2;
}
private static double ScoreMatches(IReadOnlyList<WeightedMatch> matches)
{
var score = matches
.GroupBy(match => match.Group, StringComparer.OrdinalIgnoreCase)
.Sum(group =>
{
var ordered = group.OrderBy(match => match.Start).ToArray();
return ordered[0].Weight
+ ordered.Skip(1).Take(2).Sum(match => match.Weight * 0.2);
});
var scoredPairs = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
for (var index = 1; index < matches.Count; index++)
{
if (string.Equals(
matches[index].Group,
matches[index - 1].Group,
StringComparison.OrdinalIgnoreCase))
{
continue;
}
var pair = string.Compare(
matches[index].Group,
matches[index - 1].Group,
StringComparison.OrdinalIgnoreCase) < 0
? $"{matches[index].Group}|{matches[index - 1].Group}"
: $"{matches[index - 1].Group}|{matches[index].Group}";
if (!scoredPairs.Add(pair))
{
continue;
}
var distance = matches[index].Start - matches[index - 1].Start;
var proximityFactor = distance <= 250
? 0.5
: distance <= 1_000
? 0.2
: 0;
score += Math.Min(matches[index].Weight, matches[index - 1].Weight)
* proximityFactor;
}
return score;
}
private enum MatchKind
{
ExactPhrase,
QueryToken,
RelatedPhrase,
RelatedToken
}
private sealed record WeightedMatch(
int Start,
int Length,
double Weight,
MatchKind Kind,
string Group);
private sealed record ScoredWindow(
int Start,
int Length,
double Score);
}

View File

@ -0,0 +1,42 @@
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
namespace WebAccess.MCP.WebCache;
internal static class WebCacheServiceCollectionExtensions
{
internal const string ConnectionStringName = "WebCache";
public static IServiceCollection AddWebCachePersistence(
this IServiceCollection services,
IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
services
.AddOptions<WebCacheOptions>()
.Bind(configuration.GetSection(WebCacheOptions.SectionName));
services.AddSingleton<IValidateOptions<WebCacheOptions>, WebCacheOptionsValidator>();
services.AddDbContextFactory<WebCacheDbContext>(options =>
{
var connectionString = configuration.GetConnectionString(ConnectionStringName);
if (string.IsNullOrWhiteSpace(connectionString))
{
throw new InvalidOperationException(
$"ConnectionStrings:{ConnectionStringName} is required when cache persistence is used.");
}
options.UseNpgsql(connectionString);
});
services.TryAddSingleton(
serviceProvider => new Lazy<IDbContextFactory<WebCacheDbContext>>(
serviceProvider.GetRequiredService<IDbContextFactory<WebCacheDbContext>>));
services.TryAddSingleton(TimeProvider.System);
services.TryAddSingleton<IWebCacheRepository, PostgresWebCacheRepository>();
services.TryAddSingleton<IWebCacheService, WebCacheService>();
return services;
}
}

View File

@ -0,0 +1,5 @@
namespace WebAccess.MCP.WebCache;
internal sealed class WebCacheUnavailableException(
string message,
Exception innerException) : Exception(message, innerException);

View File

@ -0,0 +1,6 @@
{
"WebCache": {
"DefaultExpiration": "01:00:00",
"ChunkSize": 4000
}
}