Files
codex_truenas_helper/ollama_remote_test.ps1
Rushabh Gosar 5d1a0ee72b Initial commit
2026-01-07 16:54:39 -08:00

562 lines
21 KiB
PowerShell

param(
[Parameter(Mandatory = $true)][string]$Model,
[string]$BaseUrl = "http://192.168.1.2:30068",
[string]$PromptPath = "prompt_crwv.txt",
[int]$Runs = 3,
[int]$NumPredict = 1200,
[int]$NumCtx = 131072,
[int]$NumBatch = 0,
[int]$NumGpuLayers = 0,
[int]$TimeoutSec = 900,
[int]$TopK = 1,
[double]$TopP = 1.0,
[int]$Seed = 42,
[double]$RepeatPenalty = 1.05,
[string]$BatchId,
[switch]$UseSchemaFormat = $false,
[switch]$EnableGpuMonitor = $true,
[string]$SshExe = "$env:SystemRoot\\System32\\OpenSSH\\ssh.exe",
[switch]$CheckProcessor = $true,
[string]$SshUser = "rushabh",
[string]$SshHost = "192.168.1.2",
[int]$SshPort = 55555,
[int]$GpuMonitorIntervalSec = 1,
[int]$GpuMonitorSeconds = 120
)
$ErrorActionPreference = "Stop"
$ProgressPreference = "SilentlyContinue"
function Normalize-Strike([object]$value) {
if ($null -eq $value) { return $null }
if ($value -is [double] -or $value -is [float] -or $value -is [int] -or $value -is [long]) {
return ([double]$value).ToString("0.################", [System.Globalization.CultureInfo]::InvariantCulture)
}
return ($value.ToString().Trim())
}
function Get-AllowedLegs([string]$promptText) {
$pattern = 'Options Chain\s*```\s*(\[[\s\S]*?\])\s*```'
$match = [regex]::Match($promptText, $pattern, [System.Text.RegularExpressions.RegexOptions]::Singleline)
if (-not $match.Success) {
throw "Options Chain JSON block not found in prompt."
}
$chains = $match.Groups[1].Value | ConvertFrom-Json
$allowedExpiry = @{}
$allowedLegs = @{}
foreach ($exp in $chains) {
$expiry = [string]$exp.expiry
if ([string]::IsNullOrWhiteSpace($expiry)) { continue }
$allowedExpiry[$expiry] = $true
foreach ($leg in $exp.liquidSet) {
if ($null -eq $leg) { continue }
if ($leg.liquid -ne $true) { continue }
$side = [string]$leg.side
$strikeNorm = Normalize-Strike $leg.strike
if (-not [string]::IsNullOrWhiteSpace($side) -and $strikeNorm) {
$key = "$expiry|$side|$strikeNorm"
$allowedLegs[$key] = $true
}
}
}
return @{ AllowedExpiry = $allowedExpiry; AllowedLegs = $allowedLegs }
}
function Test-TradeSchema($obj, $allowedExpiry, $allowedLegs) {
$errors = New-Object System.Collections.Generic.List[string]
$requiredTop = @("selectedExpiry", "expiryRationale", "strategyBias", "recommendedTrades", "whyOthersRejected", "confidenceScore")
foreach ($key in $requiredTop) {
if (-not ($obj.PSObject.Properties.Name -contains $key)) {
$errors.Add("Missing top-level key: $key")
}
}
if ($obj.strategyBias -and ($obj.strategyBias -notin @("DIRECTIONAL","VOLATILITY","NEUTRAL","NO_TRADE"))) {
$errors.Add("Invalid strategyBias: $($obj.strategyBias)")
}
if (-not [string]::IsNullOrWhiteSpace([string]$obj.selectedExpiry)) {
if (-not $allowedExpiry.ContainsKey([string]$obj.selectedExpiry)) {
$errors.Add("selectedExpiry not in provided expiries: $($obj.selectedExpiry)")
}
} else {
$errors.Add("selectedExpiry is missing or empty")
}
if ($obj.confidenceScore -ne $null) {
if (-not ($obj.confidenceScore -is [double] -or $obj.confidenceScore -is [int])) {
$errors.Add("confidenceScore is not numeric")
} elseif ($obj.confidenceScore -lt 0 -or $obj.confidenceScore -gt 100) {
$errors.Add("confidenceScore out of range 0-100")
}
}
if ($obj.recommendedTrades -eq $null) {
$errors.Add("recommendedTrades is null")
} elseif (-not ($obj.recommendedTrades -is [System.Collections.IEnumerable])) {
$errors.Add("recommendedTrades is not an array")
}
if ($obj.strategyBias -eq "NO_TRADE") {
if ($obj.recommendedTrades -and $obj.recommendedTrades.Count -gt 0) {
$errors.Add("strategyBias is NO_TRADE but recommendedTrades is not empty")
}
} else {
if (-not $obj.recommendedTrades -or $obj.recommendedTrades.Count -lt 1 -or $obj.recommendedTrades.Count -gt 3) {
$errors.Add("recommendedTrades must contain 1-3 trades")
}
}
if ($obj.whyOthersRejected -ne $null -and -not ($obj.whyOthersRejected -is [System.Collections.IEnumerable])) {
$errors.Add("whyOthersRejected is not an array")
}
if ($obj.recommendedTrades) {
foreach ($trade in $obj.recommendedTrades) {
$tradeRequired = @("name","structure","legs","greekProfile","maxRisk","maxReward","thesisAlignment","invalidation")
foreach ($tkey in $tradeRequired) {
if (-not ($trade.PSObject.Properties.Name -contains $tkey)) {
$errors.Add("Trade missing key: $tkey")
}
}
if ([string]::IsNullOrWhiteSpace([string]$trade.name)) { $errors.Add("Trade name is empty") }
if ([string]::IsNullOrWhiteSpace([string]$trade.structure)) { $errors.Add("Trade structure is empty") }
if ([string]::IsNullOrWhiteSpace([string]$trade.thesisAlignment)) { $errors.Add("Trade thesisAlignment is empty") }
if ([string]::IsNullOrWhiteSpace([string]$trade.invalidation)) { $errors.Add("Trade invalidation is empty") }
if ($trade.maxRisk -eq $null -or [string]::IsNullOrWhiteSpace([string]$trade.maxRisk)) { $errors.Add("Trade maxRisk is empty") }
if ($trade.maxReward -eq $null -or [string]::IsNullOrWhiteSpace([string]$trade.maxReward)) { $errors.Add("Trade maxReward is empty") }
if ($trade.maxRisk -is [double] -or $trade.maxRisk -is [int]) {
if ($trade.maxRisk -le 0) { $errors.Add("Trade maxRisk must be > 0") }
}
if ($trade.maxReward -is [double] -or $trade.maxReward -is [int]) {
if ($trade.maxReward -le 0) { $errors.Add("Trade maxReward must be > 0") }
}
if (-not $trade.legs -or -not ($trade.legs -is [System.Collections.IEnumerable])) {
$errors.Add("Trade legs missing or not an array")
continue
}
$legs = @($trade.legs)
$hasBuy = $false
$hasSell = $false
foreach ($leg in $trade.legs) {
$side = ([string]$leg.side).ToLowerInvariant()
$action = ([string]$leg.action).ToLowerInvariant()
$expiry = [string]$leg.expiry
$strikeNorm = Normalize-Strike $leg.strike
if ($side -notin @("call","put")) { $errors.Add("Invalid leg side: $side") }
if ($action -notin @("buy","sell")) { $errors.Add("Invalid leg action: $action") }
if (-not $allowedExpiry.ContainsKey($expiry)) { $errors.Add("Leg expiry not allowed: $expiry") }
if (-not $strikeNorm) { $errors.Add("Leg strike missing") } else {
$key = "$expiry|$side|$strikeNorm"
if (-not $allowedLegs.ContainsKey($key)) {
$errors.Add("Leg not in liquid set: $key")
}
}
if ($action -eq "buy") { $hasBuy = $true }
if ($action -eq "sell") { $hasSell = $true }
}
if ($obj.selectedExpiry -and $legs) {
foreach ($leg in $legs) {
if ([string]$leg.expiry -ne [string]$obj.selectedExpiry) {
$errors.Add("Leg expiry does not match selectedExpiry: $($leg.expiry)")
}
}
}
if ($hasSell -and -not $hasBuy) {
$errors.Add("Naked short detected: trade has sell leg(s) with no buy leg")
}
if ($trade.greekProfile) {
$gp = $trade.greekProfile
$gpRequired = @("deltaBias","gammaExposure","thetaExposure","vegaExposure")
foreach ($gkey in $gpRequired) {
if (-not ($gp.PSObject.Properties.Name -contains $gkey)) {
$errors.Add("Missing greekProfile.$gkey")
}
}
if ($gp.deltaBias -and ($gp.deltaBias -notin @("POS","NEG","NEUTRAL"))) { $errors.Add("Invalid deltaBias") }
if ($gp.gammaExposure -and ($gp.gammaExposure -notin @("HIGH","MED","LOW"))) { $errors.Add("Invalid gammaExposure") }
if ($gp.thetaExposure -and ($gp.thetaExposure -notin @("POS","NEG","LOW"))) { $errors.Add("Invalid thetaExposure") }
if ($gp.vegaExposure -and ($gp.vegaExposure -notin @("HIGH","MED","LOW"))) { $errors.Add("Invalid vegaExposure") }
if (-not $hasSell -and $gp.thetaExposure -eq "POS") {
$errors.Add("ThetaExposure POS on all-long legs")
}
} else {
$errors.Add("Missing greekProfile")
}
$structure = ([string]$trade.structure).ToLowerInvariant()
$tradeName = ([string]$trade.name).ToLowerInvariant()
$isStraddle = $structure -match "straddle" -or $tradeName -match "straddle"
$isStrangle = $structure -match "strangle" -or $tradeName -match "strangle"
$isCallDebit = ($structure -match "call") -and ($structure -match "debit") -and ($structure -match "spread")
$isPutDebit = ($structure -match "put") -and ($structure -match "debit") -and ($structure -match "spread")
if ($isStraddle -or $isStrangle) {
if ($legs.Count -ne 2) { $errors.Add("Straddle/Strangle must have exactly 2 legs") }
$callLegs = $legs | Where-Object { $_.side -eq "call" }
$putLegs = $legs | Where-Object { $_.side -eq "put" }
if ($callLegs.Count -ne 1 -or $putLegs.Count -ne 1) { $errors.Add("Straddle/Strangle must have 1 call and 1 put") }
if ($callLegs.Count -eq 1 -and $putLegs.Count -eq 1) {
$callStrike = Normalize-Strike $callLegs[0].strike
$putStrike = Normalize-Strike $putLegs[0].strike
if ($isStraddle -and $callStrike -ne $putStrike) { $errors.Add("Straddle strikes must match") }
if ($isStrangle) {
try {
if ([double]$callStrike -le [double]$putStrike) { $errors.Add("Strangle call strike must be above put strike") }
} catch {
$errors.Add("Strangle strike comparison failed")
}
}
if ($callLegs[0].action -ne "buy" -or $putLegs[0].action -ne "buy") {
$errors.Add("Straddle/Strangle must be long (buy) legs")
}
}
if ($trade.greekProfile -and $trade.greekProfile.deltaBias -and $trade.greekProfile.deltaBias -ne "NEUTRAL") {
$errors.Add("DeltaBias must be NEUTRAL for straddle/strangle")
}
}
if ($isCallDebit) {
$callLegs = $legs | Where-Object { $_.side -eq "call" }
if ($callLegs.Count -ne 2) { $errors.Add("Call debit spread must have 2 call legs") }
$buy = $callLegs | Where-Object { $_.action -eq "buy" }
$sell = $callLegs | Where-Object { $_.action -eq "sell" }
if ($buy.Count -ne 1 -or $sell.Count -ne 1) { $errors.Add("Call debit spread must have 1 buy and 1 sell") }
if ($buy.Count -eq 1 -and $sell.Count -eq 1) {
try {
if ([double](Normalize-Strike $buy[0].strike) -ge [double](Normalize-Strike $sell[0].strike)) {
$errors.Add("Call debit spread buy strike must be below sell strike")
}
} catch {
$errors.Add("Call debit spread strike comparison failed")
}
}
}
if ($isPutDebit) {
$putLegs = $legs | Where-Object { $_.side -eq "put" }
if ($putLegs.Count -ne 2) { $errors.Add("Put debit spread must have 2 put legs") }
$buy = $putLegs | Where-Object { $_.action -eq "buy" }
$sell = $putLegs | Where-Object { $_.action -eq "sell" }
if ($buy.Count -ne 1 -or $sell.Count -ne 1) { $errors.Add("Put debit spread must have 1 buy and 1 sell") }
if ($buy.Count -eq 1 -and $sell.Count -eq 1) {
try {
if ([double](Normalize-Strike $buy[0].strike) -le [double](Normalize-Strike $sell[0].strike)) {
$errors.Add("Put debit spread buy strike must be above sell strike")
}
} catch {
$errors.Add("Put debit spread strike comparison failed")
}
}
}
}
}
return $errors
}
function Parse-GpuLog {
param([string]$Path)
$summary = [ordered]@{ gpu0Used = $false; gpu1Used = $false; samples = 0; error = $null }
if (-not (Test-Path $Path)) {
$summary.error = "gpu log missing"
return $summary
}
$lines = Get-Content -Path $Path
$currentIndex = -1
$gpuIndex = -1
$inGpuUtilSamples = $false
$inUtilBlock = $false
foreach ($line in $lines) {
if ($line -match '^Timestamp') {
$gpuIndex = -1
$currentIndex = -1
$inGpuUtilSamples = $false
$inUtilBlock = $false
continue
}
if ($line -match '^GPU\\s+[0-9A-Fa-f:.]+$') {
$gpuIndex += 1
$currentIndex = $gpuIndex
$inGpuUtilSamples = $false
$inUtilBlock = $false
continue
}
if ($line -match '^\\s*Utilization\\s*$') {
$inUtilBlock = $true
continue
}
if ($line -match '^\\s*GPU Utilization Samples') {
$inGpuUtilSamples = $true
$inUtilBlock = $false
continue
}
if ($line -match '^\\s*(Memory|ENC|DEC) Utilization Samples') {
$inGpuUtilSamples = $false
$inUtilBlock = $false
continue
}
if ($inUtilBlock -and $line -match '^\\s*GPU\\s*:\\s*([0-9]+)\\s*%') {
$util = [int]$Matches[1]
if ($currentIndex -eq 0 -and $util -gt 0) { $summary.gpu0Used = $true }
if ($currentIndex -eq 1 -and $util -gt 0) { $summary.gpu1Used = $true }
$summary.samples += 1
continue
}
if ($inGpuUtilSamples -and $line -match '^\\s*Max\\s*:\\s*([0-9]+)\\s*%') {
$util = [int]$Matches[1]
if ($currentIndex -eq 0 -and $util -gt 0) { $summary.gpu0Used = $true }
if ($currentIndex -eq 1 -and $util -gt 0) { $summary.gpu1Used = $true }
$summary.samples += 1
}
}
return $summary
}
function Get-ProcessorShare {
param(
[string]$SshExePath,
[string]$Target,
[int]$Port,
[string]$ModelName
)
$result = [ordered]@{ cpuPct = $null; gpuPct = $null; raw = $null; error = $null }
try {
$out = & $SshExePath -p $Port $Target "sudo -n docker exec ix-ollama-ollama-1 ollama ps"
$line = $out | Select-String -SimpleMatch $ModelName | Select-Object -First 1
if ($null -eq $line) {
$result.error = "model not found in ollama ps"
return $result
}
$raw = $line.ToString().Trim()
$result.raw = $raw
if ($raw -match '([0-9]+)%\\/([0-9]+)%\\s+CPU\\/GPU') {
$result.cpuPct = [int]$Matches[1]
$result.gpuPct = [int]$Matches[2]
} elseif ($raw -match '([0-9]+)%\\s+GPU') {
$result.cpuPct = 0
$result.gpuPct = [int]$Matches[1]
} else {
$result.error = "CPU/GPU split not parsed"
}
} catch {
$result.error = $_.Exception.Message
}
return $result
}
$prompt = [string](Get-Content -Raw -Path $PromptPath)
$allowed = Get-AllowedLegs -promptText $prompt
$allowedExpiry = $allowed.AllowedExpiry
$allowedLegs = $allowed.AllowedLegs
if ([string]::IsNullOrWhiteSpace($BatchId)) {
$BatchId = (Get-Date).ToString("yyyyMMdd_HHmmss")
}
$outBase = Join-Path -Path (Get-Location) -ChildPath "ollama_runs_remote"
if (-not (Test-Path $outBase)) { New-Item -ItemType Directory -Path $outBase | Out-Null }
$safeModel = $Model -replace '[\\/:*?"<>|]', '_'
$batchDir = Join-Path -Path $outBase -ChildPath ("batch_{0}" -f $BatchId)
if (-not (Test-Path $batchDir)) { New-Item -ItemType Directory -Path $batchDir | Out-Null }
$outDir = Join-Path -Path $batchDir -ChildPath $safeModel
if (-not (Test-Path $outDir)) { New-Item -ItemType Directory -Path $outDir | Out-Null }
$summary = [ordered]@{
model = $Model
baseUrl = $BaseUrl
formatMode = $(if ($UseSchemaFormat) { "schema" } else { "json" })
batchId = $BatchId
gpuMonitor = [ordered]@{
enabled = [bool]$EnableGpuMonitor
sshHost = $SshHost
sshPort = $SshPort
intervalSec = $GpuMonitorIntervalSec
durationSec = $GpuMonitorSeconds
}
runs = @()
}
for ($i = 1; $i -le $Runs; $i++) {
Write-Host "Running $Model (run $i/$Runs)"
$runResult = [ordered]@{ run = $i; ok = $false; errors = @() }
$gpuJob = $null
$gpuLogPath = $null
if ($EnableGpuMonitor) {
$samples = [math]::Max(5, [int]([math]::Ceiling($GpuMonitorSeconds / [double]$GpuMonitorIntervalSec)))
$gpuLogPath = Join-Path $outDir ("gpu_run{0}.csv" -f $i)
$sshTarget = "{0}@{1}" -f $SshUser, $SshHost
$gpuJob = Start-Job -ScriptBlock {
param($sshExe, $target, $port, $samples, $interval, $logPath)
for ($s = 1; $s -le $samples; $s++) {
Add-Content -Path $logPath -Value ("=== SAMPLE {0} {1}" -f $s, (Get-Date).ToString('s'))
try {
$out = & $sshExe -p $port $target "nvidia-smi -q -d UTILIZATION"
Add-Content -Path $logPath -Value $out
} catch {
Add-Content -Path $logPath -Value ("GPU monitor error: $($_.Exception.Message)")
}
Start-Sleep -Seconds $interval
}
} -ArgumentList $SshExe, $sshTarget, $SshPort, $samples, $GpuMonitorIntervalSec, $gpuLogPath
Start-Sleep -Seconds 1
}
$format = "json"
if ($UseSchemaFormat) {
$format = @{
type = "object"
additionalProperties = $false
required = @("selectedExpiry","expiryRationale","strategyBias","recommendedTrades","whyOthersRejected","confidenceScore")
properties = @{
selectedExpiry = @{ type = "string"; minLength = 1 }
expiryRationale = @{ type = "string"; minLength = 1 }
strategyBias = @{ type = "string"; enum = @("DIRECTIONAL","VOLATILITY","NEUTRAL","NO_TRADE") }
recommendedTrades = @{
type = "array"
minItems = 0
maxItems = 3
items = @{
type = "object"
additionalProperties = $false
required = @("name","structure","legs","greekProfile","maxRisk","maxReward","thesisAlignment","invalidation")
properties = @{
name = @{ type = "string"; minLength = 1 }
structure = @{ type = "string"; minLength = 1 }
legs = @{
type = "array"
minItems = 1
maxItems = 4
items = @{
type = "object"
additionalProperties = $false
required = @("side","action","strike","expiry")
properties = @{
side = @{ type = "string"; enum = @("call","put") }
action = @{ type = "string"; enum = @("buy","sell") }
strike = @{ type = @("number","string") }
expiry = @{ type = "string"; minLength = 1 }
}
}
}
greekProfile = @{
type = "object"
additionalProperties = $false
required = @("deltaBias","gammaExposure","thetaExposure","vegaExposure")
properties = @{
deltaBias = @{ type = "string"; enum = @("POS","NEG","NEUTRAL") }
gammaExposure = @{ type = "string"; enum = @("HIGH","MED","LOW") }
thetaExposure = @{ type = "string"; enum = @("POS","NEG","LOW") }
vegaExposure = @{ type = "string"; enum = @("HIGH","MED","LOW") }
}
}
maxRisk = @{ anyOf = @(@{ type = "string"; minLength = 1 }, @{ type = "number" }) }
maxReward = @{ anyOf = @(@{ type = "string"; minLength = 1 }, @{ type = "number" }) }
thesisAlignment = @{ type = "string"; minLength = 1 }
invalidation = @{ type = "string"; minLength = 1 }
managementNotes = @{ type = "string" }
}
}
}
whyOthersRejected = @{
type = "array"
items = @{ type = "string" }
}
confidenceScore = @{ type = "number"; minimum = 0; maximum = 100 }
}
}
}
$options = @{
temperature = 0
top_k = $TopK
top_p = $TopP
seed = $Seed
repeat_penalty = $RepeatPenalty
num_ctx = $NumCtx
num_predict = $NumPredict
}
if ($NumBatch -gt 0) {
$options.num_batch = $NumBatch
}
if ($NumGpuLayers -gt 0) {
$options.num_gpu_layers = $NumGpuLayers
}
$body = @{
model = $Model
prompt = $prompt
format = $format
stream = $false
options = $options
} | ConvertTo-Json -Depth 10
try {
$resp = Invoke-RestMethod -Uri "$BaseUrl/api/generate" -Method Post -Body $body -ContentType "application/json" -TimeoutSec $TimeoutSec
} catch {
$runResult.errors = @("API error: $($_.Exception.Message)")
$summary.runs += $runResult
if ($gpuJob) { Stop-Job -Job $gpuJob | Out-Null }
continue
} finally {
if ($gpuJob) {
Wait-Job -Job $gpuJob -Timeout 5 | Out-Null
if ($gpuJob.State -eq "Running") { Stop-Job -Job $gpuJob | Out-Null }
Remove-Job -Job $gpuJob | Out-Null
}
}
$raw = [string]$resp.response
$jsonPath = Join-Path $outDir ("run{0}.json" -f $i)
Set-Content -Path $jsonPath -Value $raw -Encoding ASCII
try {
$parsed = $raw | ConvertFrom-Json
$errors = Test-TradeSchema -obj $parsed -allowedExpiry $allowedExpiry -allowedLegs $allowedLegs
if ($errors.Count -eq 0) {
$runResult.ok = $true
} else {
$runResult.errors = $errors
}
} catch {
$runResult.errors = @("Invalid JSON: $($_.Exception.Message)")
}
if ($gpuLogPath) {
$runResult.gpuLog = $gpuLogPath
$runResult.gpuUsage = Parse-GpuLog -Path $gpuLogPath
}
if ($CheckProcessor) {
$sshTarget = "{0}@{1}" -f $SshUser, $SshHost
$proc = Get-ProcessorShare -SshExePath $SshExe -Target $sshTarget -Port $SshPort -ModelName $Model
$runResult.processor = $proc
if ($proc.cpuPct -ne $null) {
$runResult.gpuOnly = ($proc.cpuPct -eq 0)
}
}
$summary.runs += $runResult
}
$summaryPath = Join-Path $outDir "summary.json"
$summary | ConvertTo-Json -Depth 6 | Set-Content -Path $summaryPath -Encoding ASCII
$summary | ConvertTo-Json -Depth 6