diff --git a/relay/playbooks/ember-tune-pc1.ps1 b/relay/playbooks/ember-tune-pc1.ps1 index 741619f3..d498f652 100644 --- a/relay/playbooks/ember-tune-pc1.ps1 +++ b/relay/playbooks/ember-tune-pc1.ps1 @@ -145,6 +145,12 @@ $rows = 0 $firstStatusAt = $null $lastMining = $null $lastEngineLine = '' +function EngineLogDump([string] $why) { + Write-Output ('===== engine log tail (' + $why + ')') + $t = Get-ChildItem -Path $sLogs -Filter 'app-*.log' -ErrorAction SilentlyContinue | Sort-Object LastWriteTime -Descending | Select-Object -First 1 + if ($t) { Get-Content -LiteralPath $t.FullName -ErrorAction SilentlyContinue | Where-Object { $_ -notmatch 'status: accepted 0 blocks' } | Select-Object -Last 80 | ForEach-Object { Write-Output (' ' + $_) } } else { Write-Output ' (no app-*.log in the scratch logs folder)' } + if (Test-Path -LiteralPath $errFile) { Write-Output '===== engine stderr'; Get-Content -LiteralPath $errFile -ErrorAction SilentlyContinue | Select-Object -Last 20 | ForEach-Object { Write-Output (' ' + $_) } } +} function EngineTail() { $t = Get-ChildItem -Path $sLogs -Filter 'app-*.log' -ErrorAction SilentlyContinue | Sort-Object LastWriteTime -Descending | Select-Object -First 1; if ($t) { $l = Get-Content -LiteralPath $t.FullName -Tail 1 -ErrorAction SilentlyContinue; if ($l) { return [string]$l } }; return '' } while (-not $p.HasExited) { Start-Sleep -Seconds 5 @@ -156,12 +162,14 @@ while (-not $p.HasExited) { } if ($firstStatusAt -and -not $lastMining -and ((Get-Date) - $firstStatusAt).TotalSeconds -gt 120) { Write-Output ('RESULT TUNE error=not_mining reason=no_card_mined_within_120_s_of_the_first_status_line last_log_line=' + ($lastEngineLine -replace '\s+', '_')) + EngineLogDump 'watchdog: not mining' EndTree $p.Id 'watchdog: not mining' Write-Output 'RESULT TUNE error=no_rows' exit 3 } if ($lastMining -and ((Get-Date) - $lastMining).TotalSeconds -gt 300) { Write-Output ('RESULT TUNE error=stopped_mining reason=every_card_idle_for_300_s last_log_line=' + ($lastEngineLine -replace '\s+', '_')) + EngineLogDump 'watchdog: stopped mining' EndTree $p.Id 'watchdog: stopped mining' Write-Output 'RESULT TUNE error=no_rows' exit 3