Run the same checks on Linux that the Windows scripts run

The product could only reach a GPU through one hard-coded route: `wsl.exe -d
Ubuntu`, a device named "RTX 4090", and launch scripts that existed only as
PowerShell. On this machine `WSL_DISTRO_NAME` is already set, so the local
shell is Linux and the wsl.exe hop was unnecessary; on any other Linux host
it is impossible. Every SGLang, vLLM and KTransformers profile was reported
"missing" here even with a working 4090.

profiles.py now asks the host what it is instead of assuming: `uses_wsl()`
is false inside WSL and only true when wsl.exe is actually on PATH,
`host_shell()` runs `bash -lc` directly or through `wsl.exe -d
settings.wsl_distro`, `gpu_available()` replaced `wsl_gpu_available()`, and
`launch_script()` picks the .sh twin on a Linux host and the .ps1 on a
Windows one. The schema field `wsl_gpu` became `gpu_available` because the
old name lied on Linux.

Training no longer fails on a card name. `cuda_bf16_training()` gates on
cuda_available plus bf16_supported, which the training environment already
records, so an RTX 5090 or a 4080 passes and a non-BF16 4090 still fails.

Fourteen .sh twins cover the six launch profiles plus install, verify,
serve, download and finetune; the six WSL-hopping .ps1 files read
ELY_EYE_WSL_DISTRO instead of Ubuntu. PowerShell parsing is not verified
here because no pwsh exists on this host.

ci.yml runs pytest, ruff, go build/vet/test/gofmt, the Vite build and a
docker build-plus-curl on every push. To make that lint gate mean anything,
the 17 pre-existing ruff findings across backend/src are cleared: 14
mechanical fixes plus three probe handlers that now name the failure
families they absorb (OSError, SubprocessError, ValueError) instead of
catching Exception.

Verification: pytest 239 passed; ruff check backend/src bench clean (17 at
HEAD); mypy on the 12 touched files 14 errors, identical at HEAD; pyright on
the same files 8 errors, identical at HEAD; bash -n clean on all 15 .sh.
This commit is contained in:
ZacharyZhang-NY committed 2026-10-08 02:12:29 -04:00
1 parent 36f40c4c59
commit 922d01202f
34 files changed
+482 -88

No files matched your search

+4 -2
View File
@@ -277,8 +277,10 @@ def create_app() -> FastAPI:
return benchmark_prediction_report(settings, settings.data_dir / "benchmark_predictions")
@app.get("/api/runtime/profiles", response_model=RuntimeProfileReport)
def runtime_profiles() -> RuntimeProfileReport:
return runtime_profile_report()
def runtime_profiles(
settings: Annotated[Settings, Depends(settings_dep)],
) -> RuntimeProfileReport:
return runtime_profile_report(settings)
@app.get("/api/runtime/adapters", response_model=AdapterLoadReport)
def runtime_adapters(settings: Annotated[Settings, Depends(settings_dep)]) -> AdapterLoadReport:
+1 -1
View File
@@ -325,7 +325,7 @@ def run_benchmark(
@app.command("runtime-profiles")
def runtime_profiles() -> None:
typer.echo(runtime_profile_report().model_dump_json(indent=2))
typer.echo(runtime_profile_report(get_settings()).model_dump_json(indent=2))
@app.command("research-modules")
+1
View File
@@ -27,6 +27,7 @@ class Settings(BaseSettings):
sglang_base_url: str = "http://127.0.0.1:8000/v1"
sglang_api_key: str = "ely-eye-local"
ui_dir: Path | None = None
wsl_distro: str = "Ubuntu"
dense_embeddings_enabled: bool = False
live_context_tokens: int = 262_144
extreme_context_tokens: int = 1_010_000
-1
View File
@@ -2,7 +2,6 @@ from __future__ import annotations
import json
from dataclasses import dataclass
from pathlib import Path
from typing import Any
import numpy as np
+1 -1
View File
@@ -64,5 +64,5 @@ class EvidenceGraphService:
def claim_node_id(question: str, answer: str) -> str:
digest = hashlib.sha256(f"{question}\n{answer}".encode("utf-8")).hexdigest()[:20]
digest = hashlib.sha256(f"{question}\n{answer}".encode()).hexdigest()[:20]
return f"claim_{digest}"
+64 -42
View File
@@ -6,9 +6,9 @@ import shutil
import subprocess
from pathlib import Path
from .config import Settings
from .schemas import RuntimeLaunchProfile, RuntimeProfileReport, RuntimeProfileValidation
EXTREME_YARN_OVERRIDE = {
"text_config": {
"rope_parameters": {
@@ -24,7 +24,7 @@ EXTREME_YARN_OVERRIDE = {
}
def runtime_profile_report(project_root: Path | None = None) -> RuntimeProfileReport:
def runtime_profile_report(settings: Settings, project_root: Path | None = None) -> RuntimeProfileReport:
root = project_root or Path.cwd()
profiles = [
RuntimeLaunchProfile(
@@ -185,29 +185,55 @@ def runtime_profile_report(project_root: Path | None = None) -> RuntimeProfileRe
required_features=["GGUF", "offline demo", "quantized KV"],
),
]
validations = validate_runtime_profiles(root, profiles)
validations = validate_runtime_profiles(root, profiles, settings)
ready = all(validation.status == "ready" for validation in validations)
return RuntimeProfileReport(
status="ready" if ready else "incomplete",
profiles=profiles,
profiles=[profile.model_copy(update={"script": launch_script(profile.script)}) for profile in profiles],
validations=validations,
)
def uses_wsl() -> bool:
"""A Windows process reaches Linux through wsl.exe; inside WSL the local shell already is Linux."""
if os.environ.get("WSL_DISTRO_NAME"):
return False
return shutil.which("wsl.exe") is not None
def launch_script(script: str) -> str:
"""A Windows host runs the .ps1; a Linux host, WSL included, runs the .sh twin."""
return script if uses_wsl() else str(Path(script).with_suffix(".sh"))
def host_shell(command: str, settings: Settings) -> list[str]:
"""Run a shell command where the launch environments live."""
if uses_wsl():
return ["wsl.exe", "-d", settings.wsl_distro, "bash", "-lc", command]
return ["bash", "-lc", command]
def host_path(root: Path) -> str:
"""The path the host shell can cd into; a Windows path is reached through the WSL mount."""
resolved = root.resolve()
return windows_path_to_wsl(resolved) if uses_wsl() else str(resolved)
def validate_runtime_profiles(
root: Path,
profiles: list[RuntimeLaunchProfile],
settings: Settings,
) -> list[RuntimeProfileValidation]:
wsl_gpu = wsl_gpu_available()
gpu = gpu_available(settings)
package_cache = {
".venv-linux": linux_package_versions(root, ".venv-linux"),
".venv-linux-vllm": linux_package_versions(root, ".venv-linux-vllm"),
".venv-linux": linux_package_versions(root, ".venv-linux", settings),
".venv-linux-vllm": linux_package_versions(root, ".venv-linux-vllm", settings),
}
probe_cache: dict[tuple[str, str, str | None], dict[str, object]] = {}
llama_server = resolve_llama_server()
validations: list[RuntimeProfileValidation] = []
for profile in profiles:
script_exists = (root / profile.script).exists()
script_exists = (root / launch_script(profile.script)).exists()
package = package_for_runtime(profile.runtime)
env_name = linux_env_for_runtime(profile.runtime)
linux_packages = package_cache.get(env_name, {})
@@ -216,24 +242,24 @@ def validate_runtime_profiles(
executable = llama_server if profile.runtime == "llama.cpp" else None
probe_key = (profile.runtime, env_name, executable)
if probe_key not in probe_cache:
probe_cache[probe_key] = runtime_launcher_probe(root, profile.runtime, env_name, executable)
probe_cache[probe_key] = runtime_launcher_probe(root, profile.runtime, env_name, executable, settings)
probe = probe_cache[probe_key]
probe_status = str(probe.get("status", "skipped"))
probe_passed = probe_status == "passed"
if profile.runtime in {"SGLang", "vLLM", "KTransformers"}:
ready = script_exists and bool(package_version) and wsl_gpu and probe_passed
ready = script_exists and bool(package_version) and gpu and probe_passed
if profile.runtime == "KTransformers":
ready = ready and bool(kt_sglang_version)
detail = (
(
f"WSL GPU ready; {package} {package_version}; "
f"GPU ready; {package} {package_version}; "
f"sglang-kt entry {kt_sglang_version}"
)
if profile.runtime == "KTransformers" and ready
else f"WSL GPU ready; {package} {package_version}; env {env_name}"
else f"GPU ready; {package} {package_version}; env {env_name}"
if ready
else (
f"script={script_exists}, wsl_gpu={wsl_gpu}, env={env_name}, "
f"script={script_exists}, gpu={gpu}, env={env_name}, "
f"{package}={package_version}, sglang={kt_sglang_version}"
)
)
@@ -253,7 +279,7 @@ def validate_runtime_profiles(
package=package,
package_version=package_version,
executable=executable,
wsl_gpu=wsl_gpu,
gpu_available=gpu,
probe_command=[str(item) for item in probe.get("command", [])],
probe_status=probe_status,
probe_exit_code=probe.get("exit_code") if isinstance(probe.get("exit_code"), int) else None,
@@ -278,7 +304,7 @@ def linux_env_for_runtime(runtime: str) -> str:
return ".venv-linux"
def linux_package_versions(root: Path, env_name: str) -> dict[str, str]:
def linux_package_versions(root: Path, env_name: str, settings: Settings) -> dict[str, str]:
script = (
"import importlib.metadata as md, json;"
"names=['sglang','vllm','ktransformers','torch','torchvision','qwen-vl-utils'];"
@@ -290,25 +316,21 @@ def linux_package_versions(root: Path, env_name: str) -> dict[str, str]:
)
try:
output = subprocess.check_output(
[
"wsl.exe",
"-d",
"Ubuntu",
"bash",
"-lc",
(
f"cd {wsl_quote(windows_path_to_wsl(root))} "
f"&& test -x {env_name}/bin/python "
f"&& {env_name}/bin/python -c {wsl_quote(script)}"
),
],
host_shell(
f"cd {wsl_quote(host_path(root))} "
f"&& test -x {env_name}/bin/python "
f"&& {env_name}/bin/python -c {wsl_quote(script)}",
settings,
),
text=True,
stderr=subprocess.DEVNULL,
timeout=20,
)
data = json.loads(output.strip())
return {str(key): str(value) for key, value in data.items()}
except Exception:
payload = json.loads(output.strip())
if not isinstance(payload, dict):
return {}
return {str(key): str(value) for key, value in payload.items()}
except (OSError, subprocess.SubprocessError, ValueError):
return {}
@@ -317,6 +339,7 @@ def runtime_launcher_probe(
runtime: str,
env_name: str,
executable: str | None,
settings: Settings,
) -> dict[str, object]:
if runtime == "SGLang":
probe_script = (
@@ -325,7 +348,7 @@ def runtime_launcher_probe(
"print(spec.origin if spec else 'missing'); "
"sys.exit(0 if spec else 1)"
)
command = wsl_python_command(root, env_name, f"-c {wsl_quote(probe_script)}")
command = host_python_command(root, env_name, f"-c {wsl_quote(probe_script)}", settings)
elif runtime == "vLLM":
probe_script = (
"import importlib.util, sys; "
@@ -333,10 +356,10 @@ def runtime_launcher_probe(
"print(spec.origin if spec else 'missing'); "
"sys.exit(0 if spec else 1)"
)
command = wsl_python_command(root, env_name, f"-c {wsl_quote(probe_script)}")
command = host_python_command(root, env_name, f"-c {wsl_quote(probe_script)}", settings)
elif runtime == "KTransformers":
probe_script = "import ktransformers, sglang; print('ktransformers and sglang import ok')"
command = wsl_python_command(root, env_name, f"-c {wsl_quote(probe_script)}")
command = host_python_command(root, env_name, f"-c {wsl_quote(probe_script)}", settings)
elif runtime == "llama.cpp" and executable:
command = [executable, "--help"]
else:
@@ -349,7 +372,7 @@ def runtime_launcher_probe(
timeout=30,
check=False,
)
except Exception as exc:
except (OSError, subprocess.SubprocessError) as exc:
return {
"status": "failed",
"command": command,
@@ -365,13 +388,13 @@ def runtime_launcher_probe(
}
def wsl_python_command(root: Path, env_name: str, python_args: str) -> list[str]:
def host_python_command(root: Path, env_name: str, python_args: str, settings: Settings) -> list[str]:
shell_command = (
f"cd {wsl_quote(windows_path_to_wsl(root))} "
f"cd {wsl_quote(host_path(root))} "
f"&& test -x {env_name}/bin/python "
f"&& {env_name}/bin/python {python_args}"
)
return ["wsl.exe", "-d", "Ubuntu", "bash", "-lc", shell_command]
return host_shell(shell_command, settings)
def compact_output(stdout: str, stderr: str, limit: int = 800) -> str:
@@ -379,18 +402,17 @@ def compact_output(stdout: str, stderr: str, limit: int = 800) -> str:
return text[:limit]
def wsl_gpu_available() -> bool:
if shutil.which("wsl.exe") is None:
return False
def gpu_available(settings: Settings) -> bool:
"""Ask the host for a CUDA-capable GPU without naming a card: any GPU the driver lists will do."""
try:
output = subprocess.check_output(
["wsl.exe", "-d", "Ubuntu", "nvidia-smi", "-L"],
host_shell("nvidia-smi -L", settings),
text=True,
stderr=subprocess.DEVNULL,
timeout=10,
)
return "GPU" in output
except Exception:
except (OSError, subprocess.SubprocessError):
return False
+1 -1
View File
@@ -232,7 +232,7 @@ class RuntimeChecks:
runtime.unload()
def _runtime_profile_check(self, proof_root: Path) -> ProofCheck:
report = runtime_profile_report()
report = runtime_profile_report(self.settings)
artifact = proof_root / "runtime_profiles.json"
artifact.write_text(report.model_dump_json(indent=2), encoding="utf-8")
required_runtimes = {"SGLang", "vLLM", "KTransformers", "llama.cpp"}
+7 -7
View File
@@ -24,6 +24,11 @@ from ..training import (
)
def cuda_bf16_training(environment: dict[str, Any]) -> bool:
"""The requirement is CUDA plus BF16 support, not one card name written down in 2025."""
return bool(environment.get("cuda_available")) and bool(environment.get("bf16_supported"))
def count_safetensor_keys(path: Path) -> int:
return int(safetensor_weight_stats(path)["tensor_count"])
@@ -173,8 +178,7 @@ def validate_training_trace(manifest: AdapterManifest, adapter_root: Path) -> di
}
)
environment = summary.get("environment") if isinstance(summary.get("environment"), dict) else {}
device_name = str(environment.get("device_name") or "")
report["cuda_training"] = bool(environment.get("cuda_available")) and "RTX 4090" in device_name
report["cuda_training"] = cuda_bf16_training(environment)
report["bf16_supported"] = bool(environment.get("bf16_supported"))
report["valid"] = (
report["trace_sha256_ok"]
@@ -282,11 +286,7 @@ def validate_training_proof(manifest: AdapterManifest, adapter_root: Path) -> di
and int(proof_weight_stats.get("nonzero_elements") or 0) > 0
and bool(proof_weight_stats.get("all_finite"))
),
"proof_cuda_bf16": (
bool(proof.get("bf16"))
and bool(proof_environment.get("cuda_available"))
and "RTX 4090" in str(proof_environment.get("device_name") or "")
),
"proof_cuda_bf16": bool(proof.get("bf16")) and cuda_bf16_training(proof_environment),
"proof_bf16_supported": bool(proof_environment.get("bf16_supported")),
"proof_contract_ok": training_payload_contract_ok(proof, manifest),
"proof_optimizer_ok": proof.get("optimizer") == "paged_adamw_8bit",
+1 -1
View File
@@ -155,7 +155,7 @@ class ResearchModuleService:
"entries": layer.entries,
"hit_rate": hit_rate,
"evict": hit_rate < 0.25 and layer.entries > 0,
"replay_token": hashlib.sha256(f"{layer.layer}:{layer.entries}".encode("utf-8")).hexdigest()[:16],
"replay_token": hashlib.sha256(f"{layer.layer}:{layer.entries}".encode()).hexdigest()[:16],
}
)
return {"policy": "hit-rate weighted replay buffer", "layers": rows}
+1 -1
View File
@@ -439,7 +439,7 @@ class RuntimeProfileValidation(BaseModel):
package: str | None = None
package_version: str | None = None
executable: str | None = None
wsl_gpu: bool = False
gpu_available: bool = False
probe_command: list[str] = Field(default_factory=list)
probe_status: Literal["passed", "failed", "skipped"] = "skipped"
probe_exit_code: int | None = None
+10 -11
View File
@@ -9,7 +9,7 @@ import shutil
import subprocess
import time
from contextlib import contextmanager
from datetime import datetime, timezone
from datetime import UTC, datetime
from importlib.metadata import PackageNotFoundError, version
from pathlib import Path
from typing import Any, TextIO
@@ -20,7 +20,6 @@ from .config import Settings, get_settings
from .db import Database
from .schemas import AdapterManifest, EvidenceAtom, TrainingAdapterKind
SYSTEM_PROMPT = (
"You are Ely-Eye. Answer from supplied evidence, return compact JSON, "
"and cite atom ids exactly."
@@ -433,7 +432,7 @@ class TrainingService:
gradient_accumulation_steps or self.settings.train_gradient_accumulation_steps
)
learning_rate = learning_rate or self.settings.train_learning_rate
started_at = datetime.now(timezone.utc)
started_at = datetime.now(UTC)
wall_start = time.perf_counter()
trace_path = output_dir / "training_trace.jsonl"
@@ -575,7 +574,7 @@ class TrainingService:
processor.save_pretrained(output_dir)
self.write_lora_config(output_dir, kind, rank, alpha, model_id)
weight_stats = safetensor_weight_stats(output_dir / "adapter_model.safetensors")
finished_at = datetime.now(timezone.utc)
finished_at = datetime.now(UTC)
wall_seconds = time.perf_counter() - wall_start
summary_path = output_dir / "training_summary.json"
proof_path = output_dir / "training_proof.json"
@@ -671,7 +670,7 @@ class TrainingService:
manifest = AdapterManifest(
adapter_id=adapter_id(kind, dataset_path, model_id),
kind=kind,
created_at=datetime.now(timezone.utc),
created_at=datetime.now(UTC),
proof_version=TRAINING_PROOF_VERSION,
training_method=TRAINING_METHOD,
precision=TRAINING_PRECISION,
@@ -756,7 +755,7 @@ class TrainingService:
output_dir.mkdir(parents=True, exist_ok=True)
max_length = max_length or self.settings.train_max_length
learning_rate = learning_rate or self.settings.train_learning_rate
started_at = datetime.now(timezone.utc)
started_at = datetime.now(UTC)
wall_start = time.perf_counter()
trace_path = output_dir / "training_trace.jsonl"
@@ -882,7 +881,7 @@ class TrainingService:
processor.save_pretrained(output_dir)
self.write_lora_config(output_dir, TrainingAdapterKind.hme_retrieval, rank, alpha, model_id)
weight_stats = safetensor_weight_stats(output_dir / "adapter_model.safetensors")
finished_at = datetime.now(timezone.utc)
finished_at = datetime.now(UTC)
wall_seconds = time.perf_counter() - wall_start
summary_path = output_dir / "training_summary.json"
proof_path = output_dir / "training_proof.json"
@@ -977,7 +976,7 @@ class TrainingService:
manifest = AdapterManifest(
adapter_id=adapter_id(TrainingAdapterKind.hme_retrieval, dataset_path, model_id),
kind=TrainingAdapterKind.hme_retrieval,
created_at=datetime.now(timezone.utc),
created_at=datetime.now(UTC),
proof_version=TRAINING_PROOF_VERSION,
training_method=TRAINING_METHOD,
precision=TRAINING_PRECISION,
@@ -1269,7 +1268,7 @@ def trainable_update_l2(before_update: dict[str, Any], named_parameters: list[tu
def require_cuda_bf16(torch_module: Any) -> None:
if not torch_module.cuda.is_available():
raise RuntimeError("PRD 12.2 training requires CUDA BF16 on the local RTX 4090.")
raise RuntimeError("training needs a CUDA device; none is available.")
bf16_supported = getattr(torch_module.cuda, "is_bf16_supported", lambda: False)
if not bool(bf16_supported()):
raise RuntimeError("PRD 12.2 training requires BF16-capable CUDA hardware.")
@@ -1598,7 +1597,7 @@ def slugify(value: str) -> str:
def adapter_id(kind: TrainingAdapterKind, dataset_path: Path, model_id: str) -> str:
raw = f"{kind.value}:{model_id}:{dataset_path.resolve()}:{datetime.now(timezone.utc).isoformat()}"
raw = f"{kind.value}:{model_id}:{dataset_path.resolve()}:{datetime.now(UTC).isoformat()}"
return f"{kind.value}_{sha256_bytes(raw.encode('utf-8'))[:12]}"
@@ -1632,7 +1631,7 @@ def sha256_bytes(data: bytes) -> str:
class _DigestWriter:
def __init__(self, digest: "hashlib._Hash") -> None:
def __init__(self, digest: hashlib._Hash) -> None:
self.digest = digest
def write(self, data: bytes) -> int:
-1
View File
@@ -7,7 +7,6 @@ from typing import Any
from .schemas import CitationReport, CompiledContext
ATOM_PATTERN = re.compile(r"[A-Za-z0-9_./\\:-]+:(?:text|image|page|frame|region)[A-Za-z0-9_./\\:#=-]*")