diff --git a/AGENTS.md b/AGENTS.md new file mode 100755 index 0000000..43a614f --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,98 @@ +# vLLM Scripts — Agent Instructions + +## Project Overview +Scripts for running large language models via vLLM on two local machines: +- **dankserver**: NVIDIA GPU + System RAM (frank-default-small) +- **dankclaw**: AMD Strix Halo Shared Memory (frank-default-large) + +## Directory Structure +``` +vllmScripts/ +├── AGENTS.md # This file — agent instructions +├── README.md # Human-readable project overview +├── deploy.sh # Automated deployment script +├── scripts/ # Executable vLLM launch scripts +│ ├── validate-vllm.sh # Validation tool +│ └── *.sh # Inference server scripts +├── tests/ # Test suite +│ └── test_validator.sh +├── docs/ # Knowledge base +│ ├── README.md +│ └── *.md # Research notes +└── failedAttempts/ # Archived scripts (for reference) +``` + +## Running Tests +```bash +# Test validator on dankserver +bash tests/test_validator.sh dankserver + +# Test validator on dankclaw +bash tests/test_validator.sh dankclaw + +# Run all tests +bash tests/test_validator.sh dankserver && bash tests/test_validator.sh dankclaw +``` + +## Deployment +```bash +# Deploy all scripts to all targets +bash deploy.sh all + +# Deploy specific script to dankserver +bash deploy.sh dankserver vllm-qwen36.sh + +# Deploy specific script to dankclaw +bash deploy.sh dankclaw vllm-thinkingcap.sh +``` + +## Git Workflow + +### Branching Strategy +- `main`: Stable, working scripts only +- `vllm-qwen36`: Active development for Qwen3.6 models +- Feature branches: `feature/` for experiments + +### Worktree Pattern +For parallel development: +```bash +# Create worktree for feature +git worktree add ../vllm-feature feature/qwen36-awq + +# When done, merge back +git worktree remove ../vllm-feature +``` + +### Commit Guidelines +- Use descriptive commit messages: `git commit -m "Add AWQ DFlash script"` +- Commit before deploying: `git add -A && git commit -m "Update for deployment"` +- Push to remote: `git push origin ` + +### Deployment Flow +1. Make changes locally +2. Run tests: `bash tests/test_validator.sh dankserver` +3. Commit: `git add -A && git commit -m "Description"` +4. Deploy: `bash deploy.sh dankserver` +5. Verify: `bash scripts/validate-vllm.sh -t dankserver` + +## Remote Servers + +### dankserver +- SSH: `dank@dankserver` +- Scripts: `~/vllm/` +- Service: `vllm-awq-qwen36.service` +- Model: frank-default-small (AWQ 4bit) +- Port: 34401 + +### dankclaw +- SSH: `dank@dankclaw` +- Scripts: `~/vllm/` +- Service: `vllm-thinkingcap.service` +- Model: frank-default-large (FP8) +- Port: 3090 + +## Troubleshooting +- Service failed: `systemctl status ` +- Container stuck: `docker stop && docker rm ` +- OOM error: Reduce `--max-model-len` or increase `--cpu-offload-gb` +- Prefix caching conflict: Remove `--enable-prefix-caching` from script diff --git a/README.md b/README.md index 4dbcdbf..410b0f2 100644 --- a/README.md +++ b/README.md @@ -4,10 +4,27 @@ Scripts for running large language models via vLLM on two local machines. ## Targets -| Host | User | Codename | Architecture | -|------|------|----------|-------------| -| dankserver | dank | DankServer | NVIDIA GPU + System RAM | -| dankclaw | dank | DankClaw | AMD Strix Halo (Shared Memory) | +| Host | User | Codename | Architecture | Service Status | +|------|------|----------|-------------|----------------| +| dankserver | dank | DankServer | NVIDIA GPU + System RAM | `vllm-awq-qwen36.service` (failed) | +| dankclaw | dank | DankClaw | AMD Strix Halo (Shared Memory) | `vllm-thinkingcap.service` (failed) | + +## File Locations + +### Local (vllmScripts/) +- `scripts/` — Active inference scripts +- `tests/` — Test suite +- `docs/` — Knowledge base +- `failedAttempts/` — Archived scripts + +### Remote (dankserver) +- `~/vllm/` — Deployed inference scripts +- `~/vllm/failedAttempts/` — Archived scripts +- `~/vLLM/` — Legacy project directory (vllm-qwen36.sh) + +### Remote (dankclaw) +- `~/vllm/` — vLLM source code (not inference scripts) +- `~/vLLM/` — Inference scripts (if deployed) ## Models diff --git a/deploy.sh b/deploy.sh new file mode 100755 index 0000000..eeb6d47 --- /dev/null +++ b/deploy.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +set -euo pipefail + +# deploy.sh — Deploy inference scripts to remote servers +# +# Usage: ./deploy.sh [TARGET] [SCRIPT] +# TARGET: dankserver, dankclaw, or all +# SCRIPT: Script name to deploy (default: all scripts in scripts/) + +TARGET="${1:-all}" +SCRIPT="${2:-}" + +# Remote configuration +declare -A REMOTE_HOSTS +REMOTE_HOSTS=( + ["dankserver"]="dank@dankserver" + ["dankclaw"]="dank@dankclaw" +) + +declare -A REMOTE_DIRS +REMOTE_DIRS=( + ["dankserver"]="/home/dank/vllm" + ["dankclaw"]="/home/dank/vllm" +) + +declare -A REMOTE_SERVICES +REMOTE_SERVICES=( + ["dankserver"]="vllm-awq-qwen36.service" + ["dankclaw"]="vllm-thinkingcap.service" +) + +deploy_to() { + local target="$1" + local remote="${REMOTE_HOSTS[$target]}" + local remote_dir="${REMOTE_DIRS[$target]}" + local service="${REMOTE_SERVICES[$target]}" + + echo "=== Deploying to $target ===" + + # Create remote directory if it doesn't exist + sshpass -p 'Assface.3241' ssh "$remote" "mkdir -p $remote_dir" + + # Deploy scripts + if [ -n "$SCRIPT" ]; then + if [ -f "scripts/$SCRIPT" ]; then + echo " Deploying scripts/$SCRIPT..." + sshpass -p 'Assface.3241' scp "scripts/$SCRIPT" "$remote:$remote_dir/$SCRIPT" + else + echo " ERROR: scripts/$SCRIPT not found" + return 1 + fi + else + for script in scripts/*.sh; do + if [ -f "$script" ]; then + local name=$(basename "$script") + echo " Deploying $name..." + sshpass -p 'Assface.3241' scp "$script" "$remote:$remote_dir/$name" + fi + done + fi + + # Make scripts executable + if [ -n "$SCRIPT" ]; then + sshpass -p 'Assface.3241' ssh "$remote" "chmod +x $remote_dir/$SCRIPT" + else + sshpass -p 'Assface.3241' ssh "$remote" "chmod +x $remote_dir/*.sh" + fi + + # Restart service if it exists + echo " Restarting $service..." + sshpass -p 'Assface.3241' ssh "$remote" "echo 'Assface.3241' | sudo -S systemctl restart $service 2>/dev/null || echo 'Service $service not found or failed'" + + echo " Deploy to $target complete" + echo "" +} + +case "$TARGET" in + dankserver|dankclaw) + deploy_to "$TARGET" + ;; + all) + for target in "${!REMOTE_HOSTS[@]}"; do + deploy_to "$target" + done + ;; + *) + echo "ERROR: Unknown target $TARGET" + echo "Usage: $0 [dankserver|dankclaw|all] [script]" + exit 1 + ;; +esac + +echo "=== Deployment complete ==="