From dea8d2df4d6fa20edd056264ac4bd9bc7799b52a Mon Sep 17 00:00:00 2001 From: finn Date: Mon, 20 Apr 2026 00:40:38 -0700 Subject: [PATCH] add sleep to prevent rclone before fuse mod, and add performance caching --- hub/onboard-spoke.sh | 223 +++++++++++++++++++++++-------------------- 1 file changed, 119 insertions(+), 104 deletions(-) diff --git a/hub/onboard-spoke.sh b/hub/onboard-spoke.sh index 6c63983..90f5a96 100755 --- a/hub/onboard-spoke.sh +++ b/hub/onboard-spoke.sh @@ -11,44 +11,51 @@ YELLOW='\033[1;33m' CYAN='\033[0;36m' NC='\033[0m' -info() { echo -e "${GREEN}[+]${NC} $*"; } -warn() { echo -e "${YELLOW}[!]${NC} $*"; } -die() { echo -e "${RED}[ERROR]${NC} $*" >&2; exit 1; } -header() { echo -e "\n${CYAN}══════════════════════════════════════════${NC}"; echo -e "${CYAN} $*${NC}"; echo -e "${CYAN}══════════════════════════════════════════${NC}"; } +info() { echo -e "${GREEN}[+]${NC} $*"; } +warn() { echo -e "${YELLOW}[!]${NC} $*"; } +die() { + echo -e "${RED}[ERROR]${NC} $*" >&2 + exit 1 +} +header() { + echo -e "\n${CYAN}══════════════════════════════════════════${NC}" + echo -e "${CYAN} $*${NC}" + echo -e "${CYAN}══════════════════════════════════════════${NC}" +} check_deps() { - local missing=() - for cmd in "$@"; do - if ! command -v "$cmd" >/dev/null 2>&1; then - missing+=("$cmd") - fi - done - if [ ${#missing[@]} -gt 0 ]; then - die "Missing required dependencies: ${missing[*]}" + local missing=() + for cmd in "$@"; do + if ! command -v "$cmd" >/dev/null 2>&1; then + missing+=("$cmd") fi + done + if [ ${#missing[@]} -gt 0 ]; then + die "Missing required dependencies: ${missing[*]}" + fi } retry_or_abort() { - local test_cmd="$1" - local fail_msg="$2" - while true; do - if eval "$test_cmd" 2>/dev/null; then - return 0 - fi - echo "" - warn "$fail_msg" - echo -e " ${YELLOW}[R]${NC} Retry ${RED}[A]${NC} Abort" - read -rp "Choice: " CHOICE - case "${CHOICE,,}" in - r) info "Retrying..." ;; - a) die "Aborted." ;; - *) warn "Press R to retry or A to abort." ;; - esac - done + local test_cmd="$1" + local fail_msg="$2" + while true; do + if eval "$test_cmd" 2>/dev/null; then + return 0 + fi + echo "" + warn "$fail_msg" + echo -e " ${YELLOW}[R]${NC} Retry ${RED}[A]${NC} Abort" + read -rp "Choice: " CHOICE + case "${CHOICE,,}" in + r) info "Retrying..." ;; + a) die "Aborted." ;; + *) warn "Press R to retry or A to abort." ;; + esac + done } if [ "$(id -u)" -eq 0 ]; then - die "Running as root — keys will be written to /root/.ssh. Run as the hub user instead." + die "Running as root — keys will be written to /root/.ssh. Run as the hub user instead." fi mkdir -p "$SSH_DIR" touch "$SSH_DIR/known_hosts" @@ -76,24 +83,24 @@ mkdir -p "$(dirname "$RCLONE_CONF")" header "Checking Tunnel" info "Verifying spoke SSH service is reachable on port $TUNNEL_PORT..." if ! timeout 5 bash -c "cat < /dev/null > /dev/tcp/localhost/$TUNNEL_PORT" 2>/dev/null; then - die "Cannot connect to port $TUNNEL_PORT on localhost — is the tunnel up?" + die "Cannot connect to port $TUNNEL_PORT on localhost — is the tunnel up?" fi info "Scanning spoke host key..." KEYSCAN=$(ssh-keyscan -p "$TUNNEL_PORT" -H localhost 2>/dev/null) [ -n "$KEYSCAN" ] || die "Spoke not reachable on port $TUNNEL_PORT — is the tunnel up?" while IFS= read -r KEYSCAN_LINE; do - KEYSCAN_KEY=$(echo "$KEYSCAN_LINE" | awk '{print $2, $3}') - if ! grep -qF "$KEYSCAN_KEY" "$SSH_DIR/known_hosts" 2>/dev/null; then - echo "$KEYSCAN_LINE" >> "$SSH_DIR/known_hosts" - fi -done <<< "$KEYSCAN" + KEYSCAN_KEY=$(echo "$KEYSCAN_LINE" | awk '{print $2, $3}') + if ! grep -qF "$KEYSCAN_KEY" "$SSH_DIR/known_hosts" 2>/dev/null; then + echo "$KEYSCAN_LINE" >>"$SSH_DIR/known_hosts" + fi +done <<<"$KEYSCAN" header "Generating Hub-to-Spoke Access Key" if [ -f "$KEY_PATH" ]; then - warn "Key $KEY_PATH already exists, skipping generation." + warn "Key $KEY_PATH already exists, skipping generation." else - ssh-keygen -t ed25519 -f "$KEY_PATH" -N "" -C "$KEY_NAME" - info "Key generated: $KEY_PATH" + ssh-keygen -t ed25519 -f "$KEY_PATH" -N "" -C "$KEY_NAME" + info "Key generated: $KEY_PATH" fi chmod 600 "$KEY_PATH" info "Permissions set: $KEY_PATH is 600" @@ -102,68 +109,72 @@ header "Installing Hub-to-Spoke Access Key on Spoke" info "Copying hub public key to spoke's authorized_keys so the hub can SSH in for rclone..." info "(You will be prompted for the $SPOKE_USER password on the spoke)" if ssh-copy-id -i "$KEY_PATH.pub" -p "$TUNNEL_PORT" "$SPOKE_USER"@localhost; then - info "Key copied." + info "Key copied." else - warn "ssh-copy-id failed — password auth may be disabled on the spoke." - warn "Manually append the hub public key to the spoke's authorized_keys:" - echo "" - echo " cat $KEY_PATH.pub" - echo " Then on the spoke, append the output to:" - echo " /home/$SPOKE_USER/.ssh/authorized_keys" - echo "" - read -rp "Press ENTER once the key has been added to the spoke..." + warn "ssh-copy-id failed — password auth may be disabled on the spoke." + warn "Manually append the hub public key to the spoke's authorized_keys:" + echo "" + echo " cat $KEY_PATH.pub" + echo " Then on the spoke, append the output to:" + echo " /home/$SPOKE_USER/.ssh/authorized_keys" + echo "" + read -rp "Press ENTER once the key has been added to the spoke..." fi header "Testing Hub-to-Spoke Key Auth" retry_or_abort \ - "ssh -i \"$KEY_PATH\" -o BatchMode=yes -o ConnectTimeout=10 -p \"$TUNNEL_PORT\" \"$SPOKE_USER\"@localhost exit" \ - "Key auth failed. Check authorized_keys on the spoke." + "ssh -i \"$KEY_PATH\" -o BatchMode=yes -o ConnectTimeout=10 -p \"$TUNNEL_PORT\" \"$SPOKE_USER\"@localhost exit" \ + "Key auth failed. Check authorized_keys on the spoke." info "Key auth to spoke successful." header "Adding rclone Remote" if grep -q "\[${SPOKE_NAME}-remote\]" "$RCLONE_CONF" 2>/dev/null; then - warn "Remote [${SPOKE_NAME}-remote] already exists in $RCLONE_CONF, skipping." + warn "Remote [${SPOKE_NAME}-remote] already exists in $RCLONE_CONF, skipping." else - [ -s "$RCLONE_CONF" ] && tail -c1 "$RCLONE_CONF" | grep -qv $'\n' && echo "" >> "$RCLONE_CONF" - python3 - "$RCLONE_CONF" "$SPOKE_NAME" "$TUNNEL_PORT" "$KEY_PATH" <<'PYEOF' + [ -s "$RCLONE_CONF" ] && tail -c1 "$RCLONE_CONF" | grep -qv $'\n' && echo "" >>"$RCLONE_CONF" + python3 - "$RCLONE_CONF" "$SPOKE_NAME" "$TUNNEL_PORT" "$KEY_PATH" <<'PYEOF' import sys path, name, port, key = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4] with open(path, 'a') as f: f.write(f"\n[{name}-remote]\ntype = sftp\nhost = localhost\nport = {port}\nkey_file = {key}\nshell_type = unix\nmd5sum_command = md5sum\nsha1sum_command = sha1sum\n") PYEOF - info "Remote [${SPOKE_NAME}-remote] added to $RCLONE_CONF." + info "Remote [${SPOKE_NAME}-remote] added to $RCLONE_CONF." fi header "Union Remote (optional)" read -rp "Add this spoke to a union remote for redundancy? [y/N]: " ADD_UNION ADD_UNION="${ADD_UNION:-n}" if [[ "${ADD_UNION,,}" == "y" ]]; then - read -rp "Union remote name [shared-union]: " UNION_NAME - UNION_NAME="${UNION_NAME:-shared-union}" - read -rp "Subfolder path on the spoke being onboarded (e.g. books, leave blank for root): " UNION_PATH - echo "" - echo "Upstream access mode for this spoke:" - echo " 0) None - full read/write (default)" - echo " 1) :ro - read only" - echo " 2) :nc - no create (read/write existing, no new files)" - echo " 3) :writeback - writeback cache" - echo "" - read -rp "Choose [0-3]: " UNION_MODE - UNION_MODE="${UNION_MODE:-0}" - case "$UNION_MODE" in - 0) UPSTREAM_TAG="" ;; - 1) UPSTREAM_TAG=":ro" ;; - 2) UPSTREAM_TAG=":nc" ;; - 3) UPSTREAM_TAG=":writeback" ;; - *) warn "Invalid choice, defaulting to full read/write."; UPSTREAM_TAG="" ;; - esac - if [ -n "$UNION_PATH" ]; then - UPSTREAM="${SPOKE_NAME}-remote:${UNION_PATH}${UPSTREAM_TAG}" - else - UPSTREAM="${SPOKE_NAME}-remote:${UPSTREAM_TAG}" - fi - if grep -q "^\[${UNION_NAME}\]" "$RCLONE_CONF" 2>/dev/null; then - ALREADY=$(python3 - "$RCLONE_CONF" "$UNION_NAME" "${SPOKE_NAME}-remote:" <<'PYEOF2' + read -rp "Union remote name [shared-union]: " UNION_NAME + UNION_NAME="${UNION_NAME:-shared-union}" + read -rp "Subfolder path on the spoke being onboarded (e.g. books, leave blank for root): " UNION_PATH + echo "" + echo "Upstream access mode for this spoke:" + echo " 0) None - full read/write (default)" + echo " 1) :ro - read only" + echo " 2) :nc - no create (read/write existing, no new files)" + echo " 3) :writeback - writeback cache" + echo "" + read -rp "Choose [0-3]: " UNION_MODE + UNION_MODE="${UNION_MODE:-0}" + case "$UNION_MODE" in + 0) UPSTREAM_TAG="" ;; + 1) UPSTREAM_TAG=":ro" ;; + 2) UPSTREAM_TAG=":nc" ;; + 3) UPSTREAM_TAG=":writeback" ;; + *) + warn "Invalid choice, defaulting to full read/write." + UPSTREAM_TAG="" + ;; + esac + if [ -n "$UNION_PATH" ]; then + UPSTREAM="${SPOKE_NAME}-remote:${UNION_PATH}${UPSTREAM_TAG}" + else + UPSTREAM="${SPOKE_NAME}-remote:${UPSTREAM_TAG}" + fi + if grep -q "^\[${UNION_NAME}\]" "$RCLONE_CONF" 2>/dev/null; then + ALREADY=$( + python3 - "$RCLONE_CONF" "$UNION_NAME" "${SPOKE_NAME}-remote:" <<'PYEOF2' import sys path, section, prefix = sys.argv[1], sys.argv[2], sys.argv[3] with open(path) as f: @@ -179,11 +190,11 @@ for line in lines: sys.exit(0) print("no") PYEOF2 -) - if [ "$ALREADY" = "yes" ]; then - warn "Upstream for ${SPOKE_NAME}-remote already in union remote [${UNION_NAME}], skipping." - else - python3 - "$RCLONE_CONF" "$UNION_NAME" "$UPSTREAM" <<'PYEOF2' + ) + if [ "$ALREADY" = "yes" ]; then + warn "Upstream for ${SPOKE_NAME}-remote already in union remote [${UNION_NAME}], skipping." + else + python3 - "$RCLONE_CONF" "$UNION_NAME" "$UPSTREAM" <<'PYEOF2' import sys path, section, upstream = sys.argv[1], sys.argv[2], sys.argv[3] with open(path) as f: @@ -201,20 +212,20 @@ for line in lines: with open(path, "w") as f: f.writelines(out) PYEOF2 - info "Added '$UPSTREAM' to union remote [${UNION_NAME}]." - fi - else - [ -s "$RCLONE_CONF" ] && tail -c1 "$RCLONE_CONF" | grep -qv $'\n' && echo "" >> "$RCLONE_CONF" - printf '\n[%s]\ntype = union\nupstreams = %s\n' "$UNION_NAME" "$UPSTREAM" >> "$RCLONE_CONF" - info "Union remote [${UNION_NAME}] created with upstream '$UPSTREAM'." + info "Added '$UPSTREAM' to union remote [${UNION_NAME}]." fi + else + [ -s "$RCLONE_CONF" ] && tail -c1 "$RCLONE_CONF" | grep -qv $'\n' && echo "" >>"$RCLONE_CONF" + printf '\n[%s]\ntype = union\nupstreams = %s\n' "$UNION_NAME" "$UPSTREAM" >>"$RCLONE_CONF" + info "Union remote [${UNION_NAME}] created with upstream '$UPSTREAM'." + fi fi header "Testing rclone Connection" if rclone lsd "${SPOKE_NAME}-remote:" --config "$RCLONE_CONF" 2>/dev/null; then - info "rclone connection to $SPOKE_NAME successful." + info "rclone connection to $SPOKE_NAME successful." else - warn "rclone test failed. Check the remote config in $RCLONE_CONF." + warn "rclone test failed. Check the remote config in $RCLONE_CONF." fi header "Registering Spoke" @@ -222,26 +233,30 @@ mkdir -p "$(dirname "$REGISTRY")" MOUNT_POINT="${HOME}/mnt/${SPOKE_NAME}" mkdir -p "$MOUNT_POINT" if grep -q "^${SPOKE_NAME} " "$REGISTRY" 2>/dev/null; then - warn "$SPOKE_NAME already in registry, updating." - grep -v "^${SPOKE_NAME} " "$REGISTRY" > "${REGISTRY}.tmp" 2>/dev/null || true - mv "${REGISTRY}.tmp" "$REGISTRY" + warn "$SPOKE_NAME already in registry, updating." + grep -v "^${SPOKE_NAME} " "$REGISTRY" >"${REGISTRY}.tmp" 2>/dev/null || true + mv "${REGISTRY}.tmp" "$REGISTRY" fi -echo "${SPOKE_NAME} ${TUNNEL_PORT} ${KEY_PATH} ${MOUNT_POINT}" >> "$REGISTRY" +echo "${SPOKE_NAME} ${TUNNEL_PORT} ${KEY_PATH} ${MOUNT_POINT}" >>"$REGISTRY" info "$SPOKE_NAME registered." header "Setting Up Auto-Mount" -MOUNT_CMD="rclone mount ${SPOKE_NAME}-remote: ${MOUNT_POINT} --config ${HOME}/.config/rclone/rclone.conf --vfs-cache-mode writes --allow-other --daemon" +#MOUNT_CMD="rclone mount ${SPOKE_NAME}-remote: ${MOUNT_POINT} --config ${HOME}/.config/rclone/rclone.conf --vfs-cache-mode writes --allow-other --daemon" +MOUNT_CMD="sleep 55 && rclone mount ${SPOKE_NAME}-remote: ${MOUNT_POINT} --config ${HOME}/.config/rclone/rclone.conf --vfs-cache-mode full --vfs-cache-max-size 2G --vfs-read-ahead 256M --allow-other --daemon" CRON_ENTRY="@reboot ${MOUNT_CMD}" EXISTING=$(crontab -l 2>/dev/null || true) if echo "$EXISTING" | grep -qF "${SPOKE_NAME}-remote:"; then - warn "Crontab entry for ${SPOKE_NAME}-remote already exists, skipping." + warn "Crontab entry for ${SPOKE_NAME}-remote already exists, skipping." else - CRONTAB_BACKUP="${HOME}/.config/tinyboard/crontab.$(date +%Y%m%d%H%M%S)" - mkdir -p "$(dirname "$CRONTAB_BACKUP")" - echo "$EXISTING" > "$CRONTAB_BACKUP" - info "Crontab backed up to $CRONTAB_BACKUP" - { echo "$EXISTING"; echo "$CRON_ENTRY"; } | crontab - - info "Auto-mount crontab entry added for ${SPOKE_NAME}." + CRONTAB_BACKUP="${HOME}/.config/tinyboard/crontab.$(date +%Y%m%d%H%M%S)" + mkdir -p "$(dirname "$CRONTAB_BACKUP")" + echo "$EXISTING" >"$CRONTAB_BACKUP" + info "Crontab backed up to $CRONTAB_BACKUP" + { + echo "$EXISTING" + echo "$CRON_ENTRY" + } | crontab - + info "Auto-mount crontab entry added for ${SPOKE_NAME}." fi info "Starting mount now..." mkdir -p "$MOUNT_POINT"