commit 728fdca878d86b01ab87dabd9908dd2ef8acd887 Author: Justin Oros Date: Mon Mar 23 16:17:54 2026 -0700 xteink-github-sync diff --git a/GitHubSync.cpp b/GitHubSync.cpp new file mode 100644 index 0000000..ba5e5b1 --- /dev/null +++ b/GitHubSync.cpp @@ -0,0 +1,177 @@ +#include "GitHubSync.h" +#include +#include +#include +#include +#include + +#define GH_PREFS_NS "github_sync" +#define GH_KEY_USER "username" +#define GH_KEY_PAT "pat" +#define GH_KEY_REPO "repo" +#define GH_KEY_BRANCH "branch" +#define GH_SHA_DIR "/.crosspoint/github_sha/" +#define GH_BOOKS_DIR "/" +#define GH_SLEEP_BMP "sleep.bmp" +#define GH_SLEEP_PATH "/sleep.bmp" +#define GH_API_BASE "https://api.github.com" + +bool GitHubSync::loadConfig(GitHubSyncConfig &cfg) { + Preferences prefs; + prefs.begin(GH_PREFS_NS, true); + cfg.username = prefs.getString(GH_KEY_USER, "").c_str(); + cfg.pat = prefs.getString(GH_KEY_PAT, "").c_str(); + cfg.repo = prefs.getString(GH_KEY_REPO, "xteink").c_str(); + cfg.branch = prefs.getString(GH_KEY_BRANCH, "main").c_str(); + prefs.end(); + return !cfg.username.empty() && !cfg.pat.empty(); +} + +void GitHubSync::saveConfig(const GitHubSyncConfig &cfg) { + Preferences prefs; + prefs.begin(GH_PREFS_NS, false); + prefs.putString(GH_KEY_USER, cfg.username.c_str()); + prefs.putString(GH_KEY_PAT, cfg.pat.c_str()); + prefs.putString(GH_KEY_REPO, cfg.repo.c_str()); + prefs.putString(GH_KEY_BRANCH, cfg.branch.c_str()); + prefs.end(); +} + +bool GitHubSync::isConfigured() { + GitHubSyncConfig cfg; + return loadConfig(cfg); +} + +const char* GitHubSync::resultMessage(GitHubSyncResult r) { + switch (r) { + case GitHubSyncResult::OK: return "GitHub sync complete"; + case GitHubSyncResult::NOT_CONFIGURED: return "GitHub sync: not configured"; + case GitHubSyncResult::NO_WIFI: return "GitHub sync: no WiFi"; + case GitHubSyncResult::AUTH_ERROR: return "GitHub sync: auth failed (check PAT)"; + case GitHubSyncResult::REPO_NOT_FOUND: return "GitHub sync: repo not found"; + case GitHubSyncResult::API_ERROR: return "GitHub sync: API error"; + case GitHubSyncResult::SD_ERROR: return "GitHub sync: SD card error"; + case GitHubSyncResult::PARSE_ERROR: return "GitHub sync: bad API response"; + default: return "GitHub sync: unknown error"; + } +} + +std::string GitHubSync::shaFilePath(const std::string &filename) { + std::string safe = filename; + for (char &c : safe) if (c == '/') c = '_'; + return std::string(GH_SHA_DIR) + safe + ".sha"; +} + +std::string GitHubSync::loadLocalSha(const std::string &filename) { + std::string path = shaFilePath(filename); + File f = SD.open(path.c_str(), FILE_READ); + if (!f) return ""; + String sha = f.readString(); + f.close(); + sha.trim(); + return sha.c_str(); +} + +void GitHubSync::saveLocalSha(const std::string &filename, const std::string &sha) { + SD.mkdir(GH_SHA_DIR); + std::string path = shaFilePath(filename); + File f = SD.open(path.c_str(), FILE_WRITE); + if (!f) return; + f.print(sha.c_str()); + f.close(); +} + +bool GitHubSync::fetchFileList(const GitHubSyncConfig &cfg, std::string &outJson, GitHubSyncResult &err) { + std::string url = std::string(GH_API_BASE) + "/repos/" + cfg.username + "/" + + cfg.repo + "/contents/?ref=" + cfg.branch; + + HTTPClient http; + http.begin(url.c_str()); + http.addHeader("Authorization", ("token " + cfg.pat).c_str()); + http.addHeader("Accept", "application/vnd.github.v3+json"); + http.addHeader("User-Agent", "CrossPoint-X4"); + + int code = http.GET(); + if (code == 401 || code == 403) { http.end(); err = GitHubSyncResult::AUTH_ERROR; return false; } + if (code == 404) { http.end(); err = GitHubSyncResult::REPO_NOT_FOUND; return false; } + if (code != 200) { http.end(); err = GitHubSyncResult::API_ERROR; return false; } + + outJson = http.getString().c_str(); + http.end(); + return true; +} + +bool GitHubSync::downloadFile(const GitHubSyncConfig &cfg, const std::string &path, const std::string &sha, GitHubSyncResult &err) { + std::string url = std::string(GH_API_BASE) + "/repos/" + cfg.username + "/" + + cfg.repo + "/contents/" + path + "?ref=" + cfg.branch; + + HTTPClient http; + http.begin(url.c_str()); + http.addHeader("Authorization", ("token " + cfg.pat).c_str()); + http.addHeader("Accept", "application/vnd.github.v3.raw"); + http.addHeader("User-Agent", "CrossPoint-X4"); + + int code = http.GET(); + if (code == 401 || code == 403) { http.end(); err = GitHubSyncResult::AUTH_ERROR; return false; } + if (code != 200) { http.end(); err = GitHubSyncResult::API_ERROR; return false; } + + std::string destPath = (path == GH_SLEEP_BMP) ? GH_SLEEP_PATH : std::string(GH_BOOKS_DIR) + path; + + File f = SD.open(destPath.c_str(), FILE_WRITE); + if (!f) { http.end(); err = GitHubSyncResult::SD_ERROR; return false; } + + WiFiClient *stream = http.getStreamPtr(); + uint8_t buf[512]; + int total = http.getSize(); + int remaining = total; + + while (http.connected() && (remaining > 0 || total == -1)) { + size_t available = stream->available(); + if (available) { + size_t read = stream->readBytes(buf, min(available, sizeof(buf))); + f.write(buf, read); + if (remaining > 0) remaining -= (int)read; + } + delay(1); + } + + f.close(); + http.end(); + saveLocalSha(path, sha); + return true; +} + +GitHubSyncResult GitHubSync::sync() { + GitHubSyncConfig cfg; + if (!loadConfig(cfg)) return GitHubSyncResult::NOT_CONFIGURED; + if (WiFi.status() != WL_CONNECTED) return GitHubSyncResult::NO_WIFI; + + std::string jsonStr; + GitHubSyncResult err = GitHubSyncResult::OK; + if (!fetchFileList(cfg, jsonStr, err)) return err; + + JsonDocument doc; + DeserializationError jsonErr = deserializeJson(doc, jsonStr); + if (jsonErr) return GitHubSyncResult::PARSE_ERROR; + + JsonArray files = doc.as(); + for (JsonObject file : files) { + std::string type = file["type"].as(); + std::string name = file["name"].as(); + std::string sha = file["sha"].as(); + + if (type != "file") continue; + + bool isEpub = name.size() > 5 && (name.substr(name.size()-5) == ".epub" || name.substr(name.size()-5) == ".EPUB"); + bool isSleep = (name == GH_SLEEP_BMP); + + if (!isEpub && !isSleep) continue; + + std::string localSha = loadLocalSha(name); + if (localSha == sha) continue; + + if (!downloadFile(cfg, name, sha, err)) return err; + } + + return GitHubSyncResult::OK; +} diff --git a/GitHubSync.h b/GitHubSync.h new file mode 100644 index 0000000..361f18a --- /dev/null +++ b/GitHubSync.h @@ -0,0 +1,38 @@ +#pragma once + +#include +#include + +struct GitHubSyncConfig { + std::string username; + std::string pat; + std::string repo; + std::string branch; +}; + +enum class GitHubSyncResult { + OK, + NOT_CONFIGURED, + NO_WIFI, + AUTH_ERROR, + REPO_NOT_FOUND, + API_ERROR, + SD_ERROR, + PARSE_ERROR +}; + +class GitHubSync { +public: + static bool loadConfig(GitHubSyncConfig &cfg); + static void saveConfig(const GitHubSyncConfig &cfg); + static bool isConfigured(); + static GitHubSyncResult sync(); + static const char* resultMessage(GitHubSyncResult r); + +private: + static bool fetchFileList(const GitHubSyncConfig &cfg, std::string &outJson, GitHubSyncResult &err); + static bool downloadFile(const GitHubSyncConfig &cfg, const std::string &path, const std::string &sha, GitHubSyncResult &err); + static std::string loadLocalSha(const std::string &filename); + static void saveLocalSha(const std::string &filename, const std::string &sha); + static std::string shaFilePath(const std::string &filename); +}; diff --git a/GitHubSyncSettingsActivity.cpp b/GitHubSyncSettingsActivity.cpp new file mode 100644 index 0000000..1b79ec5 --- /dev/null +++ b/GitHubSyncSettingsActivity.cpp @@ -0,0 +1,151 @@ +#include "GitHubSyncSettingsActivity.h" + +#include +#include + +#include "MappedInputManager.h" +#include "activities/util/KeyboardEntryActivity.h" +#include "components/UITheme.h" +#include "fontIds.h" + +namespace { +constexpr int MENU_ITEMS = 5; +const char* menuNames[MENU_ITEMS] = { + "Username", + "Token (PAT)", + "Repo", + "Branch", + "Save" +}; +} + +void GitHubSyncSettingsActivity::onEnter() { + Activity::onEnter(); + selectedIndex = 0; + requestUpdate(); +} + +void GitHubSyncSettingsActivity::onExit() { + Activity::onExit(); +} + +std::string GitHubSyncSettingsActivity::getMasked(const std::string &s) const { + return s.empty() ? "" : "••••••••"; +} + +void GitHubSyncSettingsActivity::handleSelection() { + GitHubSyncConfig cfg; + GitHubSync::loadConfig(cfg); + + if (selectedIndex == 0) { + startActivityForResult( + std::make_unique(renderer, mappedInput, "GitHub Username", + cfg.username, 64, false), + [](const ActivityResult &result) { + if (!result.isCancelled) { + GitHubSyncConfig c; + GitHubSync::loadConfig(c); + c.username = std::get(result.data).text; + GitHubSync::saveConfig(c); + } + }); + } else if (selectedIndex == 1) { + startActivityForResult( + std::make_unique(renderer, mappedInput, "Personal Access Token", + "", 128, false), + [](const ActivityResult &result) { + if (!result.isCancelled) { + const auto &text = std::get(result.data).text; + if (!text.empty()) { + GitHubSyncConfig c; + GitHubSync::loadConfig(c); + c.pat = text; + GitHubSync::saveConfig(c); + } + } + }); + } else if (selectedIndex == 2) { + startActivityForResult( + std::make_unique(renderer, mappedInput, "Repo Name", + cfg.repo, 64, false), + [](const ActivityResult &result) { + if (!result.isCancelled) { + GitHubSyncConfig c; + GitHubSync::loadConfig(c); + c.repo = std::get(result.data).text; + GitHubSync::saveConfig(c); + } + }); + } else if (selectedIndex == 3) { + startActivityForResult( + std::make_unique(renderer, mappedInput, "Branch", + cfg.branch, 32, false), + [](const ActivityResult &result) { + if (!result.isCancelled) { + GitHubSyncConfig c; + GitHubSync::loadConfig(c); + c.branch = std::get(result.data).text; + GitHubSync::saveConfig(c); + } + }); + } else if (selectedIndex == 4) { + finish(); + } +} + +void GitHubSyncSettingsActivity::loop() { + if (mappedInput.wasPressed(MappedInputManager::Button::Back)) { + finish(); + return; + } + + if (mappedInput.wasPressed(MappedInputManager::Button::Confirm)) { + handleSelection(); + return; + } + + buttonNavigator.onNext([this] { + selectedIndex = (selectedIndex + 1) % MENU_ITEMS; + requestUpdate(); + }); + + buttonNavigator.onPrevious([this] { + selectedIndex = (selectedIndex + MENU_ITEMS - 1) % MENU_ITEMS; + requestUpdate(); + }); +} + +void GitHubSyncSettingsActivity::render(RenderLock &&) { + renderer.clearScreen(); + + const auto &metrics = UITheme::getInstance().getMetrics(); + const auto pageWidth = renderer.getScreenWidth(); + const auto pageHeight = renderer.getScreenHeight(); + + GUI.drawHeader(renderer, Rect{0, metrics.topPadding, pageWidth, metrics.headerHeight}, "GitHub Sync"); + + GitHubSyncConfig cfg; + GitHubSync::loadConfig(cfg); + + const int contentTop = metrics.topPadding + metrics.headerHeight + metrics.verticalSpacing; + const int contentHeight = pageHeight - contentTop - metrics.buttonHintsHeight - metrics.verticalSpacing * 2; + + GUI.drawList( + renderer, Rect{0, contentTop, pageWidth, contentHeight}, MENU_ITEMS, + selectedIndex, + [](int index) { return std::string(menuNames[index]); }, + nullptr, nullptr, + [this, &cfg](int index) -> std::string { + if (index == 0) return cfg.username.empty() ? "(not set)" : cfg.username; + if (index == 1) return cfg.pat.empty() ? "(not set)" : getMasked(cfg.pat); + if (index == 2) return cfg.repo.empty() ? "xteink" : cfg.repo; + if (index == 3) return cfg.branch.empty() ? "main" : cfg.branch; + return ""; + }, + true); + + const auto labels = mappedInput.mapLabels(tr(STR_BACK), tr(STR_SELECT), tr(STR_DIR_UP), tr(STR_DIR_DOWN)); + GUI.drawButtonHints(renderer, labels.btn1, labels.btn2, labels.btn3, labels.btn4); + + renderer.displayBuffer(); +} diff --git a/GitHubSyncSettingsActivity.h b/GitHubSyncSettingsActivity.h new file mode 100644 index 0000000..071c574 --- /dev/null +++ b/GitHubSyncSettingsActivity.h @@ -0,0 +1,25 @@ +#pragma once + +#include "activities/Activity.h" +#include "GitHubSync.h" +#include "util/ButtonNavigator.h" + +class GitHubSyncSettingsActivity final : public Activity { +public: + explicit GitHubSyncSettingsActivity(GfxRenderer& renderer, MappedInputManager& mappedInput) + : Activity("GitHubSync", renderer, mappedInput) {} + + void onEnter() override; + void onExit() override; + void loop() override; + void render(RenderLock&&) override; + +private: + ButtonNavigator buttonNavigator; + int selectedIndex = 0; + + static constexpr int MENU_ITEMS = 5; + + void handleSelection(); + std::string getMasked(const std::string& s) const; +}; diff --git a/MANUAL_INTEGRATION.md b/MANUAL_INTEGRATION.md new file mode 100644 index 0000000..9ee8a88 --- /dev/null +++ b/MANUAL_INTEGRATION.md @@ -0,0 +1,116 @@ +# GitHub Sync — Manual Integration (Advanced) + +If you prefer not to use the auto-patcher (`patch.py`), follow these steps to integrate GitHub Sync into a CrossPoint/xteink firmware tree manually. + +## Files to copy into your CrossPoint repo + +Copy the files from this repository into your CrossPoint firmware repo at the following destinations: + +| Source (this repo) | Destination (CrossPoint repo) | +|---|---| +| `GitHubSync.h` | `include/GitHubSync.h` | +| `GitHubSync.cpp` | `src/github_sync/GitHubSync.cpp` | +| `GitHubSyncSettingsActivity.h` | `include/GitHubSyncSettingsActivity.h` | +| `GitHubSyncSettingsActivity.cpp` | `src/activities/settings/GitHubSyncSettingsActivity.cpp` | + +Notes: +- Ensure the destination folders exist (`src/github_sync/`, `src/activities/settings/`). +- Your CrossPoint repo may use slightly different folder names; the important part is that your build system compiles the `.cpp` files and that includes resolve from `include/`. + +## 1) `platformio.ini` dependency + +Make sure ArduinoJson v7 is available in `lib_deps`: + +```ini +bblanchon/ArduinoJson @ ^7 +``` + +If your project already depends on ArduinoJson, do not duplicate it. + +## 2) Call sync after WiFi connects (boot/startup path) + +Find your boot path after WiFi is connected (often near where OTA update checks happen) and add: + +```cpp +#include "GitHubSync.h" + +// After WiFi is up: +if (GitHubSync::isConfigured()) { + GitHubSyncResult result = GitHubSync::sync(); + if (result != GitHubSyncResult::OK) { + // Replace this with your project's preferred UI/logging mechanism. + // (Some CrossPoint forks use LOG_ERR; others use Serial; some show a toast.) + LOG_ERR("SYNC", "%s", GitHubSync::resultMessage(result)); + // Or: Serial.printf("[SYNC] %s\n", GitHubSync::resultMessage(result)); + } +} +``` + +Important: +- The return type is **`GitHubSyncResult`** (not `SyncResult`). +- If your firmware does not define `LOG_ERR`, use whatever logging you already use (or `Serial.printf`). + +## 3) Add a Settings entry ("GitHub Sync") + +In your settings menu implementation (commonly something like `SettingsActivity.cpp`): + +1. Include the activity: + +```cpp +#include "GitHubSyncSettingsActivity.h" +``` + +2. Add a new item in the settings list that navigates to the activity. + +The exact code depends on your UI framework, but the action should instantiate/push: + +```cpp +std::make_unique(renderer, mappedInput) +``` + +3. Add a translation string (if your project uses translations): +- Add a string ID similar to `STR_GITHUB_SYNC` +- Map it to the label `GitHub Sync` in your translation files + +## 4) (Optional) Web settings editor schema + +If your CrossPoint fork has a web settings editor that can read/write NVS keys, add these keys: + +```json +{ "ns": "github_sync", "key": "username", "label": "GitHub Username", "type": "text" }, +{ "ns": "github_sync", "key": "pat", "label": "GitHub PAT", "type": "password" }, +{ "ns": "github_sync", "key": "repo", "label": "Repo name", "type": "text" }, +{ "ns": "github_sync", "key": "branch", "label": "Branch", "type": "text" } +``` + +Namespace/keys used by firmware: +- Namespace: `github_sync` +- Keys: `username`, `pat`, `repo`, `branch` + +## 5) GitHub repo setup (for content) + +1. Create a GitHub repo (private recommended) +2. Add `.epub` files to the **repo root** +3. Optionally add `sleep.bmp` to the repo root (152x152 grayscale BMP) +4. Create a PAT with permission to read repository contents +5. On device: Settings → GitHub Sync: + - username + - PAT + - repo + - branch + +## Behavior summary + +- On boot (after WiFi connects), the device calls the GitHub Contents API for the repo root. +- For each `*.epub` and `sleep.bmp` found: + - Compare GitHub blob SHA to cached SHA stored in `/.crosspoint/github_sha/` on the SD card + - Download only when SHA differs or is missing locally +- Download destinations: + - `sleep.bmp` → `/sleep.bmp` + - `*.epub` → `/.epub` (SD card root) +- No automatic deletions are performed. + +## RAM note + +The GitHub Contents API directory listing is small (one JSON object per file). Downloads are streamed directly to SD using a small buffer. + diff --git a/README.md b/README.md new file mode 100644 index 0000000..7817c4b --- /dev/null +++ b/README.md @@ -0,0 +1,126 @@ +# xteink-github-sync + +Sync `.epub` files (and an optional `sleep.bmp`) from a GitHub repository to your CrossPoint/xteink device at boot. + +This repo contains: +1. The device-side C++ implementation (`GitHubSync.*`, `GitHubSyncSettingsActivity.*`) +2. A Python patcher (`patch.py`) that injects the feature into your CrossPoint Reader codebase and uploads the firmware (with a prompt to connect via USB first). + +## What it syncs + +- All files in the root of your GitHub repo ending in `.epub` +- `sleep.bmp` in the repo root (optional) + +The device compares GitHub blob SHAs against cached SHAs stored on the SD card and downloads only what changed. + +## GitHub repo setup + +1. Create a **private** GitHub repo (example: `xteink`) +2. Add `.epub` files to the **root** of the repo +3. Optionally add `sleep.bmp` (152x152 grayscale BMP) +4. Create a Personal Access Token (PAT): + - Token type: classic or a fine-grained token with equivalent access + - Scopes/permissions: `contents: read-only` (or equivalent for reading repository contents) +5. In the device UI (or during initial setup), configure: + - GitHub username + - PAT + - Repo name + - Branch (default: `main`) + +## How the sync works (device behavior) + +- After WiFi connects, the firmware calls the GitHub Contents API +- For each candidate file in the repo root (`*.epub` and `sleep.bmp`), it: + - Fetches the GitHub blob SHA + - Compares it to the cached SHA stored at `/.crosspoint/github_sha/` on the SD card + - Downloads the raw file if the SHA differs or is missing locally +- Download destinations on the SD card: + - `/sleep.bmp` for the sleep image + - `/.epub` for book files +- Files are **not** deleted automatically + +## Requirements (local machine) + +- `git` +- `platformio` (PlatformIO CLI, provides `pio`) +- `esptool.py` (used by PlatformIO for uploading) +- `esp-idf-nvs-partition-gen` (used to build an NVS partition for flashing credentials) +- Python packages the script may prompt to install (press `Enter` when asked): + - `platformio` + - `esptool` + - `esp-idf-nvs-partition-gen` + - `certifi` (recommended on macOS to avoid `CERTIFICATE_VERIFY_FAILED`) + +## Usage + +Run the patcher: + +```bash +python3 patch.py +``` + +Optional arguments: + +```bash +python3 patch.py /path/to/destination [usb_upload_port] +``` + +What it does: + +1. Clones/updates `crosspoint-reader` into `destination/crosspoint-reader` +2. Copies the C++ files into the correct CrossPoint locations +3. Injects the startup sync call into the CrossPoint boot path +4. Adds a “GitHub Sync” entry to the settings menu +5. Optionally writes NVS credentials (username/PAT/repo/branch) into the device +6. Prompts you to connect the device via USB before uploading +7. Runs `pio run --target upload` + +## Prompts and validation + +During setup, `patch.py` will prompt for: + +- GitHub username + - Validates reachability via `GET https://api.github.com/users/{username}` +- PAT + - Validates via `GET https://api.github.com/user` +- Repo name + - Validates via `GET https://api.github.com/repos/{owner}/{repo}` using your PAT + +Before uploading firmware, it prompts you to: +- Connect the xteink device via **USB data** (not charge-only) +- Press `Enter` once connected +- It then shows likely serial ports and PlatformIO’s device list to help you choose the correct upload port. + +## Troubleshooting + +### `CERTIFICATE_VERIFY_FAILED` talking to GitHub + +On macOS, Python can miss the system CA bundle. Recommended fix: + +```bash +pip3 install certifi +``` + +The script uses `certifi` automatically when available. + +If you *must* bypass SSL verification (insecure), you can run: + +```bash +export GITHUB_SYNC_SSL_NO_VERIFY=1 +python3 patch.py +``` + +### Upload picks the wrong serial device (e.g. Bluetooth) + +Make sure you use a **data** USB cable and that the device is connected. + +Use the prompt (or pass the upload port explicitly) so PlatformIO/esptool doesn’t auto-detect a non-ESP32 serial device. + +If you still have trouble, try: +- Unplugging Bluetooth devices temporarily +- Passing `usb_upload_port` explicitly to `patch.py` + +## Integration details + +See `INTEGRATION.md` for the file injection locations and the CrossPoint-side integration checklist. + diff --git a/patch.py b/patch.py new file mode 100644 index 0000000..549c246 --- /dev/null +++ b/patch.py @@ -0,0 +1,592 @@ +#!/usr/bin/env python3 + +import sys +import shutil +import subprocess +import getpass +import csv +import tempfile +from pathlib import Path +import glob +import os +import ssl +import urllib.request +import urllib.error +import json + +PATCH_DIR = Path(__file__).parent +CROSSPOINT_URL = "https://github.com/crosspoint-reader/crosspoint-reader.git" +NVS_NAMESPACE = "github_sync" +BOLD = "\033[1m" +GREEN = "\033[92m" +RED = "\033[91m" +YELLOW = "\033[93m" +CYAN = "\033[96m" +RESET = "\033[0m" + +def ok(msg): print(f"{GREEN} ✓ {msg}{RESET}") +def err(msg): print(f"{RED} ✗ {msg}{RESET}"); sys.exit(1) +def warn(msg): print(f"{YELLOW} ! {msg}{RESET}") +def info(msg): print(f" {msg}") + +def ensure_python_module(module_name: str, pip_package: str | None = None, *, description: str = "") -> None: + """If import fails, prompt user (Press Enter) then pip install into this Python.""" + pip_package = pip_package or module_name + try: + __import__(module_name) + return + except ImportError: + pass + desc = f" — {description}" if description else "" + warn(f"Python module '{module_name}' is not installed{desc}.") + input(f" Press Enter to install {pip_package}... ") + result = subprocess.run( + [sys.executable, "-m", "pip", "install", pip_package], + capture_output=True, text=True, + ) + if result.returncode != 0: + err(f"Failed to install {pip_package}:\n{result.stderr.strip()}") + ok(f"{pip_package} installed") + try: + __import__(module_name) + except ImportError: + err(f"'{module_name}' still not importable after install — try restarting the terminal or use the same Python as: {sys.executable}") + +_github_https_deps_ready = False + +def ensure_github_https_dependencies() -> None: + """certifi fixes macOS CERTIFICATE_VERIFY_FAILED for api.github.com unless user opted out.""" + global _github_https_deps_ready + if _github_https_deps_ready: + return + if os.environ.get("GITHUB_SYNC_SSL_NO_VERIFY", "").strip().lower() in ("1", "true", "yes", "on"): + _github_https_deps_ready = True + return + cf = os.environ.get("SSL_CERT_FILE", "").strip() + if cf and os.path.isfile(cf): + _github_https_deps_ready = True + return + try: + import certifi # noqa: F401 + _github_https_deps_ready = True + return + except ImportError: + pass + ensure_python_module( + "certifi", + description="recommended for GitHub HTTPS (fixes macOS SSL certificate errors)", + ) + _github_https_deps_ready = True + +def get_github_ssl_context(): + """ + macOS Python from python.org often lacks system CA bundle → CERTIFICATE_VERIFY_FAILED. + Fixes: pip install certifi, set SSL_CERT_FILE, run Install Certificates.command, + or (last resort) GITHUB_SYNC_SSL_NO_VERIFY=1. + """ + flag = os.environ.get("GITHUB_SYNC_SSL_NO_VERIFY", "").strip().lower() + if flag in ("1", "true", "yes", "on"): + warn("SSL verification is OFF (GITHUB_SYNC_SSL_NO_VERIFY). Only use if you trust this network.") + return ssl._create_unverified_context() + cert_file = os.environ.get("SSL_CERT_FILE", "").strip() + if cert_file and os.path.isfile(cert_file): + return ssl.create_default_context(cafile=cert_file) + try: + import certifi + return ssl.create_default_context(cafile=certifi.where()) + except ImportError: + pass + return ssl.create_default_context() + +def ssl_troubleshoot_hint(err_txt: str | None) -> str: + if not err_txt: + return "" + if "CERTIFICATE_VERIFY_FAILED" in err_txt or "SSL" in err_txt: + return ( + " Try: pip install certifi (then re-run), or run macOS " + "'Install Certificates.command' for your Python, or set SSL_CERT_FILE to a CA bundle. " + "Last resort: GITHUB_SYNC_SSL_NO_VERIFY=1 (insecure)." + ) + return "" + +def github_api_get(path: str, token: str | None = None, timeout: int = 10): + url = f"https://api.github.com{path}" + headers = { + "Accept": "application/vnd.github+json", + "User-Agent": "xteink-github-sync-patcher", + } + if token: + headers["Authorization"] = f"Bearer {token}" + req = urllib.request.Request(url, headers=headers) + ctx = get_github_ssl_context() + try: + with urllib.request.urlopen(req, timeout=timeout, context=ctx) as resp: + body = resp.read().decode("utf-8", errors="replace") + data = json.loads(body) if body else {} + return resp.status, data, None + except urllib.error.HTTPError as e: + body = e.read().decode("utf-8", errors="replace") + try: + data = json.loads(body) if body else {} + except Exception: + data = {"message": body.strip()} if body else {} + return e.code, data, None + except Exception as e: + return None, None, str(e) + +def validate_github_username(username: str) -> tuple[bool, str]: + status, data, err_txt = github_api_get(f"/users/{username}") + if err_txt: + return False, f"Could not reach GitHub API: {err_txt}{ssl_troubleshoot_hint(err_txt)}" + if status == 200: + return True, "GitHub username is reachable." + if status == 404: + return False, "Username not found on GitHub." + msg = data.get("message", "Unknown API error") if isinstance(data, dict) else "Unknown API error" + return False, f"GitHub API error ({status}): {msg}" + +def validate_pat(token: str) -> tuple[bool, str, str | None]: + status, data, err_txt = github_api_get("/user", token=token) + if err_txt: + return False, f"Could not validate PAT: {err_txt}{ssl_troubleshoot_hint(err_txt)}", None + if status == 200 and isinstance(data, dict): + login = data.get("login") + return True, f"PAT is valid (authenticated as {login}).", login + if status in (401, 403): + msg = data.get("message", "Unauthorized") if isinstance(data, dict) else "Unauthorized" + return False, f"PAT rejected: {msg}", None + msg = data.get("message", "Unknown API error") if isinstance(data, dict) else "Unknown API error" + return False, f"PAT validation failed ({status}): {msg}", None + +def validate_repo_access(owner: str, repo: str, token: str) -> tuple[bool, str]: + status, data, err_txt = github_api_get(f"/repos/{owner}/{repo}", token=token) + if err_txt: + return False, f"Could not validate repo access: {err_txt}{ssl_troubleshoot_hint(err_txt)}" + if status == 200: + return True, "Repo is reachable with this PAT." + if status == 404: + return False, "Repo not found, or PAT cannot access it." + if status in (401, 403): + msg = data.get("message", "Unauthorized") if isinstance(data, dict) else "Unauthorized" + return False, f"Access denied: {msg}" + msg = data.get("message", "Unknown API error") if isinstance(data, dict) else "Unknown API error" + return False, f"Repo validation failed ({status}): {msg}" + +def prompt_install(pip_package: str): + warn(f"{pip_package} is not installed or not available to this Python.") + input(f" Press Enter to install {pip_package}... ") + result = subprocess.run( + [sys.executable, "-m", "pip", "install", pip_package], + capture_output=True, text=True, + ) + if result.returncode != 0: + err(f"Failed to install {pip_package}:\n{result.stderr.strip()}") + ok(f"{pip_package} installed") + +def check_tool(cmd: list[str], pip_package: str, fatal: bool = True) -> bool: + try: + subprocess.run(cmd, check=True, capture_output=True) + return True + except (subprocess.CalledProcessError, FileNotFoundError): + if fatal: + prompt_install(pip_package) + try: + subprocess.run(cmd, check=True, capture_output=True) + return True + except (subprocess.CalledProcessError, FileNotFoundError): + err(f"{cmd[0]} still not found after install — check your PATH") + else: + return False + +def check_git(): + try: + subprocess.run(["git", "--version"], check=True, capture_output=True) + except (subprocess.CalledProcessError, FileNotFoundError): + err("git is not installed. Install from https://git-scm.com and re-run.") + +def check_pio(): + check_tool(["pio", "--version"], "platformio") + +def check_esptool(): + return check_tool(["esptool.py", "version"], "esptool", fatal=False) + +def check_nvs_gen() -> bool: + result = subprocess.run( + ["python3", "-m", "esp_idf_nvs_partition_gen", "--help"], + capture_output=True, text=True + ) + if result.returncode == 0: + return True + check_tool(["python3", "-m", "esp_idf_nvs_partition_gen", "--help"], "esp-idf-nvs-partition-gen", fatal=False) + result2 = subprocess.run( + ["python3", "-m", "esp_idf_nvs_partition_gen", "--help"], + capture_output=True, text=True + ) + return result2.returncode == 0 + +def clone_or_update(dest: Path) -> Path: + repo = dest / "crosspoint-reader" + if repo.exists() and (repo / ".git").exists(): + info(f"Repo already exists at {repo}, pulling latest...") + result = subprocess.run( + ["git", "-C", str(repo), "pull", "--recurse-submodules"], + capture_output=True, text=True + ) + if result.returncode != 0: + err(f"git pull failed:\n{result.stderr}") + ok("Repo updated to latest") + else: + info(f"Cloning CrossPoint into {repo}...") + result = subprocess.run( + ["git", "clone", "--recurse-submodules", CROSSPOINT_URL, str(repo)], + capture_output=True, text=True + ) + if result.returncode != 0: + err(f"git clone failed:\n{result.stderr}") + ok("Repo cloned successfully") + return repo + +def prompt_github_config() -> dict: + print(f"\n{BOLD}GitHub Sync Configuration{RESET}") + print(f"{CYAN} Press Enter to skip and configure on-device later.{RESET}\n") + + while True: + username = input(" GitHub username: ").strip() + if not username: + return {} + ensure_github_https_dependencies() + ok_user, msg = validate_github_username(username) + if ok_user: + ok(msg) + break + warn(msg) + retry = input(" Try another username? [Y/n]: ").strip().lower() + if retry == "n": + return {} + + print(f"\n{CYAN} To generate a Personal Access Token (PAT):{RESET}") + print(f"{CYAN} 1. Go to github.com -> Settings -> Developer settings{RESET}") + print(f"{CYAN} 2. Personal access tokens -> Fine-grained tokens{RESET}") + print(f"{CYAN} 3. Click 'Generate new token'{RESET}") + print(f"{CYAN} 4. Token name: xteink{RESET}") + print(f"{CYAN} 5. Expiration: No expiration{RESET}") + print(f"{CYAN} 6. Repository access: Only selected repositories -> select 'xteink'{RESET}") + print(f"{CYAN} 7. Permissions -> Add permissions -> Contents: Read-only{RESET}") + print(f"{CYAN} 8. Click 'Generate token' then copy it - GitHub only shows it once\n{RESET}") + + authenticated_login = None + while True: + pat = getpass.getpass(" Personal Access Token (PAT): ").strip() + if not pat: + return {} + ok_pat, msg, authenticated_login = validate_pat(pat) + if ok_pat: + ok(msg) + break + warn(msg) + retry = input(" Try entering PAT again? [Y/n]: ").strip().lower() + if retry == "n": + return {} + + while True: + repo = input(" Repo name [xteink]: ").strip() or "xteink" + owner_for_repo = authenticated_login or username + ok_repo, msg = validate_repo_access(owner_for_repo, repo, pat) + if ok_repo: + ok(msg) + if authenticated_login and authenticated_login != username: + warn(f"Username '{username}' differs from PAT owner '{authenticated_login}'. Repo was validated under '{owner_for_repo}'.") + break + warn(msg) + retry = input(" Try another repo name? [Y/n]: ").strip().lower() + if retry == "n": + return {} + + branch = input(" Branch [main]: ").strip() or "main" + + return {"username": username, "pat": pat, "repo": repo, "branch": branch} + +def write_nvs_partition(cfg: dict, repo: Path) -> Path: + if not check_nvs_gen(): + prompt_install("esp-idf-nvs-partition-gen") + + nvs_csv = Path(tempfile.mkdtemp()) / "github_sync_nvs.csv" + rows = [ + ["key", "type", "encoding", "value"], + [NVS_NAMESPACE, "namespace", "", ""], + ["username", "data", "string", cfg["username"]], + ["pat", "data", "string", cfg["pat"]], + ["repo", "data", "string", cfg["repo"]], + ["branch", "data", "string", cfg["branch"]], + ] + with open(nvs_csv, "w", newline="") as f: + writer = csv.writer(f) + writer.writerows(rows) + + nvs_bin = nvs_csv.with_suffix(".bin") + result = subprocess.run( + ["python3", "-m", "esp_idf_nvs_partition_gen", "generate", + str(nvs_csv), str(nvs_bin), "0x3000"], + capture_output=True, text=True + ) + if result.returncode != 0: + warn(f"nvs_partition_gen failed — enter credentials on-device via Settings -> GitHub Sync.\n{result.stderr.strip()}") + return None + + ok(f"NVS partition written to {nvs_bin}") + return nvs_bin + +def flash_nvs(nvs_bin: Path, port: str | None): + if not check_esptool(): + prompt_install("esptool") + info("Flashing NVS credentials partition...") + cmd = ["esptool.py", "--chip", "esp32c3", "write_flash", "0x9000", str(nvs_bin)] + if port: + cmd += ["--port", port] + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + warn("esptool.py flash failed — you may need to enter credentials on-device.") + warn(result.stderr.strip()) + else: + ok("Credentials flashed to device NVS") + +def copy_new_files(repo: Path): + copies = [ + (PATCH_DIR / "GitHubSync.h", + repo / "include" / "GitHubSync.h"), + (PATCH_DIR / "GitHubSyncSettingsActivity.h", + repo / "include" / "GitHubSyncSettingsActivity.h"), + (PATCH_DIR / "GitHubSync.cpp", + repo / "src" / "github_sync" / "GitHubSync.cpp"), + (PATCH_DIR / "GitHubSyncSettingsActivity.cpp", + repo / "src" / "activities" / "settings" / "GitHubSyncSettingsActivity.cpp"), + ] + for src, dst in copies: + if not src.exists(): + err(f"Patch file missing: {src}") + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, dst) + ok(f"Copied {dst.relative_to(repo)}") + +def inject_into_file(path: Path, marker: str, injection: str, after: bool = True, once: bool = True): + if not path.exists(): + warn(f"File not found, skipping: {path}") + return False + text = path.read_text() + if injection.strip() in text: + ok(f"Already patched: {path.name}") + return True + if marker not in text: + warn(f"Marker not found in {path.name}: {repr(marker)}") + warn(f" → Add manually per INTEGRATION.md") + return False + if after: + new_text = text.replace(marker, marker + "\n" + injection, 1 if once else -1) + else: + new_text = text.replace(marker, injection + "\n" + marker, 1 if once else -1) + path.write_text(new_text) + ok(f"Patched {path.name}") + return True + +def patch_main(repo: Path): + candidates = [repo / "src" / "main.cpp", repo / "src" / "Main.cpp"] + main = next((p for p in candidates if p.exists()), None) + if not main: + warn("Could not find main.cpp — add GitHub sync call manually per INTEGRATION.md") + return + + inject_into_file(main, + marker='#include "CrossPointSettings.h"', + injection='#include "GitHubSync.h"', + after=False) + + sync_injection = ( + '\n if (GitHubSync::isConfigured()) {\n' + ' GitHubSyncResult result = GitHubSync::sync();\n' + ' if (result != GitHubSyncResult::OK) {\n' + ' LOG_ERR("SYNC", "%s", GitHubSync::resultMessage(result));\n' + ' }\n' + ' }\n' + ) + + inject_into_file(main, + marker="activityManager.goToBoot();", + injection=sync_injection, + after=True) + +def patch_settings_menu(repo: Path): + # 1. Patch SettingsActivity.h — add GitHubSync to SettingAction enum + settings_h_candidates = list(repo.rglob("SettingsActivity.h")) + if not settings_h_candidates: + warn("Could not find SettingsActivity.h — add GitHubSync to SettingAction enum manually") + else: + inject_into_file(settings_h_candidates[0], + marker=" CheckForUpdates,", + injection=" GitHubSync,", + after=True) + + # 2. Patch SettingsActivity.cpp — add include, menu entry, and switch case + candidates = list(repo.rglob("SettingsActivity.cpp")) + if not candidates: + warn("Could not find SettingsActivity.cpp — add menu entry manually per INTEGRATION.md") + return + + settings = candidates[0] + + inject_into_file(settings, + marker='#include "SettingsActivity.h"', + injection='#include "GitHubSyncSettingsActivity.h"', + after=False) + + inject_into_file(settings, + marker="SettingInfo::Action(StrId::STR_CHECK_UPDATES, SettingAction::CheckForUpdates));", + injection=" systemSettings.push_back(SettingInfo::Action(StrId::STR_GITHUB_SYNC, SettingAction::GitHubSync));", + after=True) + + inject_into_file(settings, + marker="case SettingAction::CheckForUpdates:", + injection=" case SettingAction::GitHubSync:\n startActivityForResult(std::make_unique(renderer, mappedInput), resultHandler);\n break;", + after=True) + + # 3. Patch all YAML translation files — add STR_GITHUB_SYNC entry + yaml_files = list((repo / "lib" / "I18n" / "translations").glob("*.yaml")) + if not yaml_files: + warn("Could not find translation YAML files — add STR_GITHUB_SYNC manually to lib/I18n/translations/*.yaml") + else: + patched = 0 + for yf in yaml_files: + content = yf.read_text() + if 'STR_GITHUB_SYNC' in content: + ok(f"Already patched: {yf.name}") + patched += 1 + continue + lines = content.splitlines(keepends=True) + new_lines = [] + for line in lines: + new_lines.append(line) + if line.startswith("STR_CHECK_UPDATES:"): + new_lines.append('STR_GITHUB_SYNC: "GitHub Sync"\n') + if len(new_lines) > len(lines): + yf.write_text("".join(new_lines)) + ok(f"Patched {yf.name}") + patched += 1 + else: + warn(f"STR_CHECK_UPDATES not found in {yf.name} — add STR_GITHUB_SYNC manually") + ok(f"Added STR_GITHUB_SYNC to {patched} translation files") + +def patch_platformio(repo: Path): + ini = repo / "platformio.ini" + if not ini.exists(): + warn("platformio.ini not found") + return + if "ArduinoJson" in ini.read_text(): + ok("ArduinoJson already in platformio.ini") + return + inject_into_file(ini, + marker="lib_deps", + injection="\tbblanchon/ArduinoJson @ ^7", + after=True) + +def list_likely_serial_ports() -> list[str]: + patterns = [ + "/dev/cu.usb*", + "/dev/tty.usb*", + "/dev/cu.wchusb*", + "/dev/tty.wchusb*", + "/dev/cu.SLAB*", + "/dev/tty.SLAB*", + "/dev/ttyACM*", + "/dev/ttyUSB*", + ] + ports: list[str] = [] + for pattern in patterns: + ports.extend(glob.glob(pattern)) + # Keep stable order and remove duplicates. + return sorted(set(ports)) + +def prompt_for_upload_port(existing_port: str | None) -> str | None: + print(f"\n{BOLD}Ready to upload firmware{RESET}") + print(f"{CYAN} Connect your xteink device via USB now, then press Enter.{RESET}") + input(" Press Enter when connected... ") + + likely_ports = list_likely_serial_ports() + if likely_ports: + print("\n Detected likely USB serial ports:") + for p in likely_ports: + print(f" - {p}") + else: + warn("No obvious USB serial ports detected. You can still enter one manually.") + + # Show PlatformIO's own device list as extra context. + device_list = subprocess.run(["pio", "device", "list"], capture_output=True, text=True) + if device_list.returncode == 0 and device_list.stdout.strip(): + print("\n PlatformIO device list:") + print(device_list.stdout.rstrip()) + + if existing_port: + info(f"Current upload port argument: {existing_port}") + entered = input(" Upload port (Enter to keep current): ").strip() + return entered or existing_port + + entered = input(" Upload port (recommended, e.g. /dev/cu.usbmodemXXXX; Enter for auto-detect): ").strip() + if entered: + return entered + + warn("Proceeding with auto-detect. This may choose a non-USB port (like Bluetooth).") + return None + +def main(): + print(f"\n{BOLD}CrossPoint GitHub Sync Patcher{RESET}\n") + + check_git() + check_pio() + check_esptool() + + dest = Path(sys.argv[1]) if len(sys.argv) > 1 else Path.cwd() + port = sys.argv[2] if len(sys.argv) > 2 else None + + print(f"{BOLD}1. Fetching CrossPoint repo ({CROSSPOINT_URL}){RESET}") + repo = clone_or_update(dest) + info(f"Repo: {repo}\n") + + print(f"{BOLD}2. Copying new files{RESET}") + copy_new_files(repo) + + print(f"\n{BOLD}3. Patching platformio.ini{RESET}") + patch_platformio(repo) + + print(f"\n{BOLD}4. Patching main.cpp{RESET}") + patch_main(repo) + + print(f"\n{BOLD}5. Patching settings menu{RESET}") + patch_settings_menu(repo) + + print(f"\n{BOLD}6. GitHub credentials{RESET}") + cfg = prompt_github_config() + nvs_bin = None + if cfg: + nvs_bin = write_nvs_partition(cfg, repo) + + port = prompt_for_upload_port(port) + + print(f"\n{BOLD}7. Building and flashing firmware{RESET}") + info("Running: pio run --target upload") + result = subprocess.run( + ["pio", "run", "--target", "upload"] + (["--upload-port", port] if port else []), + cwd=repo + ) + if result.returncode != 0: + err("pio build/flash failed — check output above") + ok("Firmware flashed") + + if nvs_bin: + print(f"\n{BOLD}8. Flashing credentials{RESET}") + flash_nvs(nvs_bin, port) + + print(f"\n{GREEN}{BOLD}All done.{RESET}") + if not cfg: + print("No credentials entered — configure on-device via Settings → GitHub Sync.") + print() + +if __name__ == "__main__": + main()