Stream CrossPointWebServer data over JSON APIs (#97)

## Summary

* HTML files are now static, streamed directly to the client without
modification
* For any dynamic values, load via JSON APIs
* For files page, we stream the JSON content as we scan the directory to
avoid holding onto too much data

## Additional details

* We were previously building up a very large string all generated on
the X4 directly, we should be leveraging the browser
* Fixes https://github.com/daveallie/crosspoint-reader/issues/94

src/network/CrossPointWebServer.cpp

@@ -1,53 +1,19 @@
 #include "CrossPointWebServer.h"
 
+#include <ArduinoJson.h>
 #include <SD.h>
 #include <WiFi.h>
 
 #include <algorithm>
 
-#include "config.h"
-#include "html/FilesPageFooterHtml.generated.h"
-#include "html/FilesPageHeaderHtml.generated.h"
+#include "html/FilesPageHtml.generated.h"
 #include "html/HomePageHtml.generated.h"
 
 namespace {
-
 // Folders/files to hide from the web interface file browser
 // Note: Items starting with "." are automatically hidden
 const char* HIDDEN_ITEMS[] = {"System Volume Information", "XTCache"};
-const size_t HIDDEN_ITEMS_COUNT = sizeof(HIDDEN_ITEMS) / sizeof(HIDDEN_ITEMS[0]);
-
-// Helper function to escape HTML special characters to prevent XSS
-String escapeHtml(const String& input) {
-  String output;
-  output.reserve(input.length() * 1.1);  // Pre-allocate with some extra space
-
-  for (size_t i = 0; i < input.length(); i++) {
-    char c = input.charAt(i);
-    switch (c) {
-      case '&':
-        output += "&amp;";
-        break;
-      case '<':
-        output += "&lt;";
-        break;
-      case '>':
-        output += "&gt;";
-        break;
-      case '"':
-        output += "&quot;";
-        break;
-      case '\'':
-        output += "&#39;";
-        break;
-      default:
-        output += c;
-        break;
-    }
-  }
-  return output;
-}
-
+constexpr size_t HIDDEN_ITEMS_COUNT = sizeof(HIDDEN_ITEMS) / sizeof(HIDDEN_ITEMS[0]);
 }  // namespace
 
 // File listing page template - now using generated headers:
@@ -72,7 +38,7 @@ void CrossPointWebServer::begin() {
   Serial.printf("[%lu] [WEB] [MEM] Free heap before begin: %d bytes\n", millis(), ESP.getFreeHeap());
 
   Serial.printf("[%lu] [WEB] Creating web server on port %d...\n", millis(), port);
-  server = new WebServer(port);
+  server.reset(new WebServer(port));
   Serial.printf("[%lu] [WEB] [MEM] Free heap after WebServer allocation: %d bytes\n", millis(), ESP.getFreeHeap());
 
   if (!server) {
@@ -82,20 +48,22 @@ void CrossPointWebServer::begin() {
 
   // Setup routes
   Serial.printf("[%lu] [WEB] Setting up routes...\n", millis());
-  server->on("/", HTTP_GET, [this]() { handleRoot(); });
-  server->on("/status", HTTP_GET, [this]() { handleStatus(); });
-  server->on("/files", HTTP_GET, [this]() { handleFileList(); });
+  server->on("/", HTTP_GET, [this] { handleRoot(); });
+  server->on("/files", HTTP_GET, [this] { handleFileList(); });
+
+  server->on("/api/status", HTTP_GET, [this] { handleStatus(); });
+  server->on("/api/files", HTTP_GET, [this] { handleFileListData(); });
 
   // Upload endpoint with special handling for multipart form data
-  server->on("/upload", HTTP_POST, [this]() { handleUploadPost(); }, [this]() { handleUpload(); });
+  server->on("/upload", HTTP_POST, [this] { handleUploadPost(); }, [this] { handleUpload(); });
 
   // Create folder endpoint
-  server->on("/mkdir", HTTP_POST, [this]() { handleCreateFolder(); });
+  server->on("/mkdir", HTTP_POST, [this] { handleCreateFolder(); });
 
   // Delete file/folder endpoint
-  server->on("/delete", HTTP_POST, [this]() { handleDelete(); });
+  server->on("/delete", HTTP_POST, [this] { handleDelete(); });
 
-  server->onNotFound([this]() { handleNotFound(); });
+  server->onNotFound([this] { handleNotFound(); });
   Serial.printf("[%lu] [WEB] [MEM] Free heap after route setup: %d bytes\n", millis(), ESP.getFreeHeap());
 
   server->begin();
@@ -108,7 +76,8 @@ void CrossPointWebServer::begin() {
 
 void CrossPointWebServer::stop() {
   if (!running || !server) {
-    Serial.printf("[%lu] [WEB] stop() called but already stopped (running=%d, server=%p)\n", millis(), running, server);
+    Serial.printf("[%lu] [WEB] stop() called but already stopped (running=%d, server=%p)\n", millis(), running,
+                  server.get());
     return;
   }
 
@@ -128,9 +97,7 @@ void CrossPointWebServer::stop() {
   delay(50);
   Serial.printf("[%lu] [WEB] Waited 50ms before deleting server\n", millis());
 
-  delete server;
-  server = nullptr;
-
+  server.reset();
   Serial.printf("[%lu] [WEB] Web server stopped and deleted\n", millis());
   Serial.printf("[%lu] [WEB] [MEM] Free heap after delete server: %d bytes\n", millis(), ESP.getFreeHeap());
 
@@ -139,7 +106,7 @@ void CrossPointWebServer::stop() {
   Serial.printf("[%lu] [WEB] [MEM] Free heap final: %d bytes\n", millis(), ESP.getFreeHeap());
 }
 
-void CrossPointWebServer::handleClient() {
+void CrossPointWebServer::handleClient() const {
   static unsigned long lastDebugPrint = 0;
 
   // Check running flag FIRST before accessing server
@@ -162,25 +129,18 @@ void CrossPointWebServer::handleClient() {
   server->handleClient();
 }
 
-void CrossPointWebServer::handleRoot() {
-  String html = HomePageHtml;
-
-  // Replace placeholders with actual values
-  html.replace("%VERSION%", CROSSPOINT_VERSION);
-  html.replace("%IP_ADDRESS%", WiFi.localIP().toString());
-  html.replace("%FREE_HEAP%", String(ESP.getFreeHeap()));
-
-  server->send(200, "text/html", html);
+void CrossPointWebServer::handleRoot() const {
+  server->send(200, "text/html", HomePageHtml);
   Serial.printf("[%lu] [WEB] Served root page\n", millis());
 }
 
-void CrossPointWebServer::handleNotFound() {
+void CrossPointWebServer::handleNotFound() const {
   String message = "404 Not Found\n\n";
   message += "URI: " + server->uri() + "\n";
   server->send(404, "text/plain", message);
 }
 
-void CrossPointWebServer::handleStatus() {
+void CrossPointWebServer::handleStatus() const {
   String json = "{";
   json += "\"version\":\"" + String(CROSSPOINT_VERSION) + "\",";
   json += "\"ip\":\"" + WiFi.localIP().toString() + "\",";
@@ -192,26 +152,24 @@ void CrossPointWebServer::handleStatus() {
   server->send(200, "application/json", json);
 }
 
-std::vector<FileInfo> CrossPointWebServer::scanFiles(const char* path) {
-  std::vector<FileInfo> files;
-
+void CrossPointWebServer::scanFiles(const char* path, const std::function<void(FileInfo)>& callback) const {
   File root = SD.open(path);
   if (!root) {
     Serial.printf("[%lu] [WEB] Failed to open directory: %s\n", millis(), path);
-    return files;
+    return;
   }
 
   if (!root.isDirectory()) {
     Serial.printf("[%lu] [WEB] Not a directory: %s\n", millis(), path);
     root.close();
-    return files;
+    return;
   }
 
   Serial.printf("[%lu] [WEB] Scanning files in: %s\n", millis(), path);
 
   File file = root.openNextFile();
   while (file) {
-    String fileName = String(file.name());
+    auto fileName = String(file.name());
 
     // Skip hidden items (starting with ".")
     bool shouldHide = fileName.startsWith(".");
@@ -239,37 +197,24 @@ std::vector<FileInfo> CrossPointWebServer::scanFiles(const char* path) {
         info.isEpub = isEpubFile(info.name);
       }
 
-      files.push_back(info);
+      callback(info);
     }
 
     file.close();
     file = root.openNextFile();
   }
   root.close();
-
-  Serial.printf("[%lu] [WEB] Found %d items (files and folders)\n", millis(), files.size());
-  return files;
 }
 
-String CrossPointWebServer::formatFileSize(size_t bytes) {
-  if (bytes < 1024) {
-    return String(bytes) + " B";
-  } else if (bytes < 1024 * 1024) {
-    return String(bytes / 1024.0, 1) + " KB";
-  } else {
-    return String(bytes / (1024.0 * 1024.0), 1) + " MB";
-  }
-}
-
-bool CrossPointWebServer::isEpubFile(const String& filename) {
+bool CrossPointWebServer::isEpubFile(const String& filename) const {
   String lower = filename;
   lower.toLowerCase();
   return lower.endsWith(".epub");
 }
 
-void CrossPointWebServer::handleFileList() {
-  String html = FilesPageHeaderHtml;
+void CrossPointWebServer::handleFileList() const { server->send(200, "text/html", FilesPageHtml); }
 
+void CrossPointWebServer::handleFileListData() const {
   // Get current path from query string (default to root)
   String currentPath = "/";
   if (server->hasArg("path")) {
@@ -284,180 +229,35 @@ void CrossPointWebServer::handleFileList() {
     }
   }
 
-  // Get message from query string if present
-  if (server->hasArg("msg")) {
-    String msg = escapeHtml(server->arg("msg"));
-    String msgType = server->hasArg("type") ? escapeHtml(server->arg("type")) : "success";
-    html += "<div class=\"message " + msgType + "\">" + msg + "</div>";
-  }
+  server->setContentLength(CONTENT_LENGTH_UNKNOWN);
+  server->send(200, "application/json", "");
+  server->sendContent("[");
+  char output[512];
+  constexpr size_t outputSize = sizeof(output);
+  bool seenFirst = false;
+  scanFiles(currentPath.c_str(), [this, &output, seenFirst](const FileInfo& info) mutable {
+    JsonDocument doc;
+    doc["name"] = info.name;
+    doc["size"] = info.size;
+    doc["isDirectory"] = info.isDirectory;
+    doc["isEpub"] = info.isEpub;
+    const size_t written = serializeJson(doc, output, outputSize);
+    if (written >= outputSize) {
+      // JSON output truncated; skip this entry to avoid sending malformed JSON
+      Serial.printf("[%lu] [WEB] Skipping file entry with oversized JSON for name: %s\n", millis(), info.name.c_str());
+      return;
+    }
 
-  // Hidden input to store current path for JavaScript
-  html += "<input type=\"hidden\" id=\"currentPath\" value=\"" + currentPath + "\">";
-
-  // Scan files in current path first (we need counts for the header)
-  std::vector<FileInfo> files = scanFiles(currentPath.c_str());
-
-  // Count items
-  int epubCount = 0;
-  int folderCount = 0;
-  size_t totalSize = 0;
-  for (const auto& file : files) {
-    if (file.isDirectory) {
-      folderCount++;
+    if (seenFirst) {
+      server->sendContent(",");
     } else {
-      if (file.isEpub) epubCount++;
-      totalSize += file.size;
+      seenFirst = true;
     }
-  }
-
-  // Page header with inline breadcrumb and action buttons
-  html += "<div class=\"page-header\">";
-  html += "<div class=\"page-header-left\">";
-  html += "<h1>📁 File Manager</h1>";
-
-  // Inline breadcrumb
-  html += "<div class=\"breadcrumb-inline\">";
-  html += "<span class=\"sep\">/</span>";
-
-  if (currentPath == "/") {
-    html += "<span class=\"current\">🏠</span>";
-  } else {
-    html += "<a href=\"/files\">🏠</a>";
-    String pathParts = currentPath.substring(1);  // Remove leading /
-    String buildPath = "";
-    int start = 0;
-    int end = pathParts.indexOf('/');
-
-    while (start < (int)pathParts.length()) {
-      String part;
-      if (end == -1) {
-        part = pathParts.substring(start);
-        buildPath += "/" + part;
-        html += "<span class=\"sep\">/</span><span class=\"current\">" + escapeHtml(part) + "</span>";
-        break;
-      } else {
-        part = pathParts.substring(start, end);
-        buildPath += "/" + part;
-        html += "<span class=\"sep\">/</span><a href=\"/files?path=" + buildPath + "\">" + escapeHtml(part) + "</a>";
-        start = end + 1;
-        end = pathParts.indexOf('/', start);
-      }
-    }
-  }
-  html += "</div>";
-  html += "</div>";
-
-  // Action buttons
-  html += "<div class=\"action-buttons\">";
-  html += "<button class=\"action-btn upload-action-btn\" onclick=\"openUploadModal()\">";
-  html += "📤 Upload";
-  html += "</button>";
-  html += "<button class=\"action-btn folder-action-btn\" onclick=\"openFolderModal()\">";
-  html += "📁 New Folder";
-  html += "</button>";
-  html += "</div>";
-
-  html += "</div>";  // end page-header
-
-  // Contents card with inline summary
-  html += "<div class=\"card\">";
-
-  // Contents header with inline stats
-  html += "<div class=\"contents-header\">";
-  html += "<h2 class=\"contents-title\">Contents</h2>";
-  html += "<span class=\"summary-inline\">";
-  html += String(folderCount) + " folder" + (folderCount != 1 ? "s" : "") + ", ";
-  html += String(files.size() - folderCount) + " file" + ((files.size() - folderCount) != 1 ? "s" : "") + ", ";
-  html += formatFileSize(totalSize);
-  html += "</span>";
-  html += "</div>";
-
-  if (files.empty()) {
-    html += "<div class=\"no-files\">This folder is empty</div>";
-  } else {
-    html += "<table class=\"file-table\">";
-    html += "<tr><th>Name</th><th>Type</th><th>Size</th><th class=\"actions-col\">Actions</th></tr>";
-
-    // Sort files: folders first, then epub files, then other files, alphabetically within each group
-    std::sort(files.begin(), files.end(), [](const FileInfo& a, const FileInfo& b) {
-      // Folders come first
-      if (a.isDirectory != b.isDirectory) return a.isDirectory > b.isDirectory;
-      // Then sort by epub status (epubs first among files)
-      if (a.isEpub != b.isEpub) return a.isEpub > b.isEpub;
-      // Then alphabetically
-      return a.name < b.name;
-    });
-
-    for (const auto& file : files) {
-      String rowClass;
-      String icon;
-      String badge;
-      String typeStr;
-      String sizeStr;
-
-      if (file.isDirectory) {
-        rowClass = "folder-row";
-        icon = "📁";
-        badge = "<span class=\"folder-badge\">FOLDER</span>";
-        typeStr = "Folder";
-        sizeStr = "-";
-
-        // Build the path to this folder
-        String folderPath = currentPath;
-        if (!folderPath.endsWith("/")) folderPath += "/";
-        folderPath += file.name;
-
-        html += "<tr class=\"" + rowClass + "\">";
-        html += "<td><span class=\"file-icon\">" + icon + "</span>";
-        html += "<a href=\"/files?path=" + folderPath + "\" class=\"folder-link\">" + escapeHtml(file.name) + "</a>" +
-                badge + "</td>";
-        html += "<td>" + typeStr + "</td>";
-        html += "<td>" + sizeStr + "</td>";
-        // Escape quotes for JavaScript string
-        String escapedName = file.name;
-        escapedName.replace("'", "\\'");
-        String escapedPath = folderPath;
-        escapedPath.replace("'", "\\'");
-        html += "<td class=\"actions-col\"><button class=\"delete-btn\" onclick=\"openDeleteModal('" + escapedName +
-                "', '" + escapedPath + "', true)\" title=\"Delete folder\">🗑️</button></td>";
-        html += "</tr>";
-      } else {
-        rowClass = file.isEpub ? "epub-file" : "";
-        icon = file.isEpub ? "📗" : "📄";
-        badge = file.isEpub ? "<span class=\"epub-badge\">EPUB</span>" : "";
-        String ext = file.name.substring(file.name.lastIndexOf('.') + 1);
-        ext.toUpperCase();
-        typeStr = ext;
-        sizeStr = formatFileSize(file.size);
-
-        // Build file path for delete
-        String filePath = currentPath;
-        if (!filePath.endsWith("/")) filePath += "/";
-        filePath += file.name;
-
-        html += "<tr class=\"" + rowClass + "\">";
-        html += "<td><span class=\"file-icon\">" + icon + "</span>" + escapeHtml(file.name) + badge + "</td>";
-        html += "<td>" + typeStr + "</td>";
-        html += "<td>" + sizeStr + "</td>";
-        // Escape quotes for JavaScript string
-        String escapedName = file.name;
-        escapedName.replace("'", "\\'");
-        String escapedPath = filePath;
-        escapedPath.replace("'", "\\'");
-        html += "<td class=\"actions-col\"><button class=\"delete-btn\" onclick=\"openDeleteModal('" + escapedName +
-                "', '" + escapedPath + "', false)\" title=\"Delete file\">🗑️</button></td>";
-        html += "</tr>";
-      }
-    }
-
-    html += "</table>";
-  }
-
-  html += "</div>";
-
-  html += FilesPageFooterHtml;
-
-  server->send(200, "text/html", html);
+    server->sendContent(output);
+  });
+  server->sendContent("]");
+  // End of streamed response, empty chunk to signal client
+  server->sendContent("");
   Serial.printf("[%lu] [WEB] Served file listing page for path: %s\n", millis(), currentPath.c_str());
 }
 
@@ -469,7 +269,7 @@ static size_t uploadSize = 0;
 static bool uploadSuccess = false;
 static String uploadError = "";
 
-void CrossPointWebServer::handleUpload() {
+void CrossPointWebServer::handleUpload() const {
   static unsigned long lastWriteTime = 0;
   static unsigned long uploadStartTime = 0;
   static size_t lastLoggedSize = 0;
@@ -480,7 +280,7 @@ void CrossPointWebServer::handleUpload() {
     return;
   }
 
-  HTTPUpload& upload = server->upload();
+  const HTTPUpload& upload = server->upload();
 
   if (upload.status == UPLOAD_FILE_START) {
     uploadFileName = upload.filename;
@@ -533,10 +333,10 @@ void CrossPointWebServer::handleUpload() {
     Serial.printf("[%lu] [WEB] [UPLOAD] File created successfully: %s\n", millis(), filePath.c_str());
   } else if (upload.status == UPLOAD_FILE_WRITE) {
     if (uploadFile && uploadError.isEmpty()) {
-      unsigned long writeStartTime = millis();
-      size_t written = uploadFile.write(upload.buf, upload.currentSize);
-      unsigned long writeEndTime = millis();
-      unsigned long writeDuration = writeEndTime - writeStartTime;
+      const unsigned long writeStartTime = millis();
+      const size_t written = uploadFile.write(upload.buf, upload.currentSize);
+      const unsigned long writeEndTime = millis();
+      const unsigned long writeDuration = writeEndTime - writeStartTime;
 
       if (written != upload.currentSize) {
         uploadError = "Failed to write to SD card - disk may be full";
@@ -548,9 +348,9 @@ void CrossPointWebServer::handleUpload() {
 
         // Log progress every 50KB or if write took >100ms
         if (uploadSize - lastLoggedSize >= 51200 || writeDuration > 100) {
-          unsigned long timeSinceStart = millis() - uploadStartTime;
-          unsigned long timeSinceLastWrite = millis() - lastWriteTime;
-          float kbps = (uploadSize / 1024.0) / (timeSinceStart / 1000.0);
+          const unsigned long timeSinceStart = millis() - uploadStartTime;
+          const unsigned long timeSinceLastWrite = millis() - lastWriteTime;
+          const float kbps = (uploadSize / 1024.0) / (timeSinceStart / 1000.0);
 
           Serial.printf(
               "[%lu] [WEB] [UPLOAD] Progress: %d bytes (%.1f KB), %.1f KB/s, write took %lu ms, gap since last: %lu "
@@ -584,23 +384,23 @@ void CrossPointWebServer::handleUpload() {
   }
 }
 
-void CrossPointWebServer::handleUploadPost() {
+void CrossPointWebServer::handleUploadPost() const {
   if (uploadSuccess) {
     server->send(200, "text/plain", "File uploaded successfully: " + uploadFileName);
   } else {
-    String error = uploadError.isEmpty() ? "Unknown error during upload" : uploadError;
+    const String error = uploadError.isEmpty() ? "Unknown error during upload" : uploadError;
     server->send(400, "text/plain", error);
   }
 }
 
-void CrossPointWebServer::handleCreateFolder() {
+void CrossPointWebServer::handleCreateFolder() const {
   // Get folder name from form data
   if (!server->hasArg("name")) {
     server->send(400, "text/plain", "Missing folder name");
     return;
   }
 
-  String folderName = server->arg("name");
+  const String folderName = server->arg("name");
 
   // Validate folder name
   if (folderName.isEmpty()) {
@@ -643,7 +443,7 @@ void CrossPointWebServer::handleCreateFolder() {
   }
 }
 
-void CrossPointWebServer::handleDelete() {
+void CrossPointWebServer::handleDelete() const {
   // Get path from form data
   if (!server->hasArg("path")) {
     server->send(400, "text/plain", "Missing path");
@@ -651,7 +451,7 @@ void CrossPointWebServer::handleDelete() {
   }
 
   String itemPath = server->arg("path");
-  String itemType = server->hasArg("type") ? server->arg("type") : "file";
+  const String itemType = server->hasArg("type") ? server->arg("type") : "file";
 
   // Validate path
   if (itemPath.isEmpty() || itemPath == "/") {
@@ -665,7 +465,7 @@ void CrossPointWebServer::handleDelete() {
   }
 
   // Security check: prevent deletion of protected items
-  String itemName = itemPath.substring(itemPath.lastIndexOf('/') + 1);
+  const String itemName = itemPath.substring(itemPath.lastIndexOf('/') + 1);
 
   // Check if item starts with a dot (hidden/system file)
   if (itemName.startsWith(".")) {