Ravi ReddyandClaude Opus 5.5
3b371cdb28
VoiceNotes is an Arduino sketch for the Waveshare ESP32-C6-LCD-0.85 that records meetings and voice memos to microSD and turns them into notes: - Records 16 kHz mono WAV (up to 4 h) from the ES7210 dual mics, with a boosted preamp and soft-limited digital gain - Transcribes via an OpenAI-compatible speech-to-text API in ~10 min parts cut at quiet points, with retries and resume after failures - Summarizes the transcript with Claude into meeting notes (decisions, action items, topics with timestamps), streamed to and from the card - Copies the notes to Nextcloud over WebDAV - Shows recording/processing state on the 0.85" LCD and the LED ring - Works around the board's SD card failing multi-sector reads Credentials live in VoiceNotes/secrets.h, which is git-ignored; see secrets.example.h. Also adds the Microsoft CDM customer service SQL implementation (DDL, sample data, reporting queries and docs). Its README is renamed to CDM_README.md so the repository README stays at the top level. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
941 lines
34 KiB
C++
941 lines
34 KiB
C++
#include "cloud.h"
|
|
#include "app_state.h"
|
|
#include "audio.h"
|
|
#include "config.h"
|
|
#include "secrets.h"
|
|
#include <ArduinoJson.h>
|
|
#include <base64.h>
|
|
#include <NetworkClientSecure.h>
|
|
#include <SD.h>
|
|
#include <WiFi.h>
|
|
#include <errno.h>
|
|
#include <fcntl.h>
|
|
#include <sys/stat.h>
|
|
#include <unistd.h>
|
|
|
|
static volatile bool busy = false;
|
|
static char jobWav[64];
|
|
|
|
bool cloudBusy() { return busy; }
|
|
|
|
// Network diagnostics on the serial port, e.g. "[net 12.345s heap 98304] TLS connect ok"
|
|
#define NETLOG(fmt, ...) \
|
|
Serial.printf("[net %lu.%03lus heap %lu] " fmt "\n", millis() / 1000, millis() % 1000, \
|
|
(unsigned long)ESP.getFreeHeap(), ##__VA_ARGS__)
|
|
|
|
static const char *wifiStatusName(wl_status_t s) {
|
|
switch (s) {
|
|
case WL_CONNECTED: return "connected";
|
|
case WL_NO_SSID_AVAIL: return "SSID not found";
|
|
case WL_CONNECT_FAILED: return "connect failed (wrong password?)";
|
|
case WL_CONNECTION_LOST: return "connection lost";
|
|
case WL_DISCONNECTED: return "disconnected";
|
|
case WL_IDLE_STATUS: return "idle";
|
|
default: return "unknown";
|
|
}
|
|
}
|
|
|
|
static void logTlsError(NetworkClientSecure &c, const char *what) {
|
|
char msg[128] = "";
|
|
int code = c.lastError(msg, sizeof(msg));
|
|
NETLOG("%s: TLS error %d (-0x%04X) %s", what, code, code < 0 ? -code : code, msg);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Minimal HTTP/1.1 over TLS. We talk to the socket directly (rather than
|
|
// HTTPClient) so we can stream a multi-MB WAV from the SD card, wait minutes
|
|
// for a reply, and decode chunked / server-sent-event responses line by line.
|
|
// ---------------------------------------------------------------------------
|
|
static int readByteWait(NetworkClient &c, uint32_t timeoutMs) {
|
|
uint32_t t0 = millis();
|
|
while (!c.available()) {
|
|
if (!c.connected() || millis() - t0 > timeoutMs) return -1;
|
|
delay(5);
|
|
}
|
|
return c.read();
|
|
}
|
|
|
|
static bool readRawLine(NetworkClient &c, String &line, uint32_t timeoutMs) {
|
|
line = "";
|
|
for (;;) {
|
|
int b = readByteWait(c, timeoutMs);
|
|
if (b < 0) return line.length() > 0;
|
|
if (b == '\n') break;
|
|
if (b != '\r') line += (char)b;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
static bool writeAll(NetworkClient &c, const uint8_t *data, size_t len) {
|
|
uint32_t t0 = millis();
|
|
while (len) {
|
|
size_t n = c.write(data, len);
|
|
if (n == 0) {
|
|
if (!c.connected()) {
|
|
NETLOG("write failed: server closed the connection (%u bytes unsent)", (unsigned)len);
|
|
return false;
|
|
}
|
|
if (millis() - t0 > 20000) {
|
|
NETLOG("write failed: no progress for 20 s (%u bytes unsent, Wi-Fi %s, RSSI %d dBm)", (unsigned)len,
|
|
wifiStatusName(WiFi.status()), WiFi.RSSI());
|
|
return false;
|
|
}
|
|
delay(10);
|
|
continue;
|
|
}
|
|
data += n;
|
|
len -= n;
|
|
t0 = millis();
|
|
}
|
|
return true;
|
|
}
|
|
static bool writeAll(NetworkClient &c, const String &s) { return writeAll(c, (const uint8_t *)s.c_str(), s.length()); }
|
|
|
|
struct HttpHead {
|
|
int status = 0;
|
|
bool chunked = false;
|
|
long contentLength = -1;
|
|
};
|
|
|
|
// Wait (up to firstByteTimeoutMs) for the server to answer, then parse the head.
|
|
static bool readHead(NetworkClient &c, HttpHead &h, uint32_t firstByteTimeoutMs) {
|
|
uint32_t t0 = millis();
|
|
while (!c.available()) {
|
|
if (!c.connected()) {
|
|
NETLOG("no response: server closed the connection after %lu ms", millis() - t0);
|
|
return false;
|
|
}
|
|
if (millis() - t0 > firstByteTimeoutMs) {
|
|
NETLOG("no response: timed out after %lu s", firstByteTimeoutMs / 1000);
|
|
return false;
|
|
}
|
|
delay(20);
|
|
}
|
|
NETLOG("first response byte after %lu ms", millis() - t0);
|
|
String line;
|
|
if (!readRawLine(c, line, 15000)) {
|
|
NETLOG("no response: status line never arrived");
|
|
return false;
|
|
}
|
|
NETLOG("< %s", line.c_str());
|
|
int sp = line.indexOf(' ');
|
|
if (sp < 0) return false;
|
|
h.status = line.substring(sp + 1).toInt();
|
|
while (readRawLine(c, line, 15000) && line.length()) {
|
|
if (h.status >= 300) NETLOG("< %s", line.c_str()); // full headers only when something went wrong
|
|
String lower = line;
|
|
lower.toLowerCase();
|
|
if (lower.startsWith("transfer-encoding:") && lower.indexOf("chunked") > 0) h.chunked = true;
|
|
if (lower.startsWith("content-length:")) h.contentLength = lower.substring(15).toInt();
|
|
}
|
|
return true;
|
|
}
|
|
|
|
// Reads a response body, transparently removing chunked transfer encoding.
|
|
class BodyReader {
|
|
public:
|
|
BodyReader(NetworkClient &c, const HttpHead &h, uint32_t idleTimeoutMs)
|
|
: c_(c), chunked_(h.chunked), remaining_(h.contentLength), timeout_(idleTimeoutMs) {}
|
|
|
|
// Returns bytes read, 0 at end of body (or on timeout / disconnect).
|
|
size_t read(uint8_t *buf, size_t n) {
|
|
if (done_) return 0;
|
|
if (chunked_) {
|
|
if (chunkLeft_ == 0) {
|
|
String sizeLine;
|
|
if (!readRawLine(c_, sizeLine, timeout_)) return finish();
|
|
chunkLeft_ = strtol(sizeLine.c_str(), nullptr, 16);
|
|
if (chunkLeft_ == 0) return finish();
|
|
}
|
|
if (n > chunkLeft_) n = chunkLeft_;
|
|
} else if (remaining_ >= 0) {
|
|
if (remaining_ == 0) return finish();
|
|
if ((long)n > remaining_) n = remaining_;
|
|
}
|
|
size_t got = 0;
|
|
uint32_t t0 = millis();
|
|
while (got == 0) {
|
|
int avail = c_.available();
|
|
if (avail > 0) {
|
|
int r = c_.read(buf, min((size_t)avail, n));
|
|
if (r > 0) got = r;
|
|
} else if (!c_.connected() || millis() - t0 > timeout_) {
|
|
return finish();
|
|
} else {
|
|
delay(5);
|
|
}
|
|
}
|
|
if (chunked_) {
|
|
chunkLeft_ -= got;
|
|
if (chunkLeft_ == 0) {
|
|
String crlf;
|
|
readRawLine(c_, crlf, timeout_);
|
|
}
|
|
} else if (remaining_ > 0) {
|
|
remaining_ -= got;
|
|
}
|
|
return got;
|
|
}
|
|
|
|
bool readLine(String &line) {
|
|
line = "";
|
|
for (;;) {
|
|
if (pos_ == len_) {
|
|
len_ = read(buf_, sizeof(buf_));
|
|
pos_ = 0;
|
|
if (len_ == 0) return line.length() > 0;
|
|
}
|
|
char ch = (char)buf_[pos_++];
|
|
if (ch == '\n') return true;
|
|
if (ch != '\r') line += ch;
|
|
}
|
|
}
|
|
|
|
String readAll(size_t maxLen) {
|
|
String s;
|
|
uint8_t b[256];
|
|
size_t n;
|
|
while (s.length() < maxLen && (n = read(b, sizeof(b))) > 0) s.concat((const char *)b, n);
|
|
return s;
|
|
}
|
|
|
|
private:
|
|
size_t finish() { done_ = true; return 0; }
|
|
NetworkClient &c_;
|
|
bool chunked_;
|
|
long remaining_;
|
|
uint32_t timeout_;
|
|
size_t chunkLeft_ = 0;
|
|
bool done_ = false;
|
|
uint8_t buf_[512];
|
|
size_t pos_ = 0, len_ = 0;
|
|
};
|
|
|
|
static bool connectTls(NetworkClientSecure &c, const char *host) {
|
|
c.useBuiltinCACertBundle(); // verify the server certificate
|
|
c.setHandshakeTimeout(30);
|
|
NETLOG("connecting to %s:443", host);
|
|
uint32_t t0 = millis();
|
|
if (!c.connect(host, 443)) { // does its own (thread-safe) DNS lookup
|
|
logTlsError(c, "connect failed (DNS or TLS)");
|
|
return false;
|
|
}
|
|
NETLOG("TLS connect ok to %s in %lu ms", c.remoteIP().toString().c_str(), millis() - t0);
|
|
return true;
|
|
}
|
|
|
|
static bool ensureWifi() {
|
|
if (WiFi.status() != WL_CONNECTED) {
|
|
NETLOG("Wi-Fi %s, reconnecting...", wifiStatusName(WiFi.status()));
|
|
WiFi.reconnect();
|
|
for (int i = 0; i < 150 && WiFi.status() != WL_CONNECTED; i++) delay(100);
|
|
}
|
|
if (WiFi.status() != WL_CONNECTED) {
|
|
NETLOG("Wi-Fi still %s", wifiStatusName(WiFi.status()));
|
|
return false;
|
|
}
|
|
NETLOG("Wi-Fi ok: SSID \"%s\", IP %s, DNS %s, RSSI %d dBm", WiFi.SSID().c_str(),
|
|
WiFi.localIP().toString().c_str(), WiFi.dnsIP().toString().c_str(), WiFi.RSSI());
|
|
return true;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// SD reads. On this board the card fails multi-sector reads (CMD18) with an
|
|
// I/O error, while single-sector reads (CMD17) and all writes work. Arduino's
|
|
// File::read() goes through stdio, which reads ahead 4 KB at a time and so
|
|
// always triggers multi-sector reads. Calling POSIX read() with 512-byte
|
|
// requests keeps every read to a single sector.
|
|
// ---------------------------------------------------------------------------
|
|
static int sdOpenRead(const char *path) { return open((String("/sd") + path).c_str(), O_RDONLY); }
|
|
|
|
// Fills buf with up to len bytes, 512 at a time. Returns bytes read, or -1 on error.
|
|
static int sdRead(int fd, uint8_t *buf, size_t len) {
|
|
size_t got = 0;
|
|
while (got < len) {
|
|
int n = read(fd, buf + got, min((size_t)512, len - got));
|
|
if (n < 0) return got ? (int)got : -1;
|
|
if (n == 0) break;
|
|
got += n;
|
|
}
|
|
return got;
|
|
}
|
|
|
|
static String httpErrorText(const char *service, int status, const String &body) {
|
|
// Both OpenAI and Anthropic return {"error": {"message": "..."}}.
|
|
NETLOG("%s HTTP %d, body: %s", service, status, body.c_str());
|
|
JsonDocument doc;
|
|
String msg;
|
|
if (deserializeJson(doc, body) == DeserializationError::Ok) msg = doc["error"]["message"] | "";
|
|
if (msg.isEmpty()) msg = body.substring(0, 120);
|
|
return String(service) + " HTTP " + status + ": " + msg;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Step 1: speech-to-text, in parts.
|
|
//
|
|
// Long recordings are split into ~10-minute parts (each upload must stay
|
|
// under 25 MB). Each part is sent as its own small WAV: a fresh 44-byte
|
|
// header followed by a slice of the recording's PCM, streamed from the card.
|
|
// Cuts are moved to the quietest moment just before the target, so no word is
|
|
// split. Text from the end of the previous part goes along as a "prompt" so
|
|
// the speech-to-text model keeps names and spelling consistent across parts.
|
|
//
|
|
// After each part, its text is appended to the transcript and the position is
|
|
// saved to a .progress file. If a part fails (Wi-Fi drop, API error), holding
|
|
// PLUS again resumes from that part instead of starting over.
|
|
// ---------------------------------------------------------------------------
|
|
static const uint32_t BYTES_PER_SEC = SAMPLE_RATE * 2;
|
|
static const uint32_t PART_BYTES = STT_SEGMENT_SECONDS * BYTES_PER_SEC;
|
|
static const uint32_t LAST_PART_SLACK = PART_BYTES / 10; // fold a short tail (<1 min) into the last part
|
|
static const int CUT_BLOCKS = min(CUT_SEARCH_SECONDS, STT_SEGMENT_SECONDS / 4) * 10; // 0.1 s blocks searched
|
|
static_assert(44 + PART_BYTES + LAST_PART_SLACK < MAX_UPLOAD_BYTES, "parts must stay under the upload limit");
|
|
|
|
static String hms(uint32_t secs) {
|
|
char b[16];
|
|
snprintf(b, sizeof(b), "%02lu:%02lu:%02lu", secs / 3600, secs / 60 % 60, secs % 60);
|
|
return b;
|
|
}
|
|
|
|
// Last maxBytes of s, starting on a UTF-8 character boundary.
|
|
static String utf8Tail(const String &s, size_t maxBytes) {
|
|
if (s.length() <= maxBytes) return s;
|
|
size_t i = s.length() - maxBytes;
|
|
while (i < s.length() && ((uint8_t)s[i] & 0xC0) == 0x80) i++;
|
|
return s.substring(i);
|
|
}
|
|
|
|
// Pick a cut point (file offset) in the CUT_SEARCH_SECONDS before `target`:
|
|
// the middle of the quietest 0.4 s, measured in 0.1 s blocks.
|
|
static uint32_t findQuietCut(int fd, uint32_t target) {
|
|
const uint32_t BLOCK = BYTES_PER_SEC / 10;
|
|
const int NB = CUT_BLOCKS;
|
|
const uint32_t start = target - NB * BLOCK;
|
|
static uint64_t energy[CUT_BLOCKS];
|
|
static int16_t samples[BYTES_PER_SEC / 20]; // BLOCK bytes
|
|
{
|
|
SpiLock lock;
|
|
if (lseek(fd, start, SEEK_SET) < 0) return target;
|
|
for (int b = 0; b < NB; b++) {
|
|
if (sdRead(fd, (uint8_t *)samples, BLOCK) != (int)BLOCK) return target;
|
|
uint64_t e = 0;
|
|
for (uint32_t i = 0; i < BLOCK / 2; i++) e += abs(samples[i]);
|
|
energy[b] = e;
|
|
}
|
|
}
|
|
int best = 0;
|
|
uint64_t bestSum = UINT64_MAX;
|
|
for (int b = 0; b + 4 <= NB; b++) {
|
|
uint64_t sum = energy[b] + energy[b + 1] + energy[b + 2] + energy[b + 3];
|
|
if (sum < bestSum) { bestSum = sum; best = b; }
|
|
}
|
|
return start + (best + 2) * BLOCK; // even offset: always on a sample boundary
|
|
}
|
|
|
|
// Upload bytes [start, end) of the open WAV as one file; the text goes to `text`.
|
|
static bool transcribePart(int fd, uint32_t start, uint32_t end, const String &fileName, const String &prompt,
|
|
String &text, String &err) {
|
|
const uint32_t len = end - start;
|
|
const char *boundary = "----VoiceNotesFormBoundary7MA4YWxkTrZu0gW";
|
|
String pre;
|
|
pre += String("--") + boundary + "\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\n" STT_MODEL "\r\n";
|
|
pre += String("--") + boundary + "\r\nContent-Disposition: form-data; name=\"response_format\"\r\n\r\ntext\r\n";
|
|
if (!prompt.isEmpty())
|
|
pre += String("--") + boundary + "\r\nContent-Disposition: form-data; name=\"prompt\"\r\n\r\n" + prompt + "\r\n";
|
|
pre += String("--") + boundary + "\r\nContent-Disposition: form-data; name=\"file\"; filename=\"" + fileName +
|
|
"\"\r\nContent-Type: audio/wav\r\n\r\n";
|
|
String post = String("\r\n--") + boundary + "--\r\n";
|
|
uint8_t header[44];
|
|
wavHeader(header, len);
|
|
const size_t bodyLen = pre.length() + sizeof(header) + len + post.length();
|
|
|
|
NetworkClientSecure client;
|
|
if (!connectTls(client, STT_HOST)) { err = "Can't reach " STT_HOST; return false; }
|
|
|
|
String req;
|
|
req += "POST " STT_PATH " HTTP/1.1\r\n";
|
|
req += "Host: " STT_HOST "\r\n";
|
|
req += "Authorization: Bearer " STT_API_KEY "\r\n";
|
|
req += String("Content-Type: multipart/form-data; boundary=") + boundary + "\r\n";
|
|
req += "Content-Length: " + String(bodyLen) + "\r\n";
|
|
req += "Connection: close\r\n\r\n";
|
|
NETLOG("STT: POST %s (%s, %u bytes) model=" STT_MODEL ", key %.7s...", fileName.c_str(), hms(len / BYTES_PER_SEC).c_str(),
|
|
(unsigned)bodyLen, STT_API_KEY);
|
|
|
|
bool ok = writeAll(client, req) && writeAll(client, pre) && writeAll(client, header, sizeof(header));
|
|
if (!ok) NETLOG("STT: failed while sending request headers");
|
|
{
|
|
SpiLock lock;
|
|
if (ok && lseek(fd, start, SEEK_SET) < 0) { NETLOG("STT: seek failed"); ok = false; }
|
|
}
|
|
static uint8_t buf[4096];
|
|
uint32_t sent = 0, upStart = millis();
|
|
uint8_t lastLogged = 0;
|
|
app.progress = 0;
|
|
while (ok && sent < len) {
|
|
int n;
|
|
{
|
|
SpiLock lock;
|
|
n = sdRead(fd, buf, min((uint32_t)sizeof(buf), len - sent));
|
|
}
|
|
if (n <= 0) {
|
|
NETLOG("STT: SD read failed at offset %u (errno %d %s)", (unsigned)(start + sent), errno, strerror(errno));
|
|
ok = false;
|
|
break;
|
|
}
|
|
ok = writeAll(client, buf, n);
|
|
if (!ok) NETLOG("STT: upload stopped at byte %u of %u", (unsigned)sent, (unsigned)len);
|
|
sent += n;
|
|
app.progress = (uint8_t)((uint64_t)sent * 100 / len);
|
|
if (app.progress >= lastLogged + 25) {
|
|
lastLogged = app.progress - app.progress % 25;
|
|
float secs = (millis() - upStart) / 1000.0f;
|
|
NETLOG("STT: uploaded %u%% (%.0f KB/s)", app.progress, sent / 1024.0f / max(secs, 0.001f));
|
|
}
|
|
}
|
|
if (ok) ok = writeAll(client, post);
|
|
if (!ok) {
|
|
// Servers often reject early (bad key, file too big) and close mid-upload;
|
|
// if they sent a reply before closing, show it.
|
|
err = "Upload failed";
|
|
HttpHead early;
|
|
if (readHead(client, early, 3000)) {
|
|
BodyReader b(client, early, 3000);
|
|
err = httpErrorText("STT", early.status, b.readAll(600));
|
|
}
|
|
logTlsError(client, "STT upload");
|
|
return false;
|
|
}
|
|
NETLOG("STT: upload done in %.1f s, waiting for transcript", (millis() - upStart) / 1000.0f);
|
|
|
|
setState(ST_TRANSCRIBING);
|
|
HttpHead head;
|
|
if (!readHead(client, head, 5 * 60 * 1000)) { err = "No reply from STT service"; return false; }
|
|
BodyReader body(client, head, 60000);
|
|
if (head.status != 200) {
|
|
err = httpErrorText("STT", head.status, body.readAll(600));
|
|
return false;
|
|
}
|
|
text = body.readAll(256 * 1024); // ~10 min of speech is ~10 KB of text
|
|
text.trim();
|
|
NETLOG("STT: got %u bytes of text", text.length());
|
|
return true;
|
|
}
|
|
|
|
static bool readProgress(const String &path, uint32_t &offset, uint8_t &part) {
|
|
SpiLock lock;
|
|
File f = SD.open(path, FILE_READ); // tiny file: a single-sector read is fine
|
|
if (!f) return false;
|
|
String s = f.readString();
|
|
f.close();
|
|
return sscanf(s.c_str(), "%lu %hhu", &offset, &part) == 2;
|
|
}
|
|
|
|
static void writeProgress(const String &path, uint32_t offset, uint8_t part) {
|
|
SpiLock lock;
|
|
File f = SD.open(path, FILE_WRITE);
|
|
if (f) {
|
|
f.printf("%lu %u\n", offset, part);
|
|
f.close();
|
|
}
|
|
}
|
|
|
|
// The last ~200 bytes of the transcript so far, used as the prompt when resuming.
|
|
static String transcriptTail(const String &txtPath) {
|
|
SpiLock lock;
|
|
int fd = sdOpenRead(txtPath.c_str());
|
|
if (fd < 0) return "";
|
|
struct stat st;
|
|
String s;
|
|
if (fstat(fd, &st) == 0 && st.st_size > 0) {
|
|
char b[257];
|
|
lseek(fd, st.st_size > 256 ? st.st_size - 256 : 0, SEEK_SET);
|
|
int n = sdRead(fd, (uint8_t *)b, 256);
|
|
if (n > 0) s.concat(b, n);
|
|
}
|
|
close(fd);
|
|
return utf8Tail(s, 200);
|
|
}
|
|
|
|
static bool transcribe(const char *wavPath, const String &txtPath, const String &progPath, String &err) {
|
|
int fd;
|
|
uint32_t size;
|
|
{
|
|
SpiLock lock;
|
|
fd = sdOpenRead(wavPath);
|
|
struct stat st;
|
|
if (fd < 0 || fstat(fd, &st) != 0) {
|
|
NETLOG("STT: can't open %s (errno %d %s)", wavPath, errno, strerror(errno));
|
|
if (fd >= 0) close(fd);
|
|
err = "Can't open recording";
|
|
return false;
|
|
}
|
|
size = st.st_size;
|
|
}
|
|
NETLOG("STT: %s, %u bytes (%s of audio)", wavPath, (unsigned)size, hms((size - 44) / BYTES_PER_SEC).c_str());
|
|
if (size < 44 + BYTES_PER_SEC / 2) { SpiLock l; close(fd); err = "Recording too short"; return false; }
|
|
|
|
uint32_t offset = 44;
|
|
uint8_t part = 0;
|
|
String prompt;
|
|
if (readProgress(progPath, offset, part)) {
|
|
NETLOG("STT: resuming at part %u (%s into the recording)", part + 1, hms((offset - 44) / BYTES_PER_SEC).c_str());
|
|
prompt = transcriptTail(txtPath);
|
|
} else {
|
|
SpiLock lock;
|
|
SD.remove(txtPath);
|
|
}
|
|
if (offset == 44) writeProgress(progPath, offset, part);
|
|
|
|
String base = String(strrchr(wavPath, '/') + 1);
|
|
base.replace(".wav", "");
|
|
bool ok = true;
|
|
while (ok && offset < size) {
|
|
uint32_t end = size - offset <= PART_BYTES + LAST_PART_SLACK ? size : findQuietCut(fd, offset + PART_BYTES);
|
|
app.partIndex = part + 1;
|
|
app.partCount = part + 1 + (size > end ? (size - end + PART_BYTES - 1) / PART_BYTES : 0);
|
|
setState(ST_UPLOADING);
|
|
NETLOG("STT: part %u of ~%u: %s - %s", app.partIndex, app.partCount, hms((offset - 44) / BYTES_PER_SEC).c_str(),
|
|
hms((end - 44) / BYTES_PER_SEC).c_str());
|
|
|
|
String text;
|
|
ok = false;
|
|
for (int attempt = 1; attempt <= STT_MAX_ATTEMPTS && !ok; attempt++) {
|
|
if (attempt > 1) {
|
|
NETLOG("STT: retrying part %u (attempt %d of %d) after: %s", part + 1, attempt, STT_MAX_ATTEMPTS, err.c_str());
|
|
delay(5000 * (attempt - 1));
|
|
setState(ST_UPLOADING);
|
|
}
|
|
if (!ensureWifi()) { err = "Wi-Fi not connected"; continue; }
|
|
ok = transcribePart(fd, offset, end, base + "_part" + (part + 1) + ".wav", prompt, text, err);
|
|
}
|
|
if (!ok) break;
|
|
|
|
if (!text.isEmpty()) {
|
|
SpiLock lock;
|
|
File f = SD.open(txtPath, FILE_APPEND);
|
|
if (!f) { err = "Can't write transcript"; ok = false; break; }
|
|
f.printf("[%s]\n", hms((offset - 44) / BYTES_PER_SEC).c_str());
|
|
f.print(text);
|
|
f.print("\n\n");
|
|
f.close();
|
|
prompt = utf8Tail(text, 200);
|
|
}
|
|
app.received += text.length();
|
|
offset = end;
|
|
part++;
|
|
writeProgress(progPath, offset, part);
|
|
}
|
|
{ SpiLock l; close(fd); }
|
|
|
|
if (!ok) {
|
|
err = "Part " + String(app.partIndex) + "/" + String(app.partCount) + ": " + err + ". Hold PLUS to resume";
|
|
return false;
|
|
}
|
|
bool haveText;
|
|
{
|
|
SpiLock lock;
|
|
SD.remove(progPath);
|
|
haveText = SD.exists(txtPath);
|
|
}
|
|
if (!haveText) { err = "No speech detected"; return false; }
|
|
NETLOG("STT: transcript complete -> %s", txtPath.c_str());
|
|
return true;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Step 2: summarize with Claude.
|
|
//
|
|
// A two-hour transcript is ~100 KB, too big to hold in RAM next to Wi-Fi and
|
|
// TLS. So the JSON request body is streamed: the transcript is read from the
|
|
// card twice, once to measure its JSON-escaped length for Content-Length,
|
|
// then again to escape and send it in 512-byte pieces. The reply is streamed
|
|
// too (server-sent events), and each text delta is appended to the notes file.
|
|
// ---------------------------------------------------------------------------
|
|
static size_t jsonEscape(const uint8_t *in, size_t n, char *out) {
|
|
size_t o = 0;
|
|
for (size_t i = 0; i < n; i++) {
|
|
uint8_t ch = in[i];
|
|
switch (ch) {
|
|
case '"': if (out) { out[o] = '\\'; out[o + 1] = '"'; } o += 2; break;
|
|
case '\\': if (out) { out[o] = '\\'; out[o + 1] = '\\'; } o += 2; break;
|
|
case '\n': if (out) { out[o] = '\\'; out[o + 1] = 'n'; } o += 2; break;
|
|
case '\r': if (out) { out[o] = '\\'; out[o + 1] = 'r'; } o += 2; break;
|
|
case '\t': if (out) { out[o] = '\\'; out[o + 1] = 't'; } o += 2; break;
|
|
default:
|
|
if (ch < 0x20) {
|
|
if (out) snprintf(out + o, 7, "\\u%04x", ch);
|
|
o += 6;
|
|
} else {
|
|
if (out) out[o] = ch; // UTF-8 bytes pass through unchanged
|
|
o += 1;
|
|
}
|
|
}
|
|
}
|
|
return o; // with out == nullptr, just measures
|
|
}
|
|
|
|
static String jsonEscape(const String &s) {
|
|
String out;
|
|
out.reserve(s.length() + s.length() / 8 + 8);
|
|
char tmp[6 * 64 + 1];
|
|
for (size_t i = 0; i < s.length(); i += 64) {
|
|
size_t n = jsonEscape((const uint8_t *)s.c_str() + i, min((size_t)64, s.length() - i), tmp);
|
|
out.concat(tmp, n);
|
|
}
|
|
return out;
|
|
}
|
|
|
|
static const char *SYSTEM_PROMPT =
|
|
"You turn recordings of meetings and voice memos into clear, well-organized notes. The transcript "
|
|
"comes from automatic speech recognition, so expect filler words, false starts and the occasional "
|
|
"misheard word; infer the intended meaning, but don't add facts that aren't in the recording.";
|
|
|
|
static const char *NOTES_INSTRUCTIONS =
|
|
"Write notes in Markdown, scaled to the length of the recording (a short memo needs only a few lines):\n"
|
|
"- a short descriptive title as a level-1 heading\n"
|
|
"- **Summary**: one short paragraph, or up to three for a long meeting\n"
|
|
"- **Participants**: people who are named or clearly identifiable (omit if none)\n"
|
|
"- **Decisions**: what was decided (omit if nothing was)\n"
|
|
"- **Action items**: checkbox bullets with owner and due date when mentioned, or \"None\"\n"
|
|
"- **Discussion**: for longer recordings, the main topics in order, each with its key points and the "
|
|
"[hh:mm:ss] time it came up\n"
|
|
"- **Open questions**: unresolved issues (omit if none)\n"
|
|
"The transcript has no speaker labels, so only attribute a statement to someone when the transcript "
|
|
"makes it clear who said it. Reply with the notes only.";
|
|
|
|
static bool summarize(const String &txtPath, const String &mdPath, const char *wavPath, String &err) {
|
|
// Pass 1: measure the transcript and its escaped length.
|
|
static uint8_t in[512];
|
|
static char out[512 * 6];
|
|
size_t rawLen = 0, escLen = 0;
|
|
{
|
|
SpiLock lock;
|
|
int fd = sdOpenRead(txtPath.c_str());
|
|
if (fd < 0) { err = "Can't open transcript"; return false; }
|
|
int n;
|
|
while ((n = sdRead(fd, in, sizeof(in))) > 0) {
|
|
rawLen += n;
|
|
escLen += jsonEscape(in, n, nullptr);
|
|
}
|
|
close(fd);
|
|
}
|
|
NETLOG("Claude: transcript %u bytes", (unsigned)rawLen);
|
|
if (rawLen == 0) { err = "Transcript is empty"; return false; }
|
|
|
|
// The file name carries the start time, e.g. REC_2026-10-02_14-30-05.
|
|
String stamp = String(strrchr(wavPath, '/') + 1);
|
|
stamp.replace("REC_", "");
|
|
stamp.replace(".wav", "");
|
|
uint32_t secs = 0;
|
|
{
|
|
SpiLock lock;
|
|
File w = SD.open(wavPath, FILE_READ);
|
|
if (w) { secs = (w.size() - 44) / BYTES_PER_SEC; w.close(); }
|
|
}
|
|
|
|
String intro = "Below is the transcript of a recording started " + stamp + " and lasting " + hms(secs) +
|
|
". It was transcribed in parts; [hh:mm:ss] markers give the time since the recording "
|
|
"began.\n\n<transcript>\n";
|
|
String outro = String("\n</transcript>\n\n") + NOTES_INSTRUCTIONS;
|
|
|
|
String prefix;
|
|
prefix += "{\"model\":\"" CLAUDE_MODEL "\",\"max_tokens\":64000,\"stream\":true,";
|
|
prefix += "\"fallbacks\":\"default\",\"output_config\":{\"effort\":\"medium\"},";
|
|
prefix += "\"system\":\"" + jsonEscape(SYSTEM_PROMPT) + "\",";
|
|
prefix += "\"messages\":[{\"role\":\"user\",\"content\":\"" + jsonEscape(intro);
|
|
String suffix = jsonEscape(outro) + "\"}]}";
|
|
const size_t bodyLen = prefix.length() + escLen + suffix.length();
|
|
|
|
NetworkClientSecure client;
|
|
if (!connectTls(client, "api.anthropic.com")) { err = "Can't reach Claude API"; return false; }
|
|
String req;
|
|
req += "POST /v1/messages HTTP/1.1\r\n";
|
|
req += "Host: api.anthropic.com\r\n";
|
|
req += "x-api-key: " ANTHROPIC_API_KEY "\r\n";
|
|
req += "anthropic-version: 2023-06-01\r\n";
|
|
req += "anthropic-beta: server-side-fallback-2026-07-01\r\n";
|
|
req += "content-type: application/json\r\n";
|
|
req += "accept: text/event-stream\r\n";
|
|
req += "Content-Length: " + String(bodyLen) + "\r\n";
|
|
req += "Connection: close\r\n\r\n";
|
|
NETLOG("Claude: POST /v1/messages model=" CLAUDE_MODEL ", key %.7s..., body %u bytes", ANTHROPIC_API_KEY,
|
|
(unsigned)bodyLen);
|
|
|
|
// Pass 2: send prefix, escaped transcript, suffix.
|
|
bool ok = writeAll(client, req) && writeAll(client, prefix);
|
|
int fd;
|
|
{
|
|
SpiLock lock;
|
|
fd = sdOpenRead(txtPath.c_str());
|
|
}
|
|
size_t sentEsc = 0;
|
|
while (ok && fd >= 0) {
|
|
int n;
|
|
{
|
|
SpiLock lock;
|
|
n = sdRead(fd, in, sizeof(in));
|
|
}
|
|
if (n <= 0) break;
|
|
size_t m = jsonEscape(in, n, out);
|
|
ok = writeAll(client, (const uint8_t *)out, m);
|
|
sentEsc += m;
|
|
}
|
|
if (fd >= 0) { SpiLock l; close(fd); }
|
|
if (ok && sentEsc != escLen) {
|
|
NETLOG("Claude: transcript changed while sending (%u vs %u bytes)", (unsigned)sentEsc, (unsigned)escLen);
|
|
ok = false;
|
|
}
|
|
if (ok) ok = writeAll(client, suffix);
|
|
if (!ok) {
|
|
logTlsError(client, "Claude send");
|
|
err = "Send to Claude failed";
|
|
return false;
|
|
}
|
|
|
|
HttpHead head;
|
|
if (!readHead(client, head, 120000)) { err = "No reply from Claude"; return false; }
|
|
// Ping events arrive every few seconds while the model thinks, so a 90 s
|
|
// idle timeout only trips if the connection has really stalled.
|
|
BodyReader sse(client, head, 90000);
|
|
if (head.status != 200) {
|
|
err = httpErrorText("Claude", head.status, sse.readAll(600));
|
|
return false;
|
|
}
|
|
|
|
File md;
|
|
{
|
|
SpiLock lock;
|
|
md = SD.open(mdPath, FILE_WRITE);
|
|
if (!md) { err = "Can't write notes"; return false; }
|
|
}
|
|
|
|
// Keep only the fields we use from each event; thinking signatures etc. are dropped.
|
|
JsonDocument filter;
|
|
filter["type"] = true;
|
|
filter["delta"]["type"] = true;
|
|
filter["delta"]["text"] = true;
|
|
filter["delta"]["stop_reason"] = true;
|
|
filter["error"]["message"] = true;
|
|
|
|
String line, pending, stopReason;
|
|
bool gotStop = false;
|
|
size_t total = 0;
|
|
auto flush = [&]() {
|
|
if (pending.isEmpty()) return;
|
|
SpiLock lock;
|
|
md.print(pending);
|
|
pending = "";
|
|
};
|
|
|
|
while (sse.readLine(line)) {
|
|
if (!line.startsWith("data:")) continue;
|
|
JsonDocument ev;
|
|
if (deserializeJson(ev, line.c_str() + 5, DeserializationOption::Filter(filter))) continue;
|
|
const char *type = ev["type"] | "";
|
|
if (!strcmp(type, "content_block_delta") && !strcmp(ev["delta"]["type"] | "", "text_delta")) {
|
|
const char *t = ev["delta"]["text"] | "";
|
|
pending += t;
|
|
total += strlen(t);
|
|
app.received = total;
|
|
if (pending.length() > 512) flush();
|
|
} else if (!strcmp(type, "message_delta")) {
|
|
stopReason = ev["delta"]["stop_reason"] | "";
|
|
} else if (!strcmp(type, "message_stop")) {
|
|
gotStop = true;
|
|
break;
|
|
} else if (!strcmp(type, "error")) {
|
|
NETLOG("Claude stream error event: %s", line.c_str());
|
|
err = String("Claude: ") + (ev["error"]["message"] | "stream error");
|
|
break;
|
|
}
|
|
}
|
|
flush();
|
|
NETLOG("Claude: stream ended, %u chars, stop_reason=%s, message_stop=%s", (unsigned)total,
|
|
stopReason.isEmpty() ? "(none)" : stopReason.c_str(), gotStop ? "yes" : "no");
|
|
{
|
|
SpiLock lock;
|
|
md.printf("\n\n---\n_Recording: %s (%s) · Transcript: %s_\n", wavPath, hms(secs).c_str(), txtPath.c_str());
|
|
md.close();
|
|
}
|
|
|
|
if (!err.isEmpty()) return false;
|
|
if (stopReason == "refusal") { err = "Claude declined this request"; return false; }
|
|
if (!gotStop) { err = "Claude stream cut off"; return false; }
|
|
if (total == 0) { err = "Claude returned no text"; return false; }
|
|
return true;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Step 3: copy the notes to Nextcloud over WebDAV (HTTP PUT with Basic auth).
|
|
// /remote.php/webdav/ maps to the logged-in user's files, so it works whether
|
|
// NEXTCLOUD_USER is the user ID or an email login.
|
|
// ---------------------------------------------------------------------------
|
|
#ifndef NEXTCLOUD_HOST
|
|
#define NEXTCLOUD_HOST ""
|
|
#define NEXTCLOUD_USER ""
|
|
#define NEXTCLOUD_PASSWORD ""
|
|
#define NEXTCLOUD_FOLDER ""
|
|
#endif
|
|
|
|
static bool nextcloudEnabled() { return strlen(NEXTCLOUD_HOST) > 0; }
|
|
|
|
static String urlEncodePath(const String &s) {
|
|
String out;
|
|
for (size_t i = 0; i < s.length(); i++) {
|
|
char c = s[i];
|
|
if (isalnum((unsigned char)c) || strchr("-_.~/", c)) {
|
|
out += c;
|
|
} else {
|
|
char b[4];
|
|
snprintf(b, sizeof(b), "%%%02X", (uint8_t)c);
|
|
out += b;
|
|
}
|
|
}
|
|
return out;
|
|
}
|
|
|
|
// One WebDAV request. Streams the local file as the body when localPath is set.
|
|
static int webdavRequest(const char *method, const String &remotePath, const char *localPath, const char *contentType,
|
|
String &err) {
|
|
int fd = -1;
|
|
size_t len = 0;
|
|
if (localPath) {
|
|
SpiLock lock;
|
|
fd = sdOpenRead(localPath);
|
|
struct stat st;
|
|
if (fd < 0 || fstat(fd, &st) != 0) {
|
|
if (fd >= 0) close(fd);
|
|
err = String("Can't open ") + localPath;
|
|
return -1;
|
|
}
|
|
len = st.st_size;
|
|
}
|
|
NetworkClientSecure client;
|
|
if (!connectTls(client, NEXTCLOUD_HOST)) {
|
|
if (fd >= 0) { SpiLock l; close(fd); }
|
|
err = "Can't reach " NEXTCLOUD_HOST;
|
|
return -1;
|
|
}
|
|
String req = String(method) + " /remote.php/webdav/" + urlEncodePath(remotePath) + " HTTP/1.1\r\n";
|
|
req += "Host: " NEXTCLOUD_HOST "\r\n";
|
|
req += "Authorization: Basic " + base64::encode(String(NEXTCLOUD_USER ":" NEXTCLOUD_PASSWORD)) + "\r\n";
|
|
if (contentType) req += String("Content-Type: ") + contentType + "\r\n";
|
|
req += "Content-Length: " + String(len) + "\r\n";
|
|
req += "Connection: close\r\n\r\n";
|
|
NETLOG("Nextcloud: %s /remote.php/webdav/%s (%u bytes) as " NEXTCLOUD_USER, method, remotePath.c_str(), (unsigned)len);
|
|
|
|
bool ok = writeAll(client, req);
|
|
static uint8_t buf[4096];
|
|
size_t sent = 0;
|
|
while (ok && fd >= 0 && sent < len) {
|
|
int n;
|
|
{
|
|
SpiLock lock;
|
|
n = sdRead(fd, buf, sizeof(buf));
|
|
}
|
|
if (n <= 0) { ok = false; break; }
|
|
ok = writeAll(client, buf, n);
|
|
sent += n;
|
|
}
|
|
if (fd >= 0) { SpiLock l; close(fd); }
|
|
if (!ok) { err = "Upload to Nextcloud failed"; return -1; }
|
|
|
|
HttpHead head;
|
|
if (!readHead(client, head, 60000)) { err = "No reply from Nextcloud"; return -1; }
|
|
if (head.status >= 300) {
|
|
BodyReader body(client, head, 5000);
|
|
NETLOG("Nextcloud: HTTP %d, body: %s", head.status, body.readAll(400).c_str());
|
|
}
|
|
return head.status;
|
|
}
|
|
|
|
static bool nextcloudUpload(const String &localPath, const char *contentType, String &err) {
|
|
String name = String(strrchr(localPath.c_str(), '/') + 1);
|
|
String remote = String(NEXTCLOUD_FOLDER) + "/" + name;
|
|
int status = webdavRequest("PUT", remote, localPath.c_str(), contentType, err);
|
|
if (status == 404 || status == 409) { // folder missing: create it, then retry once
|
|
NETLOG("Nextcloud: folder " NEXTCLOUD_FOLDER " not found, creating it");
|
|
int mk = webdavRequest("MKCOL", NEXTCLOUD_FOLDER, nullptr, nullptr, err);
|
|
if (mk == 201 || mk == 405) status = webdavRequest("PUT", remote, localPath.c_str(), contentType, err);
|
|
}
|
|
if (status == 201 || status == 204) {
|
|
NETLOG("Nextcloud: saved %s", remote.c_str());
|
|
return true;
|
|
}
|
|
if (status == 401) err = "Nextcloud login failed - check the app password in secrets.h";
|
|
else if (status == 507) err = "Nextcloud storage is full";
|
|
else if (status > 0) err = "Nextcloud HTTP " + String(status);
|
|
return false;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
static void netTask(void *) {
|
|
String base = String(strrchr(jobWav, '/') + 1);
|
|
base.replace(".wav", "");
|
|
String txtPath = String(DIR_NOTES "/") + base + "_transcript.txt";
|
|
String progPath = String(DIR_NOTES "/") + base + "_transcript.progress";
|
|
String mdPath = String(DIR_NOTES "/") + base + "_summary.md";
|
|
String err;
|
|
|
|
// Reuse finished work: a transcript without a .progress file is complete,
|
|
// and an existing summary only needs copying to Nextcloud (e.g. when only
|
|
// that step failed last time). Delete the summary on the card to redo it.
|
|
bool transcriptDone, summaryDone;
|
|
{
|
|
SpiLock lock;
|
|
transcriptDone = SD.exists(txtPath) && !SD.exists(progPath);
|
|
summaryDone = transcriptDone && SD.exists(mdPath);
|
|
}
|
|
|
|
app.progress = 0;
|
|
app.received = 0;
|
|
app.partIndex = 0;
|
|
app.partCount = 0;
|
|
setState(ST_UPLOADING, base.c_str());
|
|
bool ok = ensureWifi();
|
|
if (!ok) err = "Wi-Fi not connected";
|
|
|
|
if (ok && !transcriptDone) ok = transcribe(jobWav, txtPath, progPath, err);
|
|
if (ok && !summaryDone) {
|
|
app.received = 0;
|
|
setState(ST_SUMMARIZING, base.c_str());
|
|
ok = summarize(txtPath, mdPath, jobWav, err);
|
|
}
|
|
|
|
bool synced = false;
|
|
if (ok && nextcloudEnabled()) {
|
|
setState(ST_SYNCING, base.c_str());
|
|
if (strlen(NEXTCLOUD_PASSWORD) == 0) {
|
|
err = "Notes are on the SD card. Add the Nextcloud app password to secrets.h";
|
|
ok = false;
|
|
} else {
|
|
ok = nextcloudUpload(mdPath, "text/markdown; charset=utf-8", err) &&
|
|
nextcloudUpload(txtPath, "text/plain; charset=utf-8", err);
|
|
if (!ok) err = "Notes are on the SD card. " + err + ". Hold PLUS to retry";
|
|
synced = ok;
|
|
}
|
|
}
|
|
|
|
if (ok) {
|
|
NETLOG("SUCCESS: notes written to %s%s", mdPath.c_str(), synced ? " and copied to Nextcloud" : "");
|
|
setState(ST_DONE, synced ? "Also saved to Nextcloud /" NEXTCLOUD_FOLDER : (base + "_summary.md").c_str());
|
|
} else {
|
|
NETLOG("FAILED: %s", err.c_str());
|
|
setState(ST_ERROR, err.c_str());
|
|
}
|
|
busy = false;
|
|
vTaskDelete(nullptr);
|
|
}
|
|
|
|
bool cloudStart(const char *wavPath) {
|
|
if (busy) return false;
|
|
busy = true;
|
|
strlcpy(jobWav, wavPath, sizeof(jobWav));
|
|
// TLS handshakes need a deep stack.
|
|
if (xTaskCreate(netTask, "net", 16384, nullptr, 2, nullptr) != pdPASS) {
|
|
busy = false;
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|