From 215a6f5d9517da8542d04203db6fedb4e1cc1da0 Mon Sep 17 00:00:00 2001 From: "Earle F. Philhower, III" Date: Thu, 30 May 2024 10:52:17 -0700 Subject: [PATCH] Allow uploading huge files to WebServer From https://github.com/espressif/arduino-esp32/pull/9440 --- .../examples/UploadHugeFile/README.md | 13 +++ .../UploadHugeFile/UploadHugeFile.ino | 92 +++++++++++++++++++ libraries/WebServer/src/HTTPServer.h | 19 ++++ libraries/WebServer/src/Parsing.cpp | 31 ++++++- libraries/WebServer/src/WebServerTemplate.h | 1 + .../WebServer/src/detail/RequestHandler.h | 9 ++ .../src/detail/RequestHandlersImpl.h | 16 ++++ 7 files changed, 177 insertions(+), 4 deletions(-) create mode 100644 libraries/WebServer/examples/UploadHugeFile/README.md create mode 100644 libraries/WebServer/examples/UploadHugeFile/UploadHugeFile.ino diff --git a/libraries/WebServer/examples/UploadHugeFile/README.md b/libraries/WebServer/examples/UploadHugeFile/README.md new file mode 100644 index 000000000..050c819dc --- /dev/null +++ b/libraries/WebServer/examples/UploadHugeFile/README.md @@ -0,0 +1,13 @@ +# Upload Huge File To Filesystem Over HTTP + +This project is an example of an HTTP server designed to facilitate the transfer of large files using the PUT method, in accordance with RFC specifications. + +### Example cURL Command + +```bash +curl -X PUT -T ./my-file.mp3 http://pico-ip/upload/my-file.mp3 +``` + +## Resources + +- RFC HTTP/1.0 - Additional Request Methods - PUT : [Link](https://datatracker.ietf.org/doc/html/rfc1945#appendix-D.1.1) diff --git a/libraries/WebServer/examples/UploadHugeFile/UploadHugeFile.ino b/libraries/WebServer/examples/UploadHugeFile/UploadHugeFile.ino new file mode 100644 index 000000000..133ca97e5 --- /dev/null +++ b/libraries/WebServer/examples/UploadHugeFile/UploadHugeFile.ino @@ -0,0 +1,92 @@ +#include +#include +#include +#include +#include + +#ifndef STASSID +#define STASSID "your-ssid" +#define STAPSK "your-password" +#endif + +const char *ssid = STASSID; +const char *password = STAPSK; + +WebServer server(80); + +File rawFile; +void handleCreate() { + server.send(200, "text/plain", ""); +} +void handleCreateProcess() { + String path = "/" + server.pathArg(0); + HTTPRaw& raw = server.raw(); + if (raw.status == RAW_START) { + if (LittleFS.exists((char *)path.c_str())) { + LittleFS.remove((char *)path.c_str()); + } + rawFile = LittleFS.open(path.c_str(), "w"); + Serial.print("Upload: START, filename: "); + Serial.println(path); + } else if (raw.status == RAW_WRITE) { + if (rawFile) { + rawFile.write(raw.buf, raw.currentSize); + } + Serial.print("Upload: WRITE, Bytes: "); + Serial.println(raw.currentSize); + } else if (raw.status == RAW_END) { + if (rawFile) { + rawFile.close(); + } + Serial.print("Upload: END, Size: "); + Serial.println(raw.totalSize); + } +} + +void returnFail(String msg) { + server.send(500, "text/plain", msg + "\r\n"); +} + +void handleNotFound() { + String message = "File Not Found\n\n"; + message += "URI: "; + message += server.uri(); + message += "\nMethod: "; + message += (server.method() == HTTP_GET) ? "GET" : "POST"; + message += "\nArguments: "; + message += server.args(); + message += "\n"; + for (uint8_t i = 0; i < server.args(); i++) { + message += " " + server.argName(i) + ": " + server.arg(i) + "\n"; + } + server.send(404, "text/plain", message); +} + +void setup(void) { + Serial.begin(115200); + + LittleFS.begin(); + + WiFi.mode(WIFI_STA); + WiFi.begin(ssid, password); + + while (WiFi.status() != WL_CONNECTED) { + delay(500); + Serial.print("."); + } + Serial.print("Connected to "); + Serial.println(ssid); + Serial.print("IP address: "); + Serial.println(WiFi.localIP()); + + server.on(UriRegex("/upload/(.*)"), HTTP_PUT, handleCreate, handleCreateProcess); + server.onNotFound(handleNotFound); + server.begin(); + Serial.println("HTTP server started"); + +} + +void loop(void) { + server.handleClient(); + delay(2);//allow the cpu to switch to other tasks +} diff --git a/libraries/WebServer/src/HTTPServer.h b/libraries/WebServer/src/HTTPServer.h index 1bc6413b6..50eb03018 100644 --- a/libraries/WebServer/src/HTTPServer.h +++ b/libraries/WebServer/src/HTTPServer.h @@ -31,6 +31,7 @@ enum HTTPUploadStatus { UPLOAD_FILE_START, UPLOAD_FILE_WRITE, UPLOAD_FILE_END, UPLOAD_FILE_ABORTED }; +enum HTTPRawStatus { RAW_START, RAW_WRITE, RAW_END, RAW_ABORTED }; enum HTTPClientStatus { HC_NONE, HC_WAIT_READ, HC_WAIT_CLOSE }; enum HTTPAuthMethod { BASIC_AUTH, DIGEST_AUTH }; @@ -40,6 +41,10 @@ enum HTTPAuthMethod { BASIC_AUTH, DIGEST_AUTH }; #define HTTP_UPLOAD_BUFLEN 1436 #endif +#ifndef HTTP_RAW_BUFLEN +#define HTTP_RAW_BUFLEN 1436 +#endif + #define HTTP_MAX_DATA_WAIT 5000 //ms to wait for the client to send the request #define HTTP_MAX_DATA_AVAILABLE_WAIT 30 //ms to wait for the client to send the request when there is another client with data available #define HTTP_MAX_POST_WAIT 5000 //ms to wait for POST data to arrive @@ -63,6 +68,16 @@ typedef struct { uint8_t buf[HTTP_UPLOAD_BUFLEN]; } HTTPUpload; + +typedef struct { + HTTPRawStatus status; + size_t totalSize; // content size + size_t currentSize; // size of data currently in buf + uint8_t buf[HTTP_UPLOAD_BUFLEN]; + void *data; // additional data +} HTTPRaw; + + #include "detail/RequestHandler.h" namespace fs { @@ -97,6 +112,9 @@ class HTTPServer { HTTPUpload& upload() { return *_currentUpload; } + HTTPRaw& raw() { + return *_currentRaw; + } String pathArg(unsigned int i); // get request path argument by number String arg(String name); // get request argument value by name @@ -257,6 +275,7 @@ class HTTPServer { RequestArgument* _postArgs; std::unique_ptr _currentUpload; + std::unique_ptr _currentRaw; int _headerKeysCount; RequestArgument* _currentHeaders; diff --git a/libraries/WebServer/src/Parsing.cpp b/libraries/WebServer/src/Parsing.cpp index 71b23b725..7d680e513 100644 --- a/libraries/WebServer/src/Parsing.cpp +++ b/libraries/WebServer/src/Parsing.cpp @@ -188,7 +188,30 @@ HTTPServer::ClientFuture HTTPServer::_parseRequest(WiFiClient* client) { } } - if (!isForm) { + if (!isForm && _currentHandler && _currentHandler->canRaw(_currentUri)) { + log_v("Parse raw"); + _currentRaw.reset(new HTTPRaw()); + _currentRaw->status = RAW_START; + _currentRaw->totalSize = 0; + _currentRaw->currentSize = 0; + log_v("Start Raw"); + _currentHandler->raw(*this, _currentUri, *_currentRaw); + _currentRaw->status = RAW_WRITE; + + while (_currentRaw->totalSize < (size_t)_clientContentLength) { + _currentRaw->currentSize = client->readBytes(_currentRaw->buf, HTTP_RAW_BUFLEN); + _currentRaw->totalSize += _currentRaw->currentSize; + if (_currentRaw->currentSize == 0) { + _currentRaw->status = RAW_ABORTED; + _currentHandler->raw(*this, _currentUri, *_currentRaw); + return CLIENT_MUST_STOP; + } + _currentHandler->raw(*this, _currentUri, *_currentRaw); + } + _currentRaw->status = RAW_END; + _currentHandler->raw(*this, _currentUri, *_currentRaw); + log_v("Finish Raw"); + } else if (!isForm) { size_t plainLength; char* plainBuf = readBytesWithTimeout(client, _clientContentLength, plainLength, HTTP_MAX_POST_WAIT); if ((int)plainLength < (int)_clientContentLength) { @@ -333,7 +356,7 @@ void HTTPServer::_uploadWriteByte(uint8_t b) { _currentUpload->buf[_currentUpload->currentSize++] = b; } -int HTTPServer::_uploadReadByte(WiFiClient* client) { +int HTTPServer::_uploadReadByte(WiFiClient * client) { int res = client->read(); if (res < 0) { // keep trying until you either read a valid byte or timeout @@ -376,7 +399,7 @@ int HTTPServer::_uploadReadByte(WiFiClient* client) { return res; } -bool HTTPServer::_parseForm(WiFiClient* client, String boundary, uint32_t len) { +bool HTTPServer::_parseForm(WiFiClient * client, String boundary, uint32_t len) { (void) len; log_v("Parse Form: Boundary: %s Length: %d", boundary.c_str(), len); String line; @@ -595,7 +618,7 @@ bool HTTPServer::_parseForm(WiFiClient* client, String boundary, uint32_t len) { return false; } -String HTTPServer::urlDecode(const String& text) { +String HTTPServer::urlDecode(const String & text) { String decoded = ""; char temp[] = "0x00"; unsigned int len = text.length(); diff --git a/libraries/WebServer/src/WebServerTemplate.h b/libraries/WebServer/src/WebServerTemplate.h index de373e8c4..d31b3f949 100644 --- a/libraries/WebServer/src/WebServerTemplate.h +++ b/libraries/WebServer/src/WebServerTemplate.h @@ -171,6 +171,7 @@ void WebServerTemplate::handleClient() { } _currentStatus = HC_NONE; _currentUpload.reset(); + _currentRaw.reset(); } if (callYield) { diff --git a/libraries/WebServer/src/detail/RequestHandler.h b/libraries/WebServer/src/detail/RequestHandler.h index f816cf1c0..f40a6ffc2 100644 --- a/libraries/WebServer/src/detail/RequestHandler.h +++ b/libraries/WebServer/src/detail/RequestHandler.h @@ -15,6 +15,10 @@ class RequestHandler { (void) uri; return false; } + virtual bool canRaw(String uri) { + (void) uri; + return false; + } virtual bool handle(HTTPServer& server, HTTPMethod requestMethod, String requestUri) { (void) server; (void) requestMethod; @@ -26,6 +30,11 @@ class RequestHandler { (void) requestUri; (void) upload; } + virtual void raw(HTTPServer& server, String requestUri, HTTPRaw& raw) { + (void) server; + (void) requestUri; + (void) raw; + } RequestHandler* next() { return _next; diff --git a/libraries/WebServer/src/detail/RequestHandlersImpl.h b/libraries/WebServer/src/detail/RequestHandlersImpl.h index 43211737a..39642b755 100644 --- a/libraries/WebServer/src/detail/RequestHandlersImpl.h +++ b/libraries/WebServer/src/detail/RequestHandlersImpl.h @@ -42,6 +42,14 @@ class FunctionRequestHandler : public RequestHandler { return true; } + bool canRaw(String requestUri) override { + (void) requestUri; + if (!_ufn || _method == HTTP_GET) { + return false; + } + + return true; + } bool handle(HTTPServer& server, HTTPMethod requestMethod, String requestUri) override { (void) server; @@ -61,6 +69,14 @@ class FunctionRequestHandler : public RequestHandler { } } + void raw(HTTPServer& server, String requestUri, HTTPRaw& raw) override { + (void)server; + (void)raw; + if (canRaw(requestUri)) { + _ufn(); + } + } + protected: HTTPServer::THandlerFunction _fn; HTTPServer::THandlerFunction _ufn;