Web server with esp8266 and esp32: multi purpose generic web server – 3

by Renzo Mischianti · Published 2 November 2020 · Updated 11 February 2024

Spread the love

Now that we understand how to serve a compressed page or image, we know all about how to create a generic web server.

WebServer Esp8266 ESP32 distribution entire site

I’m going to create a git repository with the complete example, add the sketch and I’m going to explain the single lines of code.

For esp32 you must only change this lines

#include <ESP8266WiFi.h>
#include <ESP8266WebServer.h>
#include <FS.h>
[...]
ESP8266WebServer server(80);

#include <WiFi.h>
#include <WebServer.h>
#include <SPIFFS.h>
[...]
WebServer server(80);

But be careful now on the ESP32 WiFiClient there is a big bug, you can check here, so if you try to start this server you will have some problem with many simultaneous requests. Therefore I have written a port of this sketch using the ESPAsyncWebServer library which uses AsyncTCP where this problem does not arise. But now I explain the actual solution for esp8266 and at the end of the article the equivalent solution for ESP32.

Fixed on esp32 core version 1.0.5.

Here the sketch for fully working for esp8266 and now not working with esp32.

/*
 *  WeMos D1 mini (esp8266)
 *  Simple web server that read from SPIFFS and
 *  stream on browser various type of file
 *  and manage gzip format also
 *
 *  by Mischianti Renzo <https://mischianti.org>
 *
 *  https://mischianti.org/category/tutorial/how-to-create-a-web-server-with-esp8266-and-esp32/
 *
 */
#include <ESP8266WiFi.h>
#include <ESP8266WebServer.h>
#include <FS.h>

const char* ssid = "<your-ssid>";
const char* password = "<your-passwd>";

ESP8266WebServer httpServer(80);


bool loadFromSPIFFS(String path, String dataType) {
  Serial.print("Requested page -> ");
  Serial.println(path);
  if (SPIFFS.exists(path)){
	  File dataFile = SPIFFS.open(path, "r");
	  if (!dataFile) {
		  handleNotFound();
		  return false;
	  }

	  if (httpServer.streamFile(dataFile, dataType) != dataFile.size()) {
	    Serial.println("Sent less data than expected!");
	  }else{
		  Serial.println("Page served!");
	  }

	  dataFile.close();
  }else{
	  handleNotFound();
	  return false;
  }
  return true;
}

void serverRouting();

void setup(void) {
  Serial.begin(115200);
  WiFi.mode(WIFI_STA);
  WiFi.begin(ssid, password);
  Serial.println("");

  // Wait for connection
  while (WiFi.status() != WL_CONNECTED) {
    delay(500);
    Serial.print(".");
  }

  Serial.println("");
  Serial.print("Connected to ");
  Serial.println(ssid);
  Serial.print("IP address: ");
  Serial.println(WiFi.localIP());

  Serial.print(F("Inizializing FS..."));
  if (SPIFFS.begin()){
	Serial.println(F("done."));
  }else{
	Serial.println(F("fail."));
  }

  Serial.println("Set routing for http server!");
  serverRouting();
  httpServer.begin();
  Serial.println("HTTP server started");
}

void loop(void) {
	httpServer.handleClient();
}

String getContentType(String filename){
  if(filename.endsWith(F(".htm"))) 			return F("text/html");
  else if(filename.endsWith(F(".html"))) 	return F("text/html");
  else if(filename.endsWith(F(".css"))) 	return F("text/css");
  else if(filename.endsWith(F(".js"))) 		return F("application/javascript");
  else if(filename.endsWith(F(".json"))) 	return F("application/json");
  else if(filename.endsWith(F(".png"))) 	return F("image/png");
  else if(filename.endsWith(F(".gif"))) 	return F("image/gif");
  else if(filename.endsWith(F(".jpg"))) 	return F("image/jpeg");
  else if(filename.endsWith(F(".jpeg"))) 	return F("image/jpeg");
  else if(filename.endsWith(F(".ico"))) 	return F("image/x-icon");
  else if(filename.endsWith(F(".xml"))) 	return F("text/xml");
  else if(filename.endsWith(F(".pdf"))) 	return F("application/x-pdf");
  else if(filename.endsWith(F(".zip"))) 	return F("application/x-zip");
  else if(filename.endsWith(F(".gz"))) 		return F("application/x-gzip");
  return F("text/plain");
}

bool handleFileRead(String path){
  Serial.print(F("handleFileRead: "));
  Serial.println(path);

  if(path.endsWith("/")) path += F("index.html");           // If a folder is requested, send the index file
  String contentType = getContentType(path);             	// Get the MIME type
  String pathWithGz = path + F(".gz");
  if(SPIFFS.exists(pathWithGz) || SPIFFS.exists(path)){  	// If the file exists, either as a compressed archive, or normal
    if(SPIFFS.exists(pathWithGz))                          	// If there's a compressed version available
      path += F(".gz");                                     // Use the compressed version
    fs::File file = SPIFFS.open(path, "r");                 // Open the file
    size_t sent = httpServer.streamFile(file, contentType); // Send it to the client
    file.close();                                          	// Close the file again
    Serial.println(String(F("\tSent file: ")) + path + String(F(" of size ")) + sent);
    return true;
  }
  Serial.println(String(F("\tFile Not Found: ")) + path);
  return false;                                          	// If the file doesn't exist, return false
}

void handleNotFound() {
  String message = "File Not Found\n\n";
  message += "URI: ";
  message += httpServer.uri();
  message += "\nMethod: ";
  message += (httpServer.method() == HTTP_GET) ? "GET" : "POST";
  message += "\nArguments: ";
  message += httpServer.args();
  message += "\n";

  for (uint8_t i = 0; i < httpServer.args(); i++) {
    message += " " + httpServer.argName(i) + ": " + httpServer.arg(i) + "\n";
  }

  httpServer.send(404, "text/plain", message);
}

void serverRouting() {
	  httpServer.onNotFound([]() {                              // If the client requests any URI
		  Serial.println(F("On not found"));
	    if (!handleFileRead(httpServer.uri())){                  // send it if it exists
	    	handleNotFound(); // otherwise, respond with a 404 (Not Found) error
	    }
	  });

	  Serial.println(F("Set cache!"));
	  // Serve a file with no cache so every tile It's downloaded
	  httpServer.serveStatic("/configuration.json", SPIFFS, "/configuration.json","no-cache, no-store, must-revalidate");
	  // Server all other page with long cache so browser chaching they
	  httpServer.serveStatic("/", SPIFFS, "/","max-age=31536000");


}

First you must remember, as I write here “WeMos D1 mini (esp8266), integrated SPIFFS Filesystem” or here for esp32 “ESP32: integrated SPIFFS FileSystem“, that the SPIFFS filesystem has some limitations: no folders are allowed, in article c ‘is also the procedure for uploading data with the Arduino IDE.

Given what has been said, you are going to create a site in a flat structure like this:

Flat structure for WebServer site on esp8266 in SPIFFS

with no folder and a correct file length.

The result:

Site template served from esp8266 WebServer

All the management of the files is in this code

	  httpServer.onNotFound([]() {                              // If the client requests any URI
		  Serial.println(F("On not found"));
	    if (!handleFileRead(httpServer.uri())){                  // send it if it exists
	    	handleNotFound(); // otherwise, respond with a 404 (Not Found) error
	    }
	  });

handleFileRead(httpServer.uri()) check the url requested and inside get correct mime type via the file extension.

You can open developer tools from your browser with F12 button, here you can see the resources served from esp8266 on Network tab.

Network of site template served from esp8266 WebServer

Even if you don’t write index.html It’s automatically go to that page, the code that do this is

  if(path.endsWith("/")) path += F("index.html");           // If a folder is requested, send the index file

Now as you can see the esp8266 manages many types of files, you can check what type of file It’s managed in this function:

String getContentType(String filename){
  if(filename.endsWith(F(".htm"))) 			return F("text/html");
  else if(filename.endsWith(F(".html"))) 	return F("text/html");
  else if(filename.endsWith(F(".css"))) 	return F("text/css");
  else if(filename.endsWith(F(".js"))) 		return F("application/javascript");
  else if(filename.endsWith(F(".json"))) 	return F("application/json");
  else if(filename.endsWith(F(".png"))) 	return F("image/png");
  else if(filename.endsWith(F(".gif"))) 	return F("image/gif");
  else if(filename.endsWith(F(".jpg"))) 	return F("image/jpeg");
  else if(filename.endsWith(F(".jpeg"))) 	return F("image/jpeg");
  else if(filename.endsWith(F(".ico"))) 	return F("image/x-icon");
  else if(filename.endsWith(F(".xml"))) 	return F("text/xml");
  else if(filename.endsWith(F(".pdf"))) 	return F("application/x-pdf");
  else if(filename.endsWith(F(".zip"))) 	return F("application/x-zip");
  else if(filename.endsWith(F(".gz"))) 		return F("application/x-gzip");
  return F("text/plain");
}

Cache the site

An important feature offered by the browser is the ability to cache files locally, this code gives the browser specifications:

// Serve a file with no cache so every tile It's downloaded
	  httpServer.serveStatic("/configuration.json", SPIFFS, "/configuration.json","no-cache, no-store, must-revalidate");
	  // Server all other page with long cache so browser chaching they
	  httpServer.serveStatic("/", SPIFFS, "/","max-age=31536000");

in the code you find 2 type of management, the file configuration.json is never under cache, so It’s downloaded from the device every time, all the other file under / have infinite timeout.

You can check this by keypressing the F5 button with Network Chrome tab opened, the first time for all files you have the size on Size column, the second time you have a description (disk cache).

Pay attention: if you want to reload all the content because you have do a change on file you must remove cache, if you have developer tools opened you can keep pressed the refresh browser button (Chrome) and select the menu item that specify to remove cache.

Manage gzip content

Now we can go to test an additional features, the management of gzip format file:

  String pathWithGz = path + F(".gz");
  if(SPIFFS.exists(pathWithGz) || SPIFFS.exists(path)){  	// If the file exists, either as a compressed archive, or normal
    if(SPIFFS.exists(pathWithGz))                          	// If there's a compressed version available
      path += F(".gz");                                     // Use the compressed version

after the check of mime type the code check if exist a gzipped version of the file

So now we’re going to gzip all the files:

gzip *

for windows you can use Cygwin or download gzip implementation for windows.

The result is simple

app.js			->		app.js.gz
bootstrap.min.css	->		bootstrap.min.css.gz
bootstrap.min.js	->		bootstrap.min.js.gz
bridge-theme.css	->		bridge-theme.css.gz
CREDITS.txt		->		CREDITS.txt.gz
custom.js		->		custom.js.gz
dark-blue-theme.css	->		dark-blue-theme.css.gz
dark-red-theme.css	->		dark-red-theme.css.gz
default-theme.css	->		default-theme.css.gz
favicon.ico		->		favicon.ico.gz
font-awesome.min.css	->		font-awesome.min.css.gz
green-theme.css		->		green-theme.css.gz
imagelightbox.min.css	->		imagelightbox.min.css.gz
img-1.jpeg		->		img-1.jpeg.gz
img-2.jpeg		->		img-2.jpeg.gz
img-3.jpeg		->		img-3.jpeg.gz
img-4.jpeg		->		img-4.jpeg.gz
img-5.jpeg		->		img-5.jpeg.gz
img-6.jpeg		->		img-6.jpeg.gz
img-7.jpg		->		img-7.jpg.gz
img-8.jpeg		->		img-8.jpeg.gz
img-9.jpeg		->		img-9.jpeg.gz
index.html		->		index.html.gz
jquery.appear.js	->		jquery.appear.js.gz
jquery.filterizr.min.js	->		jquery.filterizr.min.js.gz
jquery.lineProgressbar.js->	jquery.lineProgressbar.js.gz
jquery.mag-pop.min.js	->		jquery.mag-pop.min.js.gz
LICENSE.txt		->		LICENSE.txt.gz
lite-blue-theme.css	->		lite-blue-theme.css.gz
logo.png		->		logo.png.gz
magnific-popup.css	->		magnific-popup.css.gz
mailer.php		->		mailer.php.gz
markups-kevin.rar	->		markups-kevin.rar.gz
orange-theme.css	->		orange-theme.css.gz
pink-theme.css		->		pink-theme.css.gz
profile.jpg		->		profile.jpg.gz
purple-theme.css	->		purple-theme.css.gz
red-theme.css		->		red-theme.css.gz
slick.css		->		slick.css.gz
slick.min.js		->		slick.min.js.gz
style.css		->		style.css.gz
testimonials-bg.jpeg	->		testimonials-bg.jpeg.gz
typed.min.js		->		typed.min.js.gz

Now if you substitute on SPIFFS the file not compressed with the gzipped one and reload all (remember to remove cache), you can see that nothing changes visually.

But if you compare the detail of the request / (which corresponds to the index.html page) before compression

You can see in the response Header a content length of 23K, but when you upload a gzipped files at the same request you obtain this:

now the content length is 0.4Kb and there is a new parameter content encoding gzip.

So now you notice the difference, but It’s all transparent for you because browser can manage this situation and do all the work for you.

ESPAsyncWebServer for ESP32

Ok, now we are going to understand the alternative solution with ESPAsyncWebServer and AsyncTCP for ESP32. First download the library ESPAsyncWebServer and AsyncTCP.

Now here is the port of the sketch described above.

/*
 *  ESP32
 *  Simple web server with ESPAsyncWebServer and AsyncTCP that 
 *  read from SPIFFS and
 *  stream on browser various type of file
 *  and manage gzip format also
 *
 *  by Mischianti Renzo <https://mischianti.org>
 *
 *  https://mischianti.org/category/tutorial/how-to-create-a-web-server-with-esp8266-and-esp32/
 *
 */
#include <WiFi.h>
#include <AsyncTCP.h>
#include <ESPAsyncWebServer.h>
#include <SPIFFS.h>

const char* ssid = "<YOUR-SSID>";
const char* password = "<YOUR-PASSWD>";

AsyncWebServer  httpServer(80);

void handleNotFound(AsyncWebServerRequest *request);

void serverRouting();

void setup(void) {
  Serial.begin(115200);
  Serial.setDebugOutput(true);

  WiFi.mode(WIFI_STA);
  WiFi.begin(ssid, password);
  WiFi.setSleep(false);
  Serial.println("");

  // Wait for connection
  while (WiFi.status() != WL_CONNECTED) {
    delay(500);
    Serial.print(".");
  }

  Serial.println("");
  Serial.print("Connected to ");
  Serial.println(ssid);
  Serial.print("IP address: ");
  Serial.println(WiFi.localIP());

  Serial.print(F("Inizializing FS..."));
  if (SPIFFS.begin()){
	Serial.println(F("done."));
  }else{
	Serial.println(F("fail."));
  }

  Serial.println("Set routing for http server!");
  serverRouting();
  httpServer.begin();
  Serial.println("HTTP server started");
}

void loop(void) {

}

String getContentType(String filename){
  if(filename.endsWith(F(".htm"))) 			return F("text/html");
  else if(filename.endsWith(F(".html"))) 	return F("text/html");
  else if(filename.endsWith(F(".css"))) 	return F("text/css");
  else if(filename.endsWith(F(".js"))) 		return F("application/javascript");
  else if(filename.endsWith(F(".json"))) 	return F("application/json");
  else if(filename.endsWith(F(".png"))) 	return F("image/png");
  else if(filename.endsWith(F(".gif"))) 	return F("image/gif");
  else if(filename.endsWith(F(".jpg"))) 	return F("image/jpeg");
  else if(filename.endsWith(F(".jpeg"))) 	return F("image/jpeg");
  else if(filename.endsWith(F(".ico"))) 	return F("image/x-icon");
  else if(filename.endsWith(F(".xml"))) 	return F("text/xml");
  else if(filename.endsWith(F(".pdf"))) 	return F("application/x-pdf");
  else if(filename.endsWith(F(".zip"))) 	return F("application/x-zip");
  else if(filename.endsWith(F(".gz"))) 		return F("application/x-gzip");
  return F("text/plain");
}

bool handleFileRead(AsyncWebServerRequest *request, String path){
  Serial.print(F("handleFileRead: "));
  Serial.println(path);

  if(path.endsWith("/")) path += F("index.html");           // If a folder is requested, send the index file
  String contentType = getContentType(path);             	// Get the MIME type
  String pathWithGz = path + F(".gz");
  if(SPIFFS.exists(pathWithGz) || SPIFFS.exists(path)){  	// If the file exists, either as a compressed archive, or normal
	bool gzipped = false;

    if(SPIFFS.exists(pathWithGz)) {                        	// If there's a compressed version available
      path += F(".gz");                                     // Use the compressed version
      gzipped = true;
    }
    AsyncWebServerResponse *response = request->beginResponse(SPIFFS, path, contentType);
    if (gzipped){
    	response->addHeader("Content-Encoding", "gzip");
    }
    Serial.print("Real file path: ");
    Serial.println(path);

    request->send(response);

    return true;
  }
  Serial.println(String(F("\tFile Not Found: ")) + path);
  return false;                                          	// If the file doesn't exist, return false
}

void handleNotFound(AsyncWebServerRequest *request) {
  String message = "File Not Found\n\n";
  message += "URI: ";
  message += request->url();
  message += "\nMethod: ";
  message += (request->method() == HTTP_GET) ? "GET" : "POST";
  message += "\nArguments: ";
  message += request->args();
  message += "\n";

  for (uint8_t i = 0; i < request->args(); i++) {
    message += " " + request->argName(i) + ": " + request->arg(i) + "\n";
  }

  request->send(200, "text/plain", message);
}

void serverRouting() {
	  httpServer.onNotFound([](AsyncWebServerRequest *request) {                              // If the client requests any URI
		  Serial.println(F("On not found"));
	    if (!handleFileRead(request, request->url())){                  // send it if it exists
	    	handleNotFound(request); // otherwise, respond with a 404 (Not Found) error
	    }
	  });

	  Serial.println(F("Set cache!"));
	  // Serve a file with no cache so every tile It's downloaded
	  httpServer.serveStatic("/configuration.json", SPIFFS, "/configuration.json","no-cache, no-store, must-revalidate");
	  // Server all other page with long cache so browser chaching they
	  httpServer.serveStatic("/", SPIFFS, "/","max-age=31536000");
}

As you can see It’s very similar, but you must note that there aren’t instruction on loop cycle, this is because the library is asynchronous.

In the serverRouting() now we must pass the request parameter.

void serverRouting() {
	  httpServer.onNotFound([](AsyncWebServerRequest *request) {                              // If the client requests any URI
		  Serial.println(F("On not found"));
	    if (!handleFileRead(request, request->url())){                  // send it if it exists
	    	handleNotFound(request); // otherwise, respond with a 404 (Not Found) error
	    }
	  });

	  Serial.println(F("Set cache!"));
	  // Serve a file with no cache so every tile It's downloaded
	  httpServer.serveStatic("/configuration.json", SPIFFS, "/configuration.json","no-cache, no-store, must-revalidate");
	  // Server all other page with long cache so browser chaching they
	  httpServer.serveStatic("/", SPIFFS, "/","max-age=31536000");
}

and than propagate request.

The cache management It’s the same.

In the handleFileRead you can check that the stream of the file It’s quite different and you must manage the gzip format via code (in the streamFile the check of gz It was implicit).

    AsyncWebServerResponse *response = request->beginResponse(SPIFFS, path, contentType);
    if (gzipped){
    	response->addHeader("Content-Encoding", "gzip");
    }
    Serial.print("Real file path: ");
    Serial.println(path);

    request->send(response);

No other significant changes are made, but if the bug is fixed I recommend using the standard library with the previous solution.

Thanks

Code and examples on this repository GitHub

Spread the love

Tags: Async Web Server esp32 esp8266 Guida alla creazione di un Web Server Web Server

Werner Mauss says:

5 April 2021 at 18:30

The big bug in ESP Wifi Client, as mentioned in the red text box above, has been fixed, see link provided.

Reply
- Renzo Mischianti says:
  
  7 April 2021 at 11:54
  
  Yes, thanks Werner, you were right to point this out.
  Hi Renzo
  
  Reply
Carlos Vargas says:

3 July 2021 at 20:11

Thanks for sharing your work…its amazing!
Tell me how do you debug the web site without upload to ESP32? do you check the code and upload always? o its possible testing change on real time since VS for example?

Thansk

Reply
- Renzo Mischianti says:
  
  3 July 2021 at 22:51
  
  Hi Carlos,
  you can develop the client side on ide with a simple http server, and you mut point to the server side (esp32) by changing the ip of the POST and GET.
  But remember to enable CORS on esp32, refer to this article
  
  How to create a REST server on esp8266 or esp32: CORS request, OPTION and POST – Part 5
  
  Bye Renzo
  
  Reply