Arduino / esp32 / esp8266 / JSON / Library / SD Card
3

Manage JSON file with Arduino, esp32 and esp8266

by Renzo Mischianti · Published 26 January 2020 · Updated 1 December 2021

Spread the love

When you create a datalogging It’s important to structure your data, sometime a good solution can be JSON format.

JavaScript Object Notation (JSON) is an open-standard file format that uses human-readable text to transmit data objects consisting of attribute–value pairs and array data types (or any other serializable value). It is a very common data format used for asynchronous browser–server communication, including as a replacement for XML in some AJAX-style systems.

JSON is a language-independent data format. It was derived from JavaScript, but as of 2017 many programming languages include code to generate and parse JSON-format data. The official Internet media type for JSON is application/json. JSON filenames use the extension .json. (cit. wiki)

For example, you need to archive and update the value of some totals many times, you can create a structure like this:

{
    "lastUpdate": "05/06/2019 06:50:57",
    "energyLifetime": 21698620,
    "energyYearly": 1363005,
    "energyMonthly": 58660,
    "energyWeekly": 41858,
    "energyDaily": 158
}

Here an example of battery voltage log:

{
    "lastUpdate": "30/01/2019 21:24:34",
    "data": {
        "1004": 3.914468,
        "1024": 3.931694,
        "1044": 3.90479,
        "1104": 3.973645,
        "1124": 3.969726,
        "1144": 3.954823,
        "1204": 3.957871,
        "1224": 3.930581,
        "1244": 3.954048,
        "1304": 3.947516,
        "1324": 3.945629,
        "1344": 3.863081,
        "1404": 3.919597,
        "1424": 3.927387,
        "1444": 3.912968,
        "1504": 3.856597,
        "1524": 3.846629,
        "1544": 3.903871,
        "1604": 3.857226,
        "1624": 3.889839,
        "1644": 3.865693,
        "1704": 3.846145,
        "1724": 3.780726,
        "1744": 3.846677,
        "1804": 3.770323,
        "1824": 3.778887,
        "1844": 3.769597,
        "1904": 3.778693,
        "1924": 3.806177,
        "1944": 3.801145,
        "2004": 3.744049,
        "2024": 3.707661,
        "2044": 3.780871,
        "2104": 3.708484,
        "2124": 3.729726,
        "0003": 4.138742,
        "0023": 4.147887,
        "0043": 4.143387,
        "0103": 4.139806,
        "0123": 4.078258,
        "0143": 4.128,
        "0203": 4.107871,
        "0223": 4.066645,
        "0243": 4.103419,
        "0303": 4.082081,
        "0323": 4.126839,
        "0343": 4.118032,
        "0403": 4.096113,
        "0423": 4.110532,
        "0443": 4.099307,
        "0503": 4.013565,
        "0523": 4.089581,
        "0544": 4.075549,
        "0604": 4.025274,
        "0624": 4.067129,
        "0644": 3.997742,
        "0704": 3.987677,
        "0724": 3.981823,
        "0744": 4.006113,
        "0804": 4.0035,
        "0824": 3.966968,
        "0844": 4.016418,
        "0904": 3.969049,
        "0924": 4.002532,
        "0944": 3.907742
    }
}

As you can see It’s more readable than CSV or other format, and It’s more versatile.

Library

For Arduino like system exist a library that can be considered a standard, you can download It from github or Arduino IDE library management.

For this library exist a site also very informative.

How to

The usage is quite simply, the difference from previous version is that DynamicJsonDocument is no more dynamic manage of memory, so now we can use that document for all (static and dynamic).

const size_t capacity = 1024;
DynamicJsonDocument doc(capacity);

It’s importanto to calculate the capacity is the max size of your file, to have an idea of the size you need you can check here, It’s a simple calculator that from file give you the relative size.

To set up the SD you can refer to my article “How to use SD card with esp8266 and Arduino”.

In this example we write a file like:

{
  "energyLifetime": 21698620,
  "energyYearly": 1363005
}

A classic configuration file structure.

I add a comment on all relevant code.

/*
 Write JSON file to SD
 {
   "energyLifetime": 21698620,
   "energyYearly": 1363005
 }
 by Mischianti Renzo <https://mischianti.org>

 https://www.mischianti.org/
*/

#include <ArduinoJson.h>
#include <SD.h>
#include <SPI.h>

const int chipSelect = 4;

const char *filename = "/test.txt";  // <- SD library uses 8.3 filenames

// Prints the content of a file to the Serial
void printFile(const char *filename) {
  // Open file for reading
  File file = SD.open(filename);
  if (!file) {
    Serial.println(F("Failed to read file"));
    return;
  }

  // Extract each characters by one by one
  while (file.available()) {
    Serial.print((char)file.read());
  }
  Serial.println();

  // Close the file
  file.close();
}

void setup() {
  // Initialize serial port
  Serial.begin(9600);
  while (!Serial) continue;

  delay(500);

  // Initialize SD library
  while (!SD.begin(chipSelect)) {
    Serial.println(F("Failed to initialize SD library"));
    delay(1000);
  }

  SD.remove(filename);

  // Open file for writing
  File file = SD.open(filename, FILE_WRITE);
  if (!file) {
    Serial.println(F("Failed to create file"));
    return;
  }

  // Allocate a temporary JsonDocument
  // Don't forget to change the capacity to match your requirements.
  // Use arduinojson.org/v6/assistant to compute the capacity.
//  StaticJsonDocument<512> doc;
  // You can use DynamicJsonDocument as well
  DynamicJsonDocument doc(512);

  // Set the values in the document
  doc["energyLifetime"] = 21698620;
  doc["energyYearly"] = 1363005;


  // Serialize JSON to file
  if (serializeJson(doc, file) == 0) {
    Serial.println(F("Failed to write to file"));
  }

  // Close the file
  file.close();

  // Print test file
  Serial.println(F("Print test file..."));
  printFile(filename);
}

void loop() {
  // not used in this example
}

Generate an array of data, add an element every 5 seconds and update the original file.

The structure generated is like this:

{
  "millis": 10735,
  "data": [
    {
      "prevNumOfElem": 1,
      "newNumOfElem": 2
    },
    {
      "prevNumOfElem": 2,
      "newNumOfElem": 3
    },
    {
      "prevNumOfElem": 3,
      "newNumOfElem": 4
    }
  ]
}

Where millis is overrided and a new value appear on array every time.

/*
 Write JSON file to SD
 by Mischianti Renzo <https://mischianti.org>

 https://www.mischianti.org/
 */

#include <ArduinoJson.h>
#include <SD.h>
#include <SPI.h>

const int chipSelect = 4;

const char *filename = "/test.jso";  // <- SD library uses 8.3 filenames

// Prints the content of a file to the Serial
void printFile(const char *filename) {
	// Open file for reading
	File file = SD.open(filename);
	if (!file) {
		Serial.println(F("Failed to read file"));
		return;
	}

	// Extract each characters by one by one
	while (file.available()) {
		Serial.print((char) file.read());
	}
	Serial.println();

	// Close the file
	file.close();
}

void setup() {
	// Initialize serial port
	Serial.begin(9600);
	while (!Serial)
		continue;

	delay(500);

	// Initialize SD library
	while (!SD.begin(chipSelect)) {
		Serial.println(F("Failed to initialize SD library"));
		delay(1000);
	}

	Serial.println(F("SD library initialized"));

	Serial.println(F("Delete original file if exists!"));
	SD.remove(filename);

}

void loop() {
	// Allocate a temporary JsonDocument
	// Don't forget to change the capacity to match your requirements.
	// Use arduinojson.org/v6/assistant to compute the capacity.
	//  StaticJsonDocument<512> doc;
	// You can use DynamicJsonDocument as well
	DynamicJsonDocument doc(1024);

	JsonObject obj;
	// Open file
	File file = SD.open(filename);
	if (!file) {
		Serial.println(F("Failed to create file, probably not exists"));
		Serial.println(F("Create an empty one!"));
		obj = doc.to<JsonObject>();
	} else {

		DeserializationError error = deserializeJson(doc, file);
		if (error) {
			// if the file didn't open, print an error:
			Serial.println(F("Error parsing JSON "));
			Serial.println(error.c_str());

			// create an empty JSON object
			obj = doc.to<JsonObject>();
		} else {
			// GET THE ROOT OBJECT TO MANIPULATE
			obj = doc.as<JsonObject>();
		}

	}

	// close the file already loaded:
	file.close();

	obj[F("millis")] = millis();

	JsonArray data;
	// Check if exist the array
	if (!obj.containsKey(F("data"))) {
		Serial.println(F("Not find data array! Crete one!"));
		data = obj.createNestedArray(F("data"));
	} else {
		Serial.println(F("Find data array!"));
		data = obj[F("data")];
	}

	// create an object to add to the array
	JsonObject objArrayData = data.createNestedObject();

	objArrayData["prevNumOfElem"] = data.size();
	objArrayData["newNumOfElem"] = data.size() + 1;

	SD.remove(filename);

	// Open file for writing
	file = SD.open(filename, FILE_WRITE);

	// Serialize JSON to file
	if (serializeJson(doc, file) == 0) {
		Serial.println(F("Failed to write to file"));
	}

	// Close the file
	file.close();

	// Print test file
	Serial.println(F("Print test file..."));
	printFile(filename);

	delay(5000);
}

Now let’s organize the code a bit. The code in this format is unusable, but with 2 simple functions it should improve.

/*
 Write JSON file to SD
 by Renzo Mischianti <https://mischianti.org>

 https://www.mischianti.org/
 */

#include <ArduinoJson.h>
#include <SD.h>
#include <SPI.h>

const int chipSelect = 4;

const char *filename = "/test.jso";  // <- SD library uses 8.3 filenames

File myFileSDCart;

/**
 * Function to deserialize file from SD
 * by Renzo Mischianti <https://mischianti.org>
 * example:
 *  DynamicJsonDocument doc(1024);
	JsonObject obj;
	obj = getJSonFromFile(&doc, filename);
 */
JsonObject getJSonFromFile(DynamicJsonDocument *doc, String filename, bool forceCleanONJsonError = true ) {
	// open the file for reading:
	myFileSDCart = SD.open(filename);
	if (myFileSDCart) {
		// read from the file until there's nothing else in it:
//			if (myFileSDCart.available()) {
//				firstWrite = false;
//			}

		DeserializationError error = deserializeJson(*doc, myFileSDCart);
		if (error) {
			// if the file didn't open, print an error:
			Serial.print(F("Error parsing JSON "));
			Serial.println(error.c_str());

			if (forceCleanONJsonError){
				return doc->to<JsonObject>();
			}
		}

		// close the file:
		myFileSDCart.close();

		return doc->as<JsonObject>();
	} else {
		// if the file didn't open, print an error:
		Serial.print(F("Error opening (or file not exists) "));
		Serial.println(filename);

		Serial.println(F("Empty json created"));
		return doc->to<JsonObject>();
	}

}

/**
 * Function to serialize file to SD
 * by Renzo Mischianti <https://mischianti.org>
 * example:
 * boolean isSaved = saveJSonToAFile(&doc, filename);
 */
bool saveJSonToAFile(DynamicJsonDocument *doc, String filename) {
	SD.remove(filename);

	// open the file. note that only one file can be open at a time,
	// so you have to close this one before opening another.
	Serial.println(F("Open file in write mode"));
	myFileSDCart = SD.open(filename, FILE_WRITE);
	if (myFileSDCart) {
		Serial.print(F("Filename --> "));
		Serial.println(filename);

		Serial.print(F("Start write..."));

		serializeJson(*doc, myFileSDCart);

		Serial.print(F("..."));
		// close the file:
		myFileSDCart.close();
		Serial.println(F("done."));

		return true;
	} else {
		// if the file didn't open, print an error:
		Serial.print(F("Error opening "));
		Serial.println(filename);

		return false;
	}
}

// Prints the content of a file to the Serial
void printFile(const char *filename) {
	// Open file for reading
	File file = SD.open(filename);
	if (!file) {
		Serial.println(F("Failed to read file"));
		return;
	}

	// Extract each characters by one by one
	while (file.available()) {
		Serial.print((char) file.read());
	}
	Serial.println();

	// Close the file
	file.close();
}

void setup() {
	// Initialize serial port
	Serial.begin(9600);
	while (!Serial)
		continue;

	delay(500);

	// Initialize SD library
	while (!SD.begin(chipSelect)) {
		Serial.println(F("Failed to initialize SD library"));
		delay(1000);
	}

	Serial.println(F("SD library initialized"));

	Serial.println(F("Delete original file if exists!"));
	SD.remove(filename);

}

void loop() {
	// Allocate a temporary JsonDocument
	// Don't forget to change the capacity to match your requirements.
	// Use arduinojson.org/v6/assistant to compute the capacity.
	//  StaticJsonDocument<512> doc;
	// You can use DynamicJsonDocument as well
	DynamicJsonDocument doc(1024);

	JsonObject obj;
	obj = getJSonFromFile(&doc, filename);

	obj[F("millis")] = millis();

	JsonArray data;
	// Check if exist the array
	if (!obj.containsKey(F("data"))) {
		Serial.println(F("Not find data array! Crete one!"));
		data = obj.createNestedArray(F("data"));
	} else {
		Serial.println(F("Find data array!"));
		data = obj[F("data")];
	}

	// create an object to add to the array
	JsonObject objArrayData = data.createNestedObject();

	objArrayData["prevNumOfElem"] = data.size();
	objArrayData["newNumOfElem"] = data.size() + 1;


	boolean isSaved = saveJSonToAFile(&doc, filename);

	if (isSaved){
		Serial.println("File saved!");
	}else{
		Serial.println("Error on save File!");
	}

	// Print test file
	Serial.println(F("Print test file..."));
	printFile(filename);

	delay(5000);
}

Now I think it’s improved and it’s pretty clear.