Manage JSON file with Arduino, esp32 and esp8266
When you create a datalogging It’s important to structure your data, sometime a good solution can be JSON format.
JavaScript Object Notation (JSON) is an open-standard file format that uses human-readable text to transmit data objects consisting of attribute–value pairs and array data types (or any other serializable value). It is a very common data format used for asynchronous browser–server communication, including as a replacement for XML in some AJAX-style systems.
JSON is a language-independent data format. It was derived from JavaScript, but as of 2017 many programming languages include code to generate and parse JSON-format data. The official Internet media type for JSON is application/json. JSON filenames use the extension .json. (cit. wiki)
For example, you need to archive and update the value of some totals many times, you can create a structure like this:
{
"lastUpdate": "05/06/2019 06:50:57",
"energyLifetime": 21698620,
"energyYearly": 1363005,
"energyMonthly": 58660,
"energyWeekly": 41858,
"energyDaily": 158
}
Here an example of battery voltage log:
{
"lastUpdate": "30/01/2019 21:24:34",
"data": {
"1004": 3.914468,
"1024": 3.931694,
"1044": 3.90479,
"1104": 3.973645,
"1124": 3.969726,
"1144": 3.954823,
"1204": 3.957871,
"1224": 3.930581,
"1244": 3.954048,
"1304": 3.947516,
"1324": 3.945629,
"1344": 3.863081,
"1404": 3.919597,
"1424": 3.927387,
"1444": 3.912968,
"1504": 3.856597,
"1524": 3.846629,
"1544": 3.903871,
"1604": 3.857226,
"1624": 3.889839,
"1644": 3.865693,
"1704": 3.846145,
"1724": 3.780726,
"1744": 3.846677,
"1804": 3.770323,
"1824": 3.778887,
"1844": 3.769597,
"1904": 3.778693,
"1924": 3.806177,
"1944": 3.801145,
"2004": 3.744049,
"2024": 3.707661,
"2044": 3.780871,
"2104": 3.708484,
"2124": 3.729726,
"0003": 4.138742,
"0023": 4.147887,
"0043": 4.143387,
"0103": 4.139806,
"0123": 4.078258,
"0143": 4.128,
"0203": 4.107871,
"0223": 4.066645,
"0243": 4.103419,
"0303": 4.082081,
"0323": 4.126839,
"0343": 4.118032,
"0403": 4.096113,
"0423": 4.110532,
"0443": 4.099307,
"0503": 4.013565,
"0523": 4.089581,
"0544": 4.075549,
"0604": 4.025274,
"0624": 4.067129,
"0644": 3.997742,
"0704": 3.987677,
"0724": 3.981823,
"0744": 4.006113,
"0804": 4.0035,
"0824": 3.966968,
"0844": 4.016418,
"0904": 3.969049,
"0924": 4.002532,
"0944": 3.907742
}
}
As you can see It’s more readable than CSV or other format, and It’s more versatile.
Library
For Arduino like system exist a library that can be considered a standard, you can download It from github or Arduino IDE library management.
For this library exist a site also very informative.
How to
The usage is quite simply, the difference from previous version is that DynamicJsonDocument is no more dynamic manage of memory, so now we can use that document for all (static and dynamic).
const size_t capacity = 1024;
DynamicJsonDocument doc(capacity);
It’s importanto to calculate the capacity is the max size of your file, to have an idea of the size you need you can check here, It’s a simple calculator that from file give you the relative size.
To set up the SD you can refer to my article “How to use SD card with esp8266 and Arduino”.
In this example we write a file like:
{
"energyLifetime": 21698620,
"energyYearly": 1363005
}
A classic configuration file structure.
I add a comment on all relevant code.
/*
Write JSON file to SD
{
"energyLifetime": 21698620,
"energyYearly": 1363005
}
by Mischianti Renzo <https://mischianti.org>
https://www.mischianti.org/
*/
#include <ArduinoJson.h>
#include <SD.h>
#include <SPI.h>
const int chipSelect = 4;
const char *filename = "/test.txt"; // <- SD library uses 8.3 filenames
// Prints the content of a file to the Serial
void printFile(const char *filename) {
// Open file for reading
File file = SD.open(filename);
if (!file) {
Serial.println(F("Failed to read file"));
return;
}
// Extract each characters by one by one
while (file.available()) {
Serial.print((char)file.read());
}
Serial.println();
// Close the file
file.close();
}
void setup() {
// Initialize serial port
Serial.begin(9600);
while (!Serial) continue;
delay(500);
// Initialize SD library
while (!SD.begin(chipSelect)) {
Serial.println(F("Failed to initialize SD library"));
delay(1000);
}
SD.remove(filename);
// Open file for writing
File file = SD.open(filename, FILE_WRITE);
if (!file) {
Serial.println(F("Failed to create file"));
return;
}
// Allocate a temporary JsonDocument
// Don't forget to change the capacity to match your requirements.
// Use arduinojson.org/v6/assistant to compute the capacity.
// StaticJsonDocument<512> doc;
// You can use DynamicJsonDocument as well
DynamicJsonDocument doc(512);
// Set the values in the document
doc["energyLifetime"] = 21698620;
doc["energyYearly"] = 1363005;
// Serialize JSON to file
if (serializeJson(doc, file) == 0) {
Serial.println(F("Failed to write to file"));
}
// Close the file
file.close();
// Print test file
Serial.println(F("Print test file..."));
printFile(filename);
}
void loop() {
// not used in this example
}
Generate an array of data, add an element every 5 seconds and update the original file.
The structure generated is like this:
{
"millis": 10735,
"data": [
{
"prevNumOfElem": 1,
"newNumOfElem": 2
},
{
"prevNumOfElem": 2,
"newNumOfElem": 3
},
{
"prevNumOfElem": 3,
"newNumOfElem": 4
}
]
}
Where millis is overrided and a new value appear on array every time.
/*
Write JSON file to SD
by Mischianti Renzo <https://mischianti.org>
https://www.mischianti.org/
*/
#include <ArduinoJson.h>
#include <SD.h>
#include <SPI.h>
const int chipSelect = 4;
const char *filename = "/test.jso"; // <- SD library uses 8.3 filenames
// Prints the content of a file to the Serial
void printFile(const char *filename) {
// Open file for reading
File file = SD.open(filename);
if (!file) {
Serial.println(F("Failed to read file"));
return;
}
// Extract each characters by one by one
while (file.available()) {
Serial.print((char) file.read());
}
Serial.println();
// Close the file
file.close();
}
void setup() {
// Initialize serial port
Serial.begin(9600);
while (!Serial)
continue;
delay(500);
// Initialize SD library
while (!SD.begin(chipSelect)) {
Serial.println(F("Failed to initialize SD library"));
delay(1000);
}
Serial.println(F("SD library initialized"));
Serial.println(F("Delete original file if exists!"));
SD.remove(filename);
}
void loop() {
// Allocate a temporary JsonDocument
// Don't forget to change the capacity to match your requirements.
// Use arduinojson.org/v6/assistant to compute the capacity.
// StaticJsonDocument<512> doc;
// You can use DynamicJsonDocument as well
DynamicJsonDocument doc(1024);
JsonObject obj;
// Open file
File file = SD.open(filename);
if (!file) {
Serial.println(F("Failed to create file, probably not exists"));
Serial.println(F("Create an empty one!"));
obj = doc.to<JsonObject>();
} else {
DeserializationError error = deserializeJson(doc, file);
if (error) {
// if the file didn't open, print an error:
Serial.println(F("Error parsing JSON "));
Serial.println(error.c_str());
// create an empty JSON object
obj = doc.to<JsonObject>();
} else {
// GET THE ROOT OBJECT TO MANIPULATE
obj = doc.as<JsonObject>();
}
}
// close the file already loaded:
file.close();
obj[F("millis")] = millis();
JsonArray data;
// Check if exist the array
if (!obj.containsKey(F("data"))) {
Serial.println(F("Not find data array! Crete one!"));
data = obj.createNestedArray(F("data"));
} else {
Serial.println(F("Find data array!"));
data = obj[F("data")];
}
// create an object to add to the array
JsonObject objArrayData = data.createNestedObject();
objArrayData["prevNumOfElem"] = data.size();
objArrayData["newNumOfElem"] = data.size() + 1;
SD.remove(filename);
// Open file for writing
file = SD.open(filename, FILE_WRITE);
// Serialize JSON to file
if (serializeJson(doc, file) == 0) {
Serial.println(F("Failed to write to file"));
}
// Close the file
file.close();
// Print test file
Serial.println(F("Print test file..."));
printFile(filename);
delay(5000);
}
Now let’s organize the code a bit. The code in this format is unusable, but with 2 simple functions it should improve.
/*
Write JSON file to SD
by Renzo Mischianti <https://mischianti.org>
https://www.mischianti.org/
*/
#include <ArduinoJson.h>
#include <SD.h>
#include <SPI.h>
const int chipSelect = 4;
const char *filename = "/test.jso"; // <- SD library uses 8.3 filenames
File myFileSDCart;
/**
* Function to deserialize file from SD
* by Renzo Mischianti <https://mischianti.org>
* example:
* DynamicJsonDocument doc(1024);
JsonObject obj;
obj = getJSonFromFile(&doc, filename);
*/
JsonObject getJSonFromFile(DynamicJsonDocument *doc, String filename, bool forceCleanONJsonError = true ) {
// open the file for reading:
myFileSDCart = SD.open(filename);
if (myFileSDCart) {
// read from the file until there's nothing else in it:
// if (myFileSDCart.available()) {
// firstWrite = false;
// }
DeserializationError error = deserializeJson(*doc, myFileSDCart);
if (error) {
// if the file didn't open, print an error:
Serial.print(F("Error parsing JSON "));
Serial.println(error.c_str());
if (forceCleanONJsonError){
return doc->to<JsonObject>();
}
}
// close the file:
myFileSDCart.close();
return doc->as<JsonObject>();
} else {
// if the file didn't open, print an error:
Serial.print(F("Error opening (or file not exists) "));
Serial.println(filename);
Serial.println(F("Empty json created"));
return doc->to<JsonObject>();
}
}
/**
* Function to serialize file to SD
* by Renzo Mischianti <https://mischianti.org>
* example:
* boolean isSaved = saveJSonToAFile(&doc, filename);
*/
bool saveJSonToAFile(DynamicJsonDocument *doc, String filename) {
SD.remove(filename);
// open the file. note that only one file can be open at a time,
// so you have to close this one before opening another.
Serial.println(F("Open file in write mode"));
myFileSDCart = SD.open(filename, FILE_WRITE);
if (myFileSDCart) {
Serial.print(F("Filename --> "));
Serial.println(filename);
Serial.print(F("Start write..."));
serializeJson(*doc, myFileSDCart);
Serial.print(F("..."));
// close the file:
myFileSDCart.close();
Serial.println(F("done."));
return true;
} else {
// if the file didn't open, print an error:
Serial.print(F("Error opening "));
Serial.println(filename);
return false;
}
}
// Prints the content of a file to the Serial
void printFile(const char *filename) {
// Open file for reading
File file = SD.open(filename);
if (!file) {
Serial.println(F("Failed to read file"));
return;
}
// Extract each characters by one by one
while (file.available()) {
Serial.print((char) file.read());
}
Serial.println();
// Close the file
file.close();
}
void setup() {
// Initialize serial port
Serial.begin(9600);
while (!Serial)
continue;
delay(500);
// Initialize SD library
while (!SD.begin(chipSelect)) {
Serial.println(F("Failed to initialize SD library"));
delay(1000);
}
Serial.println(F("SD library initialized"));
Serial.println(F("Delete original file if exists!"));
SD.remove(filename);
}
void loop() {
// Allocate a temporary JsonDocument
// Don't forget to change the capacity to match your requirements.
// Use arduinojson.org/v6/assistant to compute the capacity.
// StaticJsonDocument<512> doc;
// You can use DynamicJsonDocument as well
DynamicJsonDocument doc(1024);
JsonObject obj;
obj = getJSonFromFile(&doc, filename);
obj[F("millis")] = millis();
JsonArray data;
// Check if exist the array
if (!obj.containsKey(F("data"))) {
Serial.println(F("Not find data array! Crete one!"));
data = obj.createNestedArray(F("data"));
} else {
Serial.println(F("Find data array!"));
data = obj[F("data")];
}
// create an object to add to the array
JsonObject objArrayData = data.createNestedObject();
objArrayData["prevNumOfElem"] = data.size();
objArrayData["newNumOfElem"] = data.size() + 1;
boolean isSaved = saveJSonToAFile(&doc, filename);
if (isSaved){
Serial.println("File saved!");
}else{
Serial.println("Error on save File!");
}
// Print test file
Serial.println(F("Print test file..."));
printFile(filename);
delay(5000);
}
Now I think it’s improved and it’s pretty clear.
