expanding the ESP32 NVS

[kdschlosser]

Checks

• I agree to follow the MicroPython Code of Conduct to ensure a safe and respectful space for everyone.

• I've searched for existing issues regarding this feature, and didn't find any.

Description

Pseudo code of the expansion.

#include <string.h>

#include "py/runtime.h"
#include "py/mperrno.h"
#include "mphalport.h"
#include "modesp32.h"
#include "nvs_flash.h"
#include "nvs.h"

#define NVS_TYPE_FLOAT  (0xFE)

// This file implements the NVS (Non-Volatile Storage) class in the esp32 module.
// It provides simple access to the NVS feature provided by ESP-IDF.

// NVS python object that represents an NVS namespace.
typedef struct _esp32_nvs_obj_t {
    mp_obj_base_t base;
    nvs_handle_t namespace;
} esp32_nvs_obj_t;

// *esp32_nvs_new allocates a python NVS object given a handle to an esp-idf namespace C obj.
STATIC esp32_nvs_obj_t *esp32_nvs_new(nvs_handle_t namespace) {
    esp32_nvs_obj_t *self = mp_obj_malloc(esp32_nvs_obj_t, &esp32_nvs_type);
    self->namespace = namespace;
    return self;
}

// esp32_nvs_print prints an NVS object, unfortunately it doesn't seem possible to extract the
// namespace string or anything else from the opaque handle provided by esp-idf.
STATIC void esp32_nvs_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_t kind) {
    // esp32_nvs_obj_t *self = MP_OBJ_TO_PTR(self_in);
    mp_printf(print, "<NVS namespace>");
}

// esp32_nvs_make_new constructs a handle to an NVS namespace.
STATIC mp_obj_t esp32_nvs_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *all_args) {
    // Check args
    mp_arg_check_num(n_args, n_kw, 1, 1, false);

    // Get requested nvs namespace
    const char *ns_name = mp_obj_str_get_str(all_args[0]);
    nvs_handle_t namespace;
    check_esp_err(nvs_open(ns_name, NVS_READWRITE, &namespace));
    return MP_OBJ_FROM_PTR(esp32_nvs_new(namespace));
}


// esp32_nvs_set_i32 sets a 32-bit integer value
STATIC mp_obj_t esp32_nvs_set(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) {
    enum { ARG_self, ARG_key, ARG_value, ARG_type };
    static const mp_arg_t allowed_args[] = {
        { MP_QSTR_self,   MP_ARG_OBJ  | MP_ARG_REQUIRED, { .u_obj = mp_const_none } },
        { MP_QSTR_key,    MP_ARG_OBJ  | MP_ARG_REQUIRED, { .u_obj = mp_const_none } },
        { MP_QSTR_value,  MP_ARG_OBJ  | MP_ARG_REQUIRED, { .u_obj = mp_const_none } },
        { MP_QSTR_type,   MP_ARG_INT  | MP_ARG_REQUIRED, { .u_int = 1 } },
    };

    mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)];
    mp_arg_parse_all(n_args, pos_args, kw_args, MP_ARRAY_SIZE(allowed_args), allowed_args, args);

    esp32_nvs_obj_t *self = MP_OBJ_TO_PTR(args[ARG_self].u_obj);
    const char *key = mp_obj_str_get_str(args[ARG_key].u_obj);
    int type = (int)mp_obj_get_int(args[ARG_type].u_obj);

    esp_err_t res;

    switch(type) {
        case NVS_TYPE_U8:
            uint8_t valu8 = (uint8_t)mp_obj_get_int(args[ARG_value].u_obj);
            res = nvs_set_u8(nvs_handle_t handle, key, valu8);
            break;
        case NVS_TYPE_I8:
            int8_t vali8 = (int8_t)mp_obj_get_int(args[ARG_value].u_obj);
            res = nvs_set_i8(self->namespace, key, vali8);
            break;
        case NVS_TYPE_U16:
            uint16_t valu16 = (uint16_t)mp_obj_get_int(args[ARG_value].u_obj);
            res = nvs_set_u16(self->namespace, key, valu16);
            break;
        case NVS_TYPE_I16:
            int16_t vali16 = (int16_t)mp_obj_get_int(args[ARG_value].u_obj);
            res = nvs_set_i16(self->namespace, key, vali16);
            break;
        case NVS_TYPE_U32:
            uint32_t valu32 = (uint32_t)mp_obj_get_int(args[ARG_value].u_obj);
            res = nvs_set_u32(self->namespace, key, valu32);
            break;
        case NVS_TYPE_I32:
            int32_t vali32 = (int32_t)mp_obj_get_int(args[ARG_value].u_obj);
            res = nvs_set_i32(self->namespace, key, vali32);
            break;
        case NVS_TYPE_U64:
            uint64_t valu64 = (uint64_t)mp_obj_get_int(args[ARG_value].u_obj);
            res = nvs_set_u64(self->namespace, key, valu64);
            break;
        case NVS_TYPE_I64:
            int64_t vali64 = (int64_t)mp_obj_get_int(args[ARG_value].u_obj);
            res = nvs_set_i64(self->namespace, key, vali64);
            break;
        case NVS_TYPE_STR:
            const char *valstr = mp_obj_str_get_str(args[ARG_value].u_obj);
            res = nvs_set_str(self->namespace, key, valstr);
            break;
        case NVS_TYPE_BLOB:
            mp_buffer_info_t valblob;
            mp_get_buffer_raise(args[ARG_value].u_obj, &valblob, MP_BUFFER_READ);
            res = nvs_set_blob(self->namespace, key, valblob.buf, valblob.len);
            break;
        case NVS_TYPE_FLOAT:
            union {float f; uint32_t i;} u;
            u.f = (float)mp_obj_float_get(args[ARG_value].u_obj);
            res = nvs_set_u32(self->namespace, key, u.i);
            break;
        default:
            res = ESP_ERR_NVS_TYPE_MISMATCH;
            break;
    }

    check_esp_err(res);
    return mp_const_none
}

MP_DEFINE_CONST_FUN_OBJ_KW(esp32_nvs_set_obj, 4, esp32_nvs_set);


STATIC mp_obj_t esp32_nvs_get(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) {

    enum { ARG_self, ARG_key, ARG_type };
    static const mp_arg_t allowed_args[] = {
        { MP_QSTR_self,   MP_ARG_OBJ  | MP_ARG_REQUIRED, { .u_obj = mp_const_none } },
        { MP_QSTR_key,    MP_ARG_OBJ  | MP_ARG_REQUIRED, { .u_obj = mp_const_none } },
        { MP_QSTR_type,   MP_ARG_INT  | MP_ARG_REQUIRED, { .u_int = 1 } },
    };

    mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)];
    mp_arg_parse_all(n_args, pos_args, kw_args, MP_ARRAY_SIZE(allowed_args), allowed_args, args);

    esp32_nvs_obj_t *self = MP_OBJ_TO_PTR(args[ARG_self].u_obj);
    const char *key = mp_obj_str_get_str(args[ARG_key].u_obj);
    int type = (int)mp_obj_get_int(args[ARG_type].u_obj);

    esp_err_t res;
    mp_obj_t ret;

    switch(type) {
        case NVS_TYPE_U8:
            uint8_t valu8;
            res = nvs_get_u8(nvs_handle_t handle, key, &valu8);
            ret = mp_obj_new_int_from_uint(valu8);
            break;
        case NVS_TYPE_I8:
            int8_t vali8;
            res = nvs_get_i8(self->namespace, key, &vali8);
            ret = mp_obj_new_int(vali8);
            break;
        case NVS_TYPE_U16:
            uint16_t valu16;
            res = nvs_get_u16(self->namespace, key, &valu16);
            ret = mp_obj_new_int_from_uint(valu16);
            break;
        case NVS_TYPE_I16:
            int16_t vali16;
            res = nvs_get_i16(self->namespace, key, &vali16);
            ret = mp_obj_new_int(vali16);
            break;
        case NVS_TYPE_U32:
            uint32_t valu32;
            res = nvs_get_u32(self->namespace, key, &valu32);
            ret = mp_obj_new_int_from_uint(valu32);
            break;
        case NVS_TYPE_I32:
            int32_t vali32;
            res = nvs_get_i32(self->namespace, key, &vali32);
            ret = mp_obj_new_int(vali32);
            break;
        case NVS_TYPE_U64:
            uint64_t valu64;
            res = nvs_get_u64(self->namespace, key, &valu64);
            ret = mp_obj_new_int_from_uint(valu64);
            break;
        case NVS_TYPE_I64:
            int64_t vali64;
            res = nvs_get_i64(self->namespace, key, &vali64);
            ret = mp_obj_new_int(vali64);
            break;
        case NVS_TYPE_STR:
            char *valstr;
            size_t len;
            nvs_get_str(self->namespace, key, NULL, &len);
            res = nvs_get_str(self->namespace, key, valstr, &len);
            ret = mp_obj_new_str(valstr, len);
            break;
        case NVS_TYPE_BLOB:
            char *valblob;
            size_t len;
            nvs_get_blob(self->namespace, key, NULL, &len);
            res = nvs_get_blob(self->namespace, key, valblob, &len);
            ret = mp_obj_new_bytes(valblob, len);
            break;
        case NVS_TYPE_FLOAT:
            union {float f; uint32_t i;} u;
            res = nvs_get_u32(self->namespace, key, &u.i);
            ret = mp_obj_new_float(u.f);
            break;
        default:
            res = ESP_ERR_NVS_TYPE_MISMATCH;
            break;
    }

    check_esp_err(res);
}

MP_DEFINE_CONST_FUN_OBJ_KW(esp32_nvs_get_obj, 3, esp32_nvs_get);


// esp32_nvs_erase_key erases one key.
STATIC mp_obj_t esp32_nvs_erase(mp_obj_t self_in, mp_obj_t key_in) {
    esp32_nvs_obj_t *self = MP_OBJ_TO_PTR(self_in);
    const char *key = mp_obj_str_get_str(key_in);
    check_esp_err(nvs_erase_key(self->namespace, key));

    return mp_const_none;
}

STATIC MP_DEFINE_CONST_FUN_OBJ_2(esp32_nvs_erase_obj, esp32_nvs_erase);


// esp32_nvs_commit commits any changes to flash.
STATIC mp_obj_t esp32_nvs_commit(mp_obj_t self_in) {
    esp32_nvs_obj_t *self = MP_OBJ_TO_PTR(self_in);
    check_esp_err(nvs_commit(self->namespace));
    return mp_const_none;
}

STATIC MP_DEFINE_CONST_FUN_OBJ_1(esp32_nvs_commit_obj, esp32_nvs_commit);


STATIC mp_obj_t esp32_nvs_close(mp_obj_t self_in) {
    esp32_nvs_obj_t *self = MP_OBJ_TO_PTR(self_in);
    nvs_close(self->namespace);
    return mp_const_none;
}

STATIC MP_DEFINE_CONST_FUN_OBJ_1(esp32_nvs_close_obj, esp32_nvs_close);


STATIC const mp_rom_map_elem_t esp32_nvs_locals_dict_table[] = {
    { MP_ROM_QSTR(MP_QSTR_TYPE_U8),    MP_ROM_INT(NVS_TYPE_U8)    },
    { MP_ROM_QSTR(MP_QSTR_TYPE_I8),    MP_ROM_INT(NVS_TYPE_I8)    },
    { MP_ROM_QSTR(MP_QSTR_TYPE_U16),   MP_ROM_INT(NVS_TYPE_U16)   },
    { MP_ROM_QSTR(MP_QSTR_TYPE_I16),   MP_ROM_INT(NVS_TYPE_I16)   },
    { MP_ROM_QSTR(MP_QSTR_TYPE_U32),   MP_ROM_INT(NVS_TYPE_U32)   },
    { MP_ROM_QSTR(MP_QSTR_TYPE_I32),   MP_ROM_INT(NVS_TYPE_I32)   },
    { MP_ROM_QSTR(MP_QSTR_TYPE_U64),   MP_ROM_INT(NVS_TYPE_U64)   },
    { MP_ROM_QSTR(MP_QSTR_TYPE_I64),   MP_ROM_INT(NVS_TYPE_I64)   },
    { MP_ROM_QSTR(MP_QSTR_TYPE_STR),   MP_ROM_INT(NVS_TYPE_STR)   },
    { MP_ROM_QSTR(MP_QSTR_TYPE_BLOB),  MP_ROM_INT(NVS_TYPE_BLOB)  },
    { MP_ROM_QSTR(MP_QSTR_TYPE_FLOAT), MP_ROM_INT(NVS_TYPE_FLOAT) },
    { MP_ROM_QSTR(MP_QSTR___del__),    MP_ROM_PTR(&esp32_nvs_close_obj)  },
    { MP_ROM_QSTR(MP_QSTR_close),      MP_ROM_PTR(&esp32_nvs_close_obj)  },
    { MP_ROM_QSTR(MP_QSTR_get),        MP_ROM_PTR(&esp32_nvs_get_obj)    },
    { MP_ROM_QSTR(MP_QSTR_set),        MP_ROM_PTR(&esp32_nvs_set_obj)    },
    { MP_ROM_QSTR(MP_QSTR_erase),      MP_ROM_PTR(&esp32_nvs_erase_obj)  },
    { MP_ROM_QSTR(MP_QSTR_commit),     MP_ROM_PTR(&esp32_nvs_commit_obj) },
    
};
STATIC MP_DEFINE_CONST_DICT(esp32_nvs_locals_dict, esp32_nvs_locals_dict_table);

MP_DEFINE_CONST_OBJ_TYPE(
    esp32_nvs_type,
    MP_QSTR_NVS,
    MP_TYPE_FLAG_NONE,
    make_new, esp32_nvs_make_new,
    print, esp32_nvs_print,
    locals_dict, &esp32_nvs_locals_dict
);

Code Size

I think the code size increase would not be very large but it would be worth doing to provide better use of the NVS and at the same time having the possibility of a reduction in memory use by having smaller python source files.

It would be nice to use all of the data types that are supported by the NVS and while this won't directly reduce the size it has the potential to reduce the ram use. Python code takes up memory when the files are loaded. The smaller the source file the lower amount of memory is used. currently there are only 2 data types that are supported. int32_t and blob. storing a uint8_t is going to take up 4 bytes of NVS space instead of 1. sure I can use bit shifting at the expense of increase memory use do to a larger python code footprint to do that. The other thing is storing negative and positive numbers together, it just won't work because there is no way to know what is negative and what is positive when separating them.

I also added support for storing floats as well. This is easily done using type punning.

Implementation

• I intend to implement this feature and would submit a Pull Request if desirable.
• I hope the MicroPython maintainers or community will implement this feature.
• I would like to Sponsor development of this feature.