ESP32: Switch to CMake-based build

A prerequisite for supporting targets other than the original ESP32.

#564

platforms/esp32/CMakeLists.txt

@@ -0,0 +1,23 @@
+cmake_minimum_required(VERSION 3.5)
+set(PYTHON_DEPS_CHECKED 1)  # Save some build time
+set(EXTRA_CFLAGS $ENV{EXTRA_CFLAGS})
+set(EXTRA_CPPFLAGS $ENV{EXTRA_CPPFLAGS})
+set(EXTRA_CXXFLAGS $ENV{EXTRA_CXXFLAGS})
+include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+project($ENV{APP})
+
+# Add binary libs.
+separate_arguments(app_bin_libs NATIVE_COMMAND "$ENV{APP_BIN_LIBS}")
+foreach(bin_lib ${app_bin_libs})
+  get_filename_component(target_name ${bin_lib} NAME_WLE)
+  add_prebuilt_library(${target_name} ${bin_lib})
+  target_link_libraries(${CMAKE_PROJECT_NAME}.elf ${target_name})
+endforeach()
+
+# ESP-IDF passes component libs multiple times to the linker.
+# This looks like a bug but is not a problem unless they are --whole-archive's.
+# Which is what we want to do, since that is the only way to make sure weak
+# functions are overridden by non-weak ones, which we definitely want to happen.
+# We wrap the link command with a script that dedupes the static libs,
+# puts them in a group and wraps with --whole-archive.
+set(CMAKE_CXX_LINK_EXECUTABLE "/mongoose-os/tools/link.py <CMAKE_CXX_COMPILER> <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")

platforms/esp32/Makefile.build

@@ -1,4 +1,5 @@
-MAKEFLAGS += --no-builtin-rules --warn-undefined-variables
+MAKEFLAGS += --no-builtin-rules --no-print-directory
+#MAKEFLAGS += --warn-undefined-variables
 .DEFAULT_GOAL := all
 
 ifdef MGOS_VARS_FILE
@@ -44,6 +45,8 @@ MGOS_ROOT_DEVTAB ?=
 MGOS_ROOT_FS_TYPE ?= LFS
 MGOS_ROOT_FS_SIZE ?= 262144
 
+export IDF_TARGET = $(APP_PLATFORM)
+
 export MGOS_ROOT_FS_SIZE
 
 # SPIFFS uses its own VFS-level encryption because flash access
@@ -63,12 +66,12 @@ include $(MGOS_PATH)/tools/mk/mgos_common.mk
 
 APP_PATH := $(CURDIR)
 MGOS_ESP_PATH ?= $(MGOS_PATH)/platforms/esp32
-MGOS_ESP_SRC_PATH ?= $(MGOS_PATH)/platforms/esp32/src
+MGOS_ESP_SRC_PATH ?= $(MGOS_PATH)/platforms/esp32/main
 
 BUILD_DIR ?= $(CURDIR)/.build
 FS_STAGING_DIR = $(BUILD_DIR)/fs
 FW_STAGING_DIR ?= $(BUILD_DIR)/fw_temp
-FW_DIR ?= $(CURDIR)/firmware
+FW_DIR ?= $(BUILD_DIR)/firmware
 GEN_DIR ?= $(BUILD_DIR)/gen
 
 export APP_SLOT_SIZE ?= 0x180000
@@ -93,115 +96,109 @@ override APP_CONF_SCHEMA := $(abspath $(APP_CONF_SCHEMA))
 override APP_EXTRA_SRCS := $(abspath $(APP_EXTRA_SRCS))
 override BUILD_DIR := $(abspath $(BUILD_DIR))
 override FW_STAGING_DIR := $(abspath $(FW_STAGING_DIR))
-override FW_DIR := $(abspath $(FW_DIR))
 override GEN_DIR := $(abspath $(GEN_DIR))
 override MGOS_PATH := $(abspath $(MGOS_PATH))
-override _MGOS_PATH = $(MGOS_PATH)
-# It seems to be impossible to override vars passed to submake from the
-# command line, so we pass them via a separate set of variables, which
-# submake then uses to override the values it received.
-export APP
-export APP_BUILD_ID
-export APP_CFLAGS
-export APP_CXXFLAGS
-export _APP_CONF_SCHEMA = $(APP_CONF_SCHEMA)
-export _APP_EXTRA_SRCS = $(APP_EXTRA_SRCS)
-export _APP_FS_FILES = $(APP_FS_FILES)
-export _APP_BIN_LIBS = $(APP_BIN_LIBS)
-export APP_PATH
-export APP_PLATFORM
-export _APP_SOURCES = $(APP_SOURCES)
-export _APP_INCLUDES = $(APP_INCLUDES)
-export APP_VERSION
-export _BUILD_DIR = $(BUILD_DIR)
-export FFI_SYMBOLS
-export _FS_FILES = $(FS_FILES)
-export _FW_DIR := $(FW_DIR)
-export _GEN_DIR := $(GEN_DIR)
-export MGOS_DEBUG_UART
-export MGOS_SRC_PATH
-export MGOS_INCLUDE_PATH
-export MGOS_ESP_PATH
-export MGOS_ESP_SRC_PATH
-export _MGOS_PATH
 
-export BUILD_INFO_JSON = $(GEN_DIR)/build_info.json
+BUILD_INFO_JSON = $(GEN_DIR)/build_info.json
 
-export GEN_BUILD_INFO_EXTRA ?=
+GEN_BUILD_INFO_EXTRA ?=
 
 MGOS_FEATURES =
 MGOS_CONF_SCHEMA =
 FS_FILES =
 
 include $(MGOS_PATH)/tools/mk/mgos_features.mk
 
-export MGOS_SRCS
-export MGOS_FEATURES
-export MGOS_CONF_SCHEMA
-export MGOS_IPATH
-
+BUILD_INFO_C = $(GEN_DIR)/build_info.c
+FFI_EXPORTS_C = $(GEN_DIR)/ffi_exports.c
+MG_BUILD_INFO_C = $(GEN_DIR)/mg_build_info.c
+MGOS_CONFIG_C = $(GEN_DIR)/mgos_config.c
+MGOS_RO_VARS_C = $(GEN_DIR)/mgos_ro_vars.c
+APP_BIN = $(BUILD_DIR)/$(APP).bin
+BOOTLOADER_BIN = $(BUILD_DIR)/bootloader/bootloader.bin
+PARTITION_TABLE_BIN = $(BUILD_DIR)/partition_table/partition-table.bin
+PARTITION_TABLE_OFFSET = 0x8000
+
+# TODO(rojer): We should really stop doing this.
+APP_SOURCE_DIRS = $(sort $(dir $(APP_SOURCES)))
+
+include $(MGOS_PATH)/tools/mk/mgos_build_info.mk
+include $(MGOS_PATH)/tools/mk/mgos_config.mk
+include $(MGOS_PATH)/tools/mk/mgos_ffi_exports.mk
 include $(MGOS_PATH)/tools/mk/mgos_mkfs.mk
 
-# Flags for other component builds go here.
 # -fno-jump-tables -fno-tree-switch-conversion:
 #    https://github.com/espressif/esp-idf/issues/1552#issuecomment-883600363
-CFLAGS += -Wall -Werror -pipe -mtext-section-literals \
+export EXTRA_CPPFLAGS = -Wall -Werror -pipe \
           -fno-jump-tables -fno-tree-switch-conversion \
-          -I$(MGOS_PATH) -I$(MGOS_SRC_PATH) -I$(MGOS_INCLUDE_PATH) \
-          $(APP_CFLAGS) $(addprefix -I ,$(APP_INCLUDES)) \
-          -DMGOS_ROOT_DEVTAB='$(MGOS_ROOT_DEVTAB)' \
-          -DMGOS_ROOT_FS_TYPE='$(MGOS_ROOT_FS_TYPE)' \
-          -DMGOS_ROOT_FS_OPTS='$(MGOS_ROOT_FS_OPTS)' \
-          -DMGOS_ROOT_FS_EXTRACT='$(MGOS_ROOT_FS_EXTRACT)'
-CXXFLAGS += -pipe -fno-exceptions -fno-rtti \
-            -fno-jump-tables -fno-tree-switch-conversion \
-            $(APP_CXXFLAGS) $(addprefix -I ,$(APP_INCLUDES))
-
-# Settings for the ESP-IDF build system
-# See docs here:
-#   https://github.com/espressif/esp-idf/blob/master/docs/build_system.rst
-
-PROJECT_NAME = $(APP)
-BUILD_DIR_BASE = $(BUILD_DIR)
-
-COMPONENT_DIRS = $(APP_PATH)/components \
-                 $(IDF_PATH)/components \
-                 $(MGOS_ESP_PATH) \
-                 $(MGOS_PATH)/third_party \
-                 /opt
+          -DMGOS_ROOT_DEVTAB=$(MGOS_ROOT_DEVTAB) \
+          -DMGOS_ROOT_FS_TYPE=$(MGOS_ROOT_FS_TYPE) \
+          -DMGOS_ROOT_FS_OPTS=$(MGOS_ROOT_FS_OPTS) \
+          -DMGOS_APP="$(APP)" -DFW_ARCHITECTURE=$(APP_PLATFORM) \
+          -DMGOS_ESP32 -include mgos_iram.h \
+          $(MGOS_FEATURES) -DMG_ENABLE_FILESYSTEM -DMG_ENABLE_DIRECTORY_LISTING \
+          -DMGOS_MAX_NUM_UARTS=3 -DMGOS_NUM_HW_TIMERS=4 \
+          $(addprefix -I,$(APP_INCLUDES) $(MGOS_IPATH) $(GEN_DIR) $(APP_SOURCE_DIRS)) \
+          -I$(IDF_PATH)/components/freertos/include/freertos
+export EXTRA_CFLAGS = $(APP_CFLAGS)
+export EXTRA_CXXFLAGS = -fno-exceptions -fno-rtti $(APP_CXXFLAGS)
+
+export APP
+export MGOS_ESP_PATH
+
+ifeq "$(VERBOSE)" "1"
+export VERBOSE
+endif
+export COLOR = 0
+
 # Explicitly specify SDK components to build.
 SDK_COMPONENTS = app_trace app_update bootloader bootloader_support cxx driver efuse esp32 \
                  esp_common esp_event esp_gdbstub esp_ipc esp_hw_support esp_netif esp_phy esp_pm \
                  esp_ringbuf esp_rom esp_system esp_timer espcoredump esptool_py \
                  freertos hal heap log lwip micro-ecc newlib nvs_flash partition_table \
                  pthread soc spi_flash vfs xtensa xtensa-debug-module
 
-MGOS_MAIN_COMPONENT ?= mosapp
+export COMPONENTS = $(SDK_COMPONENTS) main mosapp $(ESP_IDF_EXTRA_COMPONENTS)
+
+MGOS_SRCS += mgos_config_util.c mgos_core_dump.c mgos_dlsym.c mgos_event.c \
+             mgos_gpio.c mgos_init.c mgos_mmap_esp.c \
+             mgos_sys_config.c \
+             mgos_file_utils.c mgos_hw_timers.c mgos_system.c mgos_system.cpp \
+             mgos_time.c mgos_timers.c mgos_timers.cpp mgos_uart.c mgos_utils.c \
+             mgos_json_utils.cpp mgos_utils.cpp error_codes.cpp status.cpp \
+             common/cs_crc32.c common/cs_file.c common/cs_hex.c common/cs_rbuf.c common/cs_varint.c common/json_utils.c \
+             frozen/frozen.c
 
-# sort used to remove duplicates, we don't really care about order.
-COMPONENTS = $(sort $(SDK_COMPONENTS) $(MGOS_MAIN_COMPONENT) $(ESP_IDF_EXTRA_COMPONENTS))
+MGOS_ESP_SRCS = esp32_debug.c esp32_exc.c \
+                esp32_gpio.c esp32_hal.c esp32_hw_timers.c \
+                esp32_main.c esp32_uart.c \
+                esp32_nsleep100.S
 
-ESPFLASHMODE = qio
-ESPFLASHSIZE = 4MB
-ESPBAUD = 1500000
+export MGOS_SOURCES = $(addprefix $(MGOS_SRC_PATH)/,$(MGOS_SRCS)) \
+                      $(addprefix $(MGOS_ESP_SRC_PATH)/,$(MGOS_ESP_SRCS))
 
-SDKCONFIG = $(GEN_DIR)/sdkconfig
-BATCH_BUILD = 1
+$(BUILD_INFO_C): $(MGOS_SOURCES) $(APP_SOURCES)
+	$(call gen_build_info,$@,$(APP_PATH),$(APP_BUILD_ID),$(APP_VERSION),,$(BUILD_INFO_C),$(BUILD_INFO_JSON))
 
-PARTITION_TABLE_CSV_PATH = $(MGOS_ESP_PATH)/src/partitions_mgos.csv
+$(MG_BUILD_INFO_C): $(MGOS_SOURCES)
+	$(call gen_build_info,$@,$(MGOS_PATH)/fw,,,mg_,$(MG_BUILD_INFO_C),)
 
-include $(IDF_PATH)/make/project.mk
+export GEN_SOURCES = $(MGOS_CONFIG_C) $(MGOS_RO_VARS_C) $(BUILD_INFO_C) $(MG_BUILD_INFO_C) $(FFI_EXPORTS_C)
 
-$(SDKCONFIG): $(MGOS_ESP_PATH)/sdkconfig $(MANIFEST_FINAL)
-	mkdir -p $(dir $@)
-	cp $(MGOS_ESP_PATH)/sdkconfig $@
-	for kv in $(ESP_IDF_SDKCONFIG_OPTS); do echo $$kv >> $@; done
+export APP_BIN_LIBS
+export APP_SOURCES
 
-# Override defconfig and menuconfig, they don't work properly in our environment.
-defconfig menuconfig:
-	true
+SDKCONFIG_OUT = $(GEN_DIR)/sdkconfig
+SDKCONFIG_DEFAULTS_OUT = $(GEN_DIR)/sdkconfig.defaults
 
-LDFLAGS += -L$(BUILD_DIR)/src
+$(SDKCONFIG_DEFAULTS_OUT): $(MGOS_ESP_PATH)/sdkconfig.defaults $(MANIFEST_FINAL)
+	@mkdir -p $(dir $@)
+	@(cat $< && for kv in $(ESP_IDF_SDKCONFIG_OPTS); do echo $$kv; done) > $@
+	@# Force rebuild of sdkconfig, otherwise it's not updated
+	@rm -f $(SDKCONFIG_OUT)
+
+$(FFI_EXPORTS_C): $(APP_FS_FILES)
+	$(call gen_ffi_exports,$@,$(FFI_SYMBOLS),$(filter %.js,$(FS_FILES)))
 
 FW_PARTS = boot:type=boot,addr=0x1000,src=$(BOOTLOADER_BIN),encrypt=true,update=$(MGOS_UPDATE_BOOT_LOADER) \
            pt:type=pt,addr=$(PARTITION_TABLE_OFFSET),src=$(PARTITION_TABLE_BIN),encrypt=true \
@@ -215,6 +212,20 @@ include $(MGOS_PATH)/tools/mk/mgos_fw_meta.mk
 
 all: $(FW_STAGING_DIR) $(FW_DIR) $(GEN_DIR) $(FW_ZIP)
 
+cmake-out: $(SDKCONFIG_DEFAULTS_OUT) $(APP_SOURCES) $(MGOS_SOURCES) $(GEN_SOURCES) $(MANIFEST_FINAL)
+	cd $(BUILD_DIR) && \
+	  cmake -DCOMPONENTS="$(COMPONENTS)" \
+	        -DEXTRA_COMPONENT_DIRS="$(APP_PATH)/components" \
+	        -DFFI_EXPORTS_C="$(FFI_EXPORTS_C)" \
+	        -DSDKCONFIG="$(SDKCONFIG_OUT)" \
+	        -DSDKCONFIG_DEFAULTS="$(SDKCONFIG_DEFAULTS_OUT)" \
+	        $(MGOS_ESP_PATH)
+
+$(APP_BIN): cmake-out
+	$(MAKE) -C $(BUILD_DIR) all
+
+$(BOOTLOADER_BIN) $(PARTITION_TABLE_BIN): $(APP_BIN)
+
 FS_FILES += $(GEN_DIR)/conf0.json
 
 $(FW_DIR) $(FW_STAGING_DIR) $(GEN_DIR):
@@ -226,18 +237,18 @@ app-bin-check: $(APP_BIN)
 	$(eval _appslotsize := $(shell echo $$(($(APP_SLOT_SIZE)))))
 	$(eval _appsizepct := $(shell echo $$(($(_appsize) * 100 / $(_appslotsize)))))
 	$(vecho) "  App size: $(_appsize) of $(_appslotsize) ($(_appsizepct)%)"
-	[ $(_appsize) -lt $(_appslotsize) ] || { echo App is too big && exit 1; }
+	@[ $(_appsize) -lt $(_appslotsize) ] || { echo App is too big && exit 1; }
 
 $(FW_MANIFEST): app-bin-check $(BOOTLOADER_BIN) $(PARTITION_TABLE_BIN) $(FS_IMG) $(MKFS)
 
-# Deprecated since 2019/05/14.
-# This can be deleted eventually. For now we keep the file so during OTA previous one gets overwritten
-# and some space is freed.
-$(GEN_DIR)/conf0.json: all_binaries
-	$(Q) echo '{}' > $@
-
 $(FS_IMG): $(MKFS) $(APP_FS_DIRS) $(FS_FILES)
 	$(call mkfs,$(MGOS_ROOT_FS_SIZE),$(MGOS_ROOT_FS_OPTS))
 
+# Lib targets, used in lib builds.
+$(BUILD_DIR)/lib.a $(BUILD_DIR)/moslib/libmoslib.a: cmake-out
+	$(MAKE) -C $(BUILD_DIR) -f esp-idf/mosapp/CMakeFiles/__idf_mosapp.dir/build.make esp-idf/mosapp/libmosapp.a
+	@mkdir -p $(dir $@)
+	@cp $(BUILD_DIR)/esp-idf/mosapp/libmosapp.a $@
+
 clean:
-	rm -rf $(BUILD_DIR) $(FW_DIR)
+	rm -rf $(BUILD_DIR)

platforms/esp32/components/mosapp/CMakeLists.txt

@@ -0,0 +1,5 @@
+separate_arguments(app_srcs NATIVE_COMMAND "$ENV{APP_SOURCES}")
+idf_component_register(
+  SRCS ${app_srcs}
+  REQUIRES main
+)

platforms/esp32/main/CMakeLists.txt

@@ -0,0 +1,9 @@
+separate_arguments(gen_srcs NATIVE_COMMAND "$ENV{GEN_SOURCES}")
+separate_arguments(mgos_srcs NATIVE_COMMAND "$ENV{MGOS_SOURCES}")
+idf_component_register(
+  INCLUDE_DIRS "."
+  SRCS ${app_srcs} ${gen_srcs} ${mgos_srcs}
+)
+# In ffi exports file we use fake signatures: void func(void), and it conflicts
+# with the builtin functions like fopen, etc.
+set_source_files_properties(${FFI_EXPORTS_C} PROPERTIES COMPILE_FLAGS -fno-builtin)

platforms/esp32/sdk.version

@@ -1 +1 @@
-docker.io/mgos/esp32-build:4.4.1-r1
+docker.io/mgos/esp32-build:4.4.1-r3

platforms/esp32/sdkconfig → platforms/esp32/sdkconfig.defaults

@@ -3,8 +3,8 @@
 
 CONFIG_PARTITION_TABLE_SINGLE_APP=n
 CONFIG_PARTITION_TABLE_CUSTOM=y
-CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="src/partitions_mgos.csv"
-CONFIG_PARTITION_TABLE_FILENAME="src/partitions_mgos.csv"
+CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions_mgos.csv"
+CONFIG_PARTITION_TABLE_FILENAME="partitions_mgos.csv"
 CONFIG_COMPILER_OPTIMIZATION_SIZE=y
 
 CONFIG_ESP_COREDUMP_ENABLE_TO_UART=y
@@ -53,3 +53,10 @@ CONFIG_COMPILER_CXX_RTTI=y
 # Disable use of RTC for timekeeping for now. It flakes out sometimes.
 CONFIG_ESP32_TIME_SYSCALL_USE_RTC_FRC1=n
 CONFIG_ESP32_TIME_SYSCALL_USE_FRC1=y
+
+# Stop wpa-supplicant from building custom crypto.
+CONFIG_MBEDTLS_CCM_C=y
+CONFIG_MBEDTLS_CMAC_C=y
+CONFIG_MBEDTLS_NIST_KW_C=y
+
+# App-specific settings below.