Fix device test environment variables (#6229)

* Fix device test environment variables

Device tests were not connecting properly to WiFi because the
environment variables were not set when WiFi.connect was called.
This would result in tests sometimes working *if* the prior sketch run
on the ESP saved WiFi connection information and auto-connect was
enabled.  But, in most cases, the tests would simply never connect to
any WiFi and fail.

getenv() works only after BS_RUN is called (because BS_RUN handles the
actual parsing of environment variables sent from the host).

Add a "pretest" function to all tests which is called by the host test
controller only after all environment variables are set.  Move all
WiFi/etc. operations that were in each separate test's setup() into it.

So the order of operations for tests now is:
ESP:  setup()
      -> Set serial baud
      -> Call BS_RUN()
HOST: Send environment
      Send "do pretest"
ESP:  pretest()
      -> Set Wifi using env. ariables, etc. return "true" on success
HOST: Send "run test 1"
ESP:  Run 1st test, return result
HOST: Send "run test 2"
ESP:  Run 2nd test, return result
<and so forth>

If nothing is needed to be set up, just return true from the pretest
function.

All tests now run and at least connect to WiFi.  There still seem to be
some actual test errors, but not because of the WiFi/environment
variables anymore.

* Remove unneeded debug prints

* Silence esptool.py output when not in V=1 mode

Esptool-ck.exe had an option to be silent, but esptool.py doesn't so the
output is very chatty and makes looking a the run logs hard (60 lines
of esptool.py output, 3 lines of actual test reports).

Redirect esptool.py STDOUT to /dev/null unless V=1 to clear this up.

* Speed up builds massively by removing old JSON

arduino-builder checks the build.options.json file and then goes off and
pegs my CPU at 100% for over a minute on each test compile checking if
files have been modified.

Simply deleting any pre-existing options.json file causes this step to
be skipped and a quick, clean recompile is done in siginificantly less
time.

* Enable compile warnings, fix any that show up

Enable all GCC warnings when building the tests and fix any that came up
(mostly signed/unsigned, unused, and deprecated ones).

* Fix UMM_MALLOC printf crash, umm_test

Printf can now handle PROGMEM addresses, so simplify and correct the
debug printouts in umm_info and elsewhere.

Author: earlephilhower

cores/esp8266/umm_malloc/umm_malloc.cpp

@@ -520,7 +520,7 @@ extern int umm_last_fail_alloc_size;
 #endif
 
 // Macro to place constant strings into PROGMEM and print them properly
-#define printf(fmt, ...)  do { static const char fstr[] PROGMEM = fmt; char rstr[sizeof(fmt)]; for (size_t i=0; i<sizeof(rstr); i++) rstr[i] = fstr[i]; printf(rstr, ##__VA_ARGS__); } while (0)
+#define printf(fmt, ...) printf(PSTR(fmt), ## __VA_ARGS__ )
 
 /* -- dbglog {{{ */
 

tests/device/Makefile

@@ -23,6 +23,7 @@ TEST_REPORT_HTML := test_report.html
 
 ifneq ("$(V)","1")
 	SILENT = @
+	REDIR = >& /dev/null
 else
 	BUILDER_DEBUG_FLAG = -verbose
 	RUNNER_DEBUG_FLAG = -d
@@ -50,10 +51,11 @@ $(TEST_LIST):
 ifneq ("$(NO_BUILD)","1")
 	@test -n "$(ARDUINO_IDE_PATH)" || (echo "Please export ARDUINO_IDE_PATH" && exit 1)
 	@echo Compiling $(notdir $@)
+	@rm -f $(LOCAL_BUILD_DIR)/build.options.json
 	$(SILENT)$(BUILD_TOOL) -compile -logger=human \
 		-libraries "$(PWD)/libraries" \
 		-core-api-version="10608" \
-		-warnings=none \
+		-warnings=all \
 		$(BUILDER_DEBUG_FLAG) \
 		-build-path $(LOCAL_BUILD_DIR) \
 		-tools $(ARDUINO_IDE_PATH)/tools-builder \
@@ -64,24 +66,25 @@ ifneq ("$(NO_BUILD)","1")
 endif
 ifneq ("$(NO_UPLOAD)","1")
 	@test -n "$(UPLOAD_PORT)" || (echo "Failed to detect upload port, please export UPLOAD_PORT manually" && exit 1)
-	@test -e $(dir $@)/make_spiffs.py && (echo "Generating and uploading SPIFFS" && \
-		(cd $(dir $@) && $(PYTHON) ./make_spiffs.py) && \
+	@test -e $(dir $@)/make_spiffs.py && ( \
+		echo "Generating and uploading SPIFFS" && \
+		(cd $(dir $@) && $(PYTHON) ./make_spiffs.py $(REDIR) ) && \
 		$(MKSPIFFS) --create $(dir $@)data/ --size 0xFB000 \
-			--block 8192 --page 256 $(LOCAL_BUILD_DIR)/spiffs.img && \
+			--block 8192 --page 256 $(LOCAL_BUILD_DIR)/spiffs.img $(REDIR) && \
 		$(ESPTOOL) $(UPLOAD_VERBOSE_FLAG) \
 			--chip esp8266 \
 			--port $(UPLOAD_PORT) \
 			--baud $(UPLOAD_BAUD) \
 			--after no_reset \
-			write_flash 0x300000 $(LOCAL_BUILD_DIR)/spiffs.img ) \
+			write_flash 0x300000 $(LOCAL_BUILD_DIR)/spiffs.img $(REDIR) ) \
 			|| (echo "No SPIFFS to upload")
 	@echo Uploading binary
 	$(SILENT)$(ESPTOOL) $(UPLOAD_VERBOSE_FLAG)  \
 		--chip esp8266 \
 		--port $(UPLOAD_PORT) \
 		--baud $(UPLOAD_BAUD) \
 		--after no_reset \
-		write_flash 0x0 $(LOCAL_BUILD_DIR)/$(notdir $@).bin # no reset
+		write_flash 0x0 $(LOCAL_BUILD_DIR)/$(notdir $@).bin $(REDIR) # no reset
 endif
 ifneq ("$(NO_RUN)","1")
 	@test -n "$(UPLOAD_PORT)" || (echo "Failed to detect upload port, please export UPLOAD_PORT manually" && exit 1)
@@ -90,7 +93,7 @@ ifneq ("$(NO_RUN)","1")
 		--chip esp8266 \
 		--port $(UPLOAD_PORT) \
 		--baud $(UPLOAD_BAUD) \
-		read_flash_status # reset
+		read_flash_status $(REDIR) # reset
 	$(SILENT)source $(BS_DIR)/virtualenv/bin/activate && \
 		$(PYTHON) $(BS_DIR)/runner.py \
 			$(RUNNER_DEBUG_FLAG) \

tests/device/libraries/BSTest/runner.py

@@ -104,6 +104,10 @@ def run_tests(self):
                     if res != BSTestRunner.SUCCESS:
                         print('failed to set environment variables')
                         break;
+                    res = self.pretest()
+                    if res != BSTestRunner.SUCCESS:
+                        print('failed to run pretest init')
+                        break;
                     should_update_env = False
                 if name in self.mocks:
                     debug_print('setting up mocks')
@@ -198,6 +202,21 @@ def update_env(self, env_to_set):
                 return BSTestRunner.TIMEOUT
         return BSTestRunner.SUCCESS
 
+    def pretest(self):
+        # Environment now set up, call the pretest init (wifi connect, etc.)
+        self.sp.sendline('pretest');
+        timeout = 10
+        while timeout > 0:
+            res = self.sp.expect(['>>>>>bs_test_pretest result=1', EOF, TIMEOUT]) # Only expect a pass, abort testing if failure
+            if res == 0:
+                break
+            time.sleep(0.1)
+            timeout -= 0.1
+        if res != 0:
+            return BSTestRunner.TIMEOUT
+        else:
+            return BSTestRunner.SUCCESS
+
     def request_env(self, key):
         self.sp.sendline('getenv "{}"'.format(key))
         timeout = 10

tests/device/libraries/BSTest/src/BSArgs.h

@@ -69,7 +69,7 @@ inline size_t split_args(char *line, char **argv, size_t argv_size)
     const int ESCAPE = '\\';
     const int SPACE = ' ';
     split_state_t state = SS_SPACE;
-    int argc = 0;
+    size_t argc = 0;
     char *next_arg_start = line;
     char *out_ptr = line;
     for (char *in_ptr = line; argc < argv_size - 1; ++in_ptr) {
@@ -148,4 +148,4 @@ inline size_t split_args(char *line, char **argv, size_t argv_size)
 
 } // namespace protocol
 
-#endif //BS_ARGS_H
+#endif //BS_ARGS_H

tests/device/libraries/BSTest/src/BSProtocol.h

@@ -5,6 +5,8 @@
 
 #define BS_LINE_PREFIX ">>>>>bs_test_"
 
+extern bool pretest();
+
 namespace bs
 {
 namespace protocol
@@ -58,6 +60,12 @@ void output_getenv_result(IO& io, const char* key, const char* value)
     io.printf(BS_LINE_PREFIX "getenv value=\"%s\"\n", value);
 }
 
+template<typename IO>
+void output_pretest_result(IO& io, bool res)
+{
+    io.printf(BS_LINE_PREFIX "pretest result=%d\n", res?1:0);
+}
+
 template<typename IO>
 bool input_handle(IO& io, char* line_buf, size_t line_buf_size, int& test_num)
 {
@@ -87,6 +95,14 @@ bool input_handle(IO& io, char* line_buf, size_t line_buf_size, int& test_num)
         output_getenv_result(io, argv[1], (value != NULL) ? value : "");
         return false;
     }
+    if (strcmp(argv[0], "pretest") == 0) {
+        if (argc != 1) {
+            return false;
+        }
+        bool res = ::pretest();
+        output_pretest_result(io, res);
+        return false;
+    }
     /* not one of the commands, try to parse as test number */
     char* endptr;
     test_num = (int) strtol(argv[0], &endptr, 10);

tests/device/test_BearSSL/test_BearSSL.ino

@@ -33,6 +33,11 @@ void setup()
 {
     Serial.begin(115200);
     Serial.setDebugOutput(true);
+    BS_RUN(Serial);
+}
+
+bool pretest()
+{
     WiFi.persistent(false);
     WiFi.mode(WIFI_STA);
     WiFi.begin(getenv("STA_SSID"), getenv("STA_PASS"));
@@ -45,22 +50,11 @@ void setup()
     Serial.printf("Number of CA certs read: %d\n", numCerts);
     if (numCerts == 0) {
         Serial.printf("No certs found. Did you run certs-from-mozill.py and upload the SPIFFS directory before running?\n");
-        REQUIRE(1==0);
+        return false;
     }
-    BS_RUN(Serial);
-}
-
-static void stopAll()
-{
-    WiFiClient::stopAll();
-}
-
-static void disconnectWiFI()
-{
-    wifi_station_disconnect();
+    return true;
 }
 
-
 // Set time via NTP, as required for x.509 validation
 void setClock() {
   configTime(3 * 3600, 0, "pool.ntp.org", "time.nist.gov");
@@ -143,7 +137,7 @@ int run(const char *str)
     return maxUsage;
 }
 
-#define TC(x) TEST_CASE("BearSSL - Maximum stack usage < 5600 bytes @ "x".badssl.org", "[bearssl]") { REQUIRE(run(x) < 5600); }
+#define TC(x) TEST_CASE("BearSSL - Maximum stack usage < 5600 bytes @ " x ".badssl.org", "[bearssl]") { REQUIRE(run(x) < 5600); }
 
 TC("expired")
 TC("wrong.host")

tests/device/test_ClientContext/test_ClientContext.ino

@@ -23,13 +23,18 @@ void setup()
 {
     Serial.begin(115200);
     Serial.setDebugOutput(true);
+    BS_RUN(Serial);
+}
+
+bool pretest()
+{
     WiFi.persistent(false);
     WiFi.mode(WIFI_STA);
     WiFi.begin(getenv("STA_SSID"), getenv("STA_PASS"));
     while (WiFi.status() != WL_CONNECTED) {
         delay(500);
     }
-    BS_RUN(Serial);
+    return true;
 }
 
 TEST_CASE("WiFi release ClientContext", "[clientcontext]")

tests/device/test_FS/test_FS.ino

@@ -10,6 +10,10 @@ void setup()
     BS_RUN(Serial);
 }
 
+bool pretest()
+{
+    return true;
+}
 
 
 TEST_CASE("read-write test","[fs]")

tests/device/test_Print_printf/test_Print_printf.ino

@@ -9,13 +9,18 @@ void setup()
     BS_RUN(Serial);
 }
 
+bool pretest()
+{
+    return true;
+}
+
 TEST_CASE("Print::printf works for any reasonable output length", "[Print]")
 {
 
     auto test_printf = [](size_t size) {
         StreamString str;
         auto buf = new char[size + 1];
-        for (int i = 0; i < size; ++i) {
+        for (size_t i = 0; i < size; ++i) {
             buf[i] = 'a';
         }
         buf[size] = 0;

tests/device/test_WiFiClient/test_WiFiClient.ino

@@ -13,13 +13,18 @@ void setup()
 {
     Serial.begin(115200);
     Serial.setDebugOutput(true);
+    BS_RUN(Serial);
+}
+
+bool pretest()
+{
     WiFi.persistent(false);
     WiFi.mode(WIFI_STA);
     WiFi.begin(getenv("STA_SSID"), getenv("STA_PASS"));
     while (WiFi.status() != WL_CONNECTED) {
         delay(500);
     }
-    BS_RUN(Serial);
+    return true;
 }
 
 static void stopAll()
@@ -53,4 +58,4 @@ TEST_CASE("WiFi disconnect during WiFiClient::connect", "[wificlient]")
 
 void loop()
 {
-}
+}

tests/device/test_WiFiServer/test_WiFiServer.ino

@@ -10,13 +10,18 @@ BS_ENV_DECLARE();
 void setup()
 {
     Serial.begin(115200);
+    BS_RUN(Serial);
+}
+
+bool pretest()
+{
     WiFi.persistent(false);
     WiFi.begin(getenv("STA_SSID"), getenv("STA_PASS"));
     while (WiFi.status() != WL_CONNECTED) {
         delay(500);
     }
     MDNS.begin("esp8266-wfs-test");
-    BS_RUN(Serial);
+    return true;
 }
 
 TEST_CASE("Simple echo server", "[WiFiServer]")

tests/device/test_WiFi_events/test_WiFi_events.ino

@@ -11,9 +11,14 @@ void setup()
 {
     Serial.begin(115200);
     Serial.setDebugOutput(false);
+    BS_RUN(Serial);
+}
+
+bool pretest()
+{
     WiFi.persistent(false);
     WiFi.mode(WIFI_OFF);
-    BS_RUN(Serial);
+    return true;
 }
 
 static std::map<WiFiEvent_t, int> sEventsReceived;
@@ -29,10 +34,13 @@ TEST_CASE("WiFi.onEvent is called for specific events", "[wifi][events]")
     sEventsReceived[WIFI_EVENT_STAMODE_DISCONNECTED] = 0;
     sEventsReceived[WIFI_EVENT_STAMODE_GOT_IP] = 0;
 
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
     WiFi.onEvent(onWiFiEvent, WIFI_EVENT_STAMODE_CONNECTED);
     WiFi.onEvent(onWiFiEvent, WIFI_EVENT_STAMODE_DISCONNECTED);
     WiFi.onEvent(onWiFiEvent, WIFI_EVENT_STAMODE_GOT_IP);
     WiFi.onEvent(onWiFiEvent, WIFI_EVENT_ANY);
+#pragma GCC diagnostic pop
 
     WiFi.mode(WIFI_STA);
     WiFi.begin(getenv("STA_SSID"), getenv("STA_PASS"));
@@ -44,24 +52,30 @@ TEST_CASE("WiFi.onEvent is called for specific events", "[wifi][events]")
     WiFi.disconnect();
     delay(100);
     WiFi.mode(WIFI_OFF);
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
     REQUIRE(sEventsReceived[WIFI_EVENT_STAMODE_CONNECTED] == 2);
     REQUIRE(sEventsReceived[WIFI_EVENT_STAMODE_DISCONNECTED] >= 2 && sEventsReceived[WIFI_EVENT_STAMODE_DISCONNECTED] % 2 == 0);
     REQUIRE(sEventsReceived[WIFI_EVENT_STAMODE_GOT_IP] == 2);
+#pragma GCC diagnostic pop
 }
 
 TEST_CASE("STA mode events are called both when using DHCP and static config", "[wifi][events]")
 {
     String events;
 
     auto handler1 = WiFi.onStationModeConnected([&](const WiFiEventStationModeConnected& evt){
+        (void) evt;
         events += "connected,";
     });  
     auto handler2 = WiFi.onStationModeDisconnected([&](const WiFiEventStationModeDisconnected& evt){
+        (void) evt;
         if (events.length()) {
             events += "disconnected,";
         }
     });
     auto handler3 = WiFi.onStationModeGotIP([&](const WiFiEventStationModeGotIP& evt){
+        (void) evt;
         events += "got_ip,";
     });
 
@@ -104,12 +118,15 @@ TEST_CASE("Events are not called if handler is deleted", "[wifi][events]")
     String events;
 
     WiFi.onStationModeConnected([&](const WiFiEventStationModeConnected& evt){
+        (void) evt;
         events += "connected,";
     });  
     WiFi.onStationModeDisconnected([&](const WiFiEventStationModeDisconnected& evt){
+        (void) evt;
         events += "disconnected,";
     });
     WiFi.onStationModeGotIP([&](const WiFiEventStationModeGotIP& evt){
+        (void) evt;
         events += "got_ip,";
     });