From cb53587ec12c1a3a43d6586ce01038355549f2bd Mon Sep 17 00:00:00 2001 From: Ben Liao Date: Mon, 17 Jul 2023 16:55:01 -0700 Subject: [PATCH 01/13] [BUILD] Trying multiple Makefiles idea --- Makefile | 18 ++++++++++++++++++ dev_handler/Makefile | 17 ++++++++++------- 2 files changed, 28 insertions(+), 7 deletions(-) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..0695f459 --- /dev/null +++ b/Makefile @@ -0,0 +1,18 @@ +BIN = bin +BUILD = build +OBJ = $(BUILD)/obj + +COMPONENT_DIRS = dev_handler net_handler executor network_switch shm_wrapper runtime_util logger +INCLUDE_FLAGS = $(foreach dir,$(COMPONENT_DIRS),-I$(dir)) # list of folders to include in compilation commands + +BUILD_DIRS = $(BIN) $(BUILD) $(OBJ) + +OBJS_ALL = + +$(BUILD_DIRS): + mkdir -p $(BUILD_DIRS) + + + +clean: + rm -rf $(BUILD_DIRS) \ No newline at end of file diff --git a/dev_handler/Makefile b/dev_handler/Makefile index a971dcac..0285157a 100644 --- a/dev_handler/Makefile +++ b/dev_handler/Makefile @@ -1,10 +1,13 @@ -LIBFLAGS=-pthread -lrt -Wall -BIN=../bin +LIBS=-pthread -lrt -Wall -dev_handler: dev_handler.c message.c message.h ../logger/logger.c ../runtime_util/runtime_util.c ../shm_wrapper/shm_wrapper.c - gcc $^ -o $(BIN)/dev_handler $(LIBFLAGS) +# include top-level Makefile +include ../Makefile -clean: - rm -f $(BIN)/dev_handler - rm -f dev_handler +SRCS = dev_handler.c message.c ../logger/logger.c ../runtime_util/runtime_util.c ../shm_wrapper/shm_wrapper.c +HEADERS = message.h ../logger/logger.h ../runtime_util/runtime_util.h ../shm_wrapper/shm_wrapper.h + +OBJS = $(foreach file,$(SRCS),../$(OBJ)/$(shell basename $(file)).o) # list of all object files for dev handler + +dev_handler: $(SRCS) $(HEADERS) | $(BIN) + gcc $(OBJS) -o $(BIN)/$@ $(LIBS) From 700355b63d45029790f80e7c68d496eae7fe00ef Mon Sep 17 00:00:00 2001 From: Ben Liao Date: Tue, 18 Jul 2023 08:53:56 +0000 Subject: [PATCH 02/13] [BUILD] Working individual Makefiles idea with dev_handler only --- .gitmodules | 6 +- Makefile | 36 ++++++++---- Makefile_defs | 11 ++++ Makefile_old | 55 +++++++++++++++++++ dev_handler/Makefile | 20 ++++--- dev_handler/dev_handler.c | 14 ++--- .../{message.c => dev_handler_message.c} | 10 ++-- .../{message.h => dev_handler_message.h} | 4 +- executor/executor.c | 8 ++- executor/gamestate_filter.c | 2 +- executor/gamestate_filter.h | 6 +- logger/logger.c | 2 +- logger/logger.h | 4 +- net_handler/connection.c | 2 +- net_handler/connection.h | 6 +- net_handler/net_handler.c | 6 +- .../{message.c => net_handler_message.c} | 14 ++--- .../{message.h => net_handler_message.h} | 4 +- net_handler/net_util.c | 2 +- net_handler/net_util.h | 20 +++---- net_handler/protos | 1 - network_switch/network_switch.c | 6 +- protos | 1 + shm_wrapper/shm_start.c | 2 +- shm_wrapper/shm_stop.c | 2 +- shm_wrapper/shm_wrapper.c | 2 +- shm_wrapper/shm_wrapper.h | 4 +- 27 files changed, 168 insertions(+), 82 deletions(-) create mode 100644 Makefile_defs create mode 100644 Makefile_old rename dev_handler/{message.c => dev_handler_message.c} (97%) rename dev_handler/{message.h => dev_handler_message.h} (98%) rename net_handler/{message.c => net_handler_message.c} (98%) rename net_handler/{message.h => net_handler_message.h} (95%) delete mode 160000 net_handler/protos create mode 160000 protos diff --git a/.gitmodules b/.gitmodules index 27d86d02..f854f482 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ -[submodule "net_handler/protos"] - path = net_handler/protos - url = https://github.com/pioneers/protos.git +[submodule "protos"] + path = protos + url = git@github.com:pioneers/protos.git diff --git a/Makefile b/Makefile index 0695f459..e85795c1 100644 --- a/Makefile +++ b/Makefile @@ -1,18 +1,30 @@ -BIN = bin -BUILD = build +# This Makefile defines a bunch of stuff used by the lower-level Makefiles +# all file paths are from their perspective! + +CC = gcc + +BIN = ../bin +BUILD = ../build OBJ = $(BUILD)/obj -COMPONENT_DIRS = dev_handler net_handler executor network_switch shm_wrapper runtime_util logger -INCLUDE_FLAGS = $(foreach dir,$(COMPONENT_DIRS),-I$(dir)) # list of folders to include in compilation commands +# this is the name of the lower level directory (e.g. THIS_DIR in the dev_handler folder is "dev_handler") +THIS_DIR = $(strip $(shell pwd | xargs basename -z)) + +COMPONENT_DIRS = dev_handler net_handler executor network_switch shm_wrapper runtime_util logger # list of runtime component directories +OBJ_SUBDIR = $(foreach dir,$(COMPONENT_DIRS),$(OBJ)/$(dir)) # list of subfolders in $(OBJ) for runtime object files +INCLUDE = $(foreach dir,$(COMPONENT_DIRS),-I../$(dir)) # list of folders to include in compilation commands + +OBJS = $(patsubst %.c,$(OBJ)/$(THIS_DIR)/%.o,$(SRCS)) # make a list of all object files this executable is dependent on relative to this Makefile -BUILD_DIRS = $(BIN) $(BUILD) $(OBJ) +.PHONY: all clean -OBJS_ALL = +# list of build folders +BUILD_DIR = $(OBJ) $(BIN) $(BUILD) $(OBJ_SUBDIR) -$(BUILD_DIRS): - mkdir -p $(BUILD_DIRS) - +# general rule for compiling a list of source files to object files in the $(OBJ) directory +$(OBJ)/$(THIS_DIR)/%.o: %.c | $(OBJ_SUBDIR) + $(CC) $(CFLAGS) -MMD -MP $(INCLUDE) -c $< -o $@ - -clean: - rm -rf $(BUILD_DIRS) \ No newline at end of file +# general rule for making a build directory +$(BUILD_DIR): + mkdir -p $@ diff --git a/Makefile_defs b/Makefile_defs new file mode 100644 index 00000000..9330d2b9 --- /dev/null +++ b/Makefile_defs @@ -0,0 +1,11 @@ +# definitions for all Makefiles + +PY_VER = python3.10 +CFLAGS1 = $(shell pkg-config --cflags libprotobuf-c) +CFLAGS2 = $(shell $(PY_VER)-config --cflags) +CFLAGS = $(CFLAGS1) $(CFLAGS2) +CC = gcc + +BIN = bin +BUILD = build +OBJ = $(BUILD)/obj diff --git a/Makefile_old b/Makefile_old new file mode 100644 index 00000000..7aec37d6 --- /dev/null +++ b/Makefile_old @@ -0,0 +1,55 @@ +include Makefile_defs + +# protos folders +PROTO = protos +PBC_GEN = $(BUILD)/pbc_gen +PBC_OBJ = $(BUILD)/pbc_obj + +PROTOS = $(shell cd $(PROTO) && ls *.proto && cd ..) # list all the protos +PBC_SRCS = $(patsubst %.proto,$(PBC_GEN)/%.pb-c.c, $(PROTOS)) # list of all generated source files by doing a path substitution +PBC_OBJS = $(patsubst $(PBC_GEN)/%.c,$(PBC_OBJ)/%.o, $(PBC_SRCS)) # list of all generated object files by doing a path substitution + +COMPONENT_DIRS = dev_handler net_handler executor network_switch shm_wrapper runtime_util logger +OBJ_SUBDIR = $(foreach dir,$(COMPONENT_DIRS),$(OBJ)/$(dir)) # list of subfolders in $(OBJ) for runtime object files +INCLUDE = $(foreach dir,$(COMPONENT_DIRS),-I$(dir)) -I$(PBC_GEN) # list of folders to include in compilation commands + +BUILD_DIRS = $(BIN) $(BUILD) $(OBJ) $(PBC_GEN) $(PBC_OBJ) $(OBJ_SUBDIR) + +# make list of all source files and corresponding object files +SRCS_ALL = $(foreach dir,$(COMPONENT_DIRS),$(shell find $(dir) -name "*.c")) +OBJS_ALL = $(patsubst %.c,$(OBJ)/%.o, $(SRCS_ALL)) + +$(info OBJS_ALL is $(OBJS_ALL)) +$(info SRCS_ALL is $(SRCS_ALL)) +$(info INCLUDE is $(INCLUDE)) + +.PHONY: all clean protobuf obj + +.SECONDARY: + +all: protobuf obj + +protobuf: $(PBC_OBJS) + +obj: $(OBJS_ALL) + +# makes runtime object file from runtime source files +# makes the runtime object files in $(OBJ) folder +$(OBJ)/%.o: %.c $(PBC_OBJS) | $(OBJ) $(OBJ_SUBDIR) + $(CC) $(CFLAGS) $(INCLUDE) -c $< -o $@ + +# makes the generated protobuf-c object files in the $(PBC_OBJ) folder +$(PBC_OBJ)/%.o: $(PBC_GEN)/%.c | $(PBC_OBJ) + $(CC) $(CFLAGS) -c $< -o $@ + +# makes the generated protobuf-c files in the $(PBC_GEN) folder +$(PBC_GEN)/%.pb-c.c: $(PROTO)/%.proto | $(PBC_GEN) + cd protos && protoc-c --c_out $(HOME)/runtime/$(PBC_GEN) $*.proto + +# makes build directories if needed +$(BUILD_DIRS): + mkdir -p $@ + +# removes all build directories +clean: + rm -rf $(BUILD_DIRS) diff --git a/dev_handler/Makefile b/dev_handler/Makefile index 0285157a..3a336464 100644 --- a/dev_handler/Makefile +++ b/dev_handler/Makefile @@ -1,13 +1,19 @@ +# list of libraries that dev_handler needs to compile LIBS=-pthread -lrt -Wall -# include top-level Makefile +# list of source files that the target (dev_handler) depends on, relative to this folder +SRCS = dev_handler.c dev_handler_message.c ../logger/logger.c ../runtime_util/runtime_util.c ../shm_wrapper/shm_wrapper.c + include ../Makefile -SRCS = dev_handler.c message.c ../logger/logger.c ../runtime_util/runtime_util.c ../shm_wrapper/shm_wrapper.c -HEADERS = message.h ../logger/logger.h ../runtime_util/runtime_util.h ../shm_wrapper/shm_wrapper.h +all: dev_handler + +dev_handler: $(OBJS) | $(BIN) + $(CC) $(OBJS) -o $(BIN)/$@ $(LIBS) -OBJS = $(foreach file,$(SRCS),../$(OBJ)/$(shell basename $(file)).o) # list of all object files for dev handler +clean: + rm -f $(OBJS) + rm -f $(patsubst %.o,%.d,$(OBJS)) + rm -f $(BIN)/dev_handler -dev_handler: $(SRCS) $(HEADERS) | $(BIN) - gcc $(OBJS) -o $(BIN)/$@ $(LIBS) - +-include $(patsubst %.o,%.d,$(OBJS)) diff --git a/dev_handler/dev_handler.c b/dev_handler/dev_handler.c index cf9bee00..03a75449 100644 --- a/dev_handler/dev_handler.c +++ b/dev_handler/dev_handler.c @@ -5,10 +5,10 @@ #include // for POSIX terminal control definitions in serialport_open() -#include "../logger/logger.h" -#include "../runtime_util/runtime_util.h" -#include "../shm_wrapper/shm_wrapper.h" -#include "message.h" +#include +#include +#include +#include /** * Each device will have a unique port number. @@ -395,7 +395,7 @@ void relay_clean_up(relay_t* relay) { pthread_mutex_unlock(&used_ports_lock); pthread_mutex_destroy(&relay->relay_lock); pthread_cond_destroy(&relay->start_cond); - if (relay->dev_id.uid == -1) { + if (relay->dev_id.uid == (uint64_t) -1) { char port_name[MAX_PORT_NAME_SIZE]; construct_port_name(port_name, relay->is_virtual, relay->is_usb, relay->port_num); log_printf(DEBUG, "Cleaned up bad device %s\n", port_name); @@ -582,7 +582,7 @@ int receive_message(relay_t* relay, message_t* msg) { char port_name[MAX_PORT_NAME_SIZE]; construct_port_name(port_name, relay->is_virtual, relay->is_usb, relay->port_num); - if (relay->dev_id.uid == -1) { + if (relay->dev_id.uid == (uint64_t) -1) { /* Haven't verified device is lowcar yet * read() is set to timeout while waiting for an ACK (see serialport_open())*/ pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL); @@ -872,7 +872,7 @@ int main(int argc, char* argv[]) { // If SIGINT (Ctrl+C) is received, call stop() to clean up signal(SIGINT, stop); init(); - home_dir = getenv("HOME"); // set the home directory + home_dir = getenv("HOME"); // set the home directory log_printf(INFO, "DEV_HANDLER initialized."); poll_connected_devices(); return 0; diff --git a/dev_handler/message.c b/dev_handler/dev_handler_message.c similarity index 97% rename from dev_handler/message.c rename to dev_handler/dev_handler_message.c index 8115f12d..996e9b0c 100644 --- a/dev_handler/message.c +++ b/dev_handler/dev_handler_message.c @@ -1,10 +1,10 @@ -#include "message.h" +#include // ******************************** Utility ********************************* // void print_bytes(uint8_t* data, size_t len) { printf("0x"); - for (int i = 0; i < len; i++) { + for (size_t i = 0; i < len; i++) { printf("%02X ", data[i]); } printf("\n"); @@ -71,7 +71,7 @@ static int append_payload(message_t* msg, uint8_t* data, size_t len) { */ static uint8_t checksum(uint8_t* data, size_t len) { uint8_t chk = data[0]; - for (int i = 1; i < len; i++) { + for (size_t i = 1; i < len; i++) { chk ^= data[i]; } return chk; @@ -292,7 +292,7 @@ ssize_t message_to_bytes(message_t* msg, uint8_t cobs_encoded[], size_t len) { } data[0] = msg->message_id; data[1] = msg->payload_length; - for (int i = 0; i < msg->payload_length; i++) { + for (size_t i = 0; i < msg->payload_length; i++) { data[i + MESSAGE_ID_SIZE + PAYLOAD_LENGTH_SIZE] = msg->payload[i]; } data[MESSAGE_ID_SIZE + PAYLOAD_LENGTH_SIZE + msg->payload_length] = checksum(data, MESSAGE_ID_SIZE + PAYLOAD_LENGTH_SIZE + msg->payload_length); @@ -317,7 +317,7 @@ int parse_message(uint8_t data[], message_t* msg_to_fill) { // Smaller than valid message free(decoded); return 3; - } else if (ret > (MESSAGE_ID_SIZE + PAYLOAD_LENGTH_SIZE + MAX_PAYLOAD_SIZE + CHECKSUM_SIZE)) { + } else if (ret > (int)(MESSAGE_ID_SIZE + PAYLOAD_LENGTH_SIZE + MAX_PAYLOAD_SIZE + CHECKSUM_SIZE)) { // Larger than the largest valid message free(decoded); return 3; diff --git a/dev_handler/message.h b/dev_handler/dev_handler_message.h similarity index 98% rename from dev_handler/message.h rename to dev_handler/dev_handler_message.h index 0eef311e..b8d49fa4 100644 --- a/dev_handler/message.h +++ b/dev_handler/dev_handler_message.h @@ -10,8 +10,8 @@ #ifndef MESSAGE_H #define MESSAGE_H -#include "../logger/logger.h" -#include "../runtime_util/runtime_util.h" +#include +#include /* The maximum number of milliseconds to wait between each DEVICE_PING from a device * Waiting for this long will exit all threads for that device (doing cleanup as necessary) diff --git a/executor/executor.c b/executor/executor.c index b4f351e3..38bd297d 100644 --- a/executor/executor.c +++ b/executor/executor.c @@ -11,9 +11,11 @@ #include //for wait functions #include // for getting time #include //for sleep -#include "../logger/logger.h" // for runtime logger -#include "../runtime_util/runtime_util.h" //for runtime constants (TODO: consider removing relative pathname in include) -#include "../shm_wrapper/shm_wrapper.h" // Shared memory wrapper to get/send device data + +// runtime includes +#include // for runtime logger +#include //for runtime constants (TODO: consider removing relative pathname in include) +#include // Shared memory wrapper to get/send device data // Global variables to all functions and threads diff --git a/executor/gamestate_filter.c b/executor/gamestate_filter.c index b65cbd3b..1510044c 100644 --- a/executor/gamestate_filter.c +++ b/executor/gamestate_filter.c @@ -1,4 +1,4 @@ -#include "gamestate_filter.h" +#include /* Gamestates: * poison ivy: Motor velocities reversed for 10 seconds diff --git a/executor/gamestate_filter.h b/executor/gamestate_filter.h index b028061f..ff602c23 100644 --- a/executor/gamestate_filter.h +++ b/executor/gamestate_filter.h @@ -22,9 +22,9 @@ */ #include -#include "../logger/logger.h" -#include "../runtime_util/runtime_util.h" -#include "../shm_wrapper/shm_wrapper.h" +#include +#include +#include /** * Starts the gamestate handler thread so that gamestates are deactivated diff --git a/logger/logger.c b/logger/logger.c index 7ab6596b..92f11ba8 100644 --- a/logger/logger.c +++ b/logger/logger.c @@ -1,4 +1,4 @@ -#include "logger.h" +#include // bit flags for setting logger output location #define LOG_STDOUT (1 << 0) diff --git a/logger/logger.h b/logger/logger.h index 4da97d9a..e44bf7e2 100644 --- a/logger/logger.h +++ b/logger/logger.h @@ -6,7 +6,7 @@ #include // for wordexp -#include "../runtime_util/runtime_util.h" +#include #define CONFIG_FILE "logger.config" // path to logger config file #define LOG_FIFO "/tmp/log-fifo" // location of the log FIFO pipe in filesystem @@ -41,4 +41,4 @@ void logger_init(process_t process); */ void log_printf(log_level_t level, char* format, ...); -#endif \ No newline at end of file +#endif diff --git a/net_handler/connection.c b/net_handler/connection.c index 8e99ee45..fd1f3ba3 100644 --- a/net_handler/connection.c +++ b/net_handler/connection.c @@ -1,4 +1,4 @@ -#include "connection.h" +#include // used for creating and cleaning up TCP connection typedef struct { diff --git a/net_handler/connection.h b/net_handler/connection.h index 4c6bbf19..1f04ae7d 100644 --- a/net_handler/connection.h +++ b/net_handler/connection.h @@ -1,8 +1,8 @@ #ifndef TCP_CONN_H #define TCP_CONN_H -#include "message.h" -#include "net_util.h" +#include +#include /** * Start the main TCP connection control thread. Does not block. @@ -24,4 +24,4 @@ void start_tcp_conn(robot_desc_field_t client, int connfd, int send_logs); */ void stop_tcp_conn(robot_desc_field_t client); -#endif \ No newline at end of file +#endif diff --git a/net_handler/net_handler.c b/net_handler/net_handler.c index af450680..ba93ea53 100644 --- a/net_handler/net_handler.c +++ b/net_handler/net_handler.c @@ -1,6 +1,6 @@ -#include "../executor/gamestate_filter.h" -#include "connection.h" -#include "net_util.h" +#include +#include +#include /* * Sets up TCP listening socket on raspberry pi. diff --git a/net_handler/message.c b/net_handler/net_handler_message.c similarity index 98% rename from net_handler/message.c rename to net_handler/net_handler_message.c index 58003198..b1f91bac 100644 --- a/net_handler/message.c +++ b/net_handler/net_handler_message.c @@ -1,4 +1,4 @@ -#include "message.h" +#include // ******************************************* SEND MESSAGES ***************************************** // @@ -41,7 +41,7 @@ void send_log_msg(int conn_fd, FILE* log_file) { } else { // Error occurred log_printf(ERROR, "send_log_msg: Error reading from log fifo: %s", strerror(errno)); // Free loaded payload contents and the payload itself - for (int i = 0; i < log_msg.n_payload; i++) { + for (size_t i = 0; i < log_msg.n_payload; i++) { free(log_msg.payload[i]); } free(log_msg.payload); @@ -60,7 +60,7 @@ void send_log_msg(int conn_fd, FILE* log_file) { } // free all allocated memory - for (int i = 0; i < log_msg.n_payload; i++) { + for (size_t i = 0; i < log_msg.n_payload; i++) { free(log_msg.payload[i]); } free(log_msg.payload); @@ -202,7 +202,7 @@ void send_device_data(int dawn_socket_fd, uint64_t dawn_start_time) { custom->name = "CustomData"; custom->type = MAX_DEVICES; custom->uid = 2020; - for (int i = 0; i < custom->n_params; i++) { + for (size_t i = 0; i < custom->n_params; i++) { Param* param = malloc(sizeof(Param)); if (param == NULL) { log_printf(FATAL, "send_device_data: Failed to malloc"); @@ -251,8 +251,8 @@ void send_device_data(int dawn_socket_fd, uint64_t dawn_start_time) { } // free everything - for (int i = 0; i < dev_data.n_devices; i++) { - for (int j = 0; j < dev_data.devices[i]->n_params; j++) { + for (size_t i = 0; i < dev_data.n_devices; i++) { + for (size_t j = 0; j < dev_data.devices[i]->n_params; j++) { free(dev_data.devices[i]->params[j]); } free(dev_data.devices[i]->params); @@ -427,7 +427,7 @@ static int process_inputs_msg(uint8_t* buf, uint16_t len_pb) { log_printf(ERROR, "recv_new_msg: Failed to unpack UserInputs"); return -1; } - for (int i = 0; i < inputs->n_inputs; i++) { + for (size_t i = 0; i < inputs->n_inputs; i++) { Input* input = inputs->inputs[i]; // Convert Protobuf source enum to Runtime source enum robot_desc_field_t source = (input->source == SOURCE__GAMEPAD) ? GAMEPAD : KEYBOARD; diff --git a/net_handler/message.h b/net_handler/net_handler_message.h similarity index 95% rename from net_handler/message.h rename to net_handler/net_handler_message.h index f36398c0..25c89ac6 100644 --- a/net_handler/message.h +++ b/net_handler/net_handler_message.h @@ -1,4 +1,4 @@ -#include "net_util.h" +#include /* * Send a log message on the TCP connection to the client. Reads lines from the pipe until there is no more data @@ -35,4 +35,4 @@ void send_device_data(int dawn_socket_fd, uint64_t dawn_start_time); * -1 if message could not be parsed because client disconnected and connection closed * -2 if message could not be unpacked or other error */ -int recv_new_msg(int conn_fd, robot_desc_field_t client); \ No newline at end of file +int recv_new_msg(int conn_fd, robot_desc_field_t client); diff --git a/net_handler/net_util.c b/net_handler/net_util.c index a43a86a9..403e9fbd 100644 --- a/net_handler/net_util.c +++ b/net_handler/net_util.c @@ -1,4 +1,4 @@ -#include "net_util.h" +#include uint8_t* make_buf(net_msg_t msg_type, uint16_t len_pb) { uint8_t* send_buf = malloc(len_pb + 3); diff --git a/net_handler/net_util.h b/net_handler/net_util.h index 9b759b73..c3fcdcd9 100644 --- a/net_handler/net_util.h +++ b/net_handler/net_util.h @@ -13,18 +13,18 @@ #include //for read, write, close // include other runtime files -#include "../logger/logger.h" -#include "../runtime_util/runtime_util.h" -#include "../shm_wrapper/shm_wrapper.h" +#include +#include +#include // include compiled protobuf headers -#include "pbc_gen/device.pb-c.h" -#include "pbc_gen/gamestate.pb-c.h" -#include "pbc_gen/input.pb-c.h" -#include "pbc_gen/run_mode.pb-c.h" -#include "pbc_gen/start_pos.pb-c.h" -#include "pbc_gen/text.pb-c.h" -#include "pbc_gen/timestamp.pb-c.h" +#include +#include +#include +#include +#include +#include +#include #define RASPI_ADDR "127.0.0.1" // The IP address of Runtime (Raspberry Pi) that clients can request a connection to #define RASPI_TCP_PORT 8101 // Port for Runtime as a TCP socket server diff --git a/net_handler/protos b/net_handler/protos deleted file mode 160000 index 9e52c0e5..00000000 --- a/net_handler/protos +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 9e52c0e5cd7ff7b83811145dfc93dbd5ea168f84 diff --git a/network_switch/network_switch.c b/network_switch/network_switch.c index 9968778d..83f39f30 100644 --- a/network_switch/network_switch.c +++ b/network_switch/network_switch.c @@ -2,9 +2,9 @@ #include #include #include -#include "../logger/logger.h" -#include "../runtime_util/runtime_util.h" -#include "../shm_wrapper/shm_wrapper.h" +#include +#include +#include #define SLEEP_DELAY 1 diff --git a/protos b/protos new file mode 160000 index 00000000..8c15c7a7 --- /dev/null +++ b/protos @@ -0,0 +1 @@ +Subproject commit 8c15c7a7680860e0ea45d70df7e88b69c81a3984 diff --git a/shm_wrapper/shm_start.c b/shm_wrapper/shm_start.c index 42a50eeb..b0c9c49d 100644 --- a/shm_wrapper/shm_start.c +++ b/shm_wrapper/shm_start.c @@ -1,4 +1,4 @@ -#include "shm_wrapper.h" +#include char sname[SNAME_SIZE]; // being lazy here but this is for holding all the semaphore names diff --git a/shm_wrapper/shm_stop.c b/shm_wrapper/shm_stop.c index 6e63ad0e..35cd1d31 100644 --- a/shm_wrapper/shm_stop.c +++ b/shm_wrapper/shm_stop.c @@ -1,4 +1,4 @@ -#include "shm_wrapper.h" +#include char sname[SNAME_SIZE]; // being lazy here but this is for holding all the semaphore names diff --git a/shm_wrapper/shm_wrapper.c b/shm_wrapper/shm_wrapper.c index dea3f272..e5fc3662 100644 --- a/shm_wrapper/shm_wrapper.c +++ b/shm_wrapper/shm_wrapper.c @@ -1,4 +1,4 @@ -#include "shm_wrapper.h" +#include // *********************************** WRAPPER-SPECIFIC GLOBAL VARS **************************************** // diff --git a/shm_wrapper/shm_wrapper.h b/shm_wrapper/shm_wrapper.h index df1d409a..157bf588 100644 --- a/shm_wrapper/shm_wrapper.h +++ b/shm_wrapper/shm_wrapper.h @@ -6,8 +6,8 @@ #include #include // for posix shared memory -#include "../logger/logger.h" // for logger -#include "../runtime_util/runtime_util.h" // for runtime constants +#include // for logger +#include // for runtime constants // names of various objects used in shm_wrapper; should not be used outside of shm_wrapper.c, shm_start.c, and shm_stop.c #define DEV_SHM_NAME "/dev-shm" // name of shared memory block across devices From 987c73471de91b8215162395fa50e2150d9c0c4a Mon Sep 17 00:00:00 2001 From: Ben Liao Date: Tue, 18 Jul 2023 09:14:28 +0000 Subject: [PATCH 03/13] [BUILD] Working new network switch Makefile - Also added new top-level Makefile --- Makefile | 35 ++++++-------------------- Makefile_defs | 33 +++++++++++++++++++------ Makefile_old | 55 ----------------------------------------- dev_handler/Makefile | 14 ++++++++--- network_switch/Makefile | 28 +++++++++++++++------ 5 files changed, 64 insertions(+), 101 deletions(-) delete mode 100644 Makefile_old diff --git a/Makefile b/Makefile index e85795c1..1c5b30a3 100644 --- a/Makefile +++ b/Makefile @@ -1,30 +1,9 @@ -# This Makefile defines a bunch of stuff used by the lower-level Makefiles -# all file paths are from their perspective! +# Starting point for building stuff in Runtime from the root directory -CC = gcc +all: + cd dev_handler && make + cd network_switch && make -BIN = ../bin -BUILD = ../build -OBJ = $(BUILD)/obj - -# this is the name of the lower level directory (e.g. THIS_DIR in the dev_handler folder is "dev_handler") -THIS_DIR = $(strip $(shell pwd | xargs basename -z)) - -COMPONENT_DIRS = dev_handler net_handler executor network_switch shm_wrapper runtime_util logger # list of runtime component directories -OBJ_SUBDIR = $(foreach dir,$(COMPONENT_DIRS),$(OBJ)/$(dir)) # list of subfolders in $(OBJ) for runtime object files -INCLUDE = $(foreach dir,$(COMPONENT_DIRS),-I../$(dir)) # list of folders to include in compilation commands - -OBJS = $(patsubst %.c,$(OBJ)/$(THIS_DIR)/%.o,$(SRCS)) # make a list of all object files this executable is dependent on relative to this Makefile - -.PHONY: all clean - -# list of build folders -BUILD_DIR = $(OBJ) $(BIN) $(BUILD) $(OBJ_SUBDIR) - -# general rule for compiling a list of source files to object files in the $(OBJ) directory -$(OBJ)/$(THIS_DIR)/%.o: %.c | $(OBJ_SUBDIR) - $(CC) $(CFLAGS) -MMD -MP $(INCLUDE) -c $< -o $@ - -# general rule for making a build directory -$(BUILD_DIR): - mkdir -p $@ +clean: + rm -rf bin + rm -rf build diff --git a/Makefile_defs b/Makefile_defs index 9330d2b9..e85795c1 100644 --- a/Makefile_defs +++ b/Makefile_defs @@ -1,11 +1,30 @@ -# definitions for all Makefiles +# This Makefile defines a bunch of stuff used by the lower-level Makefiles +# all file paths are from their perspective! -PY_VER = python3.10 -CFLAGS1 = $(shell pkg-config --cflags libprotobuf-c) -CFLAGS2 = $(shell $(PY_VER)-config --cflags) -CFLAGS = $(CFLAGS1) $(CFLAGS2) CC = gcc -BIN = bin -BUILD = build +BIN = ../bin +BUILD = ../build OBJ = $(BUILD)/obj + +# this is the name of the lower level directory (e.g. THIS_DIR in the dev_handler folder is "dev_handler") +THIS_DIR = $(strip $(shell pwd | xargs basename -z)) + +COMPONENT_DIRS = dev_handler net_handler executor network_switch shm_wrapper runtime_util logger # list of runtime component directories +OBJ_SUBDIR = $(foreach dir,$(COMPONENT_DIRS),$(OBJ)/$(dir)) # list of subfolders in $(OBJ) for runtime object files +INCLUDE = $(foreach dir,$(COMPONENT_DIRS),-I../$(dir)) # list of folders to include in compilation commands + +OBJS = $(patsubst %.c,$(OBJ)/$(THIS_DIR)/%.o,$(SRCS)) # make a list of all object files this executable is dependent on relative to this Makefile + +.PHONY: all clean + +# list of build folders +BUILD_DIR = $(OBJ) $(BIN) $(BUILD) $(OBJ_SUBDIR) + +# general rule for compiling a list of source files to object files in the $(OBJ) directory +$(OBJ)/$(THIS_DIR)/%.o: %.c | $(OBJ_SUBDIR) + $(CC) $(CFLAGS) -MMD -MP $(INCLUDE) -c $< -o $@ + +# general rule for making a build directory +$(BUILD_DIR): + mkdir -p $@ diff --git a/Makefile_old b/Makefile_old deleted file mode 100644 index 7aec37d6..00000000 --- a/Makefile_old +++ /dev/null @@ -1,55 +0,0 @@ -include Makefile_defs - -# protos folders -PROTO = protos -PBC_GEN = $(BUILD)/pbc_gen -PBC_OBJ = $(BUILD)/pbc_obj - -PROTOS = $(shell cd $(PROTO) && ls *.proto && cd ..) # list all the protos -PBC_SRCS = $(patsubst %.proto,$(PBC_GEN)/%.pb-c.c, $(PROTOS)) # list of all generated source files by doing a path substitution -PBC_OBJS = $(patsubst $(PBC_GEN)/%.c,$(PBC_OBJ)/%.o, $(PBC_SRCS)) # list of all generated object files by doing a path substitution - -COMPONENT_DIRS = dev_handler net_handler executor network_switch shm_wrapper runtime_util logger -OBJ_SUBDIR = $(foreach dir,$(COMPONENT_DIRS),$(OBJ)/$(dir)) # list of subfolders in $(OBJ) for runtime object files -INCLUDE = $(foreach dir,$(COMPONENT_DIRS),-I$(dir)) -I$(PBC_GEN) # list of folders to include in compilation commands - -BUILD_DIRS = $(BIN) $(BUILD) $(OBJ) $(PBC_GEN) $(PBC_OBJ) $(OBJ_SUBDIR) - -# make list of all source files and corresponding object files -SRCS_ALL = $(foreach dir,$(COMPONENT_DIRS),$(shell find $(dir) -name "*.c")) -OBJS_ALL = $(patsubst %.c,$(OBJ)/%.o, $(SRCS_ALL)) - -$(info OBJS_ALL is $(OBJS_ALL)) -$(info SRCS_ALL is $(SRCS_ALL)) -$(info INCLUDE is $(INCLUDE)) - -.PHONY: all clean protobuf obj - -.SECONDARY: - -all: protobuf obj - -protobuf: $(PBC_OBJS) - -obj: $(OBJS_ALL) - -# makes runtime object file from runtime source files -# makes the runtime object files in $(OBJ) folder -$(OBJ)/%.o: %.c $(PBC_OBJS) | $(OBJ) $(OBJ_SUBDIR) - $(CC) $(CFLAGS) $(INCLUDE) -c $< -o $@ - -# makes the generated protobuf-c object files in the $(PBC_OBJ) folder -$(PBC_OBJ)/%.o: $(PBC_GEN)/%.c | $(PBC_OBJ) - $(CC) $(CFLAGS) -c $< -o $@ - -# makes the generated protobuf-c files in the $(PBC_GEN) folder -$(PBC_GEN)/%.pb-c.c: $(PROTO)/%.proto | $(PBC_GEN) - cd protos && protoc-c --c_out $(HOME)/runtime/$(PBC_GEN) $*.proto - -# makes build directories if needed -$(BUILD_DIRS): - mkdir -p $@ - -# removes all build directories -clean: - rm -rf $(BUILD_DIRS) diff --git a/dev_handler/Makefile b/dev_handler/Makefile index 3a336464..1d15c78c 100644 --- a/dev_handler/Makefile +++ b/dev_handler/Makefile @@ -4,16 +4,22 @@ LIBS=-pthread -lrt -Wall # list of source files that the target (dev_handler) depends on, relative to this folder SRCS = dev_handler.c dev_handler_message.c ../logger/logger.c ../runtime_util/runtime_util.c ../shm_wrapper/shm_wrapper.c -include ../Makefile +# specify the target (executable we want to make) +TARGET = dev_handler -all: dev_handler +all: $(TARGET) -dev_handler: $(OBJS) | $(BIN) +# include top-level Makefile +include ../Makefile_defs + +# rule to compile dev_handler +$(TARGET): $(OBJS) | $(BIN) $(CC) $(OBJS) -o $(BIN)/$@ $(LIBS) +# remove build artifacts clean: rm -f $(OBJS) rm -f $(patsubst %.o,%.d,$(OBJS)) - rm -f $(BIN)/dev_handler + rm -f $(BIN)/$(TARGET) -include $(patsubst %.o,%.d,$(OBJS)) diff --git a/network_switch/Makefile b/network_switch/Makefile index e8bfdd32..c1eb9482 100644 --- a/network_switch/Makefile +++ b/network_switch/Makefile @@ -1,11 +1,25 @@ -LIBFLAGS=-pthread -lrt -Wall -BIN=../bin +# list of libraries that network_switch needs to compile +LIBS=-pthread -lrt -Wall +# list of source files that the target (network_switch) depends on, relative to this folder +SRCS = network_switch.c ../runtime_util/runtime_util.c ../logger/logger.c ../shm_wrapper/shm_wrapper.c -network_switch: network_switch.c ../logger/logger.c ../runtime_util/runtime_util.c ../shm_wrapper/shm_wrapper.c - gcc $^ -o $(BIN)/network_switch +# specify the target (executable we want to make) +TARGET = network_switch +all: $(TARGET) + +# include top-level Makefile +include ../Makefile_defs + +# rule to compile network_switch +$(TARGET): $(OBJS) | $(BIN) + $(CC) $(OBJS) -o $(BIN)/$@ $(LIBS) + +# remove build artifacts clean: - rm -f $(BIN)/network_switch - rm -f network_switch - \ No newline at end of file + rm -f $(OBJS) + rm -f $(patsubst %.o,%.d,$(OBJS)) + rm -f $(BIN)/$(TARGET) + +-include $(patsubst %.o,%.d,$(OBJS)) From 2345b21c55667e0f7dd153e238d7c9756cdae69f Mon Sep 17 00:00:00 2001 From: Ben Liao Date: Wed, 19 Jul 2023 08:17:02 +0000 Subject: [PATCH 04/13] [BUILD] Working Makefiles for all main runtime folders! - protos moved back to original location net_handler/protos - top-level Makefile has all and clean - top-level Makefile_defs included by each Makefile - resulting lower-level Makefiles are (relatively) simple --- .gitmodules | 4 +- Makefile | 3 ++ Makefile_defs | 11 +++--- dev_handler/Makefile | 9 ++++- executor/Makefile | 53 +++++++++++++++++-------- executor/setup.py | 8 +++- net_handler/Makefile | 75 ++++++++++++++++++++++++++++-------- protos => net_handler/protos | 0 network_switch/Makefile | 9 ++++- shm_wrapper/Makefile | 37 +++++++++++++----- 10 files changed, 154 insertions(+), 55 deletions(-) rename protos => net_handler/protos (100%) diff --git a/.gitmodules b/.gitmodules index f854f482..2c03fcff 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ -[submodule "protos"] - path = protos +[submodule "net_handler/protos"] + path = net_handler/protos url = git@github.com:pioneers/protos.git diff --git a/Makefile b/Makefile index 1c5b30a3..cd89e09e 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,9 @@ all: cd dev_handler && make cd network_switch && make + cd shm_wrapper && make + cd net_handler && make + cd executor && make clean: rm -rf bin diff --git a/Makefile_defs b/Makefile_defs index e85795c1..5fb7fdca 100644 --- a/Makefile_defs +++ b/Makefile_defs @@ -12,18 +12,19 @@ THIS_DIR = $(strip $(shell pwd | xargs basename -z)) COMPONENT_DIRS = dev_handler net_handler executor network_switch shm_wrapper runtime_util logger # list of runtime component directories OBJ_SUBDIR = $(foreach dir,$(COMPONENT_DIRS),$(OBJ)/$(dir)) # list of subfolders in $(OBJ) for runtime object files -INCLUDE = $(foreach dir,$(COMPONENT_DIRS),-I../$(dir)) # list of folders to include in compilation commands +INCLUDE += $(foreach dir,$(COMPONENT_DIRS),-I../$(dir)) # list of folders to include in compilation commands OBJS = $(patsubst %.c,$(OBJ)/$(THIS_DIR)/%.o,$(SRCS)) # make a list of all object files this executable is dependent on relative to this Makefile -.PHONY: all clean - # list of build folders -BUILD_DIR = $(OBJ) $(BIN) $(BUILD) $(OBJ_SUBDIR) +BUILD_DIR += $(OBJ) $(BIN) $(BUILD) $(OBJ_SUBDIR) + +# append -MMD and -MP to CFLAGS to get the dependency files built for the object files +CFLAGS += -MMD -MP # general rule for compiling a list of source files to object files in the $(OBJ) directory $(OBJ)/$(THIS_DIR)/%.o: %.c | $(OBJ_SUBDIR) - $(CC) $(CFLAGS) -MMD -MP $(INCLUDE) -c $< -o $@ + $(CC) $(CFLAGS) $(INCLUDE) -c $< -o $@ # general rule for making a build directory $(BUILD_DIR): diff --git a/dev_handler/Makefile b/dev_handler/Makefile index 1d15c78c..9608101e 100644 --- a/dev_handler/Makefile +++ b/dev_handler/Makefile @@ -7,14 +7,19 @@ SRCS = dev_handler.c dev_handler_message.c ../logger/logger.c ../runtime_util/ru # specify the target (executable we want to make) TARGET = dev_handler +.PHONY: all clean $(TARGET) + all: $(TARGET) # include top-level Makefile include ../Makefile_defs +# resolve phony target "dev_handler" as ../bin/dev_handler +$(TARGET): $(BIN)/$(TARGET) + # rule to compile dev_handler -$(TARGET): $(OBJS) | $(BIN) - $(CC) $(OBJS) -o $(BIN)/$@ $(LIBS) +$(BIN)/$(TARGET): $(OBJS) | $(BIN) + $(CC) $(OBJS) -o $@ $(LIBS) # remove build artifacts clean: diff --git a/executor/Makefile b/executor/Makefile index c274767f..505109d9 100755 --- a/executor/Makefile +++ b/executor/Makefile @@ -1,26 +1,45 @@ -# Change this to point to your specific Python version. Need >= 3.6 -py=python3.10 +# list of libraries that executor needs to compile +LIBS=-pthread -lrt -Wall -export-dynamic -fPIC -CFLAGS=$(shell $(py)-config --cflags) -LDFLAGS=$(shell $(py)-config --ldflags) -PY_LIB=-l$(py) -LIBS=-pthread -lrt -export-dynamic -fPIC -lprotobuf-c -#-rdynamic or -export-dynamic to make it dynamically linked -BIN=../bin +# list of source files that the target (executor) depends on, relative to this folder +SRCS = executor.c gamestate_filter.c ../logger/logger.c ../runtime_util/runtime_util.c ../shm_wrapper/shm_wrapper.c -all: executor studentapi.c +# Python compilation definitions +PY_VER = python3.10 +PY_LIB = -l$(PY_VER) +CFLAGS = $(shell $(PY_VER)-config --cflags) -executor: executor.c gamestate_filter.c ../shm_wrapper/shm_wrapper.c ../logger/logger.c ../runtime_util/runtime_util.c ../net_handler/net_util.c ../net_handler/pbc_gen/text.pb-c.c - gcc $(CFLAGS) $^ -o $(BIN)/$@ $(LIBS) $(PY_LIB) +# specify the target (executable we want to make) +TARGET = executor -studentapi.c: studentapi.pyx runtime.pxd - $(py) setup.py build_ext -i +.PHONY: all clean $(TARGET) +all: $(TARGET) studentapi.c + +# include top-level Makefile +include ../Makefile_defs + +$(TARGET): $(BIN)/$(TARGET) + +# rule to compile executor +$(BIN)/$(TARGET): $(OBJS) | $(BIN) + $(CC) $(OBJS) -o $@ $(LIBS) $(PY_LIB) + +# rule to compile studentapi.c +studentapi.c: studentapi.pyx runtime.pxd setup.py + $(PY_VER) setup.py build_ext -i + +# rule to compile testapi test_api: studentapi.c test_studentapi.py - - $(py) test_studentapi.py + - $(PY_VER) test_studentapi.py +# remove build artifacts clean: - rm -f studentapi.c - rm -f $(BIN)/executor - rm -f executor + rm -f $(OBJS) + rm -f $(patsubst %.o,%.d,$(OBJS)) + rm -rf studentapi.c rm -f *.so + rm -rf build + rm -f $(BIN)/$(TARGET) + +-include $(patsubst %.o,%.d,$(OBJS)) diff --git a/executor/setup.py b/executor/setup.py index 6b7b5239..2352d6fd 100755 --- a/executor/setup.py +++ b/executor/setup.py @@ -9,6 +9,12 @@ "studentapi.pyx" ] +includedirectories = [ + "../logger", + "../shm_wrapper", + "../runtime_util" +] + if sys.platform == 'linux': libraries = ['rt'] elif sys.platform == 'darwin': @@ -19,7 +25,7 @@ setup( name="Student API", ext_modules = cythonize([ - Extension("studentapi", sources=sourcefiles, libraries=libraries) + Extension("studentapi", sources=sourcefiles, include_dirs=includedirectories, libraries=libraries) ], compiler_directives={'language_level' : '3', 'boundscheck': False, 'embedsignature': True}), zip_safe=False, ) diff --git a/net_handler/Makefile b/net_handler/Makefile index de6354b2..a02cc77d 100644 --- a/net_handler/Makefile +++ b/net_handler/Makefile @@ -1,23 +1,64 @@ -CLAGS=$(shell pkg-config --cflags libprotobuf-c) -LDFLAGS=$(shell pkg-config --libs libprotobuf-c) -LIBS=-pthread -lrt -Wall -BIN=../bin +# list of libraries that net_handler needs to compile +LIBS=-pthread -lrt -Wall -lprotobuf-c -GEN_FOLDER=pbc_gen -GEN_FILES:=$(GEN_FOLDER)/*.c +###################################### protobuf defs -all: $(GEN_FOLDER)/* net_handler +PROTO = protos +PBC_OBJ = $(BUILD)/pbc_obj +PBC_GEN = pbc_gen +BUILD_DIR = $(PBC_GEN) $(PBC_OBJ) # add PBC_GEN and PBC_OBJ to list of build directories -# makes the generated protobuf-c files in the pbc_gen folder -$(GEN_FOLDER)/%: $(wildcard protos/%.proto) - mkdir -p $(GEN_FOLDER) - cd protos && protoc-c --c_out ../$(GEN_FOLDER) $*.proto +PROTOS = $(shell cd $(PROTO) && ls *.proto && cd ..) # list all the protos +PBC_SRCS = $(patsubst %.proto,$(PBC_GEN)/%.pb-c.c, $(PROTOS)) # list of all generated source files by doing a path substitution +PBC_OBJS = $(patsubst $(PBC_GEN)/%.c,$(PBC_OBJ)/%.o, $(PBC_SRCS)) # list of all generated object files by doing a path substitution +INCLUDE = -I$(PBC_GEN) +CFLAGS += $(shell pkg-config --cflags libprotobuf-c) # append more CFLAGS for compiling protobuf to existing CFLAGS + +##################################### + +# list of source files that the target (net_handler) depends on, relative to this folder +SRCS = net_handler.c net_handler_message.c net_util.c connection.c ../executor/gamestate_filter.c \ + ../logger/logger.c ../runtime_util/runtime_util.c ../shm_wrapper/shm_wrapper.c + +# specify the target (executable we want to make) +TARGET = net_handler + +.PHONY: all clean $(TARGET) protos + +all: protos $(TARGET) + +# include top-level Makefile +include ../Makefile_defs + +# resolve phony target "protos" as make all protobuf object files +protos: $(PBC_OBJS) + +# resolve phony target "net_handler" as make ../bin/net_handler +$(TARGET): $(BIN)/$(TARGET) + +# rule to compile net_handler +$(BIN)/$(TARGET): $(OBJS) $(PBC_OBJS) | $(BIN) + $(CC) $(OBJS) $(PBC_OBJS) -o $@ $(LIBS) + +# makes the generated protobuf-c object files in the $(PBC_OBJ) folder +$(PBC_OBJ)/%.o: $(PBC_GEN)/%.c | $(PBC_OBJ) + $(CC) $(CFLAGS) -c $< -o $@ + +# makes the generated protobuf-c files in the $(PBC_GEN) folder +$(PBC_GEN)/%.pb-c.c: $(PROTO)/%.proto | $(PBC_GEN) + cd $(PROTO) && protoc-c --c_out ../$(PBC_GEN) $*.proto + +# remove build artifacts clean: - rm -rf $(GEN_FOLDER) - rm -f $(BIN)/net_handler - rm -f net_handler + rm -f $(OBJS) + rm -rf $(PBC_OBJ) + rm -f $(patsubst %.o,%.d,$(OBJS)) + rm -f $(BIN)/$(TARGET) + +# also remove generated protobufs +very-clean: clean + rm -rf $(PBC_GEN) -# makes the actual net_handler executable -net_handler: net_util.c net_handler.c message.c connection.c ../logger/logger.c ../shm_wrapper/shm_wrapper.c ../runtime_util/runtime_util.c ../executor/gamestate_filter.c - gcc $^ $(GEN_FILES) -o $(BIN)/$@ $(LIBS) $(LDFLAGS) +-include $(patsubst %.o,%.d,$(OBJS)) +-include $(patsubst %.o,%.d,$(PBC_OBJS)) diff --git a/protos b/net_handler/protos similarity index 100% rename from protos rename to net_handler/protos diff --git a/network_switch/Makefile b/network_switch/Makefile index c1eb9482..d3b6921e 100644 --- a/network_switch/Makefile +++ b/network_switch/Makefile @@ -7,14 +7,19 @@ SRCS = network_switch.c ../runtime_util/runtime_util.c ../logger/logger.c ../shm # specify the target (executable we want to make) TARGET = network_switch +.PHONY: all clean $(TARGET) + all: $(TARGET) # include top-level Makefile include ../Makefile_defs +# resolve phony target "network_switch" as ../bin/network_switch +$(TARGET): $(BIN)/$(TARGET) + # rule to compile network_switch -$(TARGET): $(OBJS) | $(BIN) - $(CC) $(OBJS) -o $(BIN)/$@ $(LIBS) +$(BIN)/$(TARGET): $(OBJS) | $(BIN) + $(CC) $(OBJS) -o $@ $(LIBS) # remove build artifacts clean: diff --git a/shm_wrapper/Makefile b/shm_wrapper/Makefile index ae86c597..fd64c840 100644 --- a/shm_wrapper/Makefile +++ b/shm_wrapper/Makefile @@ -1,16 +1,35 @@ +# list of libraries that shm_wrapper needs to compile LIBS=-pthread -lrt -Wall -FILES=shm_wrapper.c ../logger/logger.c ../runtime_util/runtime_util.c -BIN=../bin +# list of source files that the targets (shm_start and shm_stop) depend on, relative to this folder +SRCS = shm_start.c shm_stop.c shm_wrapper.c ../logger/logger.c ../runtime_util/runtime_util.c -all: shm_start shm_stop +# specify the targets (executables we want to make) +TARGETS = shm_start shm_stop -shm_start: shm_start.c $(FILES) - gcc $^ -o $(BIN)/$@ $(LIBS) +.PHONY: all clean $(TARGETS) -shm_stop: shm_stop.c $(FILES) - gcc $^ -o $(BIN)/$@ $(LIBS) +all: $(TARGETS) +# include top-level Makefile +include ../Makefile_defs + +shm_start: $(BIN)/shm_start + +shm_stop: $(BIN)/shm_stop + +# rule to compile shm_start +$(BIN)/shm_start: $(filter-out %/shm_stop.o, $(OBJS)) | $(BIN) + $(CC) $^ -o $@ $(LIBS) + +# rule to compile shm_stop +$(BIN)/shm_stop: $(filter-out %/shm_start.o, $(OBJS)) | $(BIN) + $(CC) $^ -o $@ $(LIBS) + +# remove build artifacts clean: - rm -f $(BIN)/shm_start $(BIN)/shm_stop - rm -f shm_start shm_stop + rm -f $(OBJS) + rm -f $(patsubst %.o,%.d,$(OBJS)) + rm -f $(BINS) + +-include $(patsubst %.o,%.d,$(OBJS)) From bd12817f5b75f5593b02f0e43b3b64cd7639142e Mon Sep 17 00:00:00 2001 From: Ben Liao Date: Wed, 19 Jul 2023 08:31:34 +0000 Subject: [PATCH 05/13] [FORMAT] Formatted all files to conform to standard --- dev_handler/dev_handler.c | 8 +++--- dev_handler/dev_handler_message.c | 2 +- executor/executor.c | 30 ++++++++++---------- executor/gamestate_filter.h | 2 +- lowcar/devices/LimitSwitch/LimitSwitch.cpp | 4 +-- lowcar/devices/LineFollower/LineFollower.cpp | 3 +- lowcar/devices/ServoControl/ServoControl.cpp | 1 - net_handler/net_handler.c | 2 +- network_switch/network_switch.c | 22 +++++++------- shm_wrapper/shm_wrapper.h | 4 +-- tests/client/dev_handler_client.c | 6 ++-- tests/client/dev_handler_client.h | 24 ++++++++-------- tests/client/executor_client.h | 16 +++++------ tests/client/net_handler_client.h | 2 +- tests/client/shm_client.h | 2 +- 15 files changed, 62 insertions(+), 66 deletions(-) diff --git a/dev_handler/dev_handler.c b/dev_handler/dev_handler.c index 03a75449..59a17e42 100644 --- a/dev_handler/dev_handler.c +++ b/dev_handler/dev_handler.c @@ -5,10 +5,10 @@ #include // for POSIX terminal control definitions in serialport_open() +#include #include #include #include -#include /** * Each device will have a unique port number. @@ -24,7 +24,7 @@ * These file paths may also be referred to as "port_prefix" in the code. */ #define LOWCAR_FILE_PATH "/dev/ttyACM" -#define VIRTUAL_FILE_PATH "ttyACM" // will be created in the home directory +#define VIRTUAL_FILE_PATH "ttyACM" // will be created in the home directory #define LOWCAR_USB_FILE_PATH "/dev/ttyUSB" // **************************** PRIVATE STRUCT ****************************** // @@ -93,7 +93,7 @@ uint32_t used_lowcar_usb_ports = 0; pthread_mutex_t used_ports_lock; // poll_connected_devices() and relay_clean_up() shouldn't access used_ports at the same time // String to hold the home directory path (for looking for virtual device sockets) -const char *home_dir; +const char* home_dir; #define MAX_PORT_NAME_SIZE 64 @@ -872,7 +872,7 @@ int main(int argc, char* argv[]) { // If SIGINT (Ctrl+C) is received, call stop() to clean up signal(SIGINT, stop); init(); - home_dir = getenv("HOME"); // set the home directory + home_dir = getenv("HOME"); // set the home directory log_printf(INFO, "DEV_HANDLER initialized."); poll_connected_devices(); return 0; diff --git a/dev_handler/dev_handler_message.c b/dev_handler/dev_handler_message.c index 996e9b0c..eca8aeec 100644 --- a/dev_handler/dev_handler_message.c +++ b/dev_handler/dev_handler_message.c @@ -317,7 +317,7 @@ int parse_message(uint8_t data[], message_t* msg_to_fill) { // Smaller than valid message free(decoded); return 3; - } else if (ret > (int)(MESSAGE_ID_SIZE + PAYLOAD_LENGTH_SIZE + MAX_PAYLOAD_SIZE + CHECKSUM_SIZE)) { + } else if (ret > (int) (MESSAGE_ID_SIZE + PAYLOAD_LENGTH_SIZE + MAX_PAYLOAD_SIZE + CHECKSUM_SIZE)) { // Larger than the largest valid message free(decoded); return 3; diff --git a/executor/executor.c b/executor/executor.c index 38bd297d..9030f072 100644 --- a/executor/executor.c +++ b/executor/executor.c @@ -1,21 +1,21 @@ #define PY_SSIZE_T_CLEAN -#include //for networking -#include //for POSIX threads -#include // For Python's C API -#include // Used to handle SIGTERM, SIGINT, SIGKILL -#include //for standard int types -#include //for i/o -#include //for standard utility functions (exit, sleep) -#include //for sem_t and other standard system types -#include //for unix sockets -#include //for wait functions -#include // for getting time -#include //for sleep +#include //for networking +#include //for POSIX threads +#include // For Python's C API +#include // Used to handle SIGTERM, SIGINT, SIGKILL +#include //for standard int types +#include //for i/o +#include //for standard utility functions (exit, sleep) +#include //for sem_t and other standard system types +#include //for unix sockets +#include //for wait functions +#include // for getting time +#include //for sleep // runtime includes -#include // for runtime logger -#include //for runtime constants (TODO: consider removing relative pathname in include) -#include // Shared memory wrapper to get/send device data +#include // for runtime logger +#include //for runtime constants (TODO: consider removing relative pathname in include) +#include // Shared memory wrapper to get/send device data // Global variables to all functions and threads diff --git a/executor/gamestate_filter.h b/executor/gamestate_filter.h index ff602c23..cc031415 100644 --- a/executor/gamestate_filter.h +++ b/executor/gamestate_filter.h @@ -21,10 +21,10 @@ * will flip the sign of VALUE before being written to shared memory. */ -#include #include #include #include +#include /** * Starts the gamestate handler thread so that gamestates are deactivated diff --git a/lowcar/devices/LimitSwitch/LimitSwitch.cpp b/lowcar/devices/LimitSwitch/LimitSwitch.cpp index 1f6758d9..10477376 100644 --- a/lowcar/devices/LimitSwitch/LimitSwitch.cpp +++ b/lowcar/devices/LimitSwitch/LimitSwitch.cpp @@ -17,11 +17,9 @@ size_t LimitSwitch::device_read(uint8_t param, uint8_t* data_buf) { if (param < NUM_SWITCHES && param >= 0) { data_buf[0] = (digitalRead(this->pins[param]) == LOW); return sizeof(uint8_t); - } - else { + } else { return 0; } - } void LimitSwitch::device_enable() { diff --git a/lowcar/devices/LineFollower/LineFollower.cpp b/lowcar/devices/LineFollower/LineFollower.cpp index 77c7606a..aa2ef0c4 100644 --- a/lowcar/devices/LineFollower/LineFollower.cpp +++ b/lowcar/devices/LineFollower/LineFollower.cpp @@ -13,14 +13,13 @@ LineFollower::LineFollower() : Device(DeviceType::LINE_FOLLOWER, 1) { size_t LineFollower::device_read(uint8_t param, uint8_t* data_buf) { // use data_ptr_float to shove 10-bit sensor reading into the data_buf - if (param < NUM_PINS && param >=0) { + if (param < NUM_PINS && param >= 0) { float* data_ptr_float = (float*) data_buf; *data_ptr_float = 1.0 - ((float) analogRead(this->pins[param])) / 1023.0; return sizeof(float); } else { return 0; } - } void LineFollower::device_enable() { diff --git a/lowcar/devices/ServoControl/ServoControl.cpp b/lowcar/devices/ServoControl/ServoControl.cpp index 4fa72d6f..fb6518a1 100644 --- a/lowcar/devices/ServoControl/ServoControl.cpp +++ b/lowcar/devices/ServoControl/ServoControl.cpp @@ -27,7 +27,6 @@ size_t ServoControl::device_read(uint8_t param, uint8_t* data_buf) { } else { return 0; } - } size_t ServoControl::device_write(uint8_t param, uint8_t* data_buf) { diff --git a/net_handler/net_handler.c b/net_handler/net_handler.c index ba93ea53..b2ddf3ad 100644 --- a/net_handler/net_handler.c +++ b/net_handler/net_handler.c @@ -1,5 +1,5 @@ -#include #include +#include #include /* diff --git a/network_switch/network_switch.c b/network_switch/network_switch.c index 83f39f30..15cfc695 100644 --- a/network_switch/network_switch.c +++ b/network_switch/network_switch.c @@ -1,10 +1,10 @@ +#include +#include +#include #include #include #include #include -#include -#include -#include #define SLEEP_DELAY 1 @@ -32,7 +32,7 @@ int main() { char local_router[32]; char local_password[32]; - char *local_args[5]; + char* local_args[5]; char team_router[] = "teamrouter.txt"; get_router_name_password(local_router, local_password, team_router); local_args[0] = bash_exe; @@ -43,7 +43,7 @@ int main() { char pie_router[32]; char pie_password[32]; - char *pie_args[5]; + char* pie_args[5]; char pie_router_file[] = "pierouter.txt"; get_router_name_password(pie_router, pie_password, pie_router_file); pie_args[0] = bash_exe; @@ -92,7 +92,7 @@ int main() { log_printf(ERROR, "network_switch: Failed to fork"); sleep(SLEEP_DELAY); } else if (pid == 0) { - execv(bash_exe, pie_args); // call the bash script with pioneers router arguments + execv(bash_exe, pie_args); // call the bash script with pioneers router arguments } else { waitpid(pid, &status, 0); } @@ -102,18 +102,18 @@ int main() { log_printf(ERROR, "network_switch: Failed to fork"); sleep(SLEEP_DELAY); } else if (pid == 0) { - execv(bash_exe, local_args); // call the bash script with local router arguments + execv(bash_exe, local_args); // call the bash script with local router arguments } else { waitpid(pid, &status, 0); } } - char buf[5]; + char buf[5]; FILE* exit_status = fopen("exit_status.txt", "r"); - fgets(buf, 5, exit_status); // retrieve the output of the bash script in the exit_status.txt - if (strcmp(buf, "1") == 0) { // if output is 1, we successfully connected. Set previous switch = current switch + fgets(buf, 5, exit_status); // retrieve the output of the bash script in the exit_status.txt + if (strcmp(buf, "1") == 0) { // if output is 1, we successfully connected. Set previous switch = current switch log_printf(WARN, "SUCCESSFULLY CONNECTED TO NETWORK"); prev_switch = curr_switch; - } else { // if output is not 1, we loop again to call the bash script again + } else { // if output is not 1, we loop again to call the bash script again log_printf(WARN, "FAILED TO CONNECT TO NETWORK"); } fclose(exit_status); diff --git a/shm_wrapper/shm_wrapper.h b/shm_wrapper/shm_wrapper.h index 157bf588..d283dbea 100644 --- a/shm_wrapper/shm_wrapper.h +++ b/shm_wrapper/shm_wrapper.h @@ -6,8 +6,8 @@ #include #include // for posix shared memory -#include // for logger -#include // for runtime constants +#include // for logger +#include // for runtime constants // names of various objects used in shm_wrapper; should not be used outside of shm_wrapper.c, shm_start.c, and shm_stop.c #define DEV_SHM_NAME "/dev-shm" // name of shared memory block across devices diff --git a/tests/client/dev_handler_client.c b/tests/client/dev_handler_client.c index 7ec06afa..4484a6a8 100644 --- a/tests/client/dev_handler_client.c +++ b/tests/client/dev_handler_client.c @@ -180,9 +180,9 @@ int connect_virtual_device(char* dev_name, uint64_t uid) { } else { // Parent process // Take note of child pid so we can kill it in disconnect_device() used_sockets[socket_num]->pid = pid; - - // close duplicate connection file descriptor - close(used_sockets[socket_num]->fd); + + // close duplicate connection file descriptor + close(used_sockets[socket_num]->fd); } return socket_num; } diff --git a/tests/client/dev_handler_client.h b/tests/client/dev_handler_client.h index f3bd17d3..5c0b02b4 100644 --- a/tests/client/dev_handler_client.h +++ b/tests/client/dev_handler_client.h @@ -1,18 +1,18 @@ #ifndef DEV_CLIENT_H #define DEV_CLIENT_H -#include // for errno -#include // for SIGINT (Ctrl+C) -#include // for boolean types -#include // for int with specific widths -#include // for printf() -#include // for exit() -#include // for strerr() -#include // for sockets -#include // for sockaddr_un -#include // for waitpid() -#include // for read() -#include "../../logger/logger.h" // for log_printf +#include // for errno +#include // for SIGINT (Ctrl+C) +#include // for boolean types +#include // for int with specific widths +#include // for printf() +#include // for exit() +#include // for strerr() +#include // for sockets +#include // for sockaddr_un +#include // for waitpid() +#include // for read() +#include "../../logger/logger.h" // for log_printf // Starts dev handler with "virtual" argument void start_dev_handler(); diff --git a/tests/client/executor_client.h b/tests/client/executor_client.h index 120f5431..aa97a7f3 100644 --- a/tests/client/executor_client.h +++ b/tests/client/executor_client.h @@ -1,14 +1,14 @@ #ifndef EXEC_CLIENT_H #define EXEC_CLIENT_H -#include // for errno -#include // for SIGINT -#include // for printf, stderr -#include // for malloc, free, realloc, exit -#include // for strcat, strcpy -#include // for waitpid, pid_t -#include // for execlp, fork, chdir -#include "../../logger/logger.h" // for log_printf +#include // for errno +#include // for SIGINT +#include // for printf, stderr +#include // for malloc, free, realloc, exit +#include // for strcat, strcpy +#include // for waitpid, pid_t +#include // for execlp, fork, chdir +#include "../../logger/logger.h" // for log_printf /** * Spawns the executor running student_code diff --git a/tests/client/net_handler_client.h b/tests/client/net_handler_client.h index 4395bb9d..dd2f8082 100644 --- a/tests/client/net_handler_client.h +++ b/tests/client/net_handler_client.h @@ -3,8 +3,8 @@ #include #include +#include "../../logger/logger.h" // for log_printf #include "../../net_handler/net_util.h" -#include "../../logger/logger.h" // for log_printf /** * Helper function to print out contents of a DevData message diff --git a/tests/client/shm_client.h b/tests/client/shm_client.h index 3e2e8fb0..2d1b8c0c 100644 --- a/tests/client/shm_client.h +++ b/tests/client/shm_client.h @@ -2,8 +2,8 @@ #define SHM_CLIENT_H #include +#include "../../logger/logger.h" // for log_printf #include "../../shm_wrapper/shm_wrapper.h" -#include "../../logger/logger.h" // for log_printf /** * Creates and initializes the shared memory process From 487b4cfe12f8f8a318e577c6eb6168ecec65fc09 Mon Sep 17 00:00:00 2001 From: Ben Liao Date: Wed, 19 Jul 2023 08:42:51 +0000 Subject: [PATCH 06/13] [BUILD] Added format step to CI; updated root runtime script --- .github/workflows/pull_request_ci.yml | 7 ++++++- Makefile | 2 ++ runtime | 26 ++++---------------------- 3 files changed, 12 insertions(+), 23 deletions(-) diff --git a/.github/workflows/pull_request_ci.yml b/.github/workflows/pull_request_ci.yml index 1e41f582..c4282927 100644 --- a/.github/workflows/pull_request_ci.yml +++ b/.github/workflows/pull_request_ci.yml @@ -89,6 +89,11 @@ jobs: tar -xzf protobuf-c-1.4.1.tar.gz cd protobuf-c-1.4.1 && ./configure && make && sudo make install && sudo ldconfig + # Before we build Runtime, we check the formatting + - name: Format + run: | + ./runtime format -r . + # Now that we are done installing Runtime's dependencies, we build Runtime - name: Build run: | @@ -97,4 +102,4 @@ jobs: # And finally, we test Runtime - name: Test run: | - ./runtime test \ No newline at end of file + ./runtime test diff --git a/Makefile b/Makefile index cd89e09e..93fcdfb1 100644 --- a/Makefile +++ b/Makefile @@ -8,5 +8,7 @@ all: cd executor && make clean: + # executor needs to take care of its Cython build products + cd executor && make clean rm -rf bin rm -rf build diff --git a/runtime b/runtime index f6ef02bf..98ce75c6 100755 --- a/runtime +++ b/runtime @@ -14,33 +14,15 @@ cd $RUNTIME_DIR case $1 in build) - mkdir -p bin # make the runtime/bin directory - shift 1 - pushd shm_wrapper && make $@ - popd - pushd net_handler && make $@ - popd - pushd executor && make $@ - popd - pushd dev_handler && make $@ - popd - pushd network_switch && make $@ - popd - scripts/run_clang_format.py -ir . + make ;; clean) - pushd shm_wrapper && make clean && popd - pushd net_handler && make clean && popd - pushd executor && make clean && popd - pushd dev_handler && make clean && popd - pushd tests && make clean && popd - pushd network_switch && make clean && popd + make clean rm -f /tmp/log-fifo - rm -f /tmp/challenge.sock rm -f /tmp/ttyACM* ;; - + run) scripts/run.sh ;; @@ -48,7 +30,7 @@ case $1 in stop) scripts/stop.sh ;; - + test) mkdir -p tests/bin # make the runtime/tests/bin directory shift 1 From 89a35b75c5b8defd467fedab335e743d4fe6369d Mon Sep 17 00:00:00 2001 From: Ben Liao Date: Thu, 20 Jul 2023 08:46:42 +0000 Subject: [PATCH 07/13] [BUILD] Working completely rewritten runtime/tests/Makefile - tests/Makefile works more efficiently than before - Updated runtime script to reflect changes - Fixed some compiler warning bugs in test/test.c --- runtime | 1 - tests/Makefile | 172 +++++++++++++----- tests/client/dev_handler_client.h | 24 +-- tests/client/executor_client.h | 16 +- tests/client/net_handler_client.h | 4 +- tests/client/shm_client.h | 4 +- .../virtual_devices/virtual_device_util.h | 4 +- tests/test.c | 6 +- 8 files changed, 152 insertions(+), 79 deletions(-) diff --git a/runtime b/runtime index 98ce75c6..bf25610b 100755 --- a/runtime +++ b/runtime @@ -32,7 +32,6 @@ case $1 in ;; test) - mkdir -p tests/bin # make the runtime/tests/bin directory shift 1 scripts/test.sh $@ ;; diff --git a/tests/Makefile b/tests/Makefile index 11ba15d9..f08f1742 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,77 +1,151 @@ -LIBS := -pthread -lrt -Wall -lprotobuf-c -Wno-format # Ignore formatting warnings (ex: %llu vs %lu) +# list of libraries that tests need to compile +LIBS=-pthread -lrt -Wall -lprotobuf-c -lncurses -Wno-format + +CC = gcc +CFLAGS = -MMD -MP + +###################################### protobuf defs + +# getting paths to protobuf because needed for compilation but this Makefile will NOT compile them +PROTO = ../net_handler/protos +PBC_OBJ = $(BUILD)/pbc_obj +PBC_GEN = ../net_handler/pbc_gen + +PROTOS = $(shell cd $(PROTO) && ls *.proto && cd ..) # list all the protos +PBC_SRCS = $(patsubst %.proto,$(PBC_GEN)/%.pb-c.c, $(PROTOS)) # list of all generated source files by doing a path substitution +PBC_OBJS = $(patsubst $(PBC_GEN)/%.c,$(PBC_OBJ)/%.o, $(PBC_SRCS)) # list of all generated object files by doing a path substitution + +INCLUDE = -I$(PBC_GEN) +CFLAGS += $(shell pkg-config --cflags libprotobuf-c) # append more CFLAGS for compiling protobuf to existing CFLAGS + +#################################### name directories and lists of directories + +# tests will have its own bin directory (tests/bin) BIN = bin +BUILD = ../build +OBJ = $(BUILD)/obj +THIS_DIR = tests -CC := gcc +COMPONENT_DIRS = dev_handler net_handler executor network_switch shm_wrapper runtime_util logger tests # list of runtime component directories +OBJ_SUBDIR = $(foreach dir,$(COMPONENT_DIRS),$(OBJ)/$(dir)) # list of subfolders in $(OBJ) for runtime object files +INCLUDE += $(foreach dir,$(COMPONENT_DIRS),-I../$(dir)) # list of folders to include in compilation commands -SRC_FILES := $(wildcard client/*.c test.c ../net_handler/pbc_gen/*.c ../net_handler/net_util.c ../runtime_util/runtime_util.c ../shm_wrapper/shm_wrapper.c ../logger/logger.c) -PBC_FILES := $(wildcard ../net_handler/pbc_gen/*.c) +# start making the lists of build and bin directories +SUBDIR = cli client client/virtual_devices performance integration +BUILD_DIR = $(foreach dir,$(SUBDIR),$(OBJ)/$(THIS_DIR)/$(dir)) $(OBJ) $(OBJ_SUBDIR) +BIN_DIR = $(BIN) -TEST_FILES := $(shell ls integration/*.c performance/*.c | awk -F'.' '{ print $$1 }') +##################################### make lists of files -.PHONY: all clean cli +# specify files that various executables are dependent on +UTIL_SRCS = ../runtime_util/runtime_util.c ../logger/logger.c # everybody uses these +NET_HANDLER_CLI_SRCS = cli/net_handler_cli.c client/net_handler_client.c cli/keyboard_interface.c ../net_handler/net_util.c $(UTIL_SRCS) +SHM_UI_SRCS = cli/shm_ui.c client/shm_client.c ../shm_wrapper/shm_wrapper.c $(UTIL_SRCS) +EXECUTOR_CLI_SRCS = cli/executor_cli.c client/executor_client.c $(UTIL_SRCS) +DEV_HANDLER_CLI_SRCS = client/dev_handler_client.c cli/dev_handler_cli.c $(UTIL_SRCS) +VIRTUAL_DEV_SRCS = client/virtual_devices/virtual_device_util.c ../dev_handler/dev_handler_message.c $(UTIL_SRCS) +TESTS_SRCS = test.c $(wildcard client/*.c) ../net_handler/net_util.c ../shm_wrapper/shm_wrapper.c $(UTIL_SRCS) -all: $(TEST_FILES) cli +VIRTUAL_DEVICES = $(wildcard client/virtual_devices/*Device.c) +TESTS = $(wildcard integration/* performance/*) -clean: - rm -rf $(BIN)/* +# generate lists of object files from the lists of source files above +NET_HANDLER_CLI_OBJS = $(patsubst %.c,$(OBJ)/$(THIS_DIR)/%.o,$(NET_HANDLER_CLI_SRCS)) +SHM_UI_OBJS = $(patsubst %.c,$(OBJ)/$(THIS_DIR)/%.o,$(SHM_UI_SRCS)) +EXECUTOR_CLI_OBJS = $(patsubst %.c,$(OBJ)/$(THIS_DIR)/%.o,$(EXECUTOR_CLI_SRCS)) +DEV_HANDLER_CLI_OBJS = $(patsubst %.c,$(OBJ)/$(THIS_DIR)/%.o,$(DEV_HANDLER_CLI_SRCS)) +TEST_OBJS = $(patsubst %.c,$(OBJ)/$(THIS_DIR)/%.o,$(TESTS_SRCS)) +VIRTUAL_DEV_OBJS = $(patsubst %.c,$(OBJ)/$(THIS_DIR)/%.o,$(VIRTUAL_DEV_SRCS)) + +# specify directories for virtual devices and tests executables +VIRTUAL_DEV_EXE_DIR = $(BIN)/virtual_devices +TESTS_EXE_DIR = $(BIN)/integration $(BIN)/performance +BIN_DIR += $(VIRTUAL_DEV_EXE_DIR) $(TESTS_EXE_DIR) -################################## CLI TARGETS ################################# +# specify targets (arguments you can give to make i.e. "make net_handler_cli" or "make tc_150_1" or "make GeneralTestDevice") +CLI_TARGET = net_handler_cli executor_cli dev_handler_cli shm_ui +TEST_TARGET = $(patsubst %.c,%,$(TESTS)) +TEST_TARGET_SHORT = $(foreach test_target,$(TEST_TARGET),$(shell basename $(test_target))) # just tc_xx_xx +VIRTUAL_DEV_TARGET = $(patsubst client/virtual_devices/%.c,%,$(VIRTUAL_DEVICES)) -cli: net_handler_cli shm_ui executor_cli dev_handler_cli devices +# generate lists of executable names +CLI_EXE = $(patsubst %,$(BIN)/%,$(CLI_TARGET)) +TEST_EXE = $(patsubst %,$(BIN)/%,$(TEST_TARGET)) +VIRTUAL_DEV_EXE = $(patsubst %,$(BIN)/virtual_devices/%,$(VIRTUAL_DEV_TARGET)) -net_handler_cli: cli/net_handler_cli.* client/net_handler_client.* cli/keyboard_interface.* ../net_handler/net_util.* \ - ../runtime_util/runtime_util.* ../logger/logger.c $(PBC_FILES) - $(CC) $^ -o $(BIN)/$@ $(LIBS) +# combine all source files and associated object files with each other into a list for .c -> .o rule +SRCS = $(NET_HANDLER_CLI_SRCS) $(SHM_UI_SRCS) $(EXECUTOR_CLI_SRCS) $(DEV_HANDLER_CLI_SRCS) \ + $(VIRTUAL_DEV_SRCS) $(TESTS_SRCS) $(VIRTUAL_DEVICES) $(TESTS) +OBJS = $(patsubst %.c,$(OBJ)/$(THIS_DIR)/%.o,$(SRCS)) # generate list of all object files relative to this Makefile -shm_ui: cli/shm_ui.c client/shm_client.c ../shm_wrapper/shm_wrapper.c \ - ../runtime_util/runtime_util.c ../logger/logger.c - $(CC) $^ -o $(BIN)/$@ $(LIBS) -lncurses +#################################### rules begin here -executor_cli: cli/executor_cli.* client/executor_client.* - $(CC) $^ -o $(BIN)/$@ $(LIBS) +.SECONDEXPANSION: -dev_handler_cli: client/dev_handler_client.* cli/dev_handler_cli.* - $(CC) $^ -o $(BIN)/$@ +.PHONY: all cli tests devices clean $(CLI_TARGET) $(TEST_TARGET) $(TEST_TARGET_SHORT) $(VIRTUAL_DEV_TARGET) -################################ VIRTUAL DEVICES ############################### +all: cli tests devices -VIRTUAL_DEVICE_FILES := client/virtual_devices/virtual_device_util.* ../dev_handler/message.* ../runtime_util/runtime_util.* ../logger/logger.c +#################################### rules to compile cli -VIRTUAL_DEVICES := $(shell ls client/virtual_devices/*Device.c | awk -F'.' '{ print $$1 }') +# resolve phony target "cli" to list of all cli executables +cli: $(CLI_EXE) -devices: folder $(VIRTUAL_DEVICES) - echo "virtual devices made" > bin/virtual_devices/devices +$(CLI_TARGET): $(BIN)/$$@ -folder: - mkdir -p $(BIN)/virtual_devices +# net_handler_cli is special because it needs $(PBC_OBJS) +$(BIN)/net_handler_cli: $(NET_HANDLER_CLI_OBJS) $(PBC_OBJS) | $(BIN) + $(CC) $^ -o $@ $(LIBS) -client/virtual_devices/GeneralTestDevice: client/virtual_devices/GeneralTestDevice.c $(VIRTUAL_DEVICE_FILES) - $(CC) $^ -o $(BIN)/virtual_devices/GeneralTestDevice +$(BIN)/executor_cli: $(EXECUTOR_CLI_OBJS) | $(BIN) + $(CC) $^ -o $@ $(LIBS) -client/virtual_devices/SimpleTestDevice: client/virtual_devices/SimpleTestDevice.c $(VIRTUAL_DEVICE_FILES) - $(CC) $^ -o $(BIN)/virtual_devices/SimpleTestDevice +$(BIN)/dev_handler_cli: $(DEV_HANDLER_CLI_OBJS) | $(BIN) + $(CC) $^ -o $@ $(LIBS) -client/virtual_devices/UnresponsiveTestDevice: client/virtual_devices/UnresponsiveTestDevice.c - $(CC) $^ -o $(BIN)/virtual_devices/UnresponsiveTestDevice +$(BIN)/shm_ui: $(SHM_UI_OBJS) | $(BIN) + $(CC) $^ -o $@ $(LIBS) -client/virtual_devices/ForeignTestDevice: client/virtual_devices/ForeignTestDevice.c - $(CC) $^ -o $(BIN)/virtual_devices/ForeignTestDevice +################################# general rule to compile a virtual device -client/virtual_devices/UnstableTestDevice: client/virtual_devices/UnstableTestDevice.c $(VIRTUAL_DEVICE_FILES) - $(CC) $^ -o $(BIN)/virtual_devices/UnstableTestDevice +# resolve phony target "devices" to list of all virtual device executables +devices: $(VIRTUAL_DEV_EXE) -client/virtual_devices/TimeTestDevice: client/virtual_devices/TimeTestDevice.c $(VIRTUAL_DEVICE_FILES) - $(CC) $^ -o $(BIN)/virtual_devices/TimeTestDevice +# resolve virtual device target to its corresponding executable +$(VIRTUAL_DEV_TARGET): $$(patsubst %,$(BIN)/virtual_devices/%,$$@) -client/virtual_devices/SoundDevice: client/virtual_devices/SoundDevice.c $(VIRTUAL_DEVICE_FILES) - $(CC) $^ -o $(BIN)/virtual_devices/SoundDevice +# rule to compile a virtual device executable +$(VIRTUAL_DEV_EXE): $$(patsubst $(BIN)/%,$(OBJ)/$(THIS_DIR)/client/%.o,$$@) $(VIRTUAL_DEV_OBJS) | $(VIRTUAL_DEV_EXE_DIR) + $(CC) $^ -o $@ $(LIBS) -##################################### TESTS #################################### +################################# general rule to compile a test -integration/%: $(SRC_FILES) integration/%.c - mkdir -p $(BIN)/integration - $(CC) $^ -o $(BIN)/$@ $(LIBS) +# resolve phony target "tests" to list of all test executables +tests: $(TEST_EXE) + +# resolve test target to its corresponding executable +$(TEST_TARGET): $$(patsubst %,$(BIN)/%,$$@) + +# resolve test target short to its corresponding executable +$(TEST_TARGET_SHORT): $$(filter %$$@,$(TEST_EXE)) + +# rule to compile a test +$(TEST_EXE): $$(patsubst $(BIN)/%,$(OBJ)/$(THIS_DIR)/%.o,$$@) $(TEST_OBJS) $(PBC_OBJS) | $(TESTS_EXE_DIR) + $(CC) $^ -o $@ $(LIBS) + +################################ general rule for compiling a list of source files to object files in the $(OBJ) directory + +$(OBJ)/$(THIS_DIR)/%.o: %.c | $(BUILD_DIR) + $(CC) $(CFLAGS) $(INCLUDE) -c $< -o $@ + +################################ general rule for making a build or bin directory +$(BUILD_DIR) $(BIN_DIR): + mkdir -p $@ + +# remove build artifacts +clean: + rm -f $(OBJS) + rm -f $(patsubst %.o,%.d,$(OBJS)) + rm -rf $(BIN) -performance/%: $(SRC_FILES) performance/%.c - mkdir -p $(BIN)/performance - $(CC) $^ -o $(BIN)/$@ $(LIBS) +-include $(patsubst %.o,%.d,$(OBJS)) diff --git a/tests/client/dev_handler_client.h b/tests/client/dev_handler_client.h index 5c0b02b4..7188d932 100644 --- a/tests/client/dev_handler_client.h +++ b/tests/client/dev_handler_client.h @@ -1,18 +1,18 @@ #ifndef DEV_CLIENT_H #define DEV_CLIENT_H -#include // for errno -#include // for SIGINT (Ctrl+C) -#include // for boolean types -#include // for int with specific widths -#include // for printf() -#include // for exit() -#include // for strerr() -#include // for sockets -#include // for sockaddr_un -#include // for waitpid() -#include // for read() -#include "../../logger/logger.h" // for log_printf +#include // for errno +#include // for log_printf +#include // for SIGINT (Ctrl+C) +#include // for boolean types +#include // for int with specific widths +#include // for printf() +#include // for exit() +#include // for strerr() +#include // for sockets +#include // for sockaddr_un +#include // for waitpid() +#include // for read() // Starts dev handler with "virtual" argument void start_dev_handler(); diff --git a/tests/client/executor_client.h b/tests/client/executor_client.h index aa97a7f3..c3686054 100644 --- a/tests/client/executor_client.h +++ b/tests/client/executor_client.h @@ -1,14 +1,14 @@ #ifndef EXEC_CLIENT_H #define EXEC_CLIENT_H -#include // for errno -#include // for SIGINT -#include // for printf, stderr -#include // for malloc, free, realloc, exit -#include // for strcat, strcpy -#include // for waitpid, pid_t -#include // for execlp, fork, chdir -#include "../../logger/logger.h" // for log_printf +#include // for errno +#include // for log_printf +#include // for SIGINT +#include // for printf, stderr +#include // for malloc, free, realloc, exit +#include // for strcat, strcpy +#include // for waitpid, pid_t +#include // for execlp, fork, chdir /** * Spawns the executor running student_code diff --git a/tests/client/net_handler_client.h b/tests/client/net_handler_client.h index dd2f8082..4ede0708 100644 --- a/tests/client/net_handler_client.h +++ b/tests/client/net_handler_client.h @@ -1,10 +1,10 @@ #ifndef NET_CLIENT_H #define NET_CLIENT_H +#include // for log_printf +#include #include #include -#include "../../logger/logger.h" // for log_printf -#include "../../net_handler/net_util.h" /** * Helper function to print out contents of a DevData message diff --git a/tests/client/shm_client.h b/tests/client/shm_client.h index 2d1b8c0c..f4d7d175 100644 --- a/tests/client/shm_client.h +++ b/tests/client/shm_client.h @@ -1,9 +1,9 @@ #ifndef SHM_CLIENT_H #define SHM_CLIENT_H +#include // for log_printf +#include #include -#include "../../logger/logger.h" // for log_printf -#include "../../shm_wrapper/shm_wrapper.h" /** * Creates and initializes the shared memory process diff --git a/tests/client/virtual_devices/virtual_device_util.h b/tests/client/virtual_devices/virtual_device_util.h index 38ddc0a1..a6371c4f 100644 --- a/tests/client/virtual_devices/virtual_device_util.h +++ b/tests/client/virtual_devices/virtual_device_util.h @@ -5,8 +5,8 @@ #ifndef VIRTUAL_DEV_UTIL_H #define VIRTUAL_DEV_UTIL_H -#include "../../../dev_handler/message.h" -#include "../../../runtime_util/runtime_util.h" +#include +#include /** * Builds an ACKNOWLEDGEMENT message. diff --git a/tests/test.c b/tests/test.c index 09cae616..53dfa292 100644 --- a/tests/test.c +++ b/tests/test.c @@ -833,9 +833,9 @@ void check_latency(uint64_t uid, int32_t upper_bound_latency, uint64_t start_tim print_fail(); fprintf_delimiter(stderr, "Elapsed time is negative!"); fprintf_delimiter(stderr, "Got:"); - fprintf(stderr, "Elapsed time == %d\n", param_name, elapsed_time); - fprintf(stderr, "Start time(shortened) == %d\n", param_name, start_time_shortened); - fprintf(stderr, "End time == %d\n", param_name, end_time.p_i); + fprintf(stderr, "Elapsed time == %d\n", elapsed_time); + fprintf(stderr, "Start time(shortened) == %d\n", start_time_shortened); + fprintf(stderr, "End time == %d\n", end_time.p_i); fail_test(); } print_pass(); From e0117fd688d15607822ad21c53b571deb73e9ffa Mon Sep 17 00:00:00 2001 From: Ben Liao Date: Thu, 20 Jul 2023 09:51:58 +0000 Subject: [PATCH 08/13] [BUILD] Added help text for tests/Makefile - decided top-level Makefile not needed; just use the root runtime script --- Makefile | 14 ------ dev_handler/Makefile | 2 +- executor/Makefile | 2 +- net_handler/Makefile | 2 +- network_switch/Makefile | 2 +- runtime | 10 ++++- shm_wrapper/Makefile | 2 +- tests/Makefile | 99 +++++++++++++++++++++++++++++++---------- 8 files changed, 89 insertions(+), 44 deletions(-) delete mode 100644 Makefile diff --git a/Makefile b/Makefile deleted file mode 100644 index 93fcdfb1..00000000 --- a/Makefile +++ /dev/null @@ -1,14 +0,0 @@ -# Starting point for building stuff in Runtime from the root directory - -all: - cd dev_handler && make - cd network_switch && make - cd shm_wrapper && make - cd net_handler && make - cd executor && make - -clean: - # executor needs to take care of its Cython build products - cd executor && make clean - rm -rf bin - rm -rf build diff --git a/dev_handler/Makefile b/dev_handler/Makefile index 9608101e..1c4d734f 100644 --- a/dev_handler/Makefile +++ b/dev_handler/Makefile @@ -11,7 +11,7 @@ TARGET = dev_handler all: $(TARGET) -# include top-level Makefile +# include top-level Makefile for some common variables and rules include ../Makefile_defs # resolve phony target "dev_handler" as ../bin/dev_handler diff --git a/executor/Makefile b/executor/Makefile index 505109d9..a8fc2db3 100755 --- a/executor/Makefile +++ b/executor/Makefile @@ -16,7 +16,7 @@ TARGET = executor all: $(TARGET) studentapi.c -# include top-level Makefile +# include top-level Makefile for common variables and rules include ../Makefile_defs $(TARGET): $(BIN)/$(TARGET) diff --git a/net_handler/Makefile b/net_handler/Makefile index a02cc77d..b3a10908 100644 --- a/net_handler/Makefile +++ b/net_handler/Makefile @@ -28,7 +28,7 @@ TARGET = net_handler all: protos $(TARGET) -# include top-level Makefile +# include top-level Makefile for some common variables and rules include ../Makefile_defs # resolve phony target "protos" as make all protobuf object files diff --git a/network_switch/Makefile b/network_switch/Makefile index d3b6921e..9bb88bb2 100644 --- a/network_switch/Makefile +++ b/network_switch/Makefile @@ -11,7 +11,7 @@ TARGET = network_switch all: $(TARGET) -# include top-level Makefile +# include top-level Makefile for some common variables and rules include ../Makefile_defs # resolve phony target "network_switch" as ../bin/network_switch diff --git a/runtime b/runtime index bf25610b..c6f786d3 100755 --- a/runtime +++ b/runtime @@ -14,11 +14,17 @@ cd $RUNTIME_DIR case $1 in build) - make + cd dev_handler && make && cd .. + cd net_handler && make && cd .. + cd executor && make && cd .. + cd network_switch && make && cd .. + cd shm_wrapper && make && cd .. ;; clean) - make clean + cd executor && make clean && cd .. # executor needs to take care of Cython build products + rm -rf build + rm -rf bin rm -f /tmp/log-fifo rm -f /tmp/ttyACM* ;; diff --git a/shm_wrapper/Makefile b/shm_wrapper/Makefile index fd64c840..43b93e6c 100644 --- a/shm_wrapper/Makefile +++ b/shm_wrapper/Makefile @@ -11,7 +11,7 @@ TARGETS = shm_start shm_stop all: $(TARGETS) -# include top-level Makefile +# include top-level Makefile for some common variables and rules include ../Makefile_defs shm_start: $(BIN)/shm_start diff --git a/tests/Makefile b/tests/Makefile index f08f1742..bcce8b1e 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,12 +1,15 @@ -# list of libraries that tests need to compile -LIBS=-pthread -lrt -Wall -lprotobuf-c -lncurses -Wno-format +###################################### compiler flags +# libraries needed to compile the executables +LIBS=-pthread -lrt -Wall -lprotobuf-c -lncurses -Wno-format +# specify the compiler we use (gcc) CC = gcc +# add CFLAGS to auto-generate dependency files CFLAGS = -MMD -MP ###################################### protobuf defs -# getting paths to protobuf because needed for compilation but this Makefile will NOT compile them +# getting paths to protobuf because needed for compilation but this Makefile will NOT compile them (compile them in net_handler!) PROTO = ../net_handler/protos PBC_OBJ = $(BUILD)/pbc_obj PBC_GEN = ../net_handler/pbc_gen @@ -15,12 +18,13 @@ PROTOS = $(shell cd $(PROTO) && ls *.proto && cd ..) # list all the protos PBC_SRCS = $(patsubst %.proto,$(PBC_GEN)/%.pb-c.c, $(PROTOS)) # list of all generated source files by doing a path substitution PBC_OBJS = $(patsubst $(PBC_GEN)/%.c,$(PBC_OBJ)/%.o, $(PBC_SRCS)) # list of all generated object files by doing a path substitution +# add generated protobuf headers to list of folders to include in compilation INCLUDE = -I$(PBC_GEN) -CFLAGS += $(shell pkg-config --cflags libprotobuf-c) # append more CFLAGS for compiling protobuf to existing CFLAGS +CFLAGS += $(shell pkg-config --cflags libprotobuf-c) # append more CFLAGS for compiling protobuf #################################### name directories and lists of directories -# tests will have its own bin directory (tests/bin) +# tests will have its own bin directory (tests/bin), but will use the global build directory for object files (../build) BIN = bin BUILD = ../build OBJ = $(BUILD)/obj @@ -37,19 +41,27 @@ BIN_DIR = $(BIN) ##################################### make lists of files -# specify files that various executables are dependent on +# list of source files that various executables are dependent on UTIL_SRCS = ../runtime_util/runtime_util.c ../logger/logger.c # everybody uses these NET_HANDLER_CLI_SRCS = cli/net_handler_cli.c client/net_handler_client.c cli/keyboard_interface.c ../net_handler/net_util.c $(UTIL_SRCS) SHM_UI_SRCS = cli/shm_ui.c client/shm_client.c ../shm_wrapper/shm_wrapper.c $(UTIL_SRCS) EXECUTOR_CLI_SRCS = cli/executor_cli.c client/executor_client.c $(UTIL_SRCS) DEV_HANDLER_CLI_SRCS = client/dev_handler_client.c cli/dev_handler_cli.c $(UTIL_SRCS) + +# list of source files that each virtual device has as a dependency VIRTUAL_DEV_SRCS = client/virtual_devices/virtual_device_util.c ../dev_handler/dev_handler_message.c $(UTIL_SRCS) + +# list of source files that each test has as a dependency TESTS_SRCS = test.c $(wildcard client/*.c) ../net_handler/net_util.c ../shm_wrapper/shm_wrapper.c $(UTIL_SRCS) +# list of relative paths to virtual device source files from this directory (e.g. client/virtual_devices/GeneralTestDevice.c) VIRTUAL_DEVICES = $(wildcard client/virtual_devices/*Device.c) + +# list of relative paths to test source files from this directory (e.g. integration/tc_150_1.c) TESTS = $(wildcard integration/* performance/*) -# generate lists of object files from the lists of source files above +# generate lists of object files from the lists of source files above using path substituion +# i.e. if a source file is client/net_handler_client.c, we substitute to obtain ../build/obj/tests/client/net_handler_client.o) NET_HANDLER_CLI_OBJS = $(patsubst %.c,$(OBJ)/$(THIS_DIR)/%.o,$(NET_HANDLER_CLI_SRCS)) SHM_UI_OBJS = $(patsubst %.c,$(OBJ)/$(THIS_DIR)/%.o,$(SHM_UI_SRCS)) EXECUTOR_CLI_OBJS = $(patsubst %.c,$(OBJ)/$(THIS_DIR)/%.o,$(EXECUTOR_CLI_SRCS)) @@ -64,26 +76,30 @@ BIN_DIR += $(VIRTUAL_DEV_EXE_DIR) $(TESTS_EXE_DIR) # specify targets (arguments you can give to make i.e. "make net_handler_cli" or "make tc_150_1" or "make GeneralTestDevice") CLI_TARGET = net_handler_cli executor_cli dev_handler_cli shm_ui -TEST_TARGET = $(patsubst %.c,%,$(TESTS)) -TEST_TARGET_SHORT = $(foreach test_target,$(TEST_TARGET),$(shell basename $(test_target))) # just tc_xx_xx -VIRTUAL_DEV_TARGET = $(patsubst client/virtual_devices/%.c,%,$(VIRTUAL_DEVICES)) +TEST_TARGET = $(patsubst %.c,%,$(TESTS)) # e.g. "integration/tc_150_1" +TEST_TARGET_SHORT = $(foreach test_target,$(TEST_TARGET),$(shell basename $(test_target))) # e.g. "tc_150_1" +VIRTUAL_DEV_TARGET = $(patsubst client/virtual_devices/%.c,%,$(VIRTUAL_DEVICES)) # e.g. "GeneralTestDevice" -# generate lists of executable names -CLI_EXE = $(patsubst %,$(BIN)/%,$(CLI_TARGET)) -TEST_EXE = $(patsubst %,$(BIN)/%,$(TEST_TARGET)) -VIRTUAL_DEV_EXE = $(patsubst %,$(BIN)/virtual_devices/%,$(VIRTUAL_DEV_TARGET)) +# generate lists of executable names using path substitution +CLI_EXE = $(patsubst %,$(BIN)/%,$(CLI_TARGET)) # e.g. bin/net_handler_cli +TEST_EXE = $(patsubst %,$(BIN)/%,$(TEST_TARGET)) # e.g. bin/integration/tc_150_1 +VIRTUAL_DEV_EXE = $(patsubst %,$(BIN)/virtual_devices/%,$(VIRTUAL_DEV_TARGET)) # e.g. bin/virtual_devices/GeneralTestDevice # combine all source files and associated object files with each other into a list for .c -> .o rule SRCS = $(NET_HANDLER_CLI_SRCS) $(SHM_UI_SRCS) $(EXECUTOR_CLI_SRCS) $(DEV_HANDLER_CLI_SRCS) \ $(VIRTUAL_DEV_SRCS) $(TESTS_SRCS) $(VIRTUAL_DEVICES) $(TESTS) OBJS = $(patsubst %.c,$(OBJ)/$(THIS_DIR)/%.o,$(SRCS)) # generate list of all object files relative to this Makefile -#################################### rules begin here +#################################### RULES BEGIN HERE ################################## +# we need this to evaluate any rule that uses the name of the target in its list of dependencies +# i.e. any rule that contains double dollar signs "$$" .SECONDEXPANSION: -.PHONY: all cli tests devices clean $(CLI_TARGET) $(TEST_TARGET) $(TEST_TARGET_SHORT) $(VIRTUAL_DEV_TARGET) +# declare all of these targets as phony because these are not file names that exist +.PHONY: all cli tests devices clean help $(CLI_TARGET) $(TEST_TARGET) $(TEST_TARGET_SHORT) $(VIRTUAL_DEV_TARGET) +# targets that should be made when just "make" is run in this directory all: cli tests devices #################################### rules to compile cli @@ -91,8 +107,10 @@ all: cli tests devices # resolve phony target "cli" to list of all cli executables cli: $(CLI_EXE) +# resolve phony cli target to its corresponding executable, e.g. "make net_handler_cli" -> "make bin/net_handler_cli" $(CLI_TARGET): $(BIN)/$$@ +# below are four rules to compile the CLIs (too lazy to combine them into one...) # net_handler_cli is special because it needs $(PBC_OBJS) $(BIN)/net_handler_cli: $(NET_HANDLER_CLI_OBJS) $(PBC_OBJS) | $(BIN) $(CC) $^ -o $@ $(LIBS) @@ -106,46 +124,81 @@ $(BIN)/dev_handler_cli: $(DEV_HANDLER_CLI_OBJS) | $(BIN) $(BIN)/shm_ui: $(SHM_UI_OBJS) | $(BIN) $(CC) $^ -o $@ $(LIBS) -################################# general rule to compile a virtual device +################################## rules to compile virtual device # resolve phony target "devices" to list of all virtual device executables devices: $(VIRTUAL_DEV_EXE) -# resolve virtual device target to its corresponding executable +# resolve phony virtual device target to its corresponding executable, +# e.g. "make GeneralTestDevice" -> "make bin/virtual_devices/GeneralTestDevice" $(VIRTUAL_DEV_TARGET): $$(patsubst %,$(BIN)/virtual_devices/%,$$@) # rule to compile a virtual device executable $(VIRTUAL_DEV_EXE): $$(patsubst $(BIN)/%,$(OBJ)/$(THIS_DIR)/client/%.o,$$@) $(VIRTUAL_DEV_OBJS) | $(VIRTUAL_DEV_EXE_DIR) $(CC) $^ -o $@ $(LIBS) -################################# general rule to compile a test +################################# rules to compile test # resolve phony target "tests" to list of all test executables tests: $(TEST_EXE) -# resolve test target to its corresponding executable +# resolve phony test target to its corresponding executable, +# e.g. "make integration/tc_150_1" -> "make bin/integration/tc_150_1" $(TEST_TARGET): $$(patsubst %,$(BIN)/%,$$@) -# resolve test target short to its corresponding executable +# resolve phony test target short to its corresponding executable +# e.g. "make tc_150_1" -> "make bin/integration/tc_150_1" $(TEST_TARGET_SHORT): $$(filter %$$@,$(TEST_EXE)) -# rule to compile a test +# rule to compile a test executable $(TEST_EXE): $$(patsubst $(BIN)/%,$(OBJ)/$(THIS_DIR)/%.o,$$@) $(TEST_OBJS) $(PBC_OBJS) | $(TESTS_EXE_DIR) $(CC) $^ -o $@ $(LIBS) ################################ general rule for compiling a list of source files to object files in the $(OBJ) directory +# e.g. to make "../build/obj/tests/client/net_handler_client.o", it depends on +# "client/net_handler_client.c" and the build directories must be created, then run the command +# "$(CC) $(CFLAGS) $(INCLUDE) -c $< -o $@" to compile, which translates, in this case, to the following: +# [gcc][-MMD -MP ...][-I../runtime_util -I../net_handler/pbc_gen .....] -c [client/net_handler_client.c] +# -o [../build/obj/tests/client/net_handler_client.o] $(OBJ)/$(THIS_DIR)/%.o: %.c | $(BUILD_DIR) $(CC) $(CFLAGS) $(INCLUDE) -c $< -o $@ ################################ general rule for making a build or bin directory + $(BUILD_DIR) $(BIN_DIR): mkdir -p $@ -# remove build artifacts +############################### remove build artifacts + clean: rm -f $(OBJS) rm -f $(patsubst %.o,%.d,$(OBJS)) rm -rf $(BIN) +############################## help text + +help: + @printf "\nYou can make CLIs, virtual devices, and tests using this Makefile\n\n" + @printf "\tCLIs:\n" + @printf "\t\tTo make all CLIs, run \"make cli\"\n" + @printf "\t\tTo make a specific CLI, you can run, for example, \"make net_handler_cli\"\n" + @printf "\t\tThe resulting executables will be in tests/bin/<*_cli>, e.g. \"tests/bin/net_handler_cli\"\n" + @printf "\tVirtual Devices:\n" + @printf "\t\tTo make all virtual devices, run \"make devices\"\n" + @printf "\t\tTo make a specific virtual device, you can run, for example, \"make GeneralTestDevice\"\n" + @printf "\t\tThe resulting executables will be in tests/bin/virtual_devices/*, e.g. \"tests/bin/virtual_devices/GeneralTestDevice\"\n" + @printf "\tTests:\n" + @printf "\t\tTo make all tests, run \"make tests\"\n" + @printf "\t\tTo make a specific test, you can run, for example \"make tc_150_1\" OR \"make integration/tc_150_1\"\n" + @printf "\t\tThe resulting executables will be in tests/bin/integration/* or tests/bin/performance/*,\n" + @printf "\t\t\te.g. \"tests/bin/integration/tc_150_1\" or \"tests/bin/performacne/tc_71_10\"\n" + @printf "\tAll:\n" + @printf "\t\tMake everything by running either \"make all\" or simply just \"make\"\n" + @printf "\tClean:\n" + @printf "\t\tRemove all build artificats (tests/bin, all object files, etc.) by running \"make clean\"\n\n\n" + +############################## include generated dependency files + -include $(patsubst %.o,%.d,$(OBJS)) + From c628ca44dfadfc4de949389befc536f2e2160272 Mon Sep 17 00:00:00 2001 From: Ben Liao Date: Thu, 20 Jul 2023 10:11:53 +0000 Subject: [PATCH 09/13] [MISC] Updated .gitignore, restore HTTP URL for protos submodule - Removed a bunch of irrelevant lines to reduce confusion - Having the SSH URL for protos as the module might cause problems - ... when building Runtime on Raspberry Pis so I reverted it --- .gitignore | 25 ++----------------------- .gitmodules | 2 +- executor/executor.c | 2 +- lowcar/devices/Device/Device.cpp | 2 +- net_handler/protos | 2 +- 5 files changed, 6 insertions(+), 27 deletions(-) diff --git a/.gitignore b/.gitignore index 4688eb29..064c6351 100644 --- a/.gitignore +++ b/.gitignore @@ -2,9 +2,6 @@ bin/* tests/bin/* -# ignore the tests/devices -tests/devices - # ignore the entire lowcar/Device folder lowcar/Device @@ -24,7 +21,7 @@ network_switch/exit_status.txt *.pyc *.pyo __pycache__/ -build/ +executor/build/ # VS Code .vscode @@ -38,29 +35,11 @@ build/ *.o *.obj -# Precompiled Headers -*.gch -*.pch - # Compiled Dynamic libraries *.so *.dylib *.dll *.pyd -# Fortran module files -*.mod -*.smod - -# Compiled Static libraries -*.lai -*.la -*.a -*.lib - -# Executables -*.exe -*.out -*.app - +# .DS_Store *.DS_Store diff --git a/.gitmodules b/.gitmodules index 2c03fcff..27d86d02 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "net_handler/protos"] path = net_handler/protos - url = git@github.com:pioneers/protos.git + url = https://github.com/pioneers/protos.git diff --git a/executor/executor.c b/executor/executor.c index 9030f072..6415c59e 100644 --- a/executor/executor.c +++ b/executor/executor.c @@ -14,7 +14,7 @@ // runtime includes #include // for runtime logger -#include //for runtime constants (TODO: consider removing relative pathname in include) +#include // for runtime constants #include // Shared memory wrapper to get/send device data diff --git a/lowcar/devices/Device/Device.cpp b/lowcar/devices/Device/Device.cpp index 396dad37..f76e9233 100644 --- a/lowcar/devices/Device/Device.cpp +++ b/lowcar/devices/Device/Device.cpp @@ -57,7 +57,7 @@ void Device::loop() { } else if (sts != Status::NO_DATA) { this->msngr->lowcar_printf("Error when reading message by lowcar device"); } - device_actions(); //[MOVED TO HERE] + device_actions(); // If it's been too long since we received a DEVICE_PING, disable the device // Send a message to Runtime that we will terminate the connection if (this->enabled && (this->timeout > 0) && (this->curr_time - this->last_received_ping_time >= this->timeout)) { diff --git a/net_handler/protos b/net_handler/protos index 8c15c7a7..9e52c0e5 160000 --- a/net_handler/protos +++ b/net_handler/protos @@ -1 +1 @@ -Subproject commit 8c15c7a7680860e0ea45d70df7e88b69c81a3984 +Subproject commit 9e52c0e5cd7ff7b83811145dfc93dbd5ea168f84 From 486715d5038a95524cb2a999a1e224e35eea04fd Mon Sep 17 00:00:00 2001 From: Ben Liao Date: Fri, 21 Jul 2023 03:05:49 +0000 Subject: [PATCH 10/13] [BUILD] Replaced format script with simpler, inhouse one (less bugs) - CI format job was causing issues, wrote own simpler format script --- .clang-format-ignore | 3 - .clang-tidy | 51 ---- .github/workflows/pull_request_ci.yml | 2 +- runtime | 3 +- scripts/format.sh | 62 ++++ scripts/run_clang_format.py | 421 -------------------------- 6 files changed, 65 insertions(+), 477 deletions(-) delete mode 100644 .clang-format-ignore delete mode 100644 .clang-tidy create mode 100755 scripts/format.sh delete mode 100755 scripts/run_clang_format.py diff --git a/.clang-format-ignore b/.clang-format-ignore deleted file mode 100644 index 0c5bb728..00000000 --- a/.clang-format-ignore +++ /dev/null @@ -1,3 +0,0 @@ -./lowcar/lib/* -./executor/studentapi.c -./net_handler/protos/* diff --git a/.clang-tidy b/.clang-tidy deleted file mode 100644 index 1b7bd9ef..00000000 --- a/.clang-tidy +++ /dev/null @@ -1,51 +0,0 @@ ---- -Checks: 'clang-diagnostic-*,clang-analyzer-*,google-*' -WarningsAsErrors: '' -HeaderFilterRegex: '' -AnalyzeTemporaryDtors: false -FormatStyle: file -User: pi -CheckOptions: - - key: google-build-namespaces.HeaderFileExtensions - value: ',h,hh,hpp,hxx' - - key: google-global-names-in-headers.HeaderFileExtensions - value: ',h,hh,hpp,hxx' - - key: google-readability-braces-around-statements.ShortStatementLines - value: '1' - - key: google-readability-function-size.BranchThreshold - value: '4294967295' - - key: google-readability-function-size.LineThreshold - value: '4294967295' - - key: google-readability-function-size.NestingThreshold - value: '4294967295' - - key: google-readability-function-size.ParameterThreshold - value: '4294967295' - - key: google-readability-function-size.StatementThreshold - value: '800' - - key: google-readability-function-size.VariableThreshold - value: '4294967295' - - key: google-readability-namespace-comments.ShortNamespaceLines - value: '10' - - key: google-readability-namespace-comments.SpacesBeforeComments - value: '2' - - key: google-runtime-int.SignedTypePrefix - value: int - - key: google-runtime-int.TypeSuffix - value: '' - - key: google-runtime-int.UnsignedTypePrefix - value: uint - - key: google-runtime-references.WhiteListTypes - value: '' - - key: modernize-loop-convert.MaxCopySize - value: '16' - - key: modernize-loop-convert.MinConfidence - value: reasonable - - key: modernize-loop-convert.NamingStyle - value: CamelCase - - key: modernize-pass-by-value.IncludeStyle - value: llvm - - key: modernize-replace-auto-ptr.IncludeStyle - value: llvm - - key: modernize-use-nullptr.NullMacros - value: 'NULL' -... diff --git a/.github/workflows/pull_request_ci.yml b/.github/workflows/pull_request_ci.yml index c4282927..275d4685 100644 --- a/.github/workflows/pull_request_ci.yml +++ b/.github/workflows/pull_request_ci.yml @@ -92,7 +92,7 @@ jobs: # Before we build Runtime, we check the formatting - name: Format run: | - ./runtime format -r . + ./runtime format check # Now that we are done installing Runtime's dependencies, we build Runtime - name: Build diff --git a/runtime b/runtime index c6f786d3..0b878cba 100755 --- a/runtime +++ b/runtime @@ -44,7 +44,8 @@ case $1 in format) shift 1 - scripts/run_clang_format.py $@ + scripts/format.sh $@ +# scripts/run_clang_format.py $@ ;; flash) diff --git a/scripts/format.sh b/scripts/format.sh new file mode 100755 index 00000000..7569460a --- /dev/null +++ b/scripts/format.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +# script to run formatting checks on runtime +# assume pwd is $HOME/runtime + +# function to remove output files +function clean_up { + rm -f clang_suggestions.txt + rm -f diff_output.txt +} + +# make sure to remove output.txt if Ctrl-C is pressed +trap 'clean_up' INT + +# get a list of all runtime files that we could want to check +# i.e. all .c, .cpp, and .h files +ALL_RUNTIME_FILES=$(find -name "*.c" -o -name "*.cpp" -o -name "*.h") + +# remove unwanted files +for file in $ALL_RUNTIME_FILES; do + if [[ $file == *"net_handler/pbc_gen"* ]]; then + continue + elif [[ $file == *"lowcar/lib"* ]]; then + continue + elif [[ $file == *"executor/studentapi.c"* ]]; then + continue + fi + FILES_TO_CHECK+="$file"$'\n' # append a newline to the end of each file name +done + +# at this point, $FILES_TO_CHECK has a list of files, separated by newlines, +# of files that we want to run format checks on + +exit_code=0 # set default exit code (0 = passing) + +# depending on argument, either check all files and fail if not passing, or fix all the files +if [[ $1 == "check" ]]; then + # for each file in list of files, run clang format and dump the output to a file + # then, run diff to obtain a diff between the output and the actual file + # if there is anything in the diff, the file did not pass the format check, so we set + # the exit code to 1 + for file in $FILES_TO_CHECK; do + clang-format --style=file $file > clang_suggestions.txt + diff -u $file clang_suggestions.txt > diff_output.txt + cat diff_output.txt # dumps diff output to stdout + + # sets exit code to 1 if there is a diff + if [[ $(cat diff_output.txt) != "" ]]; then + exit_code=1 + fi + done +elif [[ $1 == "fix" ]]; then + # fixes the errors in all the files + clang-format --style=file -i $FILES_TO_CHECK +else + printf "Specify one of \"check\" or \"fix\" as first and only argument! Exiting...\n" + exit_code=2 +fi + +clean_up + +exit $exit_code diff --git a/scripts/run_clang_format.py b/scripts/run_clang_format.py deleted file mode 100755 index 0122c5e8..00000000 --- a/scripts/run_clang_format.py +++ /dev/null @@ -1,421 +0,0 @@ -#!/usr/bin/env python3 -""" -A wrapper script around clang-format, suitable for linting multiple files -and to use for continuous integration. - -This is an alternative API for the clang-format command line. -It runs over multiple files and directories in parallel. -A diff output is produced and a sensible exit code is returned. - -Code copied directly from https://github.com/Sarcasm/run-clang-format. -Has a few modifications like the DefaultHelpParser and the -s STYLE option. - -""" - -from __future__ import print_function, unicode_literals - -import argparse -import codecs -import difflib -import fnmatch -import io -import errno -import multiprocessing -import os -import signal -import subprocess -import sys -import traceback - -from functools import partial - -try: - from subprocess import DEVNULL # py3k -except ImportError: - DEVNULL = open(os.devnull, "wb") - - -DEFAULT_EXTENSIONS = 'c,h,C,H,cpp,hpp,cc,hh,c++,h++,cxx,hxx,proto' -DEFAULT_CLANG_FORMAT_IGNORE = '.clang-format-ignore' - - -class ExitStatus: - SUCCESS = 0 - DIFF = 1 - TROUBLE = 2 - -def excludes_from_file(ignore_file): - excludes = [] - try: - with io.open(ignore_file, 'r', encoding='utf-8') as f: - for line in f: - if line.startswith('#'): - # ignore comments - continue - pattern = line.rstrip() - if not pattern: - # allow empty lines - continue - excludes.append(pattern) - except EnvironmentError as e: - if e.errno != errno.ENOENT: - raise - return excludes - -def list_files(files, recursive=False, extensions=None, exclude=None): - if extensions is None: - extensions = [] - if exclude is None: - exclude = [] - - out = [] - for file in files: - if recursive and os.path.isdir(file): - for dirpath, dnames, fnames in os.walk(file): - fpaths = [os.path.join(dirpath, fname) for fname in fnames] - for pattern in exclude: - # os.walk() supports trimming down the dnames list - # by modifying it in-place, - # to avoid unnecessary directory listings. - dnames[:] = [ - x for x in dnames - if - not fnmatch.fnmatch(os.path.join(dirpath, x), pattern) - ] - fpaths = [ - x for x in fpaths if not fnmatch.fnmatch(x, pattern) - ] - for f in fpaths: - ext = os.path.splitext(f)[1][1:] - if ext in extensions: - out.append(f) - else: - out.append(file) - return out - - -def make_diff(file, original, reformatted): - return list( - difflib.unified_diff( - original, - reformatted, - fromfile='{}\t(original)'.format(file), - tofile='{}\t(reformatted)'.format(file), - n=3)) - - -class DiffError(Exception): - def __init__(self, message, errs=None): - super(DiffError, self).__init__(message) - self.errs = errs or [] - - -class UnexpectedError(Exception): - def __init__(self, message, exc=None): - super(UnexpectedError, self).__init__(message) - self.formatted_traceback = traceback.format_exc() - self.exc = exc - - -def run_clang_format_diff_wrapper(args, file): - try: - ret = run_clang_format_diff(args, file) - return ret - except DiffError: - raise - except Exception as e: - raise UnexpectedError('{}: {}: {}'.format(file, e.__class__.__name__, - e), e) - - -def run_clang_format_diff(args, file): - try: - with io.open(file, 'r', encoding='utf-8') as f: - original = f.readlines() - except IOError as exc: - raise DiffError(str(exc)) - - if args.in_place: - invocation = [args.clang_format_executable, '-i', file] - else: - invocation = [args.clang_format_executable, file] - - if args.style: - invocation.extend(['-style', args.style]) - - if args.dry_run: - print(" ".join(invocation)) - return [], [] - - # Use of utf-8 to decode the process output. - # - # Hopefully, this is the correct thing to do. - # - # It's done due to the following assumptions (which may be incorrect): - # - clang-format will returns the bytes read from the files as-is, - # without conversion, and it is already assumed that the files use utf-8. - # - if the diagnostics were internationalized, they would use utf-8: - # > Adding Translations to Clang - # > - # > Not possible yet! - # > Diagnostic strings should be written in UTF-8, - # > the client can translate to the relevant code page if needed. - # > Each translation completely replaces the format string - # > for the diagnostic. - # > -- http://clang.llvm.org/docs/InternalsManual.html#internals-diag-translation - # - # It's not pretty, due to Python 2 & 3 compatibility. - encoding_py3 = {} - if sys.version_info[0] >= 3: - encoding_py3['encoding'] = 'utf-8' - - try: - proc = subprocess.Popen( - invocation, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - **encoding_py3) - except OSError as exc: - raise DiffError( - "Command '{}' failed to start: {}".format( - subprocess.list2cmdline(invocation), exc - ) - ) - proc_stdout = proc.stdout - proc_stderr = proc.stderr - if sys.version_info[0] < 3: - # make the pipes compatible with Python 3, - # reading lines should output unicode - encoding = 'utf-8' - proc_stdout = codecs.getreader(encoding)(proc_stdout) - proc_stderr = codecs.getreader(encoding)(proc_stderr) - # hopefully the stderr pipe won't get full and block the process - outs = list(proc_stdout.readlines()) - errs = list(proc_stderr.readlines()) - proc.wait() - if proc.returncode: - raise DiffError( - "Command '{}' returned non-zero exit status {}".format( - subprocess.list2cmdline(invocation), proc.returncode - ), - errs, - ) - if args.in_place: - return [], errs - return make_diff(file, original, outs), errs - - -def bold_red(s): - return '\x1b[1m\x1b[31m' + s + '\x1b[0m' - - -def colorize(diff_lines): - def bold(s): - return '\x1b[1m' + s + '\x1b[0m' - - def cyan(s): - return '\x1b[36m' + s + '\x1b[0m' - - def green(s): - return '\x1b[32m' + s + '\x1b[0m' - - def red(s): - return '\x1b[31m' + s + '\x1b[0m' - - for line in diff_lines: - if line[:4] in ['--- ', '+++ ']: - yield bold(line) - elif line.startswith('@@ '): - yield cyan(line) - elif line.startswith('+'): - yield green(line) - elif line.startswith('-'): - yield red(line) - else: - yield line - - -def print_diff(diff_lines, use_color): - if use_color: - diff_lines = colorize(diff_lines) - if sys.version_info[0] < 3: - sys.stdout.writelines((l.encode('utf-8') for l in diff_lines)) - else: - sys.stdout.writelines(diff_lines) - - -def print_trouble(prog, message, use_colors): - error_text = 'error:' - if use_colors: - error_text = bold_red(error_text) - print("{}: {} {}".format(prog, error_text, message), file=sys.stderr) - - -class DefaultHelpParser(argparse.ArgumentParser): - def error(self, message: str): - self.print_help() - sys.stderr.write(f"\nERROR: {message}\n") - sys.exit(2) - -def main(): - parser = DefaultHelpParser(description=__doc__) - parser.add_argument( - '--clang-format-executable', - metavar='EXECUTABLE', - help='path to the clang-format executable', - default='clang-format') - parser.add_argument( - '--extensions', - help='comma separated list of file extensions (default: {})'.format( - DEFAULT_EXTENSIONS), - default=DEFAULT_EXTENSIONS) - parser.add_argument( - '-r', - '--recursive', - action='store_true', - help='run recursively over directories') - parser.add_argument( - '-d', - '--dry-run', - action='store_true', - help='just print the list of files') - parser.add_argument( - '-i', - '--in-place', - action='store_true', - help='format file instead of printing differences') - parser.add_argument('files', metavar='file', nargs='+') - parser.add_argument( - '-q', - '--quiet', - action='store_true', - help="disable output, useful for the exit code") - parser.add_argument( - '-j', - metavar='N', - type=int, - default=0, - help='run N clang-format jobs in parallel' - ' (default number of cpus + 1)') - parser.add_argument( - '--color', - default='auto', - choices=['auto', 'always', 'never'], - help='show colored diff (default: auto)') - parser.add_argument( - '-e', - '--exclude', - metavar='PATTERN', - action='append', - default=[], - help='exclude paths matching the given glob-like pattern(s)' - ' from recursive search') - parser.add_argument( - '-s', - '--style', - type=str, - default='', - help='formatting style to apply (file, LLVM, Google, Chromium, Mozilla, WebKit)') - - args = parser.parse_args() - - # use default signal handling, like diff return SIGINT value on ^C - # https://bugs.python.org/issue14229#msg156446 - signal.signal(signal.SIGINT, signal.SIG_DFL) - try: - signal.SIGPIPE - except AttributeError: - # compatibility, SIGPIPE does not exist on Windows - pass - else: - signal.signal(signal.SIGPIPE, signal.SIG_DFL) - - colored_stdout = False - colored_stderr = False - if args.color == 'always': - colored_stdout = True - colored_stderr = True - elif args.color == 'auto': - colored_stdout = sys.stdout.isatty() - colored_stderr = sys.stderr.isatty() - - version_invocation = [args.clang_format_executable, str("--version")] - try: - subprocess.check_call(version_invocation, stdout=DEVNULL) - except subprocess.CalledProcessError as e: - print_trouble(parser.prog, str(e), use_colors=colored_stderr) - return ExitStatus.TROUBLE - except OSError as e: - print_trouble( - parser.prog, - "Command '{}' failed to start: {}".format( - subprocess.list2cmdline(version_invocation), e - ), - use_colors=colored_stderr, - ) - return ExitStatus.TROUBLE - - retcode = ExitStatus.SUCCESS - - excludes = excludes_from_file(DEFAULT_CLANG_FORMAT_IGNORE) - excludes.extend(args.exclude) - - files = list_files( - args.files, - recursive=args.recursive, - exclude=excludes, - extensions=args.extensions.split(',')) - - if not files: - return - - njobs = args.j - if njobs == 0: - njobs = multiprocessing.cpu_count() + 1 - njobs = min(len(files), njobs) - - if njobs == 1: - # execute directly instead of in a pool, - # less overhead, simpler stacktraces - it = (run_clang_format_diff_wrapper(args, file) for file in files) - pool = None - else: - pool = multiprocessing.Pool(njobs) - it = pool.imap_unordered( - partial(run_clang_format_diff_wrapper, args), files) - pool.close() - while True: - try: - outs, errs = next(it) - except StopIteration: - break - except DiffError as e: - print_trouble(parser.prog, str(e), use_colors=colored_stderr) - retcode = ExitStatus.TROUBLE - sys.stderr.writelines(e.errs) - except UnexpectedError as e: - print_trouble(parser.prog, str(e), use_colors=colored_stderr) - sys.stderr.write(e.formatted_traceback) - retcode = ExitStatus.TROUBLE - # stop at the first unexpected error, - # something could be very wrong, - # don't process all files unnecessarily - if pool: - pool.terminate() - break - else: - sys.stderr.writelines(errs) - if outs == []: - continue - if not args.quiet: - print_diff(outs, use_color=colored_stdout) - if retcode == ExitStatus.SUCCESS: - retcode = ExitStatus.DIFF - if pool: - pool.join() - return retcode - - -if __name__ == '__main__': - sys.exit(main()) From 11db2bb7ac9c63f3aac0fd863d380d20414a7a19 Mon Sep 17 00:00:00 2001 From: Ben Liao Date: Fri, 21 Jul 2023 03:39:22 +0000 Subject: [PATCH 11/13] [BUILD] Fixing lingering errors in CI build - Some tests are spewing -Wno-format errors that should be suppressed - To address, added $(LIBS) to .o -> .c rule in tests/Makefile - Cython is complaining about not being able to import log levels - To address, renamed runtime.pxd -> studentapi.pxd --- executor/Makefile | 2 +- executor/{runtime.pxd => studentapi.pxd} | 0 executor/studentapi.pyx | 2 +- tests/Makefile | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename executor/{runtime.pxd => studentapi.pxd} (100%) diff --git a/executor/Makefile b/executor/Makefile index a8fc2db3..ec1f2cfe 100755 --- a/executor/Makefile +++ b/executor/Makefile @@ -26,7 +26,7 @@ $(BIN)/$(TARGET): $(OBJS) | $(BIN) $(CC) $(OBJS) -o $@ $(LIBS) $(PY_LIB) # rule to compile studentapi.c -studentapi.c: studentapi.pyx runtime.pxd setup.py +studentapi.c: studentapi.pyx studentapi.pxd setup.py $(PY_VER) setup.py build_ext -i # rule to compile testapi diff --git a/executor/runtime.pxd b/executor/studentapi.pxd similarity index 100% rename from executor/runtime.pxd rename to executor/studentapi.pxd diff --git a/executor/studentapi.pyx b/executor/studentapi.pyx index 10be066a..606008a2 100755 --- a/executor/studentapi.pyx +++ b/executor/studentapi.pyx @@ -1,7 +1,7 @@ # cython: nonecheck=True # from libc.stdint cimport * -from runtime cimport * +from studentapi cimport * from cpython.mem cimport PyMem_Malloc, PyMem_Free from libc.string cimport strcmp diff --git a/tests/Makefile b/tests/Makefile index bcce8b1e..b7844b37 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -162,7 +162,7 @@ $(TEST_EXE): $$(patsubst $(BIN)/%,$(OBJ)/$(THIS_DIR)/%.o,$$@) $(TEST_OBJS) $(PBC # [gcc][-MMD -MP ...][-I../runtime_util -I../net_handler/pbc_gen .....] -c [client/net_handler_client.c] # -o [../build/obj/tests/client/net_handler_client.o] $(OBJ)/$(THIS_DIR)/%.o: %.c | $(BUILD_DIR) - $(CC) $(CFLAGS) $(INCLUDE) -c $< -o $@ + $(CC) $(CFLAGS) $(INCLUDE) -c $< -o $@ $(LIBS) ################################ general rule for making a build or bin directory From c82d403efb67eff0a576cc0bbc5aabcec30e467a Mon Sep 17 00:00:00 2001 From: Ben Liao Date: Fri, 21 Jul 2023 03:54:18 +0000 Subject: [PATCH 12/13] [TEST] Modified scripts/test.sh to take ./runtime test tc_150_1 as argument - Previously, you had to write ./runtime test integration/tc_150_1 - Also deleted provisioning and installation scripts for Runtime - Updated help text on root runtime script --- runtime | 8 ++---- scripts/install_raspi.sh | 58 -------------------------------------- scripts/provision_raspi.sh | 39 ------------------------- scripts/test.sh | 13 +++++++-- 4 files changed, 13 insertions(+), 105 deletions(-) delete mode 100755 scripts/install_raspi.sh delete mode 100755 scripts/provision_raspi.sh diff --git a/runtime b/runtime index 0b878cba..cfa06f90 100755 --- a/runtime +++ b/runtime @@ -45,7 +45,6 @@ case $1 in format) shift 1 scripts/format.sh $@ -# scripts/run_clang_format.py $@ ;; flash) @@ -66,7 +65,6 @@ case $1 in printf "commands:\n" printf "\tbuild\t\tRuns 'make' on files needed to run Runtime.\n" - printf "\t\t\tOptional: pass in 'make' arguments. (Ex: runtime make shm_wrapper)\n" printf "\tclean\t\tRuns 'make clean' for Runtime and removes Runtime files in '/tmp/'\n" @@ -75,11 +73,11 @@ case $1 in printf "\tstop\t\tStops Runtime.\n" printf "\ttest\t\tRuns all Runtime tests.\n" - printf "\t\t\tOptional: Pass in a specific test case. (Ex: runtime test integration/tc_71_14)\n" + printf "\t\t\tOptional: Pass in a specific test case. (Ex: runtime test tc_71_2)\n" printf "\tformat\t\tUses Clang to format the repo.\n" - printf "\t\t\tUse 'runtime format -ir .' to format the repo and save the changes.\n" - printf "\t\t\tRun without args to see full list of options.\n" + printf "\t\t\tUse 'runtime format fix' to format the repo and save the changes.\n" + printf "\t\tUse 'runtime format check' to check if the code in the repo conforms to format standards.\n" printf "\tflash\t\tFlashes a plugged in Arduino with Lowcar code.\n" printf "\t\t\tEx: 'runtime flash DummyDevice'\n" diff --git a/scripts/install_raspi.sh b/scripts/install_raspi.sh deleted file mode 100755 index a426463e..00000000 --- a/scripts/install_raspi.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash - -# This script/document automates *most* of the process for installing Runtime on a blank Raspberry Pi -# that has only the non-GUI Raspbian image copied onto its SD card. To copy that image, do the following: -# - https://www.raspberrypi.org/software/operating-systems/ Download the "Raspberry Pi OS Lite" -# - Unzip the zip on your computer -# - On a Mac, follow these instructions: https://www.raspberrypi.org/documentation/installation/installing-images/mac.md -# - Before ejecting, go to: -# --> cd /Volumes/boot -# --> touch ssh -# --> cd .. -# - Then eject. Those steps enable ssh'ing into the rapsberry pi - -# ssh into Raspberry Pi - -# Change date -# --> sudo date -s "20 MAR 2021 14:23:50" <-- (for example) -# Change timezone -# --> sudo raspi-config; select Localisation Options -# Connect to WiFi -# --> sudo raspi-config; select Network Options - -# update apt-get, install git, clone runtime onto Raspi -sudo apt-get update -sudo apt-get -y install git -git clone https://github.com/pioneers/runtime.git - -# install Python devtools and Cython -sudo apt-get -y install python3-dev -sudo apt-get -y install python3-pip -python3 -m pip install Cython - -# install protobuf -wget https://github.com/protocolbuffers/protobuf/releases/download/v3.15.1/protobuf-cpp-3.15.1.tar.gz -tar -xzf protobuf-cpp-3.15.1.tar.gz -cd protobuf-3.15.1 && ./configure && make && sudo make install && sudo ldconfig - -# install protobuf-c -wget https://github.com/protobuf-c/protobuf-c/releases/download/v1.3.3/protobuf-c-1.3.3.tar.gz -tar -xzf protobuf-c-1.3.3.tar.gz -cd protobuf-c-1.3.3 && ./configure && make && sudo make install && sudo ldconfig - -# install other Runtime dependencies with apt-get -sudo apt-get -y install libncurses5-dev libncursesw5-dev -sudo apt-get -y install clang-format - -# build runtime -cd runtime && ./runtime build - -# enable systemd services by doing the commands in runtime/systemd README -# (you can't put these commands in scripts) -# --> cd runtime/systemd -# --> sudo systemctl disable *.service -# --> sudo ln -s $HOME/runtime/systemd/*.service /etc/systemd/system/ -# --> sudo systemctl enable *.service - -# reboot machine: -# --> sudo reboot now \ No newline at end of file diff --git a/scripts/provision_raspi.sh b/scripts/provision_raspi.sh deleted file mode 100755 index 0413043a..00000000 --- a/scripts/provision_raspi.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -# This script/document automates *most* of the process for copying the SD card of a Raspberry Pi with a -# working Runtime on it to other SD cards for more Raspberry Pi's. - -# First step (only need to do this once): copy working SD card off of Raspberry Pi onto machine -# - On a mac, follow these instructions: https://www.raspberrypi.org/documentation/installation/installing-images/mac.md -# - IMPORTANT! Since you're copying FROM the Raspberry Pi TO your computer, for the dd command, switch the if=/ and of=/, i.e. -# --> sudo dd bs=1m if=/dev/rdiskN of=path_of_your_image.img; sync . Name the image whatever you want - -# Repeat these steps for each Raspberry Pi to be provisioned: -sudo dd bs=1m if=path_of_your_image.img of=/dev/rdiskN; sync # where path_of_your_image is from above and N is the number of the SD card in diskutil list - -# Insert SD card into Raspberry Pi, plug in and ssh in. It will have the same hostname as the original Raspberry Pi - -# Change date -# --> sudo date -s "20 MAR 2021 14:23:50" <-- (for example) -# Change timezone -# --> sudo raspi-config; select Localisation Options -# Ensure connection to WiFi (e.g. do ping google.com) - -sudo rm -rf runtime -git clone https://github.com/pioneers/runtime.git -cd runtime && ./runtime build - -# enable systemd services by doing the commands in runtime/systemd README -# (you can't put these commands in scripts) -# --> cd runtime/systemd -# --> sudo systemctl disable *.service -# --> sudo ln -s $HOME/runtime/systemd/*.service /etc/systemd/system/ -# --> sudo systemctl enable *.service - -# if using ngrok, change the authtoken in runtime/systemd/ngrok.yml - -# TODO: Put something here to change the hostname to the argument to the script -# --> sudo hostname - -# reboot machine: -# --> sudo reboot now \ No newline at end of file diff --git a/scripts/test.sh b/scripts/test.sh index 71af787a..4045c692 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -10,7 +10,7 @@ function clean_up { function sigint_handler { # if logger/logger.config.orig exists, we need to restore it if [[ -f ../logger/logger.config.orig ]]; then - clean_up + clean_up fi exit 1 } @@ -38,12 +38,19 @@ function run_tests { if [[ $? != 0 ]]; then failing_tests="$failing_tests $test" # add this test to list of failing tests failed=1 - continue + continue fi # run test printf "Running $test...\n" - ./bin/$test + + if [[ -f bin/integration/$test ]]; then + ./bin/integration/$test + elif [[ -f bin/performance/$test ]]; then + ./bin/performance/$test + else + ./bin/$test + fi # if that test failed, set failed to 1 if [[ $? != 0 ]]; then From 4c8ebec7722e337a4198fad097aad63126dee7e7 Mon Sep 17 00:00:00 2001 From: Ben Liao Date: Tue, 25 Jul 2023 18:16:32 -0700 Subject: [PATCH 13/13] [BUILD] Moved comments in Makefile_defs to above lines --- Makefile_defs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Makefile_defs b/Makefile_defs index 5fb7fdca..708053c4 100644 --- a/Makefile_defs +++ b/Makefile_defs @@ -10,11 +10,15 @@ OBJ = $(BUILD)/obj # this is the name of the lower level directory (e.g. THIS_DIR in the dev_handler folder is "dev_handler") THIS_DIR = $(strip $(shell pwd | xargs basename -z)) -COMPONENT_DIRS = dev_handler net_handler executor network_switch shm_wrapper runtime_util logger # list of runtime component directories -OBJ_SUBDIR = $(foreach dir,$(COMPONENT_DIRS),$(OBJ)/$(dir)) # list of subfolders in $(OBJ) for runtime object files -INCLUDE += $(foreach dir,$(COMPONENT_DIRS),-I../$(dir)) # list of folders to include in compilation commands - -OBJS = $(patsubst %.c,$(OBJ)/$(THIS_DIR)/%.o,$(SRCS)) # make a list of all object files this executable is dependent on relative to this Makefile +# list of runtime component directories +COMPONENT_DIRS = dev_handler net_handler executor network_switch shm_wrapper runtime_util logger + # list of subfolders in $(OBJ) for runtime object files +OBJ_SUBDIR = $(foreach dir,$(COMPONENT_DIRS),$(OBJ)/$(dir)) +# list of folders to include in compilation commands +INCLUDE += $(foreach dir,$(COMPONENT_DIRS),-I../$(dir)) + +# list of all object files this executable is dependent on relative to this Makefile +OBJS = $(patsubst %.c,$(OBJ)/$(THIS_DIR)/%.o,$(SRCS)) # list of build folders BUILD_DIR += $(OBJ) $(BIN) $(BUILD) $(OBJ_SUBDIR)