Merge pull request #56 from Servostar/47-add-tests

47 add tests

.github/workflows/build-check-sdk.yaml

@@ -2,7 +2,7 @@ name: "Build check gemstone in SDK"
 run-name: SDK build check to ${{ inputs.deploy_target }} by @${{ github.actor }}
 on: [push, pull_request]
 env:
-  SDK: 0.1.0-alma-9.3
+  SDK: 0.2.2-alpine-3.19.1
 jobs:
   build-check-sdk:
     runs-on: ubuntu-latest
@@ -11,4 +11,4 @@ jobs:
     - name: Setup SDK
       run: docker pull servostar/gemstone:sdk-"$SDK" && docker build --tag gemstone:devkit-"$SDK" .
     - name: Compile
-      run: docker run gemstone:devkit-"$SDK" make check
+      run: docker run gemstone:devkit-"$SDK" sh run-check-test.sh

.gitignore

@@ -14,3 +14,4 @@ lexer.ll.c
 parser.tab.c
 parser.tab.h
 build
+/Testing/

CMakeLists.txt

@@ -21,6 +21,15 @@ project(gemstone
     DESCRIPTION "programming language compiler"
     LANGUAGES C)
 
+set(GEMSTONE_TEST_DIR ${PROJECT_SOURCE_DIR}/tests)
+set(GEMSTONE_BINARY_DIR ${PROJECT_SOURCE_DIR}/bin)
+
+include(CTest)
+
+if(BUILD_TESTING)
+    add_subdirectory(tests)
+endif()
+
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
 # ------------------------------------------------ #
@@ -36,6 +45,11 @@ add_custom_command(OUTPUT ${LEX_GENERATED_SOURCE_FILE}
         COMMENT "generate C source file for lexer"
         VERBATIM)
 
+# remove dependency when compiling with MSVC on windows
+if (MSVC)
+    add_compile_definitions(YY_NO_UNISTD_H)
+endif()
+
 # ------------------------------------------------ #
 #  Yacc                                            #
 # ------------------------------------------------ #
@@ -56,7 +70,11 @@ add_custom_command(OUTPUT ${YACC_GENERATED_SOURCE_FILE}
 file(GLOB_RECURSE SOURCE_FILES src/*.c)
 
 # define default compile flags
-set(FLAGS -Wall -Wextra -Wconversion -Wpedantic)
+if (MSVC)
+    set(FLAGS /Wall /W3 /permissive)
+else()
+    set(FLAGS -Wall -Wextra -Wconversion -Wpedantic)
+endif()
 
 # ------------------------------------------------ #
 #  Target RELEASE                                  #
@@ -70,7 +88,14 @@ add_executable(release
 set_target_properties(release
         PROPERTIES
         OUTPUT_NAME "gsc"
-        RUNTIME_OUTPUT_DIRECTORY "bin/release")
+        RUNTIME_OUTPUT_DIRECTORY ${GEMSTONE_BINARY_DIR}/release)
+
+# FIXME: cannot compile with /O2 because of /RTC1 flag
+if (MSVC)
+    set(RELEASE_FLAGS)
+else()
+    set(RELEASE_FLAGS -m64 -O3 -fprefetch-loop-arrays -mrecip)
+endif()
 
 # compiler flags targeting a 64-bit GCC release environment
 # flags:
@@ -78,7 +103,7 @@ set_target_properties(release
 #  - O3: optimization level 3
 #  - fprefetch-loop-arrays: pre load arrays used in loops by using prefetch instruction
 #  - mrecip: make use RCPSS and RSQRTSS instructions
-target_compile_options(release PUBLIC ${FLAGS} -m64 -O3 -fprefetch-loop-arrays -mrecip)
+target_compile_options(release PUBLIC ${FLAGS} ${RELEASE_FLAGS})
 
 # add src directory as include path
 target_include_directories(release PUBLIC src)
@@ -95,10 +120,16 @@ add_executable(debug
 set_target_properties(debug
         PROPERTIES
         OUTPUT_NAME "gsc"
-        RUNTIME_OUTPUT_DIRECTORY "bin/debug")
+        RUNTIME_OUTPUT_DIRECTORY ${GEMSTONE_BINARY_DIR}/debug)
+
+if (MSVC)
+    set(DEBUG_FLAGS /DEBUG)
+else()
+    set(DEBUG_FLAGS -g)
+endif()
 
 # compiler flags targeting a GCC debug environment
-target_compile_options(debug PUBLIC ${FLAGS} -g)
+target_compile_options(debug PUBLIC ${FLAGS} ${DEBUG_FLAGS})
 
 # add src directory as include path
 target_include_directories(debug PUBLIC src)
@@ -118,11 +149,17 @@ add_executable(check
 set_target_properties(check
         PROPERTIES
         OUTPUT_NAME "gsc"
-        RUNTIME_OUTPUT_DIRECTORY "bin/check")
+        RUNTIME_OUTPUT_DIRECTORY ${GEMSTONE_BINARY_DIR}/check)
+
+if (MSVC)
+    set(CHECK_FLAGS /DEBUG /WX)
+else()
+    set(DEBUG_FLAGS -g -Werror)
+endif()
 
 # compiler flags targeting a GCC debug environment
 # extra -Werror flag to treat warnings as error to make github action fail on warning
-target_compile_options(check PUBLIC ${FLAGS} -g -Werror)
+target_compile_options(check PUBLIC ${FLAGS} ${DEBUG_FLAGS})
 
 # add src directory as include path
 target_include_directories(check PUBLIC src)

Dockerfile

@@ -1,10 +1,12 @@
-FROM servostar/gemstone:sdk-0.2.0-alpine-3.19.1
+FROM servostar/gemstone:sdk-0.2.2-alpine-3.19.1
 LABEL authors="servostar"
-LABEL version="0.2.0"
+LABEL version="0.2.2"
 LABEL description="docker image for setting up the build pipeline on SDK"
 LABEL website="https://github.com/Servostar/gemstone"
 
 COPY --chown=lorang src /home/lorang/src
+COPY --chown=lorang tests /home/lorang/tests
 COPY --chown=lorang CMakeLists.txt /home/lorang/
+COPY --chown=lorang run-check-test.sh /home/lorang/
 
 RUN cmake .

README.md

@@ -2,6 +2,47 @@
 
 Gemstone is a programming language compiler written in C with lex and yacc.
 
+## Dependencies (build)
+
+### Windows 11
+
+For setup instruction see issue #30
+
+Requires:
+- Microsoft Build Tools 2022 (includes: CMake, MSVC)
+- WinFlexBison [find it here](https://github.com/lexxmark/winflexbison) (needs to be in PATH)
+
+### GNU/Linux
+
+Requires:
+- GCC
+- CMake
+- Make
+- bison
+- flex
+
+## Writing Tests
+
+Since the project is build and configured through CMake it makes sense to rely for tests
+on CTest. All tests are located in the subfolder `tests`. In this directory is a CMakeLists.txt which specifies which tests
+are to be run. Actual tests are located in folders within tests and contain a final CMakeLists.txt which specifies what to run 
+for a single test.
+
+```
+tests
+  └─ test_group1
+      └─ CMakeLists.txt  # specify tests in this group
+      └─ ...             # test files of group 1
+      
+  └─ test_group2
+      └─ CMakeLists.txt  # specify tests in this group
+      └─ ...             # test files of group 2
+      
+  └─ CMakeLists.txt      # specify test groups to run
+  
+CMakeLists.txt           # build configuration
+```
+
 ## Development with VSCode/Codium
 
 Recommended extensions for getting a decent experience are the following:

run-check-test.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env sh
+
+# Author: Sven Vogel
+# Created: 02.05.2024
+# Description: Builds the project and runs tests
+#              Returns 0 on success and 1 when something went wrong
+
+echo "+--------------------------------------+"
+echo "| BUILDING all TARGETS                 |"
+echo "+--------------------------------------+"
+
+make -B
+if [ ! $? -eq 0 ]; then
+  echo "===> failed to build targets"
+  exit 1
+fi
+
+echo "+--------------------------------------+"
+echo "| RUNNING CODE CHECK                   |"
+echo "+--------------------------------------+"
+
+make check
+if [ ! $? -eq 0 ]; then
+  echo "===> failed code check..."
+  exit 1
+fi
+
+echo "+--------------------------------------+"
+echo "| RUNNING TESTS                        |"
+echo "+--------------------------------------+"
+
+ctest -VV --output-on-failure --schedule-random -j 4
+if [ ! $? -eq 0 ]; then
+  echo "===> failed tests..."
+  exit 1
+fi
+
+echo "+--------------------------------------+"
+echo "| COMPLETED CHECK + TESTS SUCCESSFULLY |"
+echo "+--------------------------------------+"

sdk/Dockerfile

@@ -1,11 +1,11 @@
 FROM alpine:3.19.1
 LABEL authors="servostar"
-LABEL version="0.2.0"
+LABEL version="0.2.2"
 LABEL description="base image for building the gemstone programming language compiler"
 LABEL website="https://github.com/Servostar/gemstone"
 
 # install dependencies
-RUN apk add build-base gcc make cmake bison flex
+RUN apk add build-base gcc make cmake bison flex git python3
 
 # create user for build
 RUN adduser --disabled-password lorang

src/main.c

@@ -5,6 +5,8 @@
 
 #define LOG_LEVEL LOG_LEVEL_DEBUG
 
+extern FILE* yyin;
+
 /**
  * @brief Log a debug message to inform about beginning exit procedures
  * 
@@ -14,6 +16,19 @@ void notify_exit(void)
     DEBUG("Exiting gemstone...");
 }
 
+/**
+ * @brief Closes File after compiling.
+ * 
+ */
+
+void close_file(void)
+{
+    if (NULL != yyin)
+    {
+        fclose(yyin);
+    }
+}
+
 /**
  * @brief Run compiler setup here
  * 
@@ -34,8 +49,22 @@ void setup(void)
     DEBUG("finished starting up gemstone...");
 }
 
-int main(void) {
+int main(int argc, char *argv[]) {
+
     setup();
+    atexit(close_file);
+    
+    // Check for file input as argument
+    if (2 != argc)
+    {
+        INFO("Usage: %s <filename>\n", argv[0]);
+        PANIC("No File could be found");
+    }
+    
+    // filename as first argument
+    char *filename = argv[1];
+
+    FILE *file = fopen(filename, "r");
 
     struct AST_Node_t* node = AST_new_node(AST_Branch, NULL);
 

src/sys/log.h

@@ -24,7 +24,7 @@
 // generally not defined by GCC < 11.3 and MSVC
 #ifndef __FILE_NAME__
 #if defined(_WIN32) || defined(_WIN64) || defined(_MSC_VER)
-#define __FILE_NAME__ (strrstr(__FILE__, "\\") ? strrstr(__FILE__, "\\") + 1 : __FILE__)
+#define __FILE_NAME__ (strrchr(__FILE__, '\\') ? strrchr(__FILE__, '\\') + 1 : __FILE__)
 #else
 #define __FILE_NAME__ (strrchr(__FILE__, '/') ? strrchr(__FILE__, '/') + 1 : __FILE__)
 #endif

tests/CMakeLists.txt

@@ -0,0 +1,10 @@
+include(CTest)
+
+set(PROJECT_BINARY_DIR bin)
+set(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/tests)
+set(CTEST_BINARY_DIRECTORY ${PROJECT_BINARY_DIR}/tests)
+
+# Provide test to run here or include another CMakeLists.txt
+
+add_subdirectory(logging)
+add_subdirectory(input_file)

tests/input_file/CMakeLists.txt

@@ -0,0 +1,9 @@
+include(CTest)
+
+# ------------------------------------------------------- #
+# CTEST 1
+# test if the program accepts a file as input
+
+add_test(NAME input_file_check
+        WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/bin/check
+        COMMAND python ${GEMSTONE_TEST_DIR}/input_file/test_input_file.py ${GEMSTONE_TEST_DIR}/input_file/test.gem)

tests/input_file/test.gem

@@ -0,0 +1,6 @@
+
+import "std.io"
+
+fun main {
+    print("Hello, World!!!")
+}

tests/input_file/test_input_file.py

@@ -0,0 +1,36 @@
+import os.path
+import subprocess
+import sys
+import logging
+from logging import info
+
+
+def check_accept():
+    info("testing handling of input file...")
+
+    logging.basicConfig(level=logging.INFO)
+
+    test_file_name = sys.argv[1]
+
+    p = subprocess.run(["./gsc", test_file_name], capture_output=True, text=True)
+
+    assert p.returncode == 0
+
+
+def check_abort():
+    info("testing handling of missing input file...")
+
+    logging.basicConfig(level=logging.INFO)
+
+    p = subprocess.run("./gsc", capture_output=True, text=True)
+
+    assert p.returncode == 1
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    info("check if binary exists...")
+    assert os.path.exists("./gsc")
+
+    check_accept()
+    check_abort()

tests/logging/CMakeLists.txt

@@ -0,0 +1,63 @@
+include(CTest)
+
+include_directories(${PROJECT_SOURCE_DIR}/src)
+
+# ------------------------------------------------------- #
+# CTEST 1
+# test the default output of the logger
+
+add_executable(logging_output
+        ${PROJECT_SOURCE_DIR}/src/sys/log.c
+        output.c)
+set_target_properties(logging_output
+        PROPERTIES
+        OUTPUT_NAME "output"
+        RUNTIME_OUTPUT_DIRECTORY ${GEMSTONE_BINARY_DIR}/tests/logging)
+add_test(NAME logging_output
+        WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+        COMMAND python ${GEMSTONE_TEST_DIR}/logging/test_logging.py check_output)
+
+# ------------------------------------------------------- #
+# CTEST 2
+# test the panic functionality of the logger
+
+add_executable(logging_panic
+        ${PROJECT_SOURCE_DIR}/src/sys/log.c
+        panic.c)
+set_target_properties(logging_panic
+        PROPERTIES
+        OUTPUT_NAME "panic"
+        RUNTIME_OUTPUT_DIRECTORY ${GEMSTONE_BINARY_DIR}/tests/logging)
+add_test(NAME logging_panic
+        WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+        COMMAND python ${GEMSTONE_TEST_DIR}/logging/test_logging.py check_panic)
+
+# ------------------------------------------------------- #
+# CTEST 3
+# test the ability to write to multiple output streams
+
+add_executable(logging_streams
+        ${PROJECT_SOURCE_DIR}/src/sys/log.c
+        streams.c)
+set_target_properties(logging_streams
+        PROPERTIES
+        OUTPUT_NAME "stream"
+        RUNTIME_OUTPUT_DIRECTORY ${GEMSTONE_BINARY_DIR}/tests/logging)
+add_test(NAME logging_streams
+        WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+        COMMAND python ${GEMSTONE_TEST_DIR}/logging/test_logging.py check_stream)
+
+# ------------------------------------------------------- #
+# CTEST 4
+# test compile time log level switch
+
+add_executable(logging_level
+        ${PROJECT_SOURCE_DIR}/src/sys/log.c
+        level.c)
+set_target_properties(logging_level
+        PROPERTIES
+        OUTPUT_NAME "level"
+        RUNTIME_OUTPUT_DIRECTORY ${GEMSTONE_BINARY_DIR}/tests/logging)
+add_test(NAME logging_level
+        WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+        COMMAND python ${GEMSTONE_TEST_DIR}/logging/test_logging.py check_level)

tests/logging/level.c

@@ -0,0 +1,18 @@
+//
+// Created by servostar on 5/2/24.
+//
+
+#include "sys/log.h"
+
+#define LOG_LEVEL LOG_LEVEL_WARNING
+
+int main(void) {
+    log_init();
+
+    DEBUG("logging some debug...");
+    INFO("logging some info...");
+    WARN("logging some warning...");
+    ERROR("logging some error...");
+
+    return 0;
+}

tests/logging/output.c

@@ -0,0 +1,16 @@
+//
+// Created by servostar on 5/1/24.
+//
+
+#include "sys/log.h"
+
+int main(void) {
+    log_init();
+
+    DEBUG("logging some debug...");
+    INFO("logging some info...");
+    WARN("logging some warning...");
+    ERROR("logging some error...");
+
+    return 0;
+}

tests/logging/panic.c

@@ -0,0 +1,20 @@
+//
+// Created by servostar on 5/2/24.
+//
+
+#include "sys/log.h"
+
+int main(void) {
+    log_init();
+
+    // this should appear in stderr
+    INFO("before exit");
+
+    PANIC("oooops something happened");
+
+    // this should NOT appear in stderr
+    //             ^^^
+    ERROR("after exit");
+
+    return 0;
+}

tests/logging/streams.c

@@ -0,0 +1,33 @@
+//
+// Created by servostar on 5/2/24.
+//
+
+#include "sys/log.h"
+#include <stdlib.h>
+
+static FILE* file;
+
+void close_file(void) {
+    if (file != NULL) {
+        fclose(file);
+    }
+}
+
+int main(void) {
+    log_init();
+
+    // this should appear in stderr
+    INFO("should only be in stderr");
+
+    file = fopen("tmp/test.log", "w");
+    if (file == NULL) {
+        PANIC("could not open file");
+    }
+    atexit(close_file);
+
+    log_register_stream(file);
+
+    INFO("should be in both");
+
+    return 0;
+}

tests/logging/test_logging.py

@@ -0,0 +1,124 @@
+import subprocess
+import sys
+import logging
+from logging import info, error
+import os
+
+BIN_DIR = "bin/tests/logging/"
+
+
+def run_check_output():
+    info("started check output...")
+
+    p = subprocess.run(BIN_DIR + "output", capture_output=True, text=True)
+
+    info("checking exit code...")
+
+    # check exit code
+    assert p.returncode == 0
+
+    output = p.stderr
+
+    # check if logs appear in default log output (stderr)
+    info("checking stderr...")
+
+    assert "logging some debug..." in output
+    assert "logging some info..." in output
+    assert "logging some warning..." in output
+    assert "logging some error..." in output
+
+
+def run_check_level():
+    info("started check level...")
+
+    p = subprocess.run(BIN_DIR + "level", capture_output=True, text=True)
+
+    info("checking exit code...")
+
+    # check exit code
+    assert p.returncode == 0
+
+    output = p.stderr
+
+    # check if logs appear in default log output (stderr)
+    info("checking stderr...")
+
+    assert "logging some debug..." not in output
+    assert "logging some info..." not in output
+    assert "logging some warning..." in output
+    assert "logging some error..." in output
+
+
+def run_check_panic():
+    info("started check panic...")
+
+    p = subprocess.run(BIN_DIR + "panic", capture_output=True, text=True)
+
+    info("checking exit code...")
+
+    # check exit code
+    assert p.returncode == 1
+
+    output = p.stderr
+
+    # check if logs appear (not) in default log output (stderr)
+    info("checking stderr...")
+
+    assert "before exit" in output
+    assert "oooops something happened" in output
+    assert "after exit" not in output
+
+
+def run_check_stream():
+    info("started check panic...")
+
+    info("creating temporary folder...")
+
+    if not os.path.exists("tmp"):
+        os.mkdir("tmp")
+
+    info("cleaning temporary folder...")
+
+    if os.path.exists("tmp/test.log"):
+        os.remove("tmp/test.log")
+
+    info("launching test binary...")
+
+    p = subprocess.run(BIN_DIR + "stream", capture_output=True, text=True)
+
+    info("checking exit code...")
+
+    # check exit code
+    assert p.returncode == 0
+
+    with open("tmp/test.log", "r") as file:
+        assert "should be in both" in "".join(file.readlines())
+
+    output = p.stderr
+
+    # check if logs appear (not) in default log output (stderr)
+    info("checking stderr...")
+
+    assert "should only be in stderr" in output
+    assert "should be in both" in output
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+
+    target = sys.argv[1]
+
+    info(f"starting logging test suite with target: {target}")
+
+    match target:
+        case "check_output":
+            run_check_output()
+        case "check_panic":
+            run_check_panic()
+        case "check_stream":
+            run_check_stream()
+        case "check_level":
+            run_check_level()
+        case _:
+            error(f"unknown target: {target}")
+            exit(1)