Merge pull request #49 from Servostar/47-add-tests

47 add tests

.github/workflows/build-check-sdk.yaml

@@ -2,7 +2,7 @@ name: "Build check gemstone in SDK"
 run-name: SDK build check to ${{ inputs.deploy_target }} by @${{ github.actor }}
 on: [push, pull_request]
 env:
-  SDK: 0.2.1-alpine-3.19.1
+  SDK: 0.2.2-alpine-3.19.1
 jobs:
   build-check-sdk:
     runs-on: ubuntu-latest
@@ -11,4 +11,4 @@ jobs:
     - name: Setup SDK
       run: docker pull servostar/gemstone:sdk-"$SDK" && docker build --tag gemstone:devkit-"$SDK" .
     - name: Compile
-      run: docker run gemstone:devkit-"$SDK" make check
+      run: docker run gemstone:devkit-"$SDK" sh run-check-test.sh

.gitignore

@@ -13,4 +13,5 @@ Makefile
 lexer.ll.c
 parser.tab.c
 parser.tab.h
-build
+build
+/Testing/

CMakeLists.txt

@@ -21,6 +21,15 @@ project(gemstone
     DESCRIPTION "programming language compiler"
     LANGUAGES C)
 
+set(GEMSTONE_TEST_DIR ${PROJECT_SOURCE_DIR}/tests)
+set(GEMSTONE_BINARY_DIR ${PROJECT_SOURCE_DIR}/bin)
+
+include(CTest)
+
+if(BUILD_TESTING)
+    add_subdirectory(tests)
+endif()
+
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
 # ------------------------------------------------ #
@@ -79,7 +88,7 @@ add_executable(release
 set_target_properties(release
         PROPERTIES
         OUTPUT_NAME "gsc"
-        RUNTIME_OUTPUT_DIRECTORY "bin/release")
+        RUNTIME_OUTPUT_DIRECTORY ${GEMSTONE_BINARY_DIR}/release)
 
 # FIXME: cannot compile with /O2 because of /RTC1 flag
 if (MSVC)
@@ -111,7 +120,7 @@ add_executable(debug
 set_target_properties(debug
         PROPERTIES
         OUTPUT_NAME "gsc"
-        RUNTIME_OUTPUT_DIRECTORY "bin/debug")
+        RUNTIME_OUTPUT_DIRECTORY ${GEMSTONE_BINARY_DIR}/debug)
 
 if (MSVC)
     set(DEBUG_FLAGS /DEBUG)
@@ -140,7 +149,7 @@ add_executable(check
 set_target_properties(check
         PROPERTIES
         OUTPUT_NAME "gsc"
-        RUNTIME_OUTPUT_DIRECTORY "bin/check")
+        RUNTIME_OUTPUT_DIRECTORY ${GEMSTONE_BINARY_DIR}/check)
 
 if (MSVC)
     set(CHECK_FLAGS /DEBUG /WX)

Dockerfile

@@ -1,9 +1,12 @@
-FROM servostar/gemstone:sdk-0.2.1-alpine-3.19.1
+FROM servostar/gemstone:sdk-0.2.2-alpine-3.19.1
 LABEL authors="servostar"
-LABEL version="0.2.1"
+LABEL version="0.2.2"
 LABEL description="docker image for setting up the build pipeline on SDK"
 LABEL website="https://github.com/Servostar/gemstone"
 
-RUN git clone https://github.com/Servostar/gemstone.git /home/lorang
+COPY --chown=lorang src /home/lorang/src
+COPY --chown=lorang tests /home/lorang/tests
+COPY --chown=lorang CMakeLists.txt /home/lorang/
+COPY --chown=lorang run-check-test.sh /home/lorang/
 
 RUN cmake .

README.md

@@ -21,6 +21,28 @@ Requires:
 - bison
 - flex
 
+## Writing Tests
+
+Since the project is build and configured through CMake it makes sense to rely for tests
+on CTest. All tests are located in the subfolder `tests`. In this directory is a CMakeLists.txt which specifies which tests
+are to be run. Actual tests are located in folders within tests and contain a final CMakeLists.txt which specifies what to run 
+for a single test.
+
+```
+tests
+  └─ test_group1
+      └─ CMakeLists.txt  # specify tests in this group
+      └─ ...             # test files of group 1
+      
+  └─ test_group2
+      └─ CMakeLists.txt  # specify tests in this group
+      └─ ...             # test files of group 2
+      
+  └─ CMakeLists.txt      # specify test groups to run
+  
+CMakeLists.txt           # build configuration
+```
+
 ## Development with VSCode/Codium
 
 Recommended extensions for getting a decent experience are the following:

run-check-test.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env sh
+
+# Author: Sven Vogel
+# Created: 02.05.2024
+# Description: Builds the project and runs tests
+#              Returns 0 on success and 1 when something went wrong
+
+echo "+--------------------------------------+"
+echo "| BUILDING all TARGETS                 |"
+echo "+--------------------------------------+"
+
+make -B
+if [ ! $? -eq 0 ]; then
+  echo "===> failed to build targets"
+  exit 1
+fi
+
+echo "+--------------------------------------+"
+echo "| RUNNING CODE CHECK                   |"
+echo "+--------------------------------------+"
+
+make check
+if [ ! $? -eq 0 ]; then
+  echo "===> failed code check..."
+  exit 1
+fi
+
+echo "+--------------------------------------+"
+echo "| RUNNING TESTS                        |"
+echo "+--------------------------------------+"
+
+ctest -VV --output-on-failure --schedule-random -j 4
+if [ ! $? -eq 0 ]; then
+  echo "===> failed tests..."
+  exit 1
+fi
+
+echo "+--------------------------------------+"
+echo "| COMPLETED CHECK + TESTS SUCCESSFULLY |"
+echo "+--------------------------------------+"

sdk/Dockerfile

@@ -1,11 +1,11 @@
 FROM alpine:3.19.1
 LABEL authors="servostar"
-LABEL version="0.2.1"
+LABEL version="0.2.2"
 LABEL description="base image for building the gemstone programming language compiler"
 LABEL website="https://github.com/Servostar/gemstone"
 
 # install dependencies
-RUN apk add build-base gcc make cmake bison flex git
+RUN apk add build-base gcc make cmake bison flex git python3
 
 # create user for build
 RUN adduser --disabled-password lorang

tests/CMakeLists.txt

@@ -0,0 +1,10 @@
+include(CTest)
+
+set(PROJECT_BINARY_DIR bin)
+set(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/tests)
+set(CTEST_BINARY_DIRECTORY ${PROJECT_BINARY_DIR}/tests)
+
+# Provide test to run here or include another CMakeLists.txt
+
+add_subdirectory(logging)
+add_subdirectory(input_file)

tests/input_file/CMakeLists.txt

@@ -0,0 +1,9 @@
+include(CTest)
+
+# ------------------------------------------------------- #
+# CTEST 1
+# test if the program accepts a file as input
+
+add_test(NAME input_file_check
+        WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/bin/check
+        COMMAND python ${GEMSTONE_TEST_DIR}/input_file/test_input_file.py ${GEMSTONE_TEST_DIR}/input_file/test.gem)

tests/input_file/test.gem

@@ -0,0 +1,6 @@
+
+import "std.io"
+
+fun main {
+    print("Hello, World!!!")
+}

tests/input_file/test_input_file.py

@@ -0,0 +1,36 @@
+import os.path
+import subprocess
+import sys
+import logging
+from logging import info
+
+
+def check_accept():
+    info("testing handling of input file...")
+
+    logging.basicConfig(level=logging.INFO)
+
+    test_file_name = sys.argv[1]
+
+    p = subprocess.run(["./gsc", test_file_name], capture_output=True, text=True)
+
+    assert p.returncode == 0
+
+
+def check_abort():
+    info("testing handling of missing input file...")
+
+    logging.basicConfig(level=logging.INFO)
+
+    p = subprocess.run("./gsc", capture_output=True, text=True)
+
+    assert p.returncode == 1
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    info("check if binary exists...")
+    assert os.path.exists("./gsc")
+
+    check_accept()
+    check_abort()

tests/logging/CMakeLists.txt

@@ -0,0 +1,63 @@
+include(CTest)
+
+include_directories(${PROJECT_SOURCE_DIR}/src)
+
+# ------------------------------------------------------- #
+# CTEST 1
+# test the default output of the logger
+
+add_executable(logging_output
+        ${PROJECT_SOURCE_DIR}/src/sys/log.c
+        output.c)
+set_target_properties(logging_output
+        PROPERTIES
+        OUTPUT_NAME "output"
+        RUNTIME_OUTPUT_DIRECTORY ${GEMSTONE_BINARY_DIR}/tests/logging)
+add_test(NAME logging_output
+        WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+        COMMAND python ${GEMSTONE_TEST_DIR}/logging/test_logging.py check_output)
+
+# ------------------------------------------------------- #
+# CTEST 2
+# test the panic functionality of the logger
+
+add_executable(logging_panic
+        ${PROJECT_SOURCE_DIR}/src/sys/log.c
+        panic.c)
+set_target_properties(logging_panic
+        PROPERTIES
+        OUTPUT_NAME "panic"
+        RUNTIME_OUTPUT_DIRECTORY ${GEMSTONE_BINARY_DIR}/tests/logging)
+add_test(NAME logging_panic
+        WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+        COMMAND python ${GEMSTONE_TEST_DIR}/logging/test_logging.py check_panic)
+
+# ------------------------------------------------------- #
+# CTEST 3
+# test the ability to write to multiple output streams
+
+add_executable(logging_streams
+        ${PROJECT_SOURCE_DIR}/src/sys/log.c
+        streams.c)
+set_target_properties(logging_streams
+        PROPERTIES
+        OUTPUT_NAME "stream"
+        RUNTIME_OUTPUT_DIRECTORY ${GEMSTONE_BINARY_DIR}/tests/logging)
+add_test(NAME logging_streams
+        WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+        COMMAND python ${GEMSTONE_TEST_DIR}/logging/test_logging.py check_stream)
+
+# ------------------------------------------------------- #
+# CTEST 4
+# test compile time log level switch
+
+add_executable(logging_level
+        ${PROJECT_SOURCE_DIR}/src/sys/log.c
+        level.c)
+set_target_properties(logging_level
+        PROPERTIES
+        OUTPUT_NAME "level"
+        RUNTIME_OUTPUT_DIRECTORY ${GEMSTONE_BINARY_DIR}/tests/logging)
+add_test(NAME logging_level
+        WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+        COMMAND python ${GEMSTONE_TEST_DIR}/logging/test_logging.py check_level)

tests/logging/level.c

@@ -0,0 +1,18 @@
+//
+// Created by servostar on 5/2/24.
+//
+
+#include "sys/log.h"
+
+#define LOG_LEVEL LOG_LEVEL_WARNING
+
+int main(void) {
+    log_init();
+
+    DEBUG("logging some debug...");
+    INFO("logging some info...");
+    WARN("logging some warning...");
+    ERROR("logging some error...");
+
+    return 0;
+}

tests/logging/output.c

@@ -0,0 +1,16 @@
+//
+// Created by servostar on 5/1/24.
+//
+
+#include "sys/log.h"
+
+int main(void) {
+    log_init();
+
+    DEBUG("logging some debug...");
+    INFO("logging some info...");
+    WARN("logging some warning...");
+    ERROR("logging some error...");
+
+    return 0;
+}

tests/logging/panic.c

@@ -0,0 +1,20 @@
+//
+// Created by servostar on 5/2/24.
+//
+
+#include "sys/log.h"
+
+int main(void) {
+    log_init();
+
+    // this should appear in stderr
+    INFO("before exit");
+
+    PANIC("oooops something happened");
+
+    // this should NOT appear in stderr
+    //             ^^^
+    ERROR("after exit");
+
+    return 0;
+}

tests/logging/streams.c

@@ -0,0 +1,33 @@
+//
+// Created by servostar on 5/2/24.
+//
+
+#include "sys/log.h"
+#include <stdlib.h>
+
+static FILE* file;
+
+void close_file(void) {
+    if (file != NULL) {
+        fclose(file);
+    }
+}
+
+int main(void) {
+    log_init();
+
+    // this should appear in stderr
+    INFO("should only be in stderr");
+
+    file = fopen("tmp/test.log", "w");
+    if (file == NULL) {
+        PANIC("could not open file");
+    }
+    atexit(close_file);
+
+    log_register_stream(file);
+
+    INFO("should be in both");
+
+    return 0;
+}

tests/logging/test_logging.py

@@ -0,0 +1,124 @@
+import subprocess
+import sys
+import logging
+from logging import info, error
+import os
+
+BIN_DIR = "bin/tests/logging/"
+
+
+def run_check_output():
+    info("started check output...")
+
+    p = subprocess.run(BIN_DIR + "output", capture_output=True, text=True)
+
+    info("checking exit code...")
+
+    # check exit code
+    assert p.returncode == 0
+
+    output = p.stderr
+
+    # check if logs appear in default log output (stderr)
+    info("checking stderr...")
+
+    assert "logging some debug..." in output
+    assert "logging some info..." in output
+    assert "logging some warning..." in output
+    assert "logging some error..." in output
+
+
+def run_check_level():
+    info("started check level...")
+
+    p = subprocess.run(BIN_DIR + "level", capture_output=True, text=True)
+
+    info("checking exit code...")
+
+    # check exit code
+    assert p.returncode == 0
+
+    output = p.stderr
+
+    # check if logs appear in default log output (stderr)
+    info("checking stderr...")
+
+    assert "logging some debug..." not in output
+    assert "logging some info..." not in output
+    assert "logging some warning..." in output
+    assert "logging some error..." in output
+
+
+def run_check_panic():
+    info("started check panic...")
+
+    p = subprocess.run(BIN_DIR + "panic", capture_output=True, text=True)
+
+    info("checking exit code...")
+
+    # check exit code
+    assert p.returncode == 1
+
+    output = p.stderr
+
+    # check if logs appear (not) in default log output (stderr)
+    info("checking stderr...")
+
+    assert "before exit" in output
+    assert "oooops something happened" in output
+    assert "after exit" not in output
+
+
+def run_check_stream():
+    info("started check panic...")
+
+    info("creating temporary folder...")
+
+    if not os.path.exists("tmp"):
+        os.mkdir("tmp")
+
+    info("cleaning temporary folder...")
+
+    if os.path.exists("tmp/test.log"):
+        os.remove("tmp/test.log")
+
+    info("launching test binary...")
+
+    p = subprocess.run(BIN_DIR + "stream", capture_output=True, text=True)
+
+    info("checking exit code...")
+
+    # check exit code
+    assert p.returncode == 0
+
+    with open("tmp/test.log", "r") as file:
+        assert "should be in both" in "".join(file.readlines())
+
+    output = p.stderr
+
+    # check if logs appear (not) in default log output (stderr)
+    info("checking stderr...")
+
+    assert "should only be in stderr" in output
+    assert "should be in both" in output
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+
+    target = sys.argv[1]
+
+    info(f"starting logging test suite with target: {target}")
+
+    match target:
+        case "check_output":
+            run_check_output()
+        case "check_panic":
+            run_check_panic()
+        case "check_stream":
+            run_check_stream()
+        case "check_level":
+            run_check_level()
+        case _:
+            error(f"unknown target: {target}")
+            exit(1)