external files can now be includes via @include

src/direct/mod.rs

@@ -1,6 +1,6 @@
 use std::{collections::VecDeque};
 
-use crate::{token::{Token}, builtin::modules::Module};
+use crate::{token::{Token, MessageType}, builtin::modules::Module};
 
 #[derive(Default)]
 pub struct LangSpecs {
@@ -8,7 +8,7 @@ pub struct LangSpecs {
     builtin_features: Vec<crate::builtin::modules::Module>,
     lang_version: u32,
     authors: Vec<String>,
-    embedded_files: Vec<String>,
+    embedded_files: Vec<(usize, String)>,
 }
 
 impl LangSpecs {
@@ -16,24 +16,22 @@ impl LangSpecs {
     pub fn features(&self) -> &[crate::builtin::modules::Module] {
         &self.builtin_features
     }
+
+    pub fn embedded_files(&self) -> &[(usize, String)] {
+        &self.embedded_files
+    }
 }
 
 pub fn resolve_directives(tokens: &mut VecDeque<Token>) -> LangSpecs {
     let mut specs = LangSpecs::default();
 
-    for token in tokens.iter() {
+    for (idx, token) in tokens.iter().enumerate() {
         match token {
-            Token::CompilerDirective(text, _) => parse_directive(text, &mut specs),
+            Token::CompilerDirective(text, _) => parse_directive(text, idx, &mut specs),
             _ => ()
         }
     }
 
-    // remove compiler directives from source
-    tokens.retain(|token| match token {
-        Token::CompilerDirective(_, _) => false,
-        _ => true
-    });
-
     specs
 }
 
@@ -41,7 +39,7 @@ static DIRECTIVE_REGEX_SRC: &'static str = concat!(
     r"@feature\(((?:\s*[\w]+\s*,?)*)\)",
     r"|@version\(\s*([0-9]{3})\s*\)",
     r"|@author\((.*)\)",
-    r"|@embed\((.*)\)"
+    r"|@include\((.*)\)"
 );
 
 lazy_static::lazy_static! {
@@ -58,7 +56,7 @@ pub fn from_list(text: &str) -> Vec<String> {
     vec
 }
 
-fn parse_directive(text: &str, specs: &mut LangSpecs) {
+fn parse_directive(text: &str, token_idx: usize, specs: &mut LangSpecs) {
     
     for cap in DIRECTIVE_REGEX.captures_iter(text) {
         let mut enumerator = cap.iter().enumerate();
@@ -91,8 +89,13 @@ fn parse_directive(text: &str, specs: &mut LangSpecs) {
                         return;
                     },
                     4 => {
-                        specs.embedded_files.append(&mut from_list(mat.as_str()));
-                        crate::message(crate::token::MessageType::Warning, "Embed directive not working at current state");
+                        for path in from_list(mat.as_str()).iter() {
+                            if let Ok(str) = std::fs::read_to_string(path) {
+                                specs.embedded_files.push((token_idx, str));
+                            } else {
+                                crate::message(MessageType::Warning, format!("Unable to read embedded file: {path}"));
+                            }
+                        }
                         return;
                     },
                     _ => crate::message(crate::token::MessageType::Warning, format!("unknown directive: `{}`", text)),

src/main.rs

@@ -36,6 +36,15 @@ fn compile(settings: &Settings) -> Option<(Vec<Func>, Vec<Declr>, Vec<BuiltinFun
         if let Ok(mut tokens) = tokenize(code, &mut diagnostics) {
             let specs = crate::direct::resolve_directives(&mut tokens);
 
+            // read source of every embedded file and tokenize
+            for (idx, src) in specs.embedded_files() {
+                if let Ok(em_tokens) = tokenize(src, &mut diagnostics) {
+                    for (idy, em_token) in em_tokens.into_iter().enumerate() {
+                        tokens.insert(idx + idy, em_token.to_owned());
+                    }
+                }
+            }
+
             let mut parser = Parser::new(&specs);
 
             if let Ok((funcs, declrs, builtin)) = parser.parse(&mut tokens, &mut diagnostics, &settings) {

src/parser/mod.rs

@@ -285,7 +285,7 @@ fn discover_functions(
 
         // if we have anything left it might be an error
         match &top {
-            Token::LineBreak(_) | Token::Terminator(_) => (), // valid whitespace
+            Token::LineBreak(_) | Token::Terminator(_) | Token::CompilerDirective(_,_) => (), // valid whitespace
             _ => {
                 diagnostics.set_err(&top, crate::msg::ERR22, "");
                 return Err(()) ;

src/token/mod.rs

@@ -472,7 +472,7 @@ pub struct DebugInfo {
     /// index in source string where the token ends  in the current line
     pub end: usize,
     /// line number where the line in which the token is begins
-    pub line: usize,
+    pub line: usize
 }
 
 #[derive(Debug)]