relative include paths now get resolved properly

Cargo.lock

@@ -1,169 +0,0 @@
-# This file is automatically @generated by Cargo.
-# It is not intended for manual editing.
-version = 3
-
-[[package]]
-name = "Yard"
-version = "0.1.0"
-dependencies = [
- "colored",
- "lazy_static",
- "rand",
- "regex",
-]
-
-[[package]]
-name = "aho-corasick"
-version = "0.7.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b4f55bd91a0978cbfd91c457a164bab8b4001c833b7f323132c0a4e1922dd44e"
-dependencies = [
- "memchr",
-]
-
-[[package]]
-name = "atty"
-version = "0.2.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
-dependencies = [
- "hermit-abi",
- "libc",
- "winapi",
-]
-
-[[package]]
-name = "cfg-if"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
-
-[[package]]
-name = "colored"
-version = "2.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b3616f750b84d8f0de8a58bda93e08e2a81ad3f523089b05f1dffecab48c6cbd"
-dependencies = [
- "atty",
- "lazy_static",
- "winapi",
-]
-
-[[package]]
-name = "getrandom"
-version = "0.2.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6"
-dependencies = [
- "cfg-if",
- "libc",
- "wasi",
-]
-
-[[package]]
-name = "hermit-abi"
-version = "0.1.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
-dependencies = [
- "libc",
-]
-
-[[package]]
-name = "lazy_static"
-version = "1.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
-
-[[package]]
-name = "libc"
-version = "0.2.134"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "329c933548736bc49fd575ee68c89e8be4d260064184389a5b77517cddd99ffb"
-
-[[package]]
-name = "memchr"
-version = "2.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
-
-[[package]]
-name = "ppv-lite86"
-version = "0.2.16"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872"
-
-[[package]]
-name = "rand"
-version = "0.8.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
-dependencies = [
- "libc",
- "rand_chacha",
- "rand_core",
-]
-
-[[package]]
-name = "rand_chacha"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
-dependencies = [
- "ppv-lite86",
- "rand_core",
-]
-
-[[package]]
-name = "rand_core"
-version = "0.6.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
-dependencies = [
- "getrandom",
-]
-
-[[package]]
-name = "regex"
-version = "1.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b"
-dependencies = [
- "aho-corasick",
- "memchr",
- "regex-syntax",
-]
-
-[[package]]
-name = "regex-syntax"
-version = "0.6.27"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244"
-
-[[package]]
-name = "wasi"
-version = "0.11.0+wasi-snapshot-preview1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
-
-[[package]]
-name = "winapi"
-version = "0.3.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
-dependencies = [
- "winapi-i686-pc-windows-gnu",
- "winapi-x86_64-pc-windows-gnu",
-]
-
-[[package]]
-name = "winapi-i686-pc-windows-gnu"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
-
-[[package]]
-name = "winapi-x86_64-pc-windows-gnu"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"

src/direct/mod.rs

@@ -1,6 +1,7 @@
-use std::{collections::VecDeque};
+use std::{collections::VecDeque, env, path::Path};
 
-use crate::{token::{Token}, builtin::modules::Module};
+use crate::{token::{Token, MessageType}, builtin::modules::Module};
+use crate::builtin::modules::io::print;
 
 #[derive(Default)]
 pub struct LangSpecs {
@@ -8,7 +9,7 @@ pub struct LangSpecs {
     builtin_features: Vec<crate::builtin::modules::Module>,
     lang_version: u32,
     authors: Vec<String>,
-    embedded_files: Vec<String>,
+    embedded_files: Vec<(usize, String, String)>,
 }
 
 impl LangSpecs {
@@ -16,24 +17,22 @@ impl LangSpecs {
     pub fn features(&self) -> &[crate::builtin::modules::Module] {
         &self.builtin_features
     }
+
+    pub fn embedded_files(&self) -> &[(usize, String, String)] {
+        &self.embedded_files
+    }
 }
 
-pub fn resolve_directives(tokens: &mut VecDeque<Token>) -> LangSpecs {
+pub fn resolve_directives(tokens: &mut VecDeque<Token>, origin: &String) -> LangSpecs {
     let mut specs = LangSpecs::default();
 
-    for token in tokens.iter() {
+    for (idx, token) in tokens.iter().enumerate() {
         match token {
-            Token::CompilerDirective(text, _) => parse_directive(text, &mut specs),
+            Token::CompilerDirective(text, _) => parse_directive(text, idx, &mut specs, origin),
             _ => ()
         }
     }
 
-    // remove compiler directives from source
-    tokens.retain(|token| match token {
-        Token::CompilerDirective(_, _) => false,
-        _ => true
-    });
-
     specs
 }
 
@@ -41,7 +40,7 @@ static DIRECTIVE_REGEX_SRC: &'static str = concat!(
     r"@feature\(((?:\s*[\w]+\s*,?)*)\)",
     r"|@version\(\s*([0-9]{3})\s*\)",
     r"|@author\((.*)\)",
-    r"|@embed\((.*)\)"
+    r"|@include\((.*)\)"
 );
 
 lazy_static::lazy_static! {
@@ -58,8 +57,8 @@ pub fn from_list(text: &str) -> Vec<String> {
     vec
 }
 
-fn parse_directive(text: &str, specs: &mut LangSpecs) {
-    
+fn parse_directive(text: &str, token_idx: usize, specs: &mut LangSpecs, origin: &String) {
+
     for cap in DIRECTIVE_REGEX.captures_iter(text) {
         let mut enumerator = cap.iter().enumerate();
         loop {
@@ -91,8 +90,21 @@ fn parse_directive(text: &str, specs: &mut LangSpecs) {
                         return;
                     },
                     4 => {
-                        specs.embedded_files.append(&mut from_list(mat.as_str()));
-                        crate::message(crate::token::MessageType::Warning, "Embed directive not working at current state");
+                        let cwd = Path::new(origin).parent().unwrap().canonicalize().unwrap();
+                        for path in from_list(mat.as_str()).iter() {
+                            let mut include_path = cwd.clone();
+                            include_path.push(path);
+
+                            if let Ok(path) = include_path.canonicalize() {
+                                let include_file = path.to_str().unwrap().to_string();
+                                
+                                if let Ok(str) = std::fs::read_to_string(&include_file) {
+                                    specs.embedded_files.push((token_idx, str, include_file));
+                                } else {
+                                    crate::message(MessageType::Warning, format!("Unable to read embedded file: {include_file}"));
+                                }
+                            }
+                        }
                         return;
                     },
                     _ => crate::message(crate::token::MessageType::Warning, format!("unknown directive: `{}`", text)),

src/main.rs

@@ -31,10 +31,21 @@ fn compile(settings: &Settings) -> Option<(Vec<Func>, Vec<Declr>, Vec<BuiltinFun
     for src in settings.get_source().iter() {
         let code = src.code();
 
-        let mut diagnostics = Diagnostics::new(&settings, code);
-        
+        let mut diagnostics = Diagnostics::new(&settings);
+        diagnostics.add_source_origin(code, src.path());
+
         if let Ok(mut tokens) = tokenize(code, &mut diagnostics) {
-            let specs = crate::direct::resolve_directives(&mut tokens);
+            let specs = crate::direct::resolve_directives(&mut tokens, src.path());
+
+            // read source of every embedded file and tokenize
+            for (idx, src, path) in specs.embedded_files() {
+                diagnostics.add_source_origin(src, path);
+                if let Ok(em_tokens) = tokenize(src, &mut diagnostics) {
+                    for (idy, em_token) in em_tokens.into_iter().enumerate() {
+                        tokens.insert(idx + idy, em_token.to_owned());
+                    }
+                }
+            }
 
             let mut parser = Parser::new(&specs);
 

src/parser/data.rs

@@ -4,9 +4,10 @@ use crate::conf::Settings;
 use crate::token::{DebugInfo, DebugNotice, Token, MessageType};
 use crate::Prim;
 use core::panic;
-use std::collections::VecDeque;
+use std::collections::{VecDeque, HashMap};
 use std::collections::hash_map::DefaultHasher;
 use std::hash::{Hash, Hasher};
+use std::path::Path;
 
 #[derive(Debug, Copy, Clone, PartialEq, Eq)]
 pub enum LogLvl {
@@ -24,28 +25,39 @@ impl Default for LogLvl {
     }
 }
 
-pub struct Diagnostics<'a> {
+pub struct Diagnostics {
     /// terminating factor on error
-    err: Option<DebugNotice<'a>>,
+    err: Option<DebugNotice>,
     /// additional warning and informations
     /// all non critical
-    hints: Vec<DebugNotice<'a>>,
-    /// source string
-    source: &'a str,
+    hints: Vec<DebugNotice>,
+    /// source hash and source string
+    source: HashMap<u64, (String, String)>,
     /// flags
     loglvl: LogLvl,
 }
 
-impl<'a> Diagnostics<'a> {
-    pub fn new(settings: &Settings, source: &'a str) -> Diagnostics<'a> {
+impl Diagnostics {
+    pub fn new(settings: &Settings) -> Diagnostics {
         Self {
             err: None,
             hints: vec![],
-            source,
+            source: HashMap::new(),
             loglvl: settings.loglvl()
         }
     }
 
+    pub fn add_source_origin<A>(&mut self, source: A, url: A) where A: Into<String> + Hash {
+        let mut hasher = DefaultHasher::new();
+        source.hash(&mut hasher);
+        let origin = hasher.finish();
+
+        let string = url.into();
+        let file_name = Path::new(&string).file_name().expect("not a falid file path");
+
+        self.source.insert(origin, (file_name.to_str().unwrap().to_string(), source.into()));
+    }
+
     pub fn set_err<T, S>(&mut self, source: &S, message: &'static crate::token::DebugMsg, ext: T)
     where
         T: Into<String>,
@@ -57,12 +69,20 @@ impl<'a> Diagnostics<'a> {
         }
 
         let info: DebugInfo = source.clone().into();
+        
+        let origin = self.source.get(&info.origin).unwrap();
+        let context = origin.1
+            .lines()
+            .nth(info.line)
+            .unwrap()
+            .trim();
 
         self.err = Some(DebugNotice {
             info,
             msg: message,
             ext: ext.into(),
-            source: self.source,
+            source: context.to_string(),
+            origin: origin.0.to_owned()
         });
     }
 
@@ -73,16 +93,24 @@ impl<'a> Diagnostics<'a> {
     {
         let info: DebugInfo = source.clone().into();
 
+        let origin = self.source.get(&info.origin).unwrap();
+        let context = origin.1
+            .lines()
+            .nth(info.line)
+            .unwrap()
+            .trim();
+
         self.hints.push(DebugNotice {
             info,
             msg: message,
             ext: ext.into(),
-            source: self.source,
+            source: context.to_string(),
+            origin: origin.0.to_owned()
         });
     }
 }
 
-impl<'a> std::fmt::Display for Diagnostics<'a> {
+impl std::fmt::Display for Diagnostics {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         for hint in self.hints.iter() {
             match hint.msg.typ {
@@ -214,6 +242,8 @@ impl<'a> Declr<'a> {
             }
         }
 
+        self.result_typ.hash(&mut hasher);
+
         self.uuid = hasher.finish()
     }
 

src/parser/mod.rs

@@ -285,7 +285,7 @@ fn discover_functions(
 
         // if we have anything left it might be an error
         match &top {
-            Token::LineBreak(_) | Token::Terminator(_) => (), // valid whitespace
+            Token::LineBreak(_) | Token::Terminator(_) | Token::CompilerDirective(_,_) => (), // valid whitespace
             _ => {
                 diagnostics.set_err(&top, crate::msg::ERR22, "");
                 return Err(()) ;

src/srcio/mod.rs

@@ -1,7 +1,7 @@
 use std::{fs};
 
 pub struct CodeSrc {
-    _path: String,
+    path: String,
     src: String,
 }
 
@@ -9,7 +9,7 @@ impl CodeSrc {
     
     pub fn new(path: &String) -> Result<CodeSrc, String> {
         Ok(Self {
-            _path: path.to_owned(),
+            path: path.to_owned(),
             src: Self::read_code(path)?,
         })
     }
@@ -28,4 +28,8 @@ impl CodeSrc {
     pub fn code(&self) -> &String {
         &self.src
     }
+
+    pub fn path(&self) -> &String {
+        &self.path
+    }
 }

src/token/mod.rs

@@ -1,5 +1,6 @@
 use colored::{ColoredString, Colorize};
-use std::{collections::VecDeque};
+use std::{collections::{VecDeque, hash_map::DefaultHasher}, hash::Hasher};
+use std::hash::Hash;
 
 use crate::parser::data::Diagnostics;
 
@@ -420,24 +421,26 @@ pub struct DebugMsg {
     pub msg: &'static str,
 }
 
-pub struct DebugNotice<'a> {
+pub struct DebugNotice {
     pub info: DebugInfo,
     /// generic error description
     pub msg: &'static DebugMsg,
     /// extra message which is case specific
     pub ext: String,
-    pub source: &'a str,
+    pub source: String,
+    pub origin: String,
 }
 
-impl<'a> std::fmt::Display for DebugNotice<'a> {
+impl std::fmt::Display for DebugNotice {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         // write header as:
         // `Error (56) some syntax error message in line 5:`
         f.write_fmt(format_args!(
-            "{} ({}) {} in line {}: {}\n",
+            "{} ({}) {} at: {} in line {}: {}\n",
             self.msg.typ.to_colored(),
             self.msg.code,
             self.msg.msg.bold().bright_white(),
+            self.origin,
             self.info.line + 1,
             self.ext
         ))?;
@@ -445,10 +448,6 @@ impl<'a> std::fmt::Display for DebugNotice<'a> {
         f.write_fmt(format_args!(
             " somewhere in here:\n  --> `{}`\n",
             self.source
-                .lines()
-                .nth(self.info.line)
-                .unwrap()
-                .trim()
                 .bold()
                 .bright_white()
         ))
@@ -473,6 +472,8 @@ pub struct DebugInfo {
     pub end: usize,
     /// line number where the line in which the token is begins
     pub line: usize,
+    /// string url of the source origin
+    pub origin: u64
 }
 
 #[derive(Debug)]
@@ -623,6 +624,9 @@ pub fn tokenize<'a>(source: &'a str, diagnostics: &mut Diagnostics) -> Result<Ve
 
     for cap in TOKEN_REGEX.captures_iter(source.as_ref()) {
         let mut enumerator = cap.iter().enumerate();
+        let mut hasher = DefaultHasher::new();
+        source.hash(&mut hasher);
+        let origin = hasher.finish();
         loop {
             let next = enumerator.next();
             if next.is_none() {
@@ -648,6 +652,7 @@ pub fn tokenize<'a>(source: &'a str, diagnostics: &mut Diagnostics) -> Result<Ve
                     start: mat.start() - line_start,
                     end: mat.end() - line_start,
                     line: line_count,
+                    origin
                 };
 
                 tokens.push_back(match i {

tests/lib.yard

@@ -0,0 +1,5 @@
+
+foo( {
+    
+    println("say Hello!");
+}

tests/program.yard

@@ -2,6 +2,8 @@
 @version(100)
 @author(Sven Vogel)
 
+@include(/home/servostar/Projects/Yard/tests/lib.yard)
+
 max(a:int, b:int)=int {
     despite a < b {
         yield a
@@ -10,9 +12,11 @@ max(a:int, b:int)=int {
 }
 
 # main function
-main = int {
+main() = int {
 
     println("" .. max(3,4));
 
+    foo()
+
     0
 }