Added full compilation of Cythan Stage 1

Author: ccgauche

.gitignore

@@ -8,3 +8,8 @@ Cargo.lock
 
 # These are backup files generated by rustfmt
 **/*.rs.bk
+
+
+# Added by cargo
+
+/target

Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "cythan-compiler"
+version = "0.1.0"
+authors = ["ccgauche <[email protected]>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+tokesies = {git = "https://github.com/Jeffail/tokesies"}

README.md

@@ -1,2 +1,135 @@
 # cythan-compiler
- A Rust implementation of the Cythan V3 Compiler 
+
+A Rust implementation of the Cythan V3 Compiler
+
+## How to program in Cythan Stage 1
+
+### Simplest example
+
+```rust
+0  12  2  31  556
+```
+
+As you probably know Cythan is a list of numbers, in the case of Cythan Stage 1 numbers are separated by a space.
+
+  
+
+### Variables
+
+In Cythan stage 1, compile time variables are accepted.
+
+  
+
+#### To define a variable:
+
+```rust
+var1 = (4  10)
+```
+
+  
+
+#### To use a variable:
+
+```rust
+0  20 var1 30
+```
+
+  
+
+__Note:__  *Variables in Cythan Stage 1 have no scope and are accessible from everywhere*
+
+  
+
+### Functions
+
+In Cythan Stage 1 function are computed at compile time like macros.
+
+  
+
+#### To define a function:
+
+```rust
+func1 {
+     12  20  10
+}
+```
+
+  
+
+#### To call a function:
+
+```rust
+func1()
+```
+
+  
+
+#### With arguments:
+
+```rust
+func1 {
+    12  20  10  self.0  self.3..10  self.20..40?5
+}
+
+func1(10  2  3  45  20  10)
+```
+
+  
+
+As you probably seen `self` keyword is used to represent arguments.
+
+  
+
+#### Self cheat-sheet:
+
+`self.x..y` returns all numbers from `x` to `y` if one don't exists, it will not be returned
+
+`self.x..y?z` returns all numbers from `x` to `y` if one don't exists, it will be replaced by `z`
+
+`self.x?z` returns the number `x` if it doesn't exists, it will be replaced by `z`
+
+`self.x` returns the number `x` if it doesn't exists, it will not be returned
+
+`self.x..` returns all numbers from `x` to the end of the arguments if one don't exists, it will not be returned
+
+`self.x..?z` returns all numbers from `x` to the end of the arguments if one don't exists, it will be replaced by `z`
+
+  
+
+## How to compile Cythan Stage 1
+
+From a string:
+
+```rust
+use cythan_compiler::*;
+
+tokenizer::Context::new().compute(&tokenizer::generate_tokens(string))
+```
+
+  
+
+From a file:
+
+```rust
+use cythan_compiler::*;
+
+tokenizer::Context::new().compute(&tokenizer::generate_tokens(std::fs::read_to_string("file.ct").unwrap()))
+```
+
+  
+
+Get only executable tokens from a file:
+
+```rust
+use cythan_compiler::*;
+
+let tokens = tokenizer::generate_tokens(std::fs::read_to_string("file.ct").unwrap();
+```
+
+  
+
+## Run in a cythan machine
+
+  
+
+To run your generated code into a Cythan machine use [Cythan Rust Library](https://github.com/Cypooos/Cythan-v3)

cythan-in/cythan.ct

@@ -0,0 +1,19 @@
+0 12 2 3
+
+# This is an example
+
+3 # Test
+# Test of a multi line comment!
+# de
+
+a = (2 10)
+
+b = (4 a)
+
+test_func {
+    1 3 9 6 8 9 self.0 self.0..12?1 self.1?6 b
+}
+
+test_func(23)
+
+4 25 26 23 b

src/lib.rs

@@ -0,0 +1,119 @@
+#![feature(test)]
+
+extern crate test;
+
+use test::*;
+
+pub mod tokenizer;
+
+pub use tokenizer::*;
+
+#[test]
+fn test() {
+    let file = r#"0 12 2 3
+
+    # This is an example
+    
+    3 # Test
+    # Test of a multi line comment!
+    # de
+
+    a = (2 10)
+
+    b = (4 a)
+    
+    test_func {
+        1 3 9 6 8 9 self.0 self.0..12?1 self.1?6 b
+    }
+    
+    test_func(23)
+    
+    4 25 26 23 b"#;
+
+    let tokens = tokenizer::generate_tokens(file);
+
+    println!("{:?}",tokens);
+    
+    println!("{:?}",&tokenizer::Context::new().compute(&tokens));
+
+    //assert_eq!(tokens.get(0).unwrap().term(), "hello");
+}
+
+#[bench]
+fn bench_tokenize(bench: &mut Bencher) {
+    let file = r#"0 12 2 3
+
+    # This is an example
+    
+    3 # Test
+    # Test of a multi line comment!
+    # de
+
+    a = (2 10)
+
+    b = (4 a)
+    
+    test_func {
+        1 3 9 6 8 9 self.0 self.0..12?1 self.1?6 b
+    }
+    
+    test_func(23)
+    
+    4 25 26 23 b"#;
+    bench.iter(|| {
+        let a = tokenizer::generate_tokens(file);
+    })
+}
+
+#[bench]
+fn bench_compute(bench: &mut Bencher) {
+    let file = r#"0 12 2 3
+
+    # This is an example
+    
+    3 # Test
+    # Test of a multi line comment!
+    # de
+
+    a = (2 10)
+
+    b = (4 a)
+    
+    test_func {
+        1 3 9 6 8 9 self.0 self.0..12?1 self.1?6 b
+    }
+    
+    test_func(23)
+    
+    4 25 26 23 b"#;
+    let a = tokenizer::generate_tokens(file);
+    bench.iter(|| {
+        tokenizer::Context::new().compute(&a)
+    })
+}
+
+#[bench]
+fn bench_whole(bench: &mut Bencher) {
+    let file = r#"0 12 2 3
+
+    # This is an example
+    
+    3 # Test
+    # Test of a multi line comment!
+    # de
+
+    a = (2 10)
+
+    b = (4 a)
+    
+    test_func {
+        1 3 9 6 8 9 self.0 self.0..12?1 self.1?6 b
+    }
+    
+    test_func(23)
+    
+    4 25 26 23 b"#;
+    bench.iter(|| {
+        tokenizer::Context::new().compute(&tokenizer::generate_tokens(file))
+    })
+}