feat: candid subtype check (#3171)

Builds on (in reverse order)

* #3170 feat: optimized candid subtype check (adding a global type table of types involved in candid subtype checks)
* #3151 feat: candid subtype check, during deserialization (first integration of subtype checks with deserialization)
* #3115 feat: implementation of rts candid subtyping check  (implementaton of (unextended) type table base candid subtype checks 

Since this PR includes more test and fixes to previous PR, its probably best to just review this one as a diff against master:

Over #3170, this PR:

- Introduces a stack allocated memo table allocated on entry to deserialize and shared amongst all recursive calls. The `extended` boolean flag is replaced by a possibly null memo table (`bit_rel_opt`) - the table is `null` for extended deserialization of stable values (for which subtype checks on references are just ommitted since unnecessary).

- Generalises the subtype check and bitrel cache to support sharing across multiple calls and caching of both positive and *negative* subtype test results:
- Adds 1 bit per pair of types in subtype `cache` to record true/false outcome separately from plain addition to cache. 
- Adjust each exit point that returns false to invalidate the positive subtyping assumption already in the cache (thus recording the negative result in the cache).

Unfortunately caching negative results doubles the space required to store the cache (we now need 2 bits, not 1, for every possible pair of type indices).
This subtyping between type tables with 128 entries each will consume 8K = (2 * 2 * 128 * 128/ 8) of stack space. Not great, but not terrible either.

The first, `visited` bit (bit 0 per (t1,t2) pair), is stored in unnegated form. The second, `related` bit (bit 1 per (t1,t2) type pair), is stored in negated form so that assuming the relation holds when first visited is actually a no-op (since the matrix is initialized with zero bits).

Also fixes a lurking bug where I failed to skip the value when the subtype check fails (meaning previous PR were buggy).

- [x] create the cache once on entry to deserialize and share it between calls to idl_sub
- [x] avoid explicitly assuming true by defaulting to full relations on entry (perhaps by inverting true and false bits)
- [ ] ~only produce global type table entries in unextended deserialization.~ Not worth the effort.
- [x] add basic test for recursive types
- [x] overflow check on Stack.dynamic_alloc
- [ ] idl_sub on future types.
- [x] fix broken re-initialization of bitrel on every sub type check
- [x] integrate with changes due to to_candid/from_candid #3155

Author: crusso

Changelog.md

@@ -2,6 +2,16 @@
 
 * motoko (`moc`)
 
+  * BREAKING CHANGE
+
+    Motoko now implements Candid 1.4 (dfinity/candid#311).
+
+    In particular, when deserializing an actor or function reference,
+    Motoko will now first check that the type of the deserialized reference
+    is a subtype of the expected type and act accordingly.
+
+    Very few users should be affected by this change in behaviour.
+
   * BREAKING CHANGE
 
     On the IC, the act of making a call to a canister function can fail, so that the call cannot (and will not be) performed.

nix/sources.json

@@ -6,10 +6,10 @@
         "homepage": "",
         "owner": "dfinity",
         "repo": "candid",
-        "rev": "a555d77704d691bb8f34e21a049d44ba0acee3f8",
-        "sha256": "0vn171lcadpznrl5nq2mws2zjjqj9jxyvndb2is3dixbjqyvjssx",
+        "rev": "fa27eef96c96c2f774a479622996b42c7ae6c1bd",
+        "sha256": "18zhn0iyakq1212jizc406v9x275nivdnnwx5h2130ci10d7f1ah",
         "type": "tarball",
-        "url": "https://github.com/dfinity/candid/archive/a555d77704d691bb8f34e21a049d44ba0acee3f8.tar.gz",
+        "url": "https://github.com/dfinity/candid/archive/fa27eef96c96c2f774a479622996b42c7ae6c1bd.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
     "esm": {

rts/motoko-rts-tests/src/bitrel.rs

@@ -0,0 +1,57 @@
+use motoko_rts::bitrel::BitRel;
+use motoko_rts::types::{Value, Words};
+
+pub unsafe fn test() {
+    println!("Testing bitrel ...");
+
+    const K: u32 = 128;
+
+    const N: usize = (2 * K * K * 2 / usize::BITS) as usize;
+
+    let mut cache: [u32; N] = [0xFFFFFFF; N];
+
+    assert_eq!(usize::BITS, 32);
+    for size1 in 0..K {
+        for size2 in 0..K {
+            let w = BitRel::words(size1, size2);
+            let bitrel = BitRel {
+                ptr: &mut cache[0],
+                end: &mut cache[w as usize],
+                size1: size1,
+                size2: size2,
+            };
+            bitrel.init();
+            for i in 0..size1 {
+                for j in 0..size2 {
+                    // initially unvisited
+                    assert!(!bitrel.visited(true, i, j)); // co
+                    assert!(!bitrel.visited(false, j, i)); // contra
+
+                    // initially related
+                    assert!(bitrel.related(true, i, j)); // co
+                    assert!(bitrel.related(false, j, i)); // contra
+
+                    // test visiting
+                    // co
+                    bitrel.visit(true, i, j);
+                    assert!(bitrel.visited(true, i, j));
+                    // contra
+                    bitrel.visit(false, j, i);
+                    assert!(bitrel.visited(false, j, i));
+
+                    // test refutation
+                    // co
+                    bitrel.assume(true, i, j);
+                    assert!(bitrel.related(true, i, j));
+                    bitrel.disprove(true, i, j);
+                    assert!(!bitrel.related(true, i, j));
+                    // contra
+                    bitrel.assume(false, j, i);
+                    assert!(bitrel.related(false, j, i));
+                    bitrel.disprove(false, j, i);
+                    assert!(!bitrel.related(false, j, i));
+                }
+            }
+        }
+    }
+}

rts/motoko-rts-tests/src/main.rs

@@ -2,6 +2,7 @@
 
 mod bigint;
 mod bitmap;
+mod bitrel;
 mod continuation_table;
 mod crc32;
 mod gc;
@@ -24,6 +25,7 @@ fn main() {
     unsafe {
         bigint::test();
         bitmap::test();
+        bitrel::test();
         continuation_table::test();
         crc32::test();
         gc::test();

rts/motoko-rts/src/bitrel.rs

@@ -0,0 +1,92 @@
+//! This module implements a simple subtype cache used by the compiler (in generated code)
+
+use crate::constants::WORD_SIZE;
+use crate::idl_trap_with;
+use crate::mem_utils::memzero;
+use crate::types::Words;
+
+const BITS: u32 = 2;
+
+#[repr(packed)]
+pub struct BitRel {
+    /// Pointer into the bit set
+    pub ptr: *mut u32,
+    /// Pointer to the end of the bit set
+    /// must allow at least 2 * size1 * size2 bits
+    pub end: *mut u32,
+    pub size1: u32,
+    pub size2: u32,
+}
+
+impl BitRel {
+    pub fn words(size1: u32, size2: u32) -> u32 {
+        return ((2 * size1 * size2 * BITS) + (usize::BITS - 1)) / usize::BITS;
+    }
+
+    pub unsafe fn init(&self) {
+        if (self.end as usize) < (self.ptr as usize) {
+            idl_trap_with("BitRel invalid fields");
+        };
+
+        let bytes = ((self.end as usize) - (self.ptr as usize)) as u32;
+        if bytes != BitRel::words(self.size1, self.size2) * WORD_SIZE {
+            idl_trap_with("BitRel missized");
+        };
+        memzero(self.ptr as usize, Words(bytes / WORD_SIZE));
+    }
+
+    unsafe fn locate_ptr_bit(&self, p: bool, i_j: u32, j_i: u32, bit: u32) -> (*mut u32, u32) {
+        let size1 = self.size1;
+        let size2 = self.size2;
+        let (base, i, j) = if p { (0, i_j, j_i) } else { (size1, j_i, i_j) };
+        debug_assert!(i < size1);
+        debug_assert!(j < size2);
+        debug_assert!(bit < BITS);
+        let k = ((base + i) * size2 + j) * BITS + bit;
+        let word = (k / usize::BITS) as usize;
+        let bit = (k % usize::BITS) as u32;
+        let ptr = self.ptr.add(word);
+        if ptr > self.end {
+            idl_trap_with("BitRel indices out of bounds");
+        };
+        return (ptr, bit);
+    }
+
+    unsafe fn set(&self, p: bool, i_j: u32, j_i: u32, bit: u32, v: bool) {
+        let (ptr, bit) = self.locate_ptr_bit(p, i_j, j_i, bit);
+        if v {
+            *ptr = *ptr | (1 << bit);
+        } else {
+            *ptr = *ptr & !(1 << bit);
+        }
+    }
+
+    unsafe fn get(&self, p: bool, i_j: u32, j_i: u32, bit: u32) -> bool {
+        let (ptr, bit) = self.locate_ptr_bit(p, i_j, j_i, bit);
+        let mask = 1 << bit;
+        return *ptr & mask == mask;
+    }
+
+    pub unsafe fn visited(&self, p: bool, i_j: u32, j_i: u32) -> bool {
+        self.get(p, i_j, j_i, 0)
+    }
+
+    pub unsafe fn visit(&self, p: bool, i_j: u32, j_i: u32) {
+        self.set(p, i_j, j_i, 0, true)
+    }
+
+    #[allow(dead_code)]
+    // NB: we store related bits in negated form to avoid setting on assumption
+    // This code is a nop in production code.
+    pub unsafe fn assume(&self, p: bool, i_j: u32, j_i: u32) {
+        debug_assert!(!self.get(p, i_j, j_i, 1));
+    }
+
+    pub unsafe fn related(&self, p: bool, i_j: u32, j_i: u32) -> bool {
+        !self.get(p, i_j, j_i, 1)
+    }
+
+    pub unsafe fn disprove(&self, p: bool, i_j: u32, j_i: u32) {
+        self.set(p, i_j, j_i, 1, true)
+    }
+}