Skip to content

Commit 574d5ae

Browse files
potatomashedpotatomashed
committed
Serialization, deserialization and a JSON parser
1 parent ea826f1 commit 574d5ae

36 files changed

+3279
-569
lines changed

‎CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.15)
22

33
project(
44
mlc
5-
VERSION 0.0.8
5+
VERSION 0.0.9
66
DESCRIPTION "MLC-Python"
77
LANGUAGES C CXX
88
)

‎README.md‎

Lines changed: 20 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,35 +1,38 @@
1-
MLC-Python
2-
==========
1+
<h1 align="center">
2+
<img src="https://gist.githubusercontent.com/potatomashed/632c58cc8df7df7fdd067aabb34c1ef6/raw/7900472d1f7fce520fef5fad2e47a6f5fb234d08/mlc-python-logo.svg" alt="MLC Logo" style="width:10%; height:auto;">
33

4-
🛠️ MLC is a Python-first toolkit for building ML compilers, runtimes, and compound AI systems. It enables you to define nested data structures (like compiler IRs) as roundtrippable text formats in Python syntax, with structural comparison for unit-testing and zero-copy C++ interop when needed.
4+
MLC-Python
5+
</h1>
56

6-
## 🔑 Key features
7+
* [:key: Key features](#keykey-features)
8+
* [:inbox_tray: Installation](#inbox_trayinstallation)
9+
+ [:package: Install From PyPI](#packageinstall-from-pypi)
10+
+ [:gear: Build from Source](#gearbuild-from-source)
11+
+ [:ferris_wheel: Create MLC-Python Wheels](#ferris_wheel-create-mlc-python-wheels)
712

8-
### 🐍 `mlc.ast`: Text formats in Python Syntax
13+
MLC is a Python-first toolkit that makes it more ergonomic to build AI compilers, runtimes, and compound AI systems. It provides Pythonic dataclasses with rich tooling infra, which includes:
914

10-
TBD
11-
12-
### 🏗️ `mlc.dataclasses`: Cross-Language Dataclasses
15+
- Structure-aware equality and hashing methods;
16+
- Serialization in JSON / pickle;
17+
- Text format printing and parsing in Python syntax.
1318

14-
TBD
19+
Additionally, MLC language bindings support:
1520

16-
### ⚡ `mlc.Func`: Zero-Copy Cross-Language Function Calling
17-
18-
TBD
21+
- Zero-copy bidirectional functioning calling for all MLC dataclasses.
1922

20-
### 🎯 Structural Testing for Nested Dataclasses
23+
## :key: Key features
2124

2225
TBD
2326

24-
## 📥 Installation
27+
## :inbox_tray: Installation
2528

26-
### 📦 Install From PyPI
29+
### :package: Install From PyPI
2730

2831
```bash
2932
pip install -U mlc-python
3033
```
3134

32-
### ⚙️ Build from Source
35+
### :gear: Build from Source
3336

3437
```bash
3538
python -m venv .venv
@@ -38,7 +41,7 @@ python -m pip install --verbose --editable ".[dev]"
3841
pre-commit install
3942
```
4043

41-
### 🎡 Create MLC-Python Wheels
44+
### :ferris_wheel: Create MLC-Python Wheels
4245

4346
This project uses `cibuildwheel` to build cross-platform wheels. See `.github/workflows/wheels.ym` for more details.
4447

‎cpp/c_api.cc‎

Lines changed: 18 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
#include "./registry.h"
2-
#include <cstdint>
3-
#include <cstdlib>
4-
#include <iostream>
2+
#include "mlc/core/str.h"
53

64
namespace mlc {
75
namespace registry {
@@ -22,6 +20,22 @@ using ::mlc::registry::TypeTable;
2220
namespace {
2321
thread_local Any last_error;
2422
MLC_REGISTER_FUNC("mlc.ffi.LoadDSO").set_body([](std::string name) { TypeTable::Get(nullptr)->LoadDSO(name); });
23+
MLC_REGISTER_FUNC("mlc.core.JSONParse").set_body([](AnyView json_str) {
24+
if (json_str.type_index == kMLCRawStr) {
25+
return ::mlc::core::ParseJSON(json_str.operator const char *());
26+
} else {
27+
::mlc::Str str = json_str;
28+
return ::mlc::core::ParseJSON(str);
29+
}
30+
});
31+
MLC_REGISTER_FUNC("mlc.core.JSONSerialize").set_body(::mlc::core::Serialize);
32+
MLC_REGISTER_FUNC("mlc.core.JSONDeserialize").set_body([](AnyView json_str) {
33+
if (json_str.type_index == kMLCRawStr) {
34+
return ::mlc::core::Deserialize(json_str.operator const char *());
35+
} else {
36+
return ::mlc::core::Deserialize(json_str.operator ::mlc::Str());
37+
}
38+
});
2539
} // namespace
2640

2741
MLC_API MLCAny MLCGetLastError() {
@@ -154,14 +168,7 @@ MLC_API int32_t MLCExtObjCreate(int32_t num_bytes, int32_t type_index, MLCAny *r
154168

155169
MLC_API int32_t _MLCExtObjDeleteImpl(void *objptr) {
156170
MLC_SAFE_CALL_BEGIN();
157-
MLCAny *header = reinterpret_cast<MLCAny *>(objptr);
158-
if (MLCTypeInfo *info = TypeTable::Global()->GetTypeInfo(header->type_index)) {
159-
::mlc::core::VisitTypeField(objptr, info, ::mlc::core::ExternObjDeleter{});
160-
std::free(objptr);
161-
} else {
162-
std::cerr << "Cannot find type info for type index: " << header->type_index << std::endl;
163-
std::abort();
164-
}
171+
::mlc::core::DeleteExternObject(static_cast<MLCAny *>(objptr));
165172
MLC_SAFE_CALL_END(&last_error);
166173
}
167174

‎cpp/c_api_tests.cc‎

Lines changed: 93 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -16,24 +16,6 @@ MLC_REGISTER_FUNC("mlc.testing.cxx_raw_str").set_body([](const char *x) { return
1616

1717
/**************** Reflection ****************/
1818

19-
struct ReflectionTestObj : public Object {
20-
Str x_mutable;
21-
int32_t y_immutable;
22-
23-
ReflectionTestObj(std::string x, int32_t y) : x_mutable(x), y_immutable(y) {}
24-
int32_t YPlusOne() { return y_immutable + 1; }
25-
26-
MLC_DEF_DYN_TYPE(ReflectionTestObj, Object, "mlc.testing.ReflectionTestObj");
27-
};
28-
29-
struct ReflectionTest : public ObjectRef {
30-
MLC_DEF_OBJ_REF(ReflectionTest, ReflectionTestObj, ObjectRef)
31-
.Field("x_mutable", &ReflectionTestObj::x_mutable)
32-
.FieldReadOnly("y_immutable", &ReflectionTestObj::y_immutable)
33-
.StaticFn("__init__", InitOf<ReflectionTestObj, std::string, int32_t>)
34-
.MemFn("YPlusOne", &ReflectionTestObj::YPlusOne);
35-
};
36-
3719
struct TestingCClassObj : public Object {
3820
int8_t i8;
3921
int16_t i16;
@@ -49,13 +31,64 @@ struct TestingCClassObj : public Object {
4931
UList ulist;
5032
UDict udict;
5133
Str str_;
34+
Str str_readonly;
35+
36+
List<Any> list_any;
37+
List<List<int>> list_list_int;
38+
Dict<Any, Any> dict_any_any;
39+
Dict<Str, Any> dict_str_any;
40+
Dict<Any, Str> dict_any_str;
41+
Dict<Str, List<int>> dict_str_list_int;
42+
43+
Optional<int64_t> opt_i64;
44+
Optional<double> opt_f64;
45+
Optional<void *> opt_raw_ptr;
46+
Optional<DLDataType> opt_dtype;
47+
Optional<DLDevice> opt_device;
48+
Optional<Func> opt_func;
49+
Optional<UList> opt_ulist;
50+
Optional<UDict> opt_udict;
51+
Optional<Str> opt_str;
52+
53+
Optional<List<Any>> opt_list_any;
54+
Optional<List<List<int>>> opt_list_list_int;
55+
Optional<Dict<Any, Any>> opt_dict_any_any;
56+
Optional<Dict<Str, Any>> opt_dict_str_any;
57+
Optional<Dict<Any, Str>> opt_dict_any_str;
58+
Optional<Dict<Str, List<int>>> opt_dict_str_list_int;
5259

5360
explicit TestingCClassObj(int8_t i8, int16_t i16, int32_t i32, int64_t i64, float f32, double f64, void *raw_ptr,
54-
DLDataType dtype, DLDevice device, Any any, Func func, UList ulist, UDict udict, Str str_)
61+
DLDataType dtype, DLDevice device, Any any, Func func, UList ulist, UDict udict, Str str_,
62+
Str str_readonly,
63+
//
64+
List<Any> list_any, List<List<int>> list_list_int, Dict<Any, Any> dict_any_any,
65+
Dict<Str, Any> dict_str_any, Dict<Any, Str> dict_any_str,
66+
Dict<Str, List<int>> dict_str_list_int,
67+
//
68+
Optional<int64_t> opt_i64, Optional<double> opt_f64, Optional<void *> opt_raw_ptr,
69+
Optional<DLDataType> opt_dtype, Optional<DLDevice> opt_device, Optional<Func> opt_func,
70+
Optional<UList> opt_ulist, Optional<UDict> opt_udict, Optional<Str> opt_str,
71+
//
72+
Optional<List<Any>> opt_list_any, Optional<List<List<int>>> opt_list_list_int,
73+
Optional<Dict<Any, Any>> opt_dict_any_any, Optional<Dict<Str, Any>> opt_dict_str_any,
74+
Optional<Dict<Any, Str>> opt_dict_any_str,
75+
Optional<Dict<Str, List<int>>> opt_dict_str_list_int)
5576
: i8(i8), i16(i16), i32(i32), i64(i64), f32(f32), f64(f64), raw_ptr(raw_ptr), dtype(dtype), device(device),
56-
any(any), func(func), ulist(ulist), udict(udict), str_(str_) {}
57-
58-
MLC_DEF_DYN_TYPE(ReflectionTestObj, Object, "mlc.testing.c_class");
77+
any(any), func(func), ulist(ulist), udict(udict), str_(str_), str_readonly(str_readonly),
78+
//
79+
list_any(list_any), list_list_int(list_list_int), dict_any_any(dict_any_any), dict_str_any(dict_str_any),
80+
dict_any_str(dict_any_str), dict_str_list_int(dict_str_list_int),
81+
//
82+
opt_i64(opt_i64), opt_f64(opt_f64), opt_raw_ptr(opt_raw_ptr), opt_dtype(opt_dtype), opt_device(opt_device),
83+
opt_func(opt_func), opt_ulist(opt_ulist), opt_udict(opt_udict), opt_str(opt_str),
84+
//
85+
opt_list_any(opt_list_any), opt_list_list_int(opt_list_list_int), opt_dict_any_any(opt_dict_any_any),
86+
opt_dict_str_any(opt_dict_str_any), opt_dict_any_str(opt_dict_any_str),
87+
opt_dict_str_list_int(opt_dict_str_list_int) {}
88+
89+
int64_t i64_plus_one() const { return i64 + 1; }
90+
91+
MLC_DEF_DYN_TYPE(TestingCClassObj, Object, "mlc.testing.c_class");
5992
};
6093

6194
struct TestingCClass : public ObjectRef {
@@ -74,8 +107,44 @@ struct TestingCClass : public ObjectRef {
74107
.Field("ulist", &TestingCClassObj::ulist)
75108
.Field("udict", &TestingCClassObj::udict)
76109
.Field("str_", &TestingCClassObj::str_)
77-
.StaticFn("__init__", InitOf<TestingCClassObj, int8_t, int16_t, int32_t, int64_t, float, double, void *,
78-
DLDataType, DLDevice, Any, Func, UList, UDict, Str>);
110+
.FieldReadOnly("str_readonly", &TestingCClassObj::str_readonly)
111+
//
112+
.Field("list_any", &TestingCClassObj::list_any)
113+
.Field("list_list_int", &TestingCClassObj::list_list_int)
114+
.Field("dict_any_any", &TestingCClassObj::dict_any_any)
115+
.Field("dict_str_any", &TestingCClassObj::dict_str_any)
116+
.Field("dict_any_str", &TestingCClassObj::dict_any_str)
117+
.Field("dict_str_list_int", &TestingCClassObj::dict_str_list_int)
118+
//
119+
.Field("opt_i64", &TestingCClassObj::opt_i64)
120+
.Field("opt_f64", &TestingCClassObj::opt_f64)
121+
.Field("opt_raw_ptr", &TestingCClassObj::opt_raw_ptr)
122+
.Field("opt_dtype", &TestingCClassObj::opt_dtype)
123+
.Field("opt_device", &TestingCClassObj::opt_device)
124+
.Field("opt_func", &TestingCClassObj::opt_func)
125+
.Field("opt_ulist", &TestingCClassObj::opt_ulist)
126+
.Field("opt_udict", &TestingCClassObj::opt_udict)
127+
.Field("opt_str", &TestingCClassObj::opt_str)
128+
//
129+
.Field("opt_list_any", &TestingCClassObj::opt_list_any)
130+
.Field("opt_list_list_int", &TestingCClassObj::opt_list_list_int)
131+
.Field("opt_dict_any_any", &TestingCClassObj::opt_dict_any_any)
132+
.Field("opt_dict_str_any", &TestingCClassObj::opt_dict_str_any)
133+
.Field("opt_dict_any_str", &TestingCClassObj::opt_dict_any_str)
134+
.Field("opt_dict_str_list_int", &TestingCClassObj::opt_dict_str_list_int)
135+
//
136+
.MemFn("i64_plus_one", &TestingCClassObj::i64_plus_one)
137+
.StaticFn("__init__",
138+
InitOf<TestingCClassObj, int8_t, int16_t, int32_t, int64_t, float, double, void *, DLDataType, DLDevice,
139+
Any, Func, UList, UDict, Str, Str,
140+
//
141+
List<Any>, List<List<int>>, Dict<Any, Any>, Dict<Str, Any>, Dict<Any, Str>, Dict<Str, List<int>>,
142+
//
143+
Optional<int64_t>, Optional<double>, Optional<void *>, Optional<DLDataType>, Optional<DLDevice>,
144+
Optional<Func>, Optional<UList>, Optional<UDict>, Optional<Str>,
145+
//
146+
Optional<List<Any>>, Optional<List<List<int>>>, Optional<Dict<Any, Any>>,
147+
Optional<Dict<Str, Any>>, Optional<Dict<Any, Str>>, Optional<Dict<Str, List<int>>>>);
79148
};
80149

81150
/**************** Traceback ****************/

‎cpp/registry.h‎

Lines changed: 12 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -261,65 +261,32 @@ struct _POD_REG {
261261
.MemFn("__str__", &::mlc::base::TypeTraits<std::nullptr_t>::__str__);
262262
inline static const int32_t _int = //
263263
::mlc::core::ReflectionHelper(static_cast<int32_t>(MLCTypeIndex::kMLCInt))
264+
.StaticFn("__init__", [](AnyView value) { return value.operator int64_t(); })
264265
.StaticFn("__new_ref__",
265-
[](void *_dst, int64_t value) {
266-
MLCAny **dst = reinterpret_cast<MLCAny **>(_dst);
267-
MLCAny *ret = ::mlc::PODAllocator<int64_t>::New(value);
268-
if (*dst != nullptr) {
269-
::mlc::base::DecRef(*dst);
270-
}
271-
*dst = ret;
272-
})
266+
[](void *dst, Optional<int64_t> value) { *reinterpret_cast<Optional<int64_t> *>(dst) = value; })
273267
.MemFn("__str__", &::mlc::base::TypeTraits<int64_t>::__str__);
274-
inline static const int32_t _float = //
268+
inline static const int32_t _float =
275269
::mlc::core::ReflectionHelper(static_cast<int32_t>(MLCTypeIndex::kMLCFloat))
276270
.StaticFn("__new_ref__",
277-
[](void *_dst, double value) {
278-
MLCAny **dst = reinterpret_cast<MLCAny **>(_dst);
279-
MLCAny *ret = ::mlc::PODAllocator<double>::New(value);
280-
if (*dst != nullptr) {
281-
::mlc::base::DecRef(*dst);
282-
}
283-
*dst = ret;
284-
})
271+
[](void *dst, Optional<double> value) { *reinterpret_cast<Optional<double> *>(dst) = value; })
285272
.MemFn("__str__", &::mlc::base::TypeTraits<double>::__str__);
286-
inline static const int32_t _ptr = //
273+
inline static const int32_t _ptr =
287274
::mlc::core::ReflectionHelper(static_cast<int32_t>(MLCTypeIndex::kMLCPtr))
288275
.StaticFn("__new_ref__",
289-
[](void *_dst, void *value) {
290-
MLCAny **dst = reinterpret_cast<MLCAny **>(_dst);
291-
MLCAny *ret = ::mlc::PODAllocator<void *>::New(value);
292-
if (*dst != nullptr) {
293-
::mlc::base::DecRef(*dst);
294-
}
295-
*dst = ret;
296-
})
276+
[](void *dst, Optional<void *> value) { *reinterpret_cast<Optional<void *> *>(dst) = value; })
297277
.MemFn("__str__", &::mlc::base::TypeTraits<void *>::__str__);
298-
inline static const int32_t _device = //
278+
inline static const int32_t _device =
299279
::mlc::core::ReflectionHelper(static_cast<int32_t>(MLCTypeIndex::kMLCDevice))
300280
.StaticFn("__init__", [](AnyView device) { return device.operator DLDevice(); })
301281
.StaticFn("__new_ref__",
302-
[](void *_dst, DLDevice value) {
303-
MLCAny **dst = reinterpret_cast<MLCAny **>(_dst);
304-
MLCAny *ret = ::mlc::PODAllocator<DLDevice>::New(value);
305-
if (*dst != nullptr) {
306-
::mlc::base::DecRef(*dst);
307-
}
308-
*dst = ret;
309-
})
282+
[](void *dst, Optional<DLDevice> value) { *reinterpret_cast<Optional<DLDevice> *>(dst) = value; })
310283
.MemFn("__str__", &::mlc::base::TypeTraits<DLDevice>::__str__);
311-
inline static const int32_t _dtype = //
284+
inline static const int32_t _dtype =
312285
::mlc::core::ReflectionHelper(static_cast<int32_t>(MLCTypeIndex::kMLCDataType))
313286
.StaticFn("__init__", [](AnyView dtype) { return dtype.operator DLDataType(); })
314-
.StaticFn("__new_ref__",
315-
[](void *_dst, DLDataType value) {
316-
MLCAny **dst = reinterpret_cast<MLCAny **>(_dst);
317-
MLCAny *ret = ::mlc::PODAllocator<DLDataType>::New(value);
318-
if (*dst != nullptr) {
319-
::mlc::base::DecRef(*dst);
320-
}
321-
*dst = ret;
322-
})
287+
.StaticFn(
288+
"__new_ref__",
289+
[](void *dst, Optional<DLDataType> value) { *reinterpret_cast<Optional<DLDataType> *>(dst) = value; })
323290
.MemFn("__str__", &::mlc::base::TypeTraits<DLDataType>::__str__);
324291
inline static const int32_t _str = //
325292
::mlc::core::ReflectionHelper(static_cast<int32_t>(MLCTypeIndex::kMLCRawStr))

‎include/mlc/base/optional.h‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,12 +36,15 @@ template <typename T> struct Optional {
3636
}
3737
/***** Section 2. Accessor and bool operator *****/
3838
MLC_INLINE const TObj *get() const { return ref_.get(); }
39+
MLC_INLINE const TObj *value() const { return ref_.get(); }
3940
MLC_INLINE TObj *get() { return ref_.get(); }
41+
MLC_INLINE TObj *value() { return ref_.get(); }
4042
MLC_INLINE const TObj *operator->() const { return ref_.operator->(); }
41-
MLC_INLINE TObj *operator->() { return ref_.operator->(); }
4243
MLC_INLINE const TObj &operator*() const { return ref_.operator*(); }
44+
MLC_INLINE TObj *operator->() { return ref_.operator->(); }
4345
MLC_INLINE TObj &operator*() { return ref_.operator*(); }
4446
MLC_INLINE bool defined() const { return ref_.defined(); }
47+
MLC_INLINE bool has_value() const { return ref_.defined(); }
4548
MLC_INLINE bool operator==(std::nullptr_t) const { return !defined(); }
4649
MLC_INLINE bool operator!=(std::nullptr_t) const { return defined(); }
4750
MLC_INLINE operator bool() const { return defined(); }

0 commit comments

Comments
 (0)