Skip to content

Commit d70be71

Browse files
certikcertik
authored
Merge pull request #1185 from Thirumalai-Shaktivel/compile_packages
Compile Simple Python package
2 parents 4b8d1ac + 2d5f401 commit d70be71

File tree

11 files changed

+163
-30
lines changed

11 files changed

+163
-30
lines changed

‎integration_tests/CMakeLists.txt

+1
Original file line numberDiff line numberDiff line change
@@ -194,6 +194,7 @@ RUN(NAME test_dict_05 LABELS cpython llvm)
194194
RUN(NAME test_for_loop LABELS cpython llvm c)
195195
RUN(NAME modules_01 LABELS cpython llvm)
196196
RUN(NAME modules_02 LABELS cpython llvm)
197+
RUN(NAME test_import_01 LABELS cpython llvm)
197198
RUN(NAME test_math LABELS cpython llvm)
198199
RUN(NAME test_numpy_01 LABELS cpython llvm c)
199200
RUN(NAME test_numpy_02 LABELS cpython llvm c)

‎integration_tests/test_import/__init__.py

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
from .test_import_1 import print_a, print_b
2+
from .test_import_2 import print_c

‎integration_tests/test_import/test_import_1.py

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
def print_a() -> str:
2+
return "A"
3+
4+
def print_b() -> str:
5+
return "B"

‎integration_tests/test_import/test_import_2.py

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
def print_c() -> str:
2+
return "C"

‎integration_tests/test_import_01.py

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
import test_import
2+
from test_import import print_a
3+
4+
print(print_a())
5+
print(test_import.print_b())
6+
print(test_import.print_c())

‎src/bin/lpython.cpp

+28-14
Original file line numberDiff line numberDiff line change
@@ -164,7 +164,8 @@ int emit_asr(const std::string &infile,
164164
diagnostics.diagnostics.clear();
165165
LFortran::Result<LFortran::ASR::TranslationUnit_t*>
166166
r = LFortran::LPython::python_ast_to_asr(al, *ast, diagnostics, true,
167-
compiler_options.disable_main, compiler_options.symtab_only, infile);
167+
compiler_options.disable_main, compiler_options.symtab_only, infile,
168+
compiler_options.import_path);
168169
std::cerr << diagnostics.render(input, lm, compiler_options);
169170
if (!r.ok) {
170171
LFORTRAN_ASSERT(diagnostics.has_error())
@@ -207,7 +208,8 @@ int emit_cpp(const std::string &infile,
207208
diagnostics.diagnostics.clear();
208209
LFortran::Result<LFortran::ASR::TranslationUnit_t*>
209210
r1 = LFortran::LPython::python_ast_to_asr(al, *ast, diagnostics, true,
210-
compiler_options.disable_main, compiler_options.symtab_only, infile);
211+
compiler_options.disable_main, compiler_options.symtab_only, infile,
212+
compiler_options.import_path);
211213
std::cerr << diagnostics.render(input, lm, compiler_options);
212214
if (!r1.ok) {
213215
LFORTRAN_ASSERT(diagnostics.has_error())
@@ -248,7 +250,8 @@ int emit_c(const std::string &infile,
248250
diagnostics.diagnostics.clear();
249251
LFortran::Result<LFortran::ASR::TranslationUnit_t*>
250252
r1 = LFortran::LPython::python_ast_to_asr(al, *ast, diagnostics, true,
251-
compiler_options.disable_main, compiler_options.symtab_only, infile);
253+
compiler_options.disable_main, compiler_options.symtab_only, infile,
254+
compiler_options.import_path);
252255
std::cerr << diagnostics.render(input, lm, compiler_options);
253256
if (!r1.ok) {
254257
LFORTRAN_ASSERT(diagnostics.has_error())
@@ -289,7 +292,8 @@ int emit_wat(const std::string &infile,
289292
diagnostics.diagnostics.clear();
290293
LFortran::Result<LFortran::ASR::TranslationUnit_t*>
291294
r1 = LFortran::LPython::python_ast_to_asr(al, *ast, diagnostics, true,
292-
compiler_options.disable_main, compiler_options.symtab_only, infile);
295+
compiler_options.disable_main, compiler_options.symtab_only, infile,
296+
compiler_options.import_path);
293297
std::cerr << diagnostics.render(input, lm, compiler_options);
294298
if (!r1.ok) {
295299
LFORTRAN_ASSERT(diagnostics.has_error())
@@ -333,7 +337,8 @@ int get_symbols (const std::string &infile,
333337
LFortran::LPython::AST::ast_t* ast = r1.result;
334338
LFortran::Result<LFortran::ASR::TranslationUnit_t*>
335339
x = LFortran::LPython::python_ast_to_asr(al, *ast, diagnostics, true,
336-
compiler_options.disable_main, compiler_options.symtab_only, infile);
340+
compiler_options.disable_main, compiler_options.symtab_only,
341+
infile, compiler_options.import_path);
337342
if (!x.ok) {
338343
std::cout << "{}\n";
339344
return 0;
@@ -426,7 +431,8 @@ int get_errors (const std::string &infile,
426431
LFortran::LPython::AST::ast_t* ast = r1.result;
427432
LFortran::Result<LFortran::ASR::TranslationUnit_t*>
428433
r = LFortran::LPython::python_ast_to_asr(al, *ast, diagnostics, true,
429-
compiler_options.disable_main, compiler_options.symtab_only, infile);
434+
compiler_options.disable_main, compiler_options.symtab_only,
435+
infile, compiler_options.import_path);
430436
}
431437
std::vector<LFortran::LPython::error_highlight> diag_lists;
432438
LFortran::LPython::error_highlight h;
@@ -538,7 +544,8 @@ int emit_llvm(const std::string &infile,
538544
diagnostics.diagnostics.clear();
539545
LFortran::Result<LFortran::ASR::TranslationUnit_t*>
540546
r1 = LFortran::LPython::python_ast_to_asr(al, *ast, diagnostics, true,
541-
compiler_options.disable_main, compiler_options.symtab_only, infile);
547+
compiler_options.disable_main, compiler_options.symtab_only, infile,
548+
compiler_options.import_path);
542549
std::cerr << diagnostics.render(input, lm, compiler_options);
543550
if (!r1.ok) {
544551
LFORTRAN_ASSERT(diagnostics.has_error())
@@ -596,7 +603,8 @@ int compile_python_to_object_file(
596603
LFortran::Result<LFortran::ASR::TranslationUnit_t*>
597604
r1 = LFortran::LPython::python_ast_to_asr(al, *ast, diagnostics,
598605
!(arg_c && compiler_options.disable_main),
599-
compiler_options.disable_main, compiler_options.symtab_only, infile);
606+
compiler_options.disable_main, compiler_options.symtab_only, infile,
607+
compiler_options.import_path);
600608
auto ast_to_asr_end = std::chrono::high_resolution_clock::now();
601609
times.push_back(std::make_pair("AST to ASR", std::chrono::duration<double, std::milli>(ast_to_asr_end - ast_to_asr_start).count()));
602610
std::cerr << diagnostics.render(input, lm, compiler_options);
@@ -679,7 +687,8 @@ int compile_to_binary_wasm(
679687
auto ast_to_asr_start = std::chrono::high_resolution_clock::now();
680688
LFortran::Result<LFortran::ASR::TranslationUnit_t*>
681689
r1 = LFortran::LPython::python_ast_to_asr(al, *ast, diagnostics, true,
682-
compiler_options.disable_main, compiler_options.symtab_only, infile);
690+
compiler_options.disable_main, compiler_options.symtab_only, infile,
691+
compiler_options.import_path);
683692
auto ast_to_asr_end = std::chrono::high_resolution_clock::now();
684693
times.push_back(std::make_pair("AST to ASR", std::chrono::duration<double, std::milli>(ast_to_asr_end - ast_to_asr_start).count()));
685694
std::cerr << diagnostics.render(input, lm, compiler_options);
@@ -927,7 +936,8 @@ EMSCRIPTEN_KEEPALIVE char* emit_asr_from_source(char *input) {
927936
auto casted_ast = (LFortran::LPython::AST::ast_t*)ast.result;
928937
LFortran::Result<LFortran::ASR::TranslationUnit_t*>
929938
asr = LFortran::LPython::python_ast_to_asr(al, *casted_ast, diagnostics, true,
930-
compiler_options.disable_main, compiler_options.symtab_only, "input");
939+
compiler_options.disable_main, compiler_options.symtab_only, "input",
940+
compiler_options.import_path);
931941
out = diagnostics.render(input, lm, compiler_options);
932942
if (asr.ok) {
933943
out += LFortran::pickle(*asr.result, compiler_options.use_colors, compiler_options.indent,
@@ -946,7 +956,8 @@ EMSCRIPTEN_KEEPALIVE char* emit_wat_from_source(char *input) {
946956
auto casted_ast = (LFortran::LPython::AST::ast_t*)ast.result;
947957
LFortran::Result<LFortran::ASR::TranslationUnit_t*>
948958
asr = LFortran::LPython::python_ast_to_asr(al, *casted_ast, diagnostics, true,
949-
compiler_options.disable_main, compiler_options.symtab_only, "input");
959+
compiler_options.disable_main, compiler_options.symtab_only, "input",
960+
compiler_options.import_path);
950961
out = diagnostics.render(input, lm, compiler_options);
951962
if (asr.ok) {
952963
LFortran::Result<LFortran::Vec<uint8_t>>
@@ -974,7 +985,8 @@ EMSCRIPTEN_KEEPALIVE char* emit_cpp_from_source(char *input) {
974985
auto casted_ast = (LFortran::LPython::AST::ast_t*)ast.result;
975986
LFortran::Result<LFortran::ASR::TranslationUnit_t*>
976987
asr = LFortran::LPython::python_ast_to_asr(al, *casted_ast, diagnostics, true,
977-
compiler_options.disable_main, compiler_options.symtab_only, "input");
988+
compiler_options.disable_main, compiler_options.symtab_only, "input",
989+
compiler_options.import_path);
978990
out = diagnostics.render(input, lm, compiler_options);
979991
if (asr.ok) {
980992
auto res = LFortran::asr_to_cpp(al, *asr.result, diagnostics,
@@ -1013,7 +1025,8 @@ EMSCRIPTEN_KEEPALIVE char* emit_wasm_from_source(char *input) {
10131025
auto casted_ast = (LFortran::LPython::AST::ast_t*)ast.result;
10141026
LFortran::Result<LFortran::ASR::TranslationUnit_t*>
10151027
asr = LFortran::LPython::python_ast_to_asr(al, *casted_ast, diagnostics, true,
1016-
compiler_options.disable_main, compiler_options.symtab_only, "input");
1028+
compiler_options.disable_main, compiler_options.symtab_only, "input",
1029+
compiler_options.import_path);
10171030
out = diagnostics.render(input, lm, compiler_options);
10181031
if (asr.ok) {
10191032
LFortran::Result<LFortran::Vec<uint8_t>>
@@ -1114,7 +1127,8 @@ int main(int argc, char *argv[])
11141127
// app.add_flag("-E", arg_E, "Preprocess only; do not compile, assemble or link");
11151128
// app.add_option("-l", arg_l, "Link library option");
11161129
// app.add_option("-L", arg_L, "Library path option");
1117-
// app.add_option("-I", arg_I, "Include path")->allow_extra_args(false);
1130+
app.add_option("-I", compiler_options.import_path, "Specify the path"
1131+
"to look for the module")->allow_extra_args(false);
11181132
// app.add_option("-J", arg_J, "Where to save mod files");
11191133
// app.add_flag("-g", arg_g, "Compile with debugging information");
11201134
// app.add_option("-D", compiler_options.c_preprocessor_defines, "Define <macro>=<value> (or 1 if <value> omitted)")->allow_extra_args(false);

‎src/libasr/utils.h

+1
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ struct CompilerOptions {
3737
bool implicit_typing = false;
3838
bool implicit_interface = false;
3939
std::string target = "";
40+
std::string import_path = "";
4041
Platform platform;
4142

4243
CompilerOptions () : platform{get_platform()} {};

‎src/lpython/semantics/python_ast_to_asr.cpp

+81-15
Original file line numberDiff line numberDiff line change
@@ -254,7 +254,7 @@ ASR::TranslationUnit_t* compile_module_till_asr(Allocator& al,
254254
LFortran::LocationManager lm;
255255
lm.in_filename = infile;
256256
Result<ASR::TranslationUnit_t*> r2 = python_ast_to_asr(al, *ast,
257-
diagnostics, false, true, false, infile);
257+
diagnostics, false, true, false, infile, "");
258258
// TODO: Uncomment once a check is added for ensuring
259259
// that module.py file hasn't changed between
260260
// builds.
@@ -315,6 +315,7 @@ ASR::Module_t* load_module(Allocator &al, SymbolTable *symtab,
315315
bool compile_module = true;
316316
ASR::TranslationUnit_t* mod1 = nullptr;
317317
std::string input;
318+
std::string module_dir_name = "";
318319
bool found = set_module_path(infile0c, rl_path, infile,
319320
path_used, input, ltypes, enum_py);
320321
if( !found ) {
@@ -332,7 +333,8 @@ ASR::Module_t* load_module(Allocator &al, SymbolTable *symtab,
332333
return nullptr;
333334
}
334335
if (!found) {
335-
err("Could not find the module '" + infile0 + "'", loc);
336+
err("Could not find the module '" + infile0 + "'. If an import path "
337+
"is available, please use the `-I` option to specify it", loc);
336338
}
337339
if (ltypes) return nullptr;
338340

@@ -342,7 +344,15 @@ ASR::Module_t* load_module(Allocator &al, SymbolTable *symtab,
342344

343345
// insert into `symtab`
344346
std::vector<std::pair<std::string, ASR::Module_t*>> children_modules;
345-
ASRUtils::extract_module_python(*mod1, children_modules, module_name);
347+
if (module_name == "__init__") {
348+
// remove `__init__.py`
349+
module_dir_name = infile.substr(0, infile.find_last_of('/'));
350+
// assign module directory name
351+
module_dir_name = module_dir_name.substr(module_dir_name.find_last_of('/') + 1);
352+
ASRUtils::extract_module_python(*mod1, children_modules, module_dir_name);
353+
} else {
354+
ASRUtils::extract_module_python(*mod1, children_modules, module_name);
355+
}
346356
ASR::Module_t* mod2 = nullptr;
347357
for( auto& a: children_modules ) {
348358
std::string a_name = a.first;
@@ -351,6 +361,10 @@ ASR::Module_t* load_module(Allocator &al, SymbolTable *symtab,
351361
a_mod->m_name = s2c(al, module_name);
352362
a_mod->m_intrinsic = intrinsic;
353363
mod2 = a_mod;
364+
} else if (a_name == module_dir_name) {
365+
a_mod->m_name = s2c(al, module_dir_name);
366+
a_mod->m_intrinsic = intrinsic;
367+
mod2 = a_mod;
354368
}
355369
symtab->add_symbol(a_name, (ASR::symbol_t*)a_mod);
356370
a_mod->m_symtab->parent = symtab;
@@ -436,9 +450,18 @@ ASR::symbol_t* import_from_module(Allocator &al, ASR::Module_t *m, SymbolTable *
436450
ASR::accessType::Public
437451
);
438452
return ASR::down_cast<ASR::symbol_t>(v);
453+
} else if (ASR::is_a<ASR::ExternalSymbol_t>(*t)) {
454+
ASR::ExternalSymbol_t *es = ASR::down_cast<ASR::ExternalSymbol_t>(t);
455+
SymbolTable *symtab = current_scope;
456+
while (symtab->parent != nullptr) symtab = symtab->parent;
457+
ASR::symbol_t *sym = symtab->get_symbol(es->m_module_name);
458+
ASR::Module_t *m = ASR::down_cast<ASR::Module_t>(sym);
459+
460+
return import_from_module(al, m, symtab, es->m_name,
461+
cur_sym_name, new_sym_name, loc);
439462
} else {
440-
throw SemanticError("Only Subroutines, Functions and Variables are currently supported in 'import'",
441-
loc);
463+
throw SemanticError("Only Subroutines, Functions, Variables and "
464+
"ExternalSymbol are currently supported in 'import'", loc);
442465
}
443466
LFORTRAN_ASSERT(false);
444467
return nullptr;
@@ -475,6 +498,7 @@ class CommonVisitor : public AST::BaseVisitor<Struct> {
475498
IntrinsicNodeHandler intrinsic_node_handler;
476499
std::map<int, ASR::symbol_t*> &ast_overload;
477500
std::string parent_dir;
501+
std::string import_path;
478502
Vec<ASR::stmt_t*> *current_body;
479503
ASR::ttype_t* ann_assign_target_type;
480504

@@ -485,9 +509,10 @@ class CommonVisitor : public AST::BaseVisitor<Struct> {
485509

486510
CommonVisitor(Allocator &al, SymbolTable *symbol_table,
487511
diag::Diagnostics &diagnostics, bool main_module,
488-
std::map<int, ASR::symbol_t*> &ast_overload, std::string parent_dir)
512+
std::map<int, ASR::symbol_t*> &ast_overload, std::string parent_dir,
513+
std::string import_path)
489514
: diag{diagnostics}, al{al}, current_scope{symbol_table}, main_module{main_module},
490-
ast_overload{ast_overload}, parent_dir{parent_dir},
515+
ast_overload{ast_overload}, parent_dir{parent_dir}, import_path{import_path},
491516
current_body{nullptr}, ann_assign_target_type{nullptr} {
492517
current_module_dependencies.reserve(al, 4);
493518
}
@@ -2912,8 +2937,10 @@ class SymbolTableVisitor : public CommonVisitor<SymbolTableVisitor> {
29122937

29132938
SymbolTableVisitor(Allocator &al, SymbolTable *symbol_table,
29142939
diag::Diagnostics &diagnostics, bool main_module,
2915-
std::map<int, ASR::symbol_t*> &ast_overload, std::string parent_dir)
2916-
: CommonVisitor(al, symbol_table, diagnostics, main_module, ast_overload, parent_dir), is_derived_type{false} {}
2940+
std::map<int, ASR::symbol_t*> &ast_overload, std::string parent_dir,
2941+
std::string import_path)
2942+
: CommonVisitor(al, symbol_table, diagnostics, main_module, ast_overload,
2943+
parent_dir, import_path), is_derived_type{false} {}
29172944

29182945

29192946
ASR::symbol_t* resolve_symbol(const Location &loc, const std::string &sub_name) {
@@ -3206,6 +3233,24 @@ class SymbolTableVisitor : public CommonVisitor<SymbolTableVisitor> {
32063233
st = st->parent;
32073234
}
32083235
bool ltypes, enum_py;
3236+
if (msym != "ltypes") {
3237+
if (import_path != "" &&
3238+
!path_exits(paths[0] + '/' + msym + ".py")) {
3239+
paths = {import_path};
3240+
if (parent_dir != "") paths[0] += '/' + parent_dir;
3241+
if(is_directory(paths[0])) {
3242+
paths[0] += '/' + msym;
3243+
msym = "__init__";
3244+
}
3245+
} else if (is_directory(msym)) {
3246+
paths = {rl_path, msym};
3247+
msym = "__init__";
3248+
} else if (paths[1] != ""
3249+
&& is_directory(paths[1] + '/' + msym)) {
3250+
paths[1] += '/' + msym;
3251+
msym = "__init__";
3252+
}
3253+
}
32093254
t = (ASR::symbol_t*)(load_module(al, st,
32103255
msym, x.base.base.loc, false, paths, ltypes, enum_py,
32113256
[&](const std::string &msg, const Location &loc) { throw SemanticError(msg, loc); }
@@ -3251,6 +3296,24 @@ class SymbolTableVisitor : public CommonVisitor<SymbolTableVisitor> {
32513296
}
32523297
for (auto &mod_sym : mods) {
32533298
bool ltypes, enum_py;
3299+
if (mod_sym != "ltypes") {
3300+
if (import_path != "" &&
3301+
!path_exits(paths[0] + '/' + mod_sym + ".py")) {
3302+
paths = {import_path};
3303+
if (parent_dir != "") paths[0] += '/' + parent_dir;
3304+
if(is_directory(paths[0])) {
3305+
paths[0] += '/' + mod_sym;
3306+
mod_sym = "__init__";
3307+
}
3308+
} else if (is_directory(mod_sym)) {
3309+
paths = {rl_path, mod_sym};
3310+
mod_sym = "__init__";
3311+
} else if (paths[1] != ""
3312+
&& is_directory(paths[1] + '/' + mod_sym)) {
3313+
paths[1] += '/' + mod_sym;
3314+
mod_sym = "__init__";
3315+
}
3316+
}
32543317
t = (ASR::symbol_t*)(load_module(al, st,
32553318
mod_sym, x.base.base.loc, false, paths, ltypes, enum_py,
32563319
[&](const std::string &msg, const Location &loc) { throw SemanticError(msg, loc); }
@@ -3355,9 +3418,11 @@ class SymbolTableVisitor : public CommonVisitor<SymbolTableVisitor> {
33553418

33563419
Result<ASR::asr_t*> symbol_table_visitor(Allocator &al, const AST::Module_t &ast,
33573420
diag::Diagnostics &diagnostics, bool main_module,
3358-
std::map<int, ASR::symbol_t*> &ast_overload, std::string parent_dir)
3421+
std::map<int, ASR::symbol_t*> &ast_overload, std::string parent_dir,
3422+
std::string import_path)
33593423
{
3360-
SymbolTableVisitor v(al, nullptr, diagnostics, main_module, ast_overload, parent_dir);
3424+
SymbolTableVisitor v(al, nullptr, diagnostics, main_module, ast_overload,
3425+
parent_dir, import_path);
33613426
try {
33623427
v.visit_Module(ast);
33633428
} catch (const SemanticError &e) {
@@ -3383,8 +3448,8 @@ class BodyVisitor : public CommonVisitor<BodyVisitor> {
33833448

33843449
BodyVisitor(Allocator &al, ASR::asr_t *unit, diag::Diagnostics &diagnostics,
33853450
bool main_module, std::map<int, ASR::symbol_t*> &ast_overload)
3386-
: CommonVisitor(al, nullptr, diagnostics, main_module, ast_overload, ""), asr{unit},
3387-
gotoids{0}
3451+
: CommonVisitor(al, nullptr, diagnostics, main_module, ast_overload, "", ""),
3452+
asr{unit}, gotoids{0}
33883453
{}
33893454

33903455
// Transforms statements to a list of ASR statements
@@ -5524,15 +5589,16 @@ std::string get_parent_dir(const std::string &path) {
55245589

55255590
Result<ASR::TranslationUnit_t*> python_ast_to_asr(Allocator &al,
55265591
AST::ast_t &ast, diag::Diagnostics &diagnostics, bool main_module,
5527-
bool disable_main, bool symtab_only, std::string file_path)
5592+
bool disable_main, bool symtab_only, std::string file_path,
5593+
std::string import_path)
55285594
{
55295595
std::map<int, ASR::symbol_t*> ast_overload;
55305596
std::string parent_dir = get_parent_dir(file_path);
55315597
AST::Module_t *ast_m = AST::down_cast2<AST::Module_t>(&ast);
55325598

55335599
ASR::asr_t *unit;
55345600
auto res = symbol_table_visitor(al, *ast_m, diagnostics, main_module,
5535-
ast_overload, parent_dir);
5601+
ast_overload, parent_dir, import_path);
55365602
if (res.ok) {
55375603
unit = res.result;
55385604
} else {

0 commit comments

Comments
 (0)