Merge pull request #999 from czgdp1807/mod01

Saving LPython's intrinsic modules as ``pyc`` files

Author: czgdp1807

CMakeLists.txt

@@ -80,6 +80,9 @@ set(WITH_LFORTRAN_BINARY_MODFILES YES
 set(WITH_RUNTIME_LIBRARY YES
     CACHE BOOL "Compile and install the runtime library")
 
+set(WITH_INTRINSIC_MODULES no
+    CACHE BOOL "Compile intrinsic modules to .pyc (ASR) at build time")
+
 # Find ZLIB with our custom finder before including LLVM since the finder for LLVM
 # might search for ZLIB again and find the shared libraries instead of the static ones
 find_package(StaticZLIB REQUIRED)

src/bin/CMakeLists.txt

@@ -57,3 +57,31 @@ set_target_properties(lpython PROPERTIES
     RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/$<0:>
     LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/$<0:>
     ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/$<0:>)
+
+if (WITH_INTRINSIC_MODULES)
+    macro(LPYTHON_COMPILE_MODULE name)
+        add_custom_command(
+            OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/../runtime/${name}.pyc
+            COMMAND ${CMAKE_CURRENT_BINARY_DIR}/lpython
+            ARGS --disable-main -c ${CMAKE_CURRENT_SOURCE_DIR}/../runtime/${name}.py -o ${name}.o
+            WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../runtime
+            DEPENDS lpython ${CMAKE_CURRENT_SOURCE_DIR}/../runtime/${name}.py ${ARGN}
+            COMMENT "LPython Compiling ${name}.py")
+    endmacro(LPYTHON_COMPILE_MODULE)
+
+    LPYTHON_COMPILE_MODULE(lpython_intrinsic_numpy)
+    LPYTHON_COMPILE_MODULE(lpython_builtin)
+
+    add_custom_target(lpython_intrinsics
+        ALL
+        DEPENDS
+        ${CMAKE_CURRENT_BINARY_DIR}/../runtime/lpython_intrinsic_numpy.pyc
+        ${CMAKE_CURRENT_BINARY_DIR}/../runtime/lpython_builtin.pyc
+    )
+
+    install(
+        FILES ${CMAKE_CURRENT_BINARY_DIR}/../runtime/lpython_intrinsic_numpy.pyc
+            ${CMAKE_CURRENT_BINARY_DIR}/../runtime/lpython_builtin.pyc
+        DESTINATION share/lfortran/lib
+    )
+endif()

src/bin/lpython.cpp

@@ -512,7 +512,7 @@ int compile_python_to_object_file(
         const std::string &runtime_library_dir,
         LCompilers::PassManager& pass_manager,
         CompilerOptions &compiler_options,
-        bool time_report)
+        bool time_report, bool arg_c=false)
 {
     Allocator al(4*1024);
     LFortran::diag::Diagnostics diagnostics;
@@ -540,7 +540,8 @@ int compile_python_to_object_file(
     diagnostics.diagnostics.clear();
     auto ast_to_asr_start = std::chrono::high_resolution_clock::now();
     LFortran::Result<LFortran::ASR::TranslationUnit_t*>
-        r1 = LFortran::LPython::python_ast_to_asr(al, *ast, diagnostics, true,
+        r1 = LFortran::LPython::python_ast_to_asr(al, *ast, diagnostics,
+            !(arg_c && compiler_options.disable_main),
             compiler_options.disable_main, compiler_options.symtab_only, infile);
     auto ast_to_asr_end = std::chrono::high_resolution_clock::now();
     times.push_back(std::make_pair("AST to ASR", std::chrono::duration<double, std::milli>(ast_to_asr_end - ast_to_asr_start).count()));
@@ -551,6 +552,12 @@ int compile_python_to_object_file(
         return 2;
     }
     LFortran::ASR::TranslationUnit_t* asr = r1.result;
+    if( compiler_options.disable_main ) {
+        int err = LFortran::LPython::save_pyc_files(*asr, infile);
+        if( err ) {
+            return err;
+        }
+    }
     diagnostics.diagnostics.clear();
 
     // ASR -> LLVM
@@ -1065,7 +1072,8 @@ int main(int argc, char *argv[])
         if (arg_c) {
             if (backend == Backend::llvm) {
 #ifdef HAVE_LFORTRAN_LLVM
-                return compile_python_to_object_file(arg_file, outfile, runtime_library_dir, lpython_pass_manager, compiler_options, time_report);
+                return compile_python_to_object_file(arg_file, outfile, runtime_library_dir, lpython_pass_manager, compiler_options, time_report,
+                                                     arg_c);
 #else
                 std::cerr << "The -c option requires the LLVM backend to be enabled. Recompile with `WITH_LLVM=yes`." << std::endl;
                 return 1;

src/libasr/modfile.cpp

@@ -12,25 +12,8 @@ namespace LFortran {
 
 const std::string lfortran_modfile_type_string = "LFortran Modfile";
 
-// The save_modfile() and load_modfile() must stay consistent. What is saved
-// must be loaded in exactly the same order.
-
-/*
-    Saves the module into a binary stream.
-
-    That stream can be saved to a mod file by the caller.
-    The sections in the file/stream are saved using write_string(), so they
-    can be efficiently read by the loader and ignored if needed.
-
-    Comments below show some possible future improvements to the mod format.
-*/
-std::string save_modfile(const ASR::TranslationUnit_t &m) {
-    LFORTRAN_ASSERT(m.m_global_scope->get_scope().size()== 1);
-    for (auto &a : m.m_global_scope->get_scope()) {
-        LFORTRAN_ASSERT(ASR::is_a<ASR::Module_t>(*a.second));
-        if ((bool&)a) { } // Suppress unused warning in Release mode
-    }
-#ifdef WITH_LFORTRAN_BINARY_MODFILES
+inline void save_asr(const ASR::TranslationUnit_t &m, std::string& asr_string) {
+    #ifdef WITH_LFORTRAN_BINARY_MODFILES
     BinaryWriter b;
 #else
     TextWriter b;
@@ -54,11 +37,40 @@ std::string save_modfile(const ASR::TranslationUnit_t &m) {
     // Full ASR:
     b.write_string(serialize(m));
 
-    return b.get_str();
+    asr_string = b.get_str();
 }
 
-ASR::TranslationUnit_t* load_modfile(Allocator &al, const std::string &s,
-        bool load_symtab_id, SymbolTable &symtab) {
+// The save_modfile() and load_modfile() must stay consistent. What is saved
+// must be loaded in exactly the same order.
+
+/*
+    Saves the module into a binary stream.
+
+    That stream can be saved to a mod file by the caller.
+    The sections in the file/stream are saved using write_string(), so they
+    can be efficiently read by the loader and ignored if needed.
+
+    Comments below show some possible future improvements to the mod format.
+*/
+std::string save_modfile(const ASR::TranslationUnit_t &m) {
+    LFORTRAN_ASSERT(m.m_global_scope->get_scope().size()== 1);
+    for (auto &a : m.m_global_scope->get_scope()) {
+        LFORTRAN_ASSERT(ASR::is_a<ASR::Module_t>(*a.second));
+        if ((bool&)a) { } // Suppress unused warning in Release mode
+    }
+
+    std::string asr_string;
+    save_asr(m, asr_string);
+    return asr_string;
+}
+
+std::string save_pycfile(const ASR::TranslationUnit_t &m) {
+    std::string asr_string;
+    save_asr(m, asr_string);
+    return asr_string;
+}
+
+inline void load_serialised_asr(const std::string &s, std::string& asr_binary) {
 #ifdef WITH_LFORTRAN_BINARY_MODFILES
     BinaryReader b(s);
 #else
@@ -72,11 +84,27 @@ ASR::TranslationUnit_t* load_modfile(Allocator &al, const std::string &s,
     if (version != LFORTRAN_VERSION) {
         throw LCompilersException("Incompatible format: LFortran Modfile was generated using version '" + version + "', but current LFortran version is '" + LFORTRAN_VERSION + "'");
     }
-    std::string asr_binary = b.read_string();
+    asr_binary = b.read_string();
+}
+
+ASR::TranslationUnit_t* load_modfile(Allocator &al, const std::string &s,
+        bool load_symtab_id, SymbolTable &symtab) {
+    std::string asr_binary;
+    load_serialised_asr(s, asr_binary);
     ASR::asr_t *asr = deserialize_asr(al, asr_binary, load_symtab_id, symtab);
 
     ASR::TranslationUnit_t *tu = ASR::down_cast2<ASR::TranslationUnit_t>(asr);
     return tu;
 }
 
+ASR::TranslationUnit_t* load_pycfile(Allocator &al, const std::string &s,
+        bool load_symtab_id) {
+    std::string asr_binary;
+    load_serialised_asr(s, asr_binary);
+    ASR::asr_t *asr = deserialize_asr(al, asr_binary, load_symtab_id);
+
+    ASR::TranslationUnit_t *tu = ASR::down_cast2<ASR::TranslationUnit_t>(asr);
+    return tu;
+}
+
 } // namespace LFortran

src/libasr/modfile.h

@@ -8,10 +8,15 @@ namespace LFortran {
     // Save a module to a modfile
     std::string save_modfile(const ASR::TranslationUnit_t &m);
 
+    std::string save_pycfile(const ASR::TranslationUnit_t &m);
+
     // Load a module from a modfile
     ASR::TranslationUnit_t* load_modfile(Allocator &al, const std::string &s,
         bool load_symtab_id, SymbolTable &symtab);
 
+    ASR::TranslationUnit_t* load_pycfile(Allocator &al, const std::string &s,
+        bool load_symtab_id);
+
 }
 
 #endif // LFORTRAN_MODFILE_H

src/libasr/serialization.cpp

@@ -307,7 +307,12 @@ void fix_external_symbols(ASR::TranslationUnit_t &unit,
 }
 
 ASR::asr_t* deserialize_asr(Allocator &al, const std::string &s,
-        bool load_symtab_id, SymbolTable &external_symtab) {
+        bool load_symtab_id, SymbolTable & /*external_symtab*/) {
+    return deserialize_asr(al, s, load_symtab_id);
+}
+
+ASR::asr_t* deserialize_asr(Allocator &al, const std::string &s,
+        bool load_symtab_id) {
     ASRDeserializationVisitor v(al, s, load_symtab_id);
     ASR::asr_t *node = v.deserialize_node();
     ASR::TranslationUnit_t *tu = ASR::down_cast2<ASR::TranslationUnit_t>(node);
@@ -319,9 +324,6 @@ ASR::asr_t* deserialize_asr(Allocator &al, const std::string &s,
 
     LFORTRAN_ASSERT(asr_verify(*tu, false));
 
-    // Suppress a warning for now
-    if ((bool&)external_symtab) {}
-
     return node;
 }
 

src/libasr/serialization.h

@@ -9,6 +9,8 @@ namespace LFortran {
     std::string serialize(const ASR::TranslationUnit_t &unit);
     ASR::asr_t* deserialize_asr(Allocator &al, const std::string &s,
             bool load_symtab_id, SymbolTable &symtab);
+    ASR::asr_t* deserialize_asr(Allocator &al, const std::string &s,
+            bool load_symtab_id);
 
     void fix_external_symbols(ASR::TranslationUnit_t &unit,
             SymbolTable &external_symtab);