Merge pull request #558 from highperformancecoder/feature-librarian

Feature librarian

Author: highperformancecoder

Doxyversion

@@ -1 +1 @@
-PROJECT_NAME=Minsky: 3.19.0-beta.1
+PROJECT_NAME=Minsky: 3.19.0-beta.2

Makefile

@@ -162,7 +162,7 @@ PREFIX=/usr/local
 # directory
 MODLINK=$(LIBMODS:%=$(ECOLAB_HOME)/lib/%)
 MODEL_OBJS=autoLayout.o cairoItems.o canvas.o CSVDialog.o dataOp.o equationDisplay.o godleyIcon.o godleyTable.o godleyTableWindow.o grid.o group.o item.o intOp.o lasso.o lock.o minsky.o operation.o operationRS.o operationRS1.o  operationRS2.o phillipsDiagram.o plotWidget.o port.o pubTab.o ravelWrap.o renderNativeWindow.o selection.o sheet.o SVGItem.o switchIcon.o userFunction.o userFunction_units.o variableInstanceList.o variable.o variablePane.o windowInformation.o wire.o 
-ENGINE_OBJS=clipboard.o derivative.o equationDisplayRender.o equations.o evalGodley.o evalOp.o flowCoef.o \
+ENGINE_OBJS=clipboard.o databaseIngestor.o derivative.o equationDisplayRender.o equations.o evalGodley.o evalOp.o flowCoef.o \
 	godleyExport.o latexMarkup.o valueId.o variableValue.o node_latex.o node_matlab.o CSVParser.o \
 	minskyTensorOps.o mdlReader.o saver.o rungeKutta.o
 SCHEMA_OBJS=schema3.o schema2.o schema1.o schema0.o schemaHelper.o variableType.o \
@@ -305,7 +305,7 @@ endif
 LIBS+=	-LRavelCAPI -lravelCAPI -LRavelCAPI/civita -lcivita \
 	-lboost_system$(BOOST_EXT) -lboost_regex$(BOOST_EXT) \
 	-lboost_date_time$(BOOST_EXT) -lboost_program_options$(BOOST_EXT) \
-	-lboost_filesystem$(BOOST_EXT) -lboost_thread$(BOOST_EXT) -lgsl -lgslcblas -lssl -lcrypto
+	-lboost_filesystem$(BOOST_EXT) -lboost_thread$(BOOST_EXT) -lsoci_core -lgsl -lgslcblas -lssl -lcrypto
 
 ifdef MXE
 LIBS+=-lcrypt32 -lbcrypt -lshcore

RESTService/addon.cc

@@ -242,6 +242,7 @@ namespace minsky
         if (reset_flag()) requestReset();
 
         civita::ITensor::cancel(false);
+        ravelCAPI::Ravel::cancel(false);
         // disable quoting wide characters in UTF-8 strings
         auto result=write(registry.process(command, arguments)->asBuffer(),json5_parser::raw_utf8);
         commandHook(command,arguments);
@@ -594,6 +595,7 @@ struct MinskyAddon: public Addon<MinskyAddon>
   Value cancelProgress(const Napi::CallbackInfo& info) {
     *addOnMinsky.progressState.cancel=true;
     civita::ITensor::cancel(true);
+    ravelCAPI::Ravel::cancel(true);
     return info.Env().Null();
   }
 

RESTService/pyminsky.cc

@@ -97,6 +97,8 @@ namespace pyminsky
   }
   CLASSDESC_ADD_FUNCTION(findObject);
   CLASSDESC_ADD_FUNCTION(findVariable);
+  using minsky::DataSpec;
+  CLASSDESC_DECLARE_TYPE(DataSpec);
 }
 
 CLASSDESC_PYTHON_MODULE(pyminsky);

RESTService/typescriptAPI.cc

@@ -7,15 +7,21 @@
 #include "bookmark.h"
 #include "bookmark.tcd"
 #include "cairoSurfaceImage.tcd"
+#include "cairoRenderer.tcd"
 #include "callableFunction.tcd"
 #include "canvas.tcd"
+#define CLASSDESC_typescriptAPI___CAPIRenderer
+#include "capiRenderer.tcd"
 #include "CSVDialog.tcd"
 #include "CSVParser.tcd"
+#include "CSVTools.tcd"
 #include "constMap.tcd"
 #include "dataSpecSchema.tcd"
 #include "dataOp.h"
 #include "dataOp.tcd"
+#include "databaseIngestor.tcd"
 #include "dimension.tcd"
+#include "dynamicRavelCAPI.tcd"
 #include "engNotation.tcd"
 #include "equationDisplay.tcd"
 #include "evalGodley.tcd"
@@ -242,10 +248,12 @@ int main()
   api.addClass<Bookmark>();
   api.addClass<Canvas::ZoomCrop>();
   api.addClass<civita::Dimension>();
+  api.addClass<civita::NamedDimension>();
   api.addClass<civita::Hypercube>();
   api.addClass<civita::Index>();
   api.addClass<civita::ITensor>();
   api.addClass<civita::XVector>();
+  api.addClass<CSVDialog>();
   api.addClass<DataSpecSchema>();
   api.addClass<ecolab::Plot::LineStyle>();
   api.addClass<EngNotation>();
@@ -254,8 +262,11 @@ int main()
   api.addClass<HandleLockInfo>();
   api.addClass<Port>();
   api.addClass<PubItem>();
+  api.addClass<ravel::DataSpec>();
   api.addClass<ravel::HandleState>();
   api.addClass<ravel::RavelState>();
+  api.addClass<ravelCAPI::Database>();
+  api.addClass<ravelCAPI::Ravel>();
   api.addClass<Units>();
   api.addClass<VariablePaneCell>();
   api.addClass<VariableValue>();
@@ -299,6 +310,7 @@ int main()
   cout << "class minsky__GodleyIcon__MoveCellArgs {}\n";
   cout << "class minsky__RenderNativeWindow__RenderFrameArgs {}\n";
   cout << "class minsky__VariableType__TypeT {}\n";
+  cout << "class CAPIRenderer  {}\n";
   cout << "class civita__ITensor__Args {}\n";
   cout << "class classdesc__json_pack_t {}\n";
   cout << "class classdesc__pack_t {}\n";

RavelCAPI

@@ -1 +1 @@
-\author{Version 3.19.0-beta.1}
+\author{Version 3.19.0-beta.2}

doc/version.tex

@@ -20,6 +20,7 @@
 #include "minsky.h"
 #include "CSVParser.h"
 
+#include "CSVTools.rcd"
 #include "CSVParser.rcd"
 #include "dataSpecSchema.rcd"
 #include "dimension.rcd"
@@ -36,156 +37,15 @@
 
 using namespace minsky;
 using namespace std;
+using ravel::Parser;
+using ravel::SpaceSeparatorParser;
+using ravel::getWholeLine;
 
 #include <boost/type_traits.hpp>
 #include <boost/tokenizer.hpp>
 #include <boost/token_functions.hpp>
 #include <boost/pool/pool.hpp>
 
-namespace escapedListSeparator
-{
-  // pinched from boost::escape_list_separator, and modified to not throw
-  template <class Char,
-            class Traits = BOOST_DEDUCED_TYPENAME std::basic_string<Char>::traits_type >
-  class EscapedListSeparator {
-
-  private:
-    typedef std::basic_string<Char,Traits> string_type;
-    struct char_eq {
-      Char e_;
-      char_eq(Char e):e_(e) { }
-      bool operator()(Char c) {
-        return Traits::eq(e_,c);
-      }
-    };
-    string_type  escape_;
-    string_type  c_;
-    string_type  quote_;
-    bool last_;
-
-    bool is_escape(Char e) {
-      const char_eq f(e);
-      return std::find_if(escape_.begin(),escape_.end(),f)!=escape_.end();
-    }
-    bool is_c(Char e) {
-      const char_eq f(e);
-      return std::find_if(c_.begin(),c_.end(),f)!=c_.end();
-    }
-    bool is_quote(Char e) {
-      const char_eq f(e);
-      return std::find_if(quote_.begin(),quote_.end(),f)!=quote_.end();
-    }
-    template <typename iterator, typename Token>
-    void do_escape(iterator& next,iterator end,Token& tok) {
-      if (++next >= end)
-        // don't throw, but pass on verbatim
-        tok+=escape_.front();
-      if (Traits::eq(*next,'n')) {
-        tok+='\n';
-        return;
-      }
-      if (is_quote(*next)) {
-        tok+=*next;
-        return;
-      }
-      if (is_c(*next)) {
-        tok+=*next;
-        return;
-      }
-      if (is_escape(*next)) {
-        tok+=*next;
-        return;
-      }
-      // don't throw, but pass on verbatim
-      tok+=escape_.front()+*next;
-    }
-
-  public:
-
-    explicit EscapedListSeparator(Char  e = '\\',
-                                  Char c = ',',Char  q = '\"')
-      : escape_(1,e), c_(1,c), quote_(1,q), last_(false) { }
-
-    EscapedListSeparator(string_type e, string_type c, string_type q)
-      : escape_(e), c_(c), quote_(q), last_(false) { }
-
-    void reset() {last_=false;}
-
-    template <typename InputIterator, typename Token>
-    bool operator()(InputIterator& next,InputIterator end,Token& tok) {
-      bool bInQuote = false;
-      tok = Token();
-
-      if (next >= end) {
-        next=end; // reset next in case it has adavanced beyond
-        if (last_) {
-          last_ = false;
-          return true;
-        }
-        return false;
-      }
-      last_ = false;
-      while (next < end) {
-        if (is_escape(*next)) {
-          do_escape(next,end,tok);
-        }
-        else if (is_c(*next)) {
-          if (!bInQuote) {
-            // If we are not in quote, then we are done
-            ++next;
-            // The last character was a c, that means there is
-            // 1 more blank field
-            last_ = true;
-            return true;
-          }
-          tok+=*next;
-        }
-        else if (is_quote(*next)) {
-          bInQuote=!bInQuote;
-        }
-        else {
-          tok += *next;
-        }
-        ++next;
-      }
-      return true;
-    }
-  };
-}
-using Parser=escapedListSeparator::EscapedListSeparator<char>;
-
-typedef boost::tokenizer<Parser> Tokenizer;
-
-struct SpaceSeparatorParser
-{
-  char escape, quote;
-  SpaceSeparatorParser(char escape='\\', char sep=' ', char quote='"'):
-    escape(escape), quote(quote) {}
-  template <class I>
-  bool operator()(I& next, I end, std::string& tok)
-  {
-    tok.clear();
-    bool quoted=false;
-    while (next!=end)
-      {
-        if (*next==escape)
-          tok+=*(++next);
-        else if (*next==quote)
-          quoted=!quoted;
-        else if (!quoted && isspace(*next))
-          {
-            while (isspace(*next)) ++next;
-            return true;
-          }
-        else
-          tok+=*next;
-        ++next;
-      }
-    return !tok.empty();
-  }
-  void reset() {}
-};
-
 namespace
 {
   /// An any with cached hash
@@ -383,6 +243,7 @@ void DataSpec::setDataArea(size_t row, size_t col)
     dataCols.erase(i);
   for (unsigned i=m_nColAxes; i<numCols && i<maxColumn; ++i)
     dataCols.insert(i);
+  toSchema();
 }
 
 
@@ -573,49 +434,6 @@ void DataSpec::populateFromRavelMetadata(const std::string& metadata, const stri
 
 namespace minsky
 {
-  // handle DOS files with '\r' '\n' line terminators
-  void chomp(string& buf)
-  {
-    if (!buf.empty() && buf.back()=='\r')
-      buf.erase(buf.size()-1);
-  }
-  
-  // gets a line, accounting for quoted newlines
-  bool getWholeLine(istream& input, string& line, const DataSpec& spec)
-  {
-    line.clear();
-    bool r=getline(input,line).good();
-    chomp(line);
-    while (r)
-      {
-        int quoteCount=0;
-        for (auto i: line)
-          if (i==spec.quote)
-            ++quoteCount;
-        if (quoteCount%2==0) break; // data line correctly terminated
-        string buf;
-        r=getline(input,buf).good(); // read next line and append
-        chomp(buf);
-        line+=buf;
-      }
-    escapeDoubledQuotes(line,spec);
-    return r || !line.empty();
-  }
-
-  void escapeDoubledQuotes(std::string& line,const DataSpec& spec)
-  {
-    // replace doubled quotes with escape quote
-    for (size_t i=1; i<line.size(); ++i)
-      if (line[i]==spec.quote && line[i-1]==spec.quote &&
-          ((i==1 && (i==line.size()-1|| line[i+1]!=spec.quote)) ||                                       // deal with leading ""
-           (i>1 &&
-            ((line[i-2]!=spec.quote && line[i-2]!=spec.escape &&
-              (line[i-2]!=spec.separator || i==line.size()-1|| line[i+1]!=spec.quote))  // deal with ,''
-             ||            // deal with "" middle or end
-             (line[i-2]==spec.quote && (i==2 || line[i-3]==spec.separator || line[i-3]==spec.escape)))))) // deal with leading """
-        line[i-1]=spec.escape;
-  }
-
   /// handle reporting errors in loadValueFromCSVFileT when loading files
   struct OnError
   {

engine/CSVParser.cc

@@ -83,6 +83,21 @@ namespace minsky
       guessFromStream(is, std::filesystem::file_size(fileName));
     }
 
+    // converts this to the ravel data spec. TODO - can we eliminate, or at least eviscerate this type?
+    operator ravel::DataSpec() const {
+      ravel::DataSpec r; static_cast<ravel::CSVSpec&>(r)=*this;
+      r.dataRowOffset=dataRowOffset; // TODO: non-normalisation between dataRowOffset and m_nRowAxes causes problems...
+      r.headerRow=headerRow;
+      r.mergeDelimiters=mergeDelimiters;
+      r.counter=counter;
+      r.dontFail=dontFail;
+      r.dimensionCols=dimensionCols;
+      r.dataCols=dataCols;
+      for (size_t i=0; i<std::min(dimensions.size(), dimensionNames.size()); ++i)
+        r.dimensions.emplace_back(dimensionNames[i],dimensions[i]);
+      return r;
+    }
+
     /// populates this spec from a "RavelHypercube" entry, \a row is the row being read, used to set the headerRow attribute
     /// \a If horizontalName is one of the dimensions, data is written in a tabular format
     void populateFromRavelMetadata(const std::string& metadata, const std::string& horizontalName, std::size_t row);
@@ -123,8 +138,6 @@ namespace minsky
   /// load a variableValue from a stream according to data spec
   void loadValueFromCSVFile(VariableValue&, std::istream& input, const DataSpec&);
 
-  /// replace doubled quotes with escaped quotes
-  void escapeDoubledQuotes(std::string&,const DataSpec&);
 }
 
 #include "CSVParser.cd"