From c8622b1c35d6d15557ca6fb99f9e597e728bf66c Mon Sep 17 00:00:00 2001 From: jangorecki Date: Tue, 20 Mar 2018 10:53:52 +0530 Subject: [PATCH] minor fixes to integration and README update --- .gitlab-ci.yml | 4 ++-- README.md | 18 +----------------- 2 files changed, 3 insertions(+), 19 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b94bc45..cc44e6d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -137,7 +137,7 @@ integration: - echo 'test.jobs<-c("test-r-release"="data.cube","test-r-release-cran"="data.cube","test-r-3.1.0-cran"="data.cube")' > integration.R - echo 'lib.copy<-function(lib.from, repodir="bus/integration/cran"){ pkgs.from<-list.dirs(lib.from, recursive=FALSE); pkgs.to<-list.dirs(lib.to<-file.path(repodir,"library"), recursive=FALSE); pkg.copy<-function(pkg.from, lib.to){ pkg<-basename(pkg.from); dir.create(file.path(lib.to, pkg), recursive=TRUE); lib.dirs<-intersect(c("html","doc"), all.lib.dirs<-list.dirs(pkg.from, full.names=FALSE)); ans1<-setNames(file.copy(file.path(pkg.from, lib.dirs), file.path(lib.to, pkg), recursive=TRUE), lib.dirs); lib.files<-setdiff(list.files(pkg.from), all.lib.dirs); ans2<-setNames(file.copy(file.path(pkg.from, lib.files), file.path(lib.to, pkg)), lib.files); all(ans1, ans2)}; pkgs.from.new<-pkgs.from[!basename(pkgs.from) %in% basename(pkgs.to)]; setNames(sapply(pkgs.from.new, pkg.copy, lib.to=lib.to), basename(pkgs.from.new)) }' >> integration.R - echo 'doc.copy<-function(repodir="bus/integration/cran"){ cp1<-c("COPYING","AUTHORS","THANKS"); ans1<-setNames(file.copy(file.path(R.home("doc"), cp1), file.path(repodir, "doc", cp1)), cp1); cp2<-c("html","manual"); ans2<-setNames(file.copy(file.path(R.home("doc"), cp2), file.path(repodir,"doc"), recursive=TRUE), cp2); c(ans1, ans2) }' >> integration.R - - echo 'check.copy<-function(job, repodir="bus/integration/cran"){ dir.create(job.checks<-file.path(repodir, "web", "checks", pkg<-"data.table", job), recursive=TRUE); all(file.copy(file.path("bus", sprintf("%s/%s.Rcheck", job, pkg), c("00install.out","00check.log")), job.checks)) }' >> integration.R + - echo 'check.copy<-function(job, repodir="bus/integration/cran"){ dir.create(job.checks<-file.path(repodir, "web", "checks", pkg<-"data.cube", job), recursive=TRUE); all(file.copy(file.path("bus", sprintf("%s/%s.Rcheck", job, pkg), c("00install.out","00check.log")), job.checks)) }' >> integration.R - echo 'pdf.copy<-function(job, repodir="bus/integration/cran"){ dir.create(pkg.to<-file.path(repodir,"web","packages",pkg<-"data.cube"), recursive=TRUE); file.copy(file.path("bus", job, sprintf("%s.Rcheck", pkg), sprintf("%s-manual.pdf",pkg)), to=file.path(pkg.to, sprintf("%s.pdf",pkg))) }' >> integration.R - echo 'check.test<-function(job) { check<-readLines(file.path("bus", job, sprintf("%s.Rcheck", pkg<-"data.cube"), "00check.log")); check[length(check)] }' >> integration.R #- echo '' >> integration.R @@ -151,7 +151,7 @@ integration: # install all pkgs to render html and double check successful installation of all devel packages - mkdir -p /tmp/opencran/library /tmp/opencran/doc/html - Rscript -e 'install.packages("data.cube", dependencies=TRUE, lib="/tmp/opencran/library", repos=file.path("file:",normalizePath("bus/integration/cran")), INSTALL_opts="--html", quiet=TRUE)' - - Rscript -e 'sapply("data.table", packageVersion, lib.loc="/tmp/opencran/library", simplify=FALSE)' + - Rscript -e 'sapply("data.cube", packageVersion, lib.loc="/tmp/opencran/library", simplify=FALSE)' # R docs, html, css, icons - Rscript -e 'source("integration.R"); doc.copy(repodir="/tmp/opencran")' # Update packages.html, rewrite file:/ to relative path diff --git a/README.md b/README.md index 4fa8fb2..66df852 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,6 @@ In-memory *OLAP cubes* R data type. Uses high performance C-implemented [data.ta - [ ] bind *grouping* dimension to retain cube normalization and avoid double counting - [x] for *pivot* use `format`/`as.data.table` with `dcast.data.table` API - [x] direct access to *data.cube* child classes and attributes -- [ ] data.cube queries processing metadata [logging to database](https://gitlab.com/jangorecki/logR) - [ ] query optimization - [ ] use [blazingly fast](https://jangorecki.github.io/blog/2015-11-23/data.table-index.html) [data.table indices](https://rawgit.com/wiki/Rdatatable/data.table/vignettes/datatable-secondary-indices-and-auto-indexing.html) - [ ] use [data.table#1377](https://github.com/Rdatatable/data.table/issues/1377) grouping sets @@ -28,8 +27,7 @@ In-memory *OLAP cubes* R data type. Uses high performance C-implemented [data.ta ```r install.packages("data.cube", repos = paste0("https://", c( "jangorecki.gitlab.io/data.cube", - "Rdatatable.github.io/data.table", - "cran.rstudio.com" + "cloud.r-project.org" ))) ``` @@ -37,20 +35,6 @@ install.packages("data.cube", repos = paste0("https://", c( Read [manual](https://jangorecki.gitlab.io/data.cube/library/data.cube/html/00Index.html) and check [*Subset and aggregate multidimensional data with data.cube*](https://jangorecki.gitlab.io/data.cube/library/data.cube/doc/sub-.data.cube.html) vignette. -## Advanced - -### client-server - -Running as a services with data.cube can be run using [Rserve: TCP/IP or local sockets](https://github.com/s-u/Rserve), [httpuv: HTTP and WebSocket server](https://github.com/rstudio/httpuv) or [svSocket: R socket server](https://github.com/SciViews/svSocket). -Parsing [MDX](https://en.wikipedia.org/wiki/MultiDimensional_eXpressions) queries or [XMLA](https://en.wikipedia.org/wiki/XML_for_Analysis) requests would be nice extension but is not on the roadmap currently. - -# Interesting reading - -- [Should OLAP databases be denormalized for read performance?](http://stackoverflow.com/q/4394183/2490497) -- [OLAP Operation in R](https://dzone.com/articles/olap-operation-r) + [r-script](https://gist.github.com/jangorecki/4aa6218b6011360338f2) -- [data.table 2E9 rows grouping benchmark](https://github.com/Rdatatable/data.table/wiki/Benchmarks-%3A-Grouping) -- [data.table vs python, big data, MPP, databases](https://github.com/szilard/benchm-databases) - # Contact `j.gorecki@wit.edu.pl`