diff --git a/pipelines/.gitignore b/pipelines/.gitignore index 1467783..55bd52f 100644 --- a/pipelines/.gitignore +++ b/pipelines/.gitignore @@ -62,5 +62,8 @@ docs/_build/ # luigi local log /luigi-log +# .vagrant files +/.vagrant + # PyBuilder target/ diff --git a/pipelines/README.md b/pipelines/README.md index 848ca65..ce9cfc6 100644 --- a/pipelines/README.md +++ b/pipelines/README.md @@ -1,21 +1,26 @@ # JOSA Session -This repo will help you get started with our session with both vagrant and docker +This repo will help you get started with our session using docker 1.7 + ### Installation for Session 1 -You need to be running vagrant and virtualbox. -Clone the repo to your computer +On your linux machine - vagrant up +Download Pentaho community edition using the following link +http://community.pentaho.com/projects/data-integration/ + + ## navigate to your cloned repo and use the below command. + unzip pentaho-jobs.zip + +Open the exercise files ### Installation for Session 2 You need to be running docker 1.7 + ```sh -$ docker build . +$ docker build -t josa-ds . ``` You need to get the new image id that was created from the docker build. @@ -44,7 +49,7 @@ Expected result CONTAINER ID IMAGE COMMAND ... PORTS 9f544db853f3 "/bin/sh -c 'exec /s ... 22/tcp, 3306/tcp, 8082/tcp -To SSH into the container (Not recommended - Only use to view post compute result logs) +To SSH into the container (Not Recommended) docker inspect @@ -55,8 +60,11 @@ $ enter pass: The password is -> "u" ``` -### Demonizing Tornado/luigi static visualiser - docker exec -i luigid --background --logdir /var/log/luigi/ +### Demonizing Tornado/luigi static visualiser and central scheduler + docker exec -i luigid --background --logdir /var/log/luigi/ + +### Running the main command (Languages Count) + docker exec -i bash -c "cd /home/ubuntu/josa/pipelines && PYTHONPATH=. luigi --module josagit AggregateLanguages" -### Running the main command - docker exec -i PYTHONPATH=. luigi --module josagit AggregateLanguages +### Viewing the output from docker (Languages count job) + docker exec -i less /home/ubuntu/josa/pipelines/cleaned-data/aggregated-languages.json diff --git a/pipelines/pentaho-jobs.zip b/pipelines/pentaho-jobs.zip new file mode 100644 index 0000000..5381008 Binary files /dev/null and b/pipelines/pentaho-jobs.zip differ