From 48b3721719dc34bb5a433ddaea098f54d30ee99f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ivan=20Verg=C3=A9s?= Date: Mon, 1 Mar 2021 10:36:08 +0100 Subject: [PATCH] add link parser --- README.md | 75 +++++++++++++++++++++++++++++++++---------- proposal_answers.rake | 1 + script_helpers.rb | 6 ++++ 3 files changed, 65 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index a3c4887..506649f 100644 --- a/README.md +++ b/README.md @@ -33,23 +33,68 @@ o.host="localhost" o.save ``` -1: Importing massive answers for proposals ------------------------------------------- +Importing massive answers for proposals +--------------------------------------- + +This script answers a list of proposals, it requires an excel with headers with this format: + +``` +ID, status, text_ca, text_es, ... +123, accepted, una resposta, una respuesta, ... +``` + +or + +``` +id, state, answer/ca, answer/es, ... +123, acceptada, una resposta, una respuesta, ... +``` + +> NOTE: +> - Component setting "Publish proposal answers immediately" is honored, so, if activated, emails and notifications will be sent. Otherwise will just introduce the answers in the database. +> - Answers are are parsed to highlight links automatically and convert line breaks to `
`; + +### Usage: Copy the script into your `lib/tasks` of your Decidim installation: ``` cd lib/tasks -wget -qO- https://github.com/Platoniq/decidim-scripts/archive/0.2.tar.gz | tar --transform 's/^decidim-scripts-0.2//' -xvz +wget -qO- https://github.com/Platoniq/decidim-scripts/archive/0.3.tar.gz | tar --transform 's/^decidim-scripts-0.3//' -xvz ``` Run in local (as testing): ``` -bin/rails "proposals:batch:answer[admin@example.org,./pam-ciutat.csv]" +bin/rails "proposals:batch:answer[admin@example.org,./proposal-answers.csv]" +``` + +Massively geolocating proposals +------------------------------- + +This script geolocates a list of proposals, it requires an excel with headers with this format: + +``` +ID, address, ... +123, some place, ... +``` + +or + +``` +i, adreça, ... +123, some place, ... +``` + +> NOTE: +> - Component setting "Geocoding enabled" is honored, so, if not activated, proposals won't be modified. + +``` +bin/rails proposals:batch:geoloc[admin@example.org,./proposal-geolocs.csv] ``` Running in heroku: +------------------ In Heroku you need to temporary copy the relevant files into a running container first. As there's no "copy" command to do that you can use the service https://transfer.sh with this trick: @@ -58,17 +103,15 @@ As there's no "copy" command to do that you can use the service https://transfer 1. From your computer, send the files to transfer.sh: ``` -curl --upload-file pam-ciutat.csv https://transfer.sh/pam-ciutat.csv -H "Max-Days: 1" -curl --upload-file pam-districtes.csv https://transfer.sh/pam-districte.csv -H "Max-Days: 1" -curl --upload-file geolocs.csv https://transfer.sh/geolocs.csv -H "Max-Days: 1" +curl --upload-file proposal-answerss.csv https://transfer.sh/proposal-answers.csv -H "Max-Days: 1" +curl --upload-file proposal-geolocs.csv https://transfer.sh/proposal-geolocs.csv -H "Max-Days: 1" ``` Which will return (for instance) the download addresses: ``` -https://transfer.sh/23lG7/pam-ciutat.csv -https://transfer.sh/YwYAN/pam-districte.csv -https://transfer.sh/x3hUa/geolocs.csv +https://transfer.sh/YwYAN/proposal-answers.csv +https://transfer.sh/x3hUa/proposal-geolocs.csv ``` 2. Login into a shell session in heroku @@ -80,15 +123,14 @@ heroku run bash 3. Download the files inside the dyno: ``` -wget https://transfer.sh/23lG7/pam-ciutat.csv -wget https://transfer.sh/YwYAN/pam-districte.csv -wget https://transfer.sh/x3hUa/golocs.csv +wget https://transfer.sh/YwYAN/proposal-answers.csv +wget https://transfer.sh/x3hUa/proposal-geolocs.csv ``` 4. Download the scripts into the `lib/tasks` folder: ``` -wget -qO- https://github.com/Platoniq/decidim-scripts/archive/0.2.tar.gz | tar --transform 's/^decidim-scripts-0.2//' -xvz -C lib/tasks +wget -qO- https://github.com/Platoniq/decidim-scripts/archive/0.3.tar.gz | tar --transform 's/^decidim-scripts-0.3//' -xvz -C lib/tasks ``` 5. Run the script inside the shell session (2nd terminal): @@ -101,9 +143,8 @@ bin/rails proposals:batch:geoloc or: ``` -bin/rails proposals:batch:answer[admin@example.org,./pam-ciutat.csv] -bin/rails proposals:batch:answer[admin@example.org,./pam-districtes.csv] -bin/rails proposals:batch:geoloc[admin@example.org,./geolocs.csv] +bin/rails proposals:batch:answer[admin@example.org,./proposal-answerss.csv] +bin/rails proposals:batch:geoloc[admin@example.org,./proposal-geolocs.csv] ``` NOTE: in case the program runs out of memory in Redis, just execute it again after a minute or so to allow for the queue to empty. \ No newline at end of file diff --git a/proposal_answers.rake b/proposal_answers.rake index 4788e1b..98eb5a1 100644 --- a/proposal_answers.rake +++ b/proposal_answers.rake @@ -30,6 +30,7 @@ namespace :proposals do raise_if_field_not_found(:state, values) raise_if_field_not_found(:answer, values) values[:state] = normalize_state(values[:state]) # throws UnprocessableError if fails + values[:answer] = parse_links(values[:answer]) @admin = admin @values = values @proposal = proposal_from_id(values[:id]) diff --git a/script_helpers.rb b/script_helpers.rb index 22a276f..3f98f50 100644 --- a/script_helpers.rb +++ b/script_helpers.rb @@ -107,6 +107,12 @@ def normalize_state(state) end end + def parse_links(texts) + texts.map do |lang, text| + [lang, Decidim::ContentRenderers::LinkRenderer.new(text.strip).render.gsub("\n", "
")] + end.to_h + end + def show_help puts HELP_TEXT end