From f70cb2a4757c08979c1795d5a0261e589b830893 Mon Sep 17 00:00:00 2001 From: Enrico Fasoli Date: Thu, 4 Oct 2018 12:07:15 +0200 Subject: [PATCH] allow user to check progress and submit another crawler request --- src/components/Reviews/index.js | 8 +- src/containers/CrawlerStatus/index.js | 73 ++++++++++ src/containers/ImportReviews/ImportReviews.js | 96 +++++++++++++ src/containers/ImportReviews/form.js | 34 +++++ src/containers/ImportReviews/index.js | 80 +++++++---- src/containers/Layout/MainLayout.js | 2 + src/containers/Reputation/index.js | 15 +- src/containers/ReviewsIWrote/index.js | 4 +- .../SignupWizard/WizardSteps/Step2.js | 49 +------ src/containers/Wallet/index.js | 19 --- src/store/modules/data/crawler.js | 130 +++++++++--------- 11 files changed, 334 insertions(+), 176 deletions(-) create mode 100644 src/containers/CrawlerStatus/index.js create mode 100644 src/containers/ImportReviews/ImportReviews.js create mode 100644 src/containers/ImportReviews/form.js delete mode 100644 src/containers/Wallet/index.js diff --git a/src/components/Reviews/index.js b/src/components/Reviews/index.js index 27ea257..8070185 100644 --- a/src/components/Reviews/index.js +++ b/src/components/Reviews/index.js @@ -17,7 +17,7 @@ const styles = { class Reviews extends Component { render() { - const { classes, reviews = [], loading, crawling = false, onLoadMoreReviews, canLoadMore, ...otherProps } = this.props + const { classes, reviews = [], loading, onLoadMoreReviews, canLoadMore, ...otherProps } = this.props const empty = !reviews || !reviews.length return @@ -30,17 +30,17 @@ class Reviews extends Component { } - {!loading && (empty || crawling) && + {!loading && empty && } title='There is nothing here yet' - subheader={crawling ? 'Your reviews from other platforms are still being imported. Once done, they will be available to you on this page shortly after.' : 'No reviews to display yet'} + subheader='No reviews to display yet' /> } - {!loading && !crawling && Array.isArray(reviews) && reviews.map((review, index) => { + {!loading && Array.isArray(reviews) && reviews.map((review, index) => { return {review.loading && this.props.push('/import') + + render() { + const { classes, loading, starting, running } = this.props + const subtitle = + loading ? 'Checking Status...' + : starting ? 'Starting import process...' + : running ? 'An import operation is in progress' + : 'No import process running for your DID' + return ( + + + } + title='Import Reviews' + subheader={subtitle} + /> + + + + + + ) + } +} + +const mapStateToProps = state => ({ + loading: state.data.crawler.loading, + starting: state.data.crawler.starting, + running: state.data.crawler.running, +}) + +const mapDispatchToProps = { + pollCrawlerStatus, + push +} + +export default compose( + withStyles(styles), + connect(mapStateToProps, mapDispatchToProps) +)(CrawlerStatus) \ No newline at end of file diff --git a/src/containers/ImportReviews/ImportReviews.js b/src/containers/ImportReviews/ImportReviews.js new file mode 100644 index 0000000..d629c07 --- /dev/null +++ b/src/containers/ImportReviews/ImportReviews.js @@ -0,0 +1,96 @@ +import React, { Component } from 'react' +// Components +import { Grid, CircularProgress } from '@material-ui/core' +import RegularCard from 'components/MaterialDashboardPro/RegularCard' +import NavPills from 'components/MaterialDashboardPro/NavPills' +import InfoArea from 'components/MaterialDashboardPro/InfoArea' +// Icons +import Person from '@material-ui/icons/Person'; +import Business from '@material-ui/icons/Business'; +// Form +import ImportReviewsForm from './form' +// Redux +import { connect } from 'react-redux' +import { pollCrawlerStatus } from 'store/modules/data/crawler' + +class ImportReviews extends Component { + + componentDidMount() { + this.props.pollCrawlerStatus() + } + + render () { + const { running, starting, loading } = this.props + + if (loading || starting) { + return ( + + ) + } else if (running) { + return ( + + ) + } else { + return ( + To begin, simply enter your email & password for any of the sites below on which you have an active profile.

, +

We extract, merge and decentrally store your reputation in a portable format so you own and control it.

+ ]} + content={} + /> + ) + }, + { + tabButton: 'Businesses', + tabIcon: Business, + tabContent: ( + To begin, simply enter your email & password for any of the sites below on which you have an active profile.

, +

We extract, merge and decentrally store your reputation in a portable format so you own and control it.

+ ]} + content={} + /> + ) + } + ]} + /> + +

Chlu guarantees that no information submitted from this form is ever stored on our system

+

By submitting this form you acknowledge you are entitled to invoke your data access rights and + data portability rights under European GDPR legislation. +

+ + ) + } + } +} + +const mapStateToProps = state => ({ + loading: state.data.crawler.loading, + starting: state.data.crawler.starting, + running: state.data.crawler.running, +}) + +const mapDispatchToProps = { + pollCrawlerStatus +} + +export default connect(mapStateToProps, mapDispatchToProps)(ImportReviews) \ No newline at end of file diff --git a/src/containers/ImportReviews/form.js b/src/containers/ImportReviews/form.js new file mode 100644 index 0000000..4e42225 --- /dev/null +++ b/src/containers/ImportReviews/form.js @@ -0,0 +1,34 @@ +import React, { Component } from 'react' +// components +import { withStyles, Button } from '@material-ui/core' +// Forms +import BusinessCrawlerForm from './businessCrawlerForm'; +import IndividualsCrawlerForm from './individualsCrawlerForm'; +// redux +import { reduxForm } from 'redux-form' +import { compose } from 'recompose' + +const styles = { + button: { + marginLeft: 'auto' + } +} + +class ImportReviewsForm extends Component { + render () { + const { classes, handleSubmit, userType, ...otherProps } = this.props + const isBusiness = userType === 'business' + const Form = isBusiness ? BusinessCrawlerForm : IndividualsCrawlerForm + + return
+ +
+ } +} + +export default compose( + withStyles(styles), + reduxForm({ + form: 'import-reviews' + }) +)(ImportReviewsForm) \ No newline at end of file diff --git a/src/containers/ImportReviews/index.js b/src/containers/ImportReviews/index.js index 66de9ef..796aa19 100644 --- a/src/containers/ImportReviews/index.js +++ b/src/containers/ImportReviews/index.js @@ -1,34 +1,58 @@ import React, { Component } from 'react' -// Components -import { LinearProgress, Grid } from '@material-ui/core' -// Forms -import BusinessCrawlerForm from './businessCrawlerForm'; -import IndividualsCrawlerForm from './individualsCrawlerForm'; -// redux -import { reduxForm } from 'redux-form' +import { Button, Grid, Card, CardContent, withStyles, CardActions } from '@material-ui/core' +import ImportReviews from './ImportReviews' +// Redux +import { connect } from 'react-redux' +import { compose } from 'recompose' +import { importReviews } from 'store/modules/data/crawler' -class ImportReviews extends Component { - render () { - const { handleSubmit, crawlerRunning, userType, ...otherProps } = this.props - const isBusiness = userType === 'business' - const Form = isBusiness ? BusinessCrawlerForm : IndividualsCrawlerForm +const styles = { + root: { + padding: '20px' + }, + button: { + marginLeft: 'auto' + } +} - if (crawlerRunning) { - return ( - - - - - - ) - } else { - return
- -
- } +class ImportReviewsPage extends Component { + render() { + const { classes, importReviews, running, starting, loading } = this.props + return ( + + + + + + + { !loading && !running && } + + + + ) } } -export default reduxForm({ - form: 'import-reviews' -})(ImportReviews) \ No newline at end of file +const mapStateToProps = state => ({ + running: state.data.crawler.running, + starting: state.data.crawler.starting, + loading: state.data.crawler.loading +}) + +const mapDispatchToProps = { + importReviews +} + +export default compose( + withStyles(styles), + connect(mapStateToProps, mapDispatchToProps) +)(ImportReviewsPage) \ No newline at end of file diff --git a/src/containers/Layout/MainLayout.js b/src/containers/Layout/MainLayout.js index dfb5d2b..b8907f7 100644 --- a/src/containers/Layout/MainLayout.js +++ b/src/containers/Layout/MainLayout.js @@ -23,6 +23,7 @@ import Settings from '../Settings' import Profile from '../Profile' import Terms from '../Terms' import Search from '../Search' +import ImportReviews from '../ImportReviews' import ReviewContainer from '../Review' class MainLayout extends Component { @@ -62,6 +63,7 @@ class MainLayout extends Component { } /> {emptyWallet && } + diff --git a/src/containers/Reputation/index.js b/src/containers/Reputation/index.js index 976c863..35ec149 100644 --- a/src/containers/Reputation/index.js +++ b/src/containers/Reputation/index.js @@ -1,18 +1,15 @@ import React, { Component } from 'react' - // components import Reviews from 'components/Reviews' - +import CrawlerStatus from 'containers/CrawlerStatus' // redux import { readMyReputation } from 'store/modules/data/reputation' -import { pollCrawlerProgress } from 'store/modules/data/crawler' import { connect } from 'react-redux' class Reputation extends Component { componentDidMount() { if (!this.props.loading) this.props.readMyReputation() - this.props.pollCrawlerProgress() } loadMore = () => { @@ -20,13 +17,13 @@ class Reputation extends Component { } render() { - const { reviews, loading, loadingPage, canLoadMore, crawling } = this.props + const { reviews, loading, loadingPage, canLoadMore } = this.props return (
+ @@ -39,13 +36,11 @@ const mapStateToProps = state => ({ reviews: state.data.reputation.reviews, loading: state.data.reputation.loading, loadingPage: state.data.reputation.loadingPage, - canLoadMore: state.data.reputation.canLoadMore, - crawling: state.data.crawler.running + canLoadMore: state.data.reputation.canLoadMore }) const mapDispatchToProps = { - readMyReputation, - pollCrawlerProgress + readMyReputation } export default connect(mapStateToProps, mapDispatchToProps)(Reputation) diff --git a/src/containers/ReviewsIWrote/index.js b/src/containers/ReviewsIWrote/index.js index f496c2e..193a267 100644 --- a/src/containers/ReviewsIWrote/index.js +++ b/src/containers/ReviewsIWrote/index.js @@ -19,7 +19,7 @@ class ReviewsIWrote extends Component { render() { const { reviews, loading, loadingPage, canLoadMore, error } = this.props - return ( + return (
- ) +
) } } diff --git a/src/containers/SignupWizard/WizardSteps/Step2.js b/src/containers/SignupWizard/WizardSteps/Step2.js index 90e03d4..812e884 100644 --- a/src/containers/SignupWizard/WizardSteps/Step2.js +++ b/src/containers/SignupWizard/WizardSteps/Step2.js @@ -4,20 +4,16 @@ import React from 'react'; import { Grid, InputAdornment, CircularProgress } from '@material-ui/core' // custom components -import RegularCard from 'components/MaterialDashboardPro/RegularCard' -import NavPills from 'components/MaterialDashboardPro/NavPills' import PictureUpload from 'components/MaterialDashboardPro/PictureUpload' import CustomInput from 'components/MaterialDashboardPro/CustomInput' import Button from 'components/MaterialDashboardPro/Button' import InfoArea from 'components/MaterialDashboardPro/InfoArea' -import ImportReviews from 'containers/ImportReviews' +import ImportReviews from 'containers/ImportReviews/ImportReviews' // icons -import Person from '@material-ui/icons/Person'; import AccountBox from '@material-ui/icons/AccountBox'; import Web from '@material-ui/icons/Web'; import StarHalf from '@material-ui/icons/StarHalf'; -import Business from '@material-ui/icons/Business'; import Face from '@material-ui/icons/Face' import DoneIcon from '@material-ui/icons/Done' @@ -245,48 +241,7 @@ class Step3 extends React.Component { /> - - To begin, simply enter your email & password for any of the sites below on which you have an active profile.

, -

We extract, merge and decentrally store your reputation in a portable format so you own and control it.

- ]} - content={} - /> - ) - }, - { - tabButton: 'Businesses', - tabIcon: Business, - tabContent: ( - To begin, simply enter your email & password for any of the sites below on which you have an active profile.

, -

We extract, merge and decentrally store your reputation in a portable format so you own and control it.

- ]} - content={} - /> - ) - } - ]} - /> - - - -

Chlu guarantees that no information submitted from this form is ever stored on our system

-

By submitting this form you acknowledge you are entitled to invoke your data access rights and - data portability rights under European GDPR legislation. -

- - +
) } diff --git a/src/containers/Wallet/index.js b/src/containers/Wallet/index.js deleted file mode 100644 index 5998a24..0000000 --- a/src/containers/Wallet/index.js +++ /dev/null @@ -1,19 +0,0 @@ -import React from 'react' -import { Route, Switch, Redirect } from 'react-router' -import ImportWallet from './Import/index' -import { connect } from 'react-redux'; - -function WalletRouter({ wallet }) { - const emptyWallet = !wallet || !wallet.did - return - {!emptyWallet && } - - - -} - -export default connect( - state => ({ - wallet: state.data.wallet - }) -)(WalletRouter) \ No newline at end of file diff --git a/src/store/modules/data/crawler.js b/src/store/modules/data/crawler.js index b7b3a17..d6c235a 100644 --- a/src/store/modules/data/crawler.js +++ b/src/store/modules/data/crawler.js @@ -1,5 +1,4 @@ import { createAction, handleActions } from 'redux-actions' -import { readMyReputation } from './reputation' // Helpers import { get } from 'lodash' import { getChluAPIClient } from 'helpers/chlu' @@ -7,28 +6,36 @@ import { createDAGNode } from 'chlu-api-client/src/utils/ipfs' import { getFormValues } from 'redux-form' // Constants -const CRAWLER_START = 'crawler/START' -const CRAWLER_ERROR = 'crawler/ERROR' -const CRAWLER_FINISH = 'crawler/FINISH' +const SET_CRAWLER_STATUS = 'crawler/STATUS' +const SET_POLLING = 'crawler/POLLING' +const SET_LOADING = 'crawler/LOADING' +const SET_STARTING = 'crawler/STARTING' const API_URL = process.env.REACT_APP_CHLU_PUBLISH_URL || 'https://publish.chlu.io' -const CRAWLER_PROGRESS_POLL_INTERVAL = 5000 // ms +const POLL_INTERVAL = 5000 // ms function getInitialState() { return { + starting: false, + loading: false, running: false, + jobs: [], + shouldPoll: true, + polling: false, error: null } } -export const crawlerError = createAction(CRAWLER_ERROR) -const startCrawlerAction = createAction(CRAWLER_START) -export const finishCrawler = createAction(CRAWLER_FINISH) +const setCrawlerStatus = createAction(SET_CRAWLER_STATUS) +const setPolling = createAction(SET_POLLING) +const setLoading = createAction(SET_LOADING) +const setStarting = createAction(SET_STARTING) export function startCrawler(type, url, username, password) { return async (dispatch, getState) => { const state = getState() try { + dispatch(setStarting(true)) const signedInDid = get(state, 'data.wallet.did', null) const signedOutDid = get(state, 'components.createWallet.walletCreated.did', null) const did = signedInDid || signedOutDid @@ -47,14 +54,15 @@ export function startCrawler(type, url, username, password) { 'Content-Type': 'application/json' }, }) - const responseJson = await response.json() console.log('Crawlers running:') console.log(responseJson) } catch (error) { console.log(error) + // TODO: error handling } + dispatch(setStarting(false)) } } @@ -79,82 +87,72 @@ export function importReviews() { } } -export function pollCrawlerProgress() { +export function pollCrawlerStatus() { return async (dispatch, getState) => { const state = getState() const didId = get(state, 'data.wallet.did.publicDidDocument.id', null) - - if (!didId) { - console.error("Cannot read crawler progress without DID ID.") - return + const shouldPoll = get(state, 'data.crawler.shouldPoll', false) + const polling = get(state, 'data.crawler.polling', false) + if (shouldPoll && didId && !polling) { + dispatch(setPolling(true)) + let first = true + do { + if (first) dispatch(setLoading(true)) + await dispatch(getCrawlerStatus(didId)) + await sleep(POLL_INTERVAL) + if (first) dispatch(setLoading(false)) + first = false + } while (shouldPoll) + dispatch(setPolling(false)) } - let isBackendCrawlingInProgress = false - let wasBackendCrawlingInProgress = false - - do { - isBackendCrawlingInProgress = hasInProgressJobs(await getCrawlerStatus(didId)) - - console.log('crawler running state: ' + isBackendCrawlingInProgress) - - if (isBackendCrawlingInProgress) { - dispatch(startCrawlerAction()) + } +} - // Wait a bit before polling again. - await new Promise(resolve => window.setTimeout(resolve, CRAWLER_PROGRESS_POLL_INTERVAL)) - } else { - if (wasBackendCrawlingInProgress) { - // If the crawler was running on the backend before, we should assume new reviews are available now. - readMyReputation()(dispatch, getState) +function getCrawlerStatus(didId) { + return async dispatch => { + try { + const response = await fetch(`${API_URL}/api/v1/crawl/${didId}`, { + headers: { + 'Accept': 'application/json', + 'Content-Type': 'application/json' } - - dispatch(finishCrawler()) - } - - wasBackendCrawlingInProgress = isBackendCrawlingInProgress - } while (isBackendCrawlingInProgress) - } + }) + const responseJson = await response.json() + if (responseJson.rows) dispatch(setCrawlerStatus(responseJson.rows)) + } catch (err) { + // TODO: error handling + console.error(err) + } + } } -function hasInProgressJobs(crawlerStatus) { - for (const status of crawlerStatus.rows) { - if (status.status === 'CREATED' || status.status === 'RUNNING') return true +function hasRunningJobs(jobs) { + for (const job of jobs) { + if (job.status === 'RUNNING' || job.status === 'CREATED') return true } return false } -async function getCrawlerStatus(didId) { - try { - const response = await fetch(`${API_URL}/api/v1/crawl/${didId}`, { - headers: { - 'Accept': 'application/json', - 'Content-Type': 'application/json' - } - }) - - const responseJson = await response.json() - console.log(responseJson) - return responseJson - } catch (err) { - console.error(err) - return [] - } +function sleep (ms) { + return new Promise(resolve => setTimeout(resolve, ms)) } export default handleActions({ - [CRAWLER_START]: state => ({ + [SET_POLLING]: (state, { payload: polling }) => ({ ...state, - running: true, - error: null + polling }), - [CRAWLER_ERROR]: (state, { payload: error }) => ({ + [SET_CRAWLER_STATUS]: (state, { payload: jobs }) => { + const running = hasRunningJobs(jobs) + return { ...state, jobs, running } + }, + [SET_STARTING]: (state, { payload: starting }) => ({ ...state, - running: false, - error + starting }), - [CRAWLER_FINISH]: state => ({ + [SET_LOADING]: (state, { payload: loading }) => ({ ...state, - running: false, - error: null - }) + loading + }), }, getInitialState())