Merge branch 'get-all-scrappe' of nikoladiligent/scraper-react into master

.env

-
-REACT_APP_BASE_API_URL=https://portalgatewayapi-q1.bullioninternational.info/
+REACT_APP_BASE_API_URL=http://localhost:3333/

.vscode/launch.json

+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "type": "chrome",
+      "request": "launch",
+      "name": "Debug in Chrome",
+      "url": "http://localhost:3000",
+      "webRoot": "${workspaceRoot}",
+      "runtimeArgs": [
+        "--disable-web-security",
+        "--ignore-certificate-errors"
+      ],
+      "sourceMaps": true,
+      "sourceMapPathOverrides": {
+        "webpack:///./dist/applications/*": "${workspaceRoot}/src/*"
+      }
+    },
+  ]
+}

package.json

     "lodash.isempty": "^4.4.0",
     "owasp-password-strength-test": "^1.3.0",
     "react": "^17.0.2",
+    "react-autocomplete": "^1.8.1",
     "react-dom": "^17.0.2",
     "react-helmet-async": "^1.0.9",
     "react-i18next": "^11.10.0",

src/App.js

 import { Helmet } from 'react-helmet-async';
 import i18next from 'i18next';
 import history from './store/utils/history';
-import Header from './components/Header/Header';
+// import Header from './components/Header/Header';
 // import Sidebar from './components/Sidebar/Sidebar';
 import AppRoutes from './AppRoutes';
 
         {i18next.t('app.title')}
       </title>
     </Helmet>
-        <>
-            <Header />
-            {/* <Sidebar /> */}
-            <AppRoutes />
-        </>
+    <>
+      {/* <Header /> */}
+      {/* <Sidebar /> */}
+      <AppRoutes />
+    </>
   </Router>
 
-  );
+);
 
 export default App;

src/assets/styles/_variables.scss

 $base-font-size: 16px;
 $base-font-size-md: 14px;
 
-$font-family: 'Avenir Next';
+$font-family: "Source Sans Pro",-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol";
 
 // Colors
 $color-primary: #024f86;

src/components/CreateScrapeRequest/CreateScrapeRequest.js

-import React, {useState} from 'react';
-import { useTranslation } from 'react-i18next';
+import React, { useState } from 'react';
 import Select from 'react-select'
-import { prices, beds, types, lifeStyles  } from '../../constants/filters';
+import { prices, beds, types, lifeStyles } from '../../constants/filters';
 import './CreateScrapeRequest.scss'
+import PropTypes from 'prop-types';
+import { states } from '../../constants/states'
+import Autocomplete from 'react-autocomplete';
+import { useTranslation } from 'react-i18next';
+
+export function matchStateToTerm(state, value) {
+    return (
+        state.label.toLowerCase().indexOf(value.toLowerCase()) !== -1
+    )
+}
+const CreateScrapeRequest = ({ handleRequest }) => {
+
+    const [requestObject, setRequestObject] = useState({ location: '' })
+    const { t } = useTranslation();
 
-const CreateScrapeRequest = () => {
 
-    const { t } = useTranslation();  
-    const [inputLocation, setLocation] = useState('')
-    const [priceFilter, setPriceFilter] = useState('')
-    const [bedFilter, setBedFilter] = useState('')
-    const [typeFilter, setTypeFilter] = useState('')
-    const [lifeStyleFilter, setLifeStyleFilter] = useState('')
-   
     const handleChangePriceType = async selectedOption => {
-        setPriceFilter(selectedOption)
+        setRequestObject(s => ({ ...s, price: selectedOption.value }))
     };
     const handleChangeBedType = async selectedOption => {
-        setBedFilter(selectedOption)
+        setRequestObject(s => ({ ...s, beds: selectedOption.value }))
     };
     const handleChangeFilterType = async selectedOption => {
-        setTypeFilter(selectedOption)
+        setRequestObject(s => ({ ...s, type: selectedOption.value }))
     };
 
     const handleChangeLifeStyleType = async selectedOption => {
-        setLifeStyleFilter(selectedOption)
+        setRequestObject(s => ({ ...s, lifestyle: selectedOption.value }))
     };
 
-    function handleSubmit (event){
-        event.preventDefault()
-        console.log(inputLocation);
-        console.log(priceFilter);
-        console.log(bedFilter);
-        console.log(typeFilter);
-        console.log(lifeStyleFilter);
-        
-    }
-  return (
-    <div className="card card-primary">
-        <div className="card-header">
-        <h3 className="card-title">{t('createScrapeRequest.Title')}</h3>
-        </div>      
-        <form >
-        <div className="card-body">
-            <div className="row">
-            <div className="col-md-3">
-                <div className="form-group">
-                    <input type="text" className="form-control input-field cursor-pointer" value={inputLocation} placeholder={t('createScrapeRequest.LocationPlaceholder')} onChange={e => setLocation(e.target.value)}/>
+    console.log("requestObject", requestObject)
+    return (
+        <div className="card card-primary">
+            <form >
+                <div className="card-body">
+                    <div className="row">
+                        {/* <div className="col-md-3">
+                            <div className="form-group">
+                                <input type="text" className="form-control input-field cursor-pointer" value={requestObject.location} placeholder={t('createScrapeRequest.LocationPlaceholder')} onChange={e => setRequestObject(s => ({ ...s, location: e.target.value }))} />
+                            </div>
+                        </div> */}
+                        <div className="col-md-3">
+                            <div className="form-group">
+                                <Autocomplete
+                                    menuStyle={{
+                                        padding: ".375rem .75rem",
+                                        fontSize: '1rem',
+                                        fontWeight: '400',
+                                        lineHeight: 1.5,
+                                        color: "#495057",
+                                        backgroundColor: '#fff',
+                                        backgroundClip: "padding-box",
+                                        border: "1px solid #ced4da",
+                                        borderRadius: ".25rem",
+                                        boxShadow: "inset 0 0 0 transparent",
+                                        transition: " border-color .15s ease-in-out,box-shadow .15s ease-in-out",
+                                        background: 'rgba(255, 255, 255, 0.9)',
+                                        position: 'fixed',
+                                        overflow: 'auto',
+                                        maxHeight: '50%',
+                                        zIndex: 2001
+                                    }}
+                                    renderInput={(props) => <input {...props} type="text" className="form-control input-field cursor-pointer" placeholder={t('createScrapeRequest.LocationPlaceholder')} />}
+                                    getItemValue={(item) => item.label}
+                                    items={states.map(s => ({ label: s }))}
+                                    renderItem={(item, isHighlighted) =>
+                                        <div style={{ background: isHighlighted ? 'lightgray' : 'white', }}>
+                                            {item.label}
+                                        </div>
+                                    }
+                                    shouldItemRender={matchStateToTerm}
+                                    value={requestObject.location}
+                                    onChange={(e, val) => setRequestObject(s => ({ ...s, location: val }))}
+                                    onSelect={(val) => setRequestObject(s => ({ ...s, location: val }))}
+                                />
+                            </div>
+                        </div>
+                        <div className="col-md-2">
+                            <div className="form-group">
+                                <Select options={prices} value={{ name: requestObject.price, label: requestObject.price }} onChange={handleChangePriceType} />
+                            </div>
+                        </div>
+                        <div className="col-md-2">
+                            <div className="form-group">
+                                <Select className="cursor-pointer" value={{ name: requestObject.beds, label: requestObject.beds }} options={beds} onChange={handleChangeBedType} />
+                            </div>
+                        </div>
+                        <div className="col-md-2">
+                            <Select className="cursor-pointer" value={{ name: requestObject.type, label: requestObject.type }} options={types} onChange={handleChangeFilterType} />
+                        </div>
+                        <div className="col-md-2">
+                            <Select className="cursor-pointer" value={{ name: requestObject.lifestyle, label: requestObject.lifestyle }} options={lifeStyles} onChange={handleChangeLifeStyleType} />
+                        </div>
+                        <div className="col-md-1">
+                            <button type="button" onClick={() => { handleRequest(requestObject); setRequestObject({ location: '' }) }} className="btn btn-outline-primary cursor-pointer">Request</button>
+                        </div>
+                    </div>
                 </div>
-            </div>
-            <div className="col-md-2">
-                <div className="form-group">
-                    <Select options={prices} onChange={handleChangePriceType} />
-                </div>
-            </div>
-            <div className="col-md-2">
-                <div className="form-group">
-                <Select className="cursor-pointer"  options={beds} onChange={handleChangeBedType}/>                    
-                </div>
-            </div>
-            <div className="col-md-2">               
-                <Select  className="cursor-pointer" options={types} onChange={handleChangeFilterType}/>                                  
-            </div>
-            <div className="col-md-2">                
-                <Select  className="cursor-pointer" options={lifeStyles} onChange={handleChangeLifeStyleType}/>
-            </div>
-            <div className="col-md-1">
-                <button   type="submit" onClick={handleSubmit} className="btn btn-outline-primary cursor-pointer">Request</button>
-            </div>
-            </div>
+            </form>
         </div>
-        </form>
-    </div>
-          
-  );
+
+    );
 };
 
-CreateScrapeRequest.propTypes = {};
+CreateScrapeRequest.propTypes = {
+    handleRequest: PropTypes.func
+};
 
 export default CreateScrapeRequest;

src/components/ScrapeRequest/ScrapeRequest.js

 import React from 'react';
 import { useTranslation } from 'react-i18next';
 import PropTypes from 'prop-types';
+import ScrappeStatus from './ScrappeStatus';
+import { SCRAPE_RESULTS_PAGE } from '../../constants/pages'
+import { Link } from 'react-router-dom';
 
-const ScrapeRequest = ({ scrape }) => {
+const ScrapeRequest = ({ scrape, index, handleExecute }) => {
     const { t } = useTranslation();
 
-    function handleSubmit(event) {
-        event.preventDefault();
-    }
     return (
         <tr>
             <td>
                 <p>
-                </p><h3>{scrape.Title}</h3>
-                <span className="text-muted">Count {scrape.Count} +</span>
+                </p><h3><Link to={{
+                    pathname: SCRAPE_RESULTS_PAGE,
+                    id: scrape._id
+                }}>#{index} {scrape.location}</Link></h3>
+                <span className="text-muted">Count {scrape.count} +</span>
                 <span> | </span>
-                <span className="text-muted">{t('scrapeRequest.EstimatedTime')} {scrape.EstimatedTime}</span>
+                <span className="text-muted">{t('scrapeRequest.EstimatedTime')} {(new Date(scrape.estimate)).toLocaleString()}</span>
                 <span> | </span>
-                {t('scrapeRequest.ViewScrape')} <a href="scrappe.html"> {scrape.Url}</a>
+                {t('scrapeRequest.ViewScrape')} <a href={scrape.sourceUrl}>{scrape.sourceUrl}</a>
                 <p></p>
                 <p>
                 </p>
             </td>
             <td>
-                {scrape.Filters.map(element => (
-                    <span key={element.id} className="badge bg-primary m-1">{element.type}</span>
+                {scrape.filters.map(element => (
+                    element.value && <span key={element.value} className="badge bg-primary m-1">{element.value}</span>
                 ))}
             </td>
-            <td><button type="submit" className="btn btn-xs btn-block btn-primary" onClick={handleSubmit}><i className="fa fa-bell"></i>{t('common.execute')}</button>
+            <td>
+                <ScrappeStatus status={scrape.status} id={scrape._id} handleExecute={handleExecute} />
             </td>
         </tr>
     );
 }
 
 ScrapeRequest.propTypes = {
-    scrape: PropTypes.object
+    scrape: PropTypes.object,
+    index: PropTypes.number,
+    handleExecute: PropTypes.func
 };
 
 export default ScrapeRequest;

src/components/ScrapeRequest/ScrappeStatus.jsx

+import React from 'react';
+import PropTypes from 'prop-types';
+import { useTranslation } from 'react-i18next';
+
+
+const ScrappeStatus = ({ status, handleExecute, id }) => {
+  const { t } = useTranslation();
+  console.log("id", id)
+  if (status === 'requested')
+    return <button type="submit" className="btn btn-sm btn-block btn-primary" onClick={() => handleExecute(id)}><i className="fa fa-bell"></i>{t('common.execute')}</button>
+  else if (status === 'done')
+    return <span className='badge bg-success text-lg'>Done</span>
+  else
+    return <span className='badge bg-danger text-lg'>Pending</span>
+
+}
+
+ScrappeStatus.propTypes = {
+  status: PropTypes.string,
+  id: PropTypes.string,
+  handleExecute: PropTypes.func
+};
+export default ScrappeStatus;

src/components/ScrapeRequests/ScrapeRequests.js

 import React from 'react';
 import { useTranslation } from 'react-i18next';
+import PropTypes from 'prop-types';
 import ScrapeRequest from '../ScrapeRequest/ScrapeRequest';
+import './ScrappeRequests.scss'
 
-const scrape = { Title: "#1 Chicago, IL", Count: "200", EstimatedTime: "20/7/2021 20:30AM", Url: "https://www.apartments.com/chicago-il/", Filters: [{ id: 1, type: "prices" }, { id: 2, type: "beds" }, { id: 3, type: "type" }, { id: 4, type: "lifestyle" }] };
-const ScrapeRequests = () => {
-
+// const scrape = { Title: "#1 Chicago, IL", Count: "200", EstimatedTime: "20/7/2021 20:30AM", Url: "https://www.apartments.com/chicago-il/", Filters: [{ id: 1, type: "prices" }, { id: 2, type: "beds" }, { id: 3, type: "type" }, { id: 4, type: "lifestyle" }] };
+const ScrapeRequests = ({ scrappes, handleExecute }) => {
   const { t } = useTranslation();
-  return (
 
+  return (
     <div className="card">
-      <div className="card-header">
-        <h3 className="card-title">{t('scrapeRequests.Title')}</h3>
-      </div>
 
-      <div className="card-body p-0">
-        <table className="table table-sm">
-          <thead>
-            <tr>
-              <th className='font-weight-bold'>{t('scrapeRequests.Columns.Scrape')}</th>
-              <th>{t('scrapeRequests.Columns.Filters')}</th>
-              <th>{t('scrapeRequests.Columns.Status')}</th>
-            </tr>
-          </thead>
-          <tbody>
-            <ScrapeRequest scrape={scrape} />
-            {/* <tr>
-                      <td>
-                        <p>
-                          </p><h3> </h3>
-                          <span className="text-muted">Count 200+</span>
-                          <span> | </span>
-                          <span className="text-muted">Estimated time 20/7/2021 20:30AM</span>
-                          <span> | </span>
-                          view scrape origin <a href="scrappe.html">https://www.apartments.com/chicago-il/</a>
-                        <p></p>
-                        <p>
-                        </p>
-                      </td>
-                      <td>
-                        <span className="badge bg-primary">price</span>
-                        <span className="badge bg-primary">beds</span>
-                        <span className="badge bg-primary">type</span>
-                        <span className="badge bg-primary">lifestyle</span>
-                      </td>
-                      <td><button className="btn btn-xs btn-block btn-primary"><i className="fa fa-bell"></i>Execute</button>
-                      </td>
-                    </tr>
-                    <tr>
-                      <td>
-                        <h3>#2 New York, NY </h3>
-                        <span className="text-muted">Count 200+</span>
-                        <span> | </span>
-                        <span className="text-muted">Estimated time 20/7/2021 20:30AM</span>
-                        <span> | </span>
-                        view scrape origin <a href="scrappe.html">https://www.apartments.com/chicago-il/</a>
-                      </td>
-                      <td>
-                        <span className="badge bg-primary">price</span>
-                        <span className="badge bg-primary">beds</span>
-                        <span className="badge bg-primary">type</span>
-                        <span className="badge bg-primary">lifestyle</span>
-                      </td>
-                      <td><span className="badge bg-warning">Pending</span></td>
-                    </tr>
-                    
-                     <tr>
-                      <td>
-                        <h3>#3 New York, NY </h3>
-                        <span className="text-muted">Count 200+</span>
-                        <span> | </span>
-                        <span className="text-muted">Estimated time 20/7/2021 20:30AM</span>
-                        <span> | </span>
-                        view scrape origin <a href="scrappe.html">https://www.apartments.com/chicago-il/</a>
-                      </td>
-                      <td>
-                        <span className="badge bg-primary">price</span>
-                        <span className="badge bg-primary">beds</span>
-                        <span className="badge bg-primary">type</span>
-                        <span className="badge bg-primary">lifestyle</span>
-                      </td>
-                      <td><span className="badge bg-success">Done</span></td>
-                    </tr>
-                     */}
-          </tbody>
-        </table>
-      </div>
+      {scrappes.length === 0 ? <tbody><tr><td><span className='center-align'>Nothing to show</span></td></tr></tbody> :
+        <div className="card-body p-0">
+          <table className="table table-sm">
+            <thead>
+              <tr>
+                <th className='text-black-50 font-weight-bold'>{t('scrapeRequests.Columns.Scrape')}</th>
+                <th className='text-black-50 font-weight-bold'>{t('scrapeRequests.Columns.Filters')}</th>
+                <th className='text-black-50 font-weight-bold'>{t('scrapeRequests.Columns.Status')}</th>
+              </tr>
+            </thead>
+            <tbody>
+              {scrappes.map((scrape, i) => <ScrapeRequest handleExecute={handleExecute} index={i + 1} key={scrape._id} scrape={scrape} />)}
+            </tbody>
+          </table>
+        </div>}
     </div>
   );
 }
 
+ScrapeRequests.propTypes = {
+  handleExecute: PropTypes.func,
+  scrappes: PropTypes.array
+};
 export default ScrapeRequests;

src/components/ScrapeRequests/ScrappeRequests.scss

+.center-align {
+  text-align: center;
+  align-items: center;
+  justify-content: center;
+}

src/components/ScrappeDetails/ScrappeDetails.js

+import React from 'react';
+import PropTypes from 'prop-types';
+import { useTranslation } from 'react-i18next';
+import ScrappeStatus from '../../components/ScrapeRequest/ScrappeStatus'
+const ScrappeDetails = (details) => {
+    const {t} = useTranslation();
+    return (
+            (details.details) ? 
+                <section>
+                    <h2>Scrappe Details</h2>                    
+                    <br/>
+                    <div className="row">
+                        <div className="com-md-4">
+                            {
+                                (details.details.location) ?
+                                    <h3>{details.details.location}</h3>
+                                    : ''}
+                            {
+                                (details.details.estimate) ?
+                                    <span className="text-muted">{t('scrapeRequest.EstimatedTime')} {(new Date(details.details.estimate)).toLocaleString()}</span>
+                                    : ''}
+                        </div>
+                        <div className="col-md-4">
+                            {
+                                (details.details.filters && details.details.filters.length > 0) ?
+                                    <div className="filters-cont">
+                                        <h3>Filters</h3>
+                                        {details.details.filters.map((filter,i) => <span className="badge bg-primary m-1" key={i}>{filter.name}</span>) }
+                                    </div>
+                                    :'' }
+                        </div>
+                        <div className="col-md-1">
+                                {details.details.status ? <ScrappeStatus status = {details.details.status} /> : '' }
+                        </div>
+                    </div>
+                    
+                </section>
+                : ''
+            
+        
+    );
+
+}
+
+ScrappeDetails.propTypes = {
+    details: PropTypes.object
+};
+
+export default ScrappeDetails;

src/components/ScrappeResult/ScrapeResult.js

+import React from 'react';
+import PropTypes from 'prop-types';
+
+const ScrappeResult = (result) => {
+
+    return (
+        <div>
+            Result
+        </div>
+    );
+
+}
+
+ScrappeResult.propTypes = {
+    result: PropTypes.object
+};
+
+export default ScrappeResult;

src/helpers/CheckProperty.js

+export default function checkProperty(data){
+    (data !== undefined && data !==null)? true: false
+}

src/pages/HomePage/HomePage.js

-import React from 'react';
-// import { Link } from 'react-router-dom';
+import React, { useEffect, useState } from 'react';
 import CreateScrapeRequest from '../../components/CreateScrapeRequest/CreateScrapeRequest';
 import ScrapeRequests from '../../components/ScrapeRequests/ScrapeRequests';
+import { createScrappes, executeScrappes, getAllScrappes } from '../../request/scrappe';
+import { useTranslation } from 'react-i18next';
 
 const HomePage = () => {
+  const [scrappes, setScrappes] = useState([])
+  const { t } = useTranslation();
+  useEffect(() => {
+    getAllScrappes().then(res => setScrappes(res.data))
+    const interval = setInterval(() => {
+      getAllScrappes().then(res => setScrappes(res.data))
+    }, 10000);
+    return () => clearInterval(interval)
+  }, [])
+
+
+  async function handleRequest(reqObj) {
+    console.log(reqObj)
+    const res = await createScrappes(reqObj)
+    if (res.status === 200) {
+      getAllScrappes().then(res => setScrappes(res.data))
+    }
+  }
+
+  async function handleExecute(id) {
+    const res = await executeScrappes(id)
+    if (res.status === 204) {
+      getAllScrappes().then(res => setScrappes(res.data))
+    }
+  }
 
   return (
     <>
-      <CreateScrapeRequest />
-      <ScrapeRequests />
-      {/* <Link to='/scrape-results' >See results</Link> */}
+      <h1 className="lead text-center" style={{ fontSize: '80px' }}>{t('scrapeRequests.Columns.Scrape')} </h1>
+      <hr></hr>
+      <h2 className="lead text-center text-muted" style={{ fontSize: '40px' }}>https://www.apartments.com/</h2>
+      <br></br>
+      <CreateScrapeRequest handleRequest={handleRequest} />
+      <ScrapeRequests scrappes={scrappes} handleExecute={handleExecute} />
     </>
   );
 };

src/pages/ScrapeResults/ScrapeResultsPage.js

-import React from 'react';
+import React, { useEffect, useState } from 'react';
+import { getByIdScrappe } from '../../request/scrappe';
+import ScrappeDetails from '../../components/ScrappeDetails/ScrappeDetails'
+//import ScrappeResult from '../../components/ScrappeResult/ScrappeResult'
+import PropTypes from 'prop-types';
 
-const ScrapeResultsPage = () => {
+const ScrapeResultsPage = ({ location }) => {
+
+  const [scrappeResults, setScrappeResults] = useState()
+  const [scrappeDetails, setScrappeDetails] = useState()
+  useEffect(() => {
+    getByIdScrappe(location.id)
+    .then(res => {
+        setScrappeDetails(res.data)
+    if(res.data.status==='done')
+    setScrappeResults(res.data.result)})
+  }, [setScrappeResults])
+  console.log("scrappeDetails", scrappeDetails)
+  console.log("scrappeResults", scrappeResults)
+ 
   return (
-    <div className="c-error-page">
-        <div className="c-error-page__content">
-        Scrape results page
-        </div>
-    </div>
+    <>
+        {/* ScrappeDetail */}
+        {console.log(scrappeDetails)}
+     
+        <ScrappeDetails details = {scrappeDetails} />
+        {/* {(scrappeResults !==undefined) 
+                    ? scrappeResults.map((result, i) => <ScrappeResult key={i} result = {result} />) 
+                    :''
+        */}
+     </> 
   );
 };
 
-ScrapeResultsPage.propTypes = {};
+ScrapeResultsPage.propTypes = {
+  location: PropTypes.object
+};
 
 export default ScrapeResultsPage;

src/request/apiEndpoints.js

 export default {
+  scrappe: {
+    getAll: 'scrapes',
+    getById: 'scrapes/{id}',
+    create: 'scrapes/',
+    execute: 'scrapes/{id}/execute'
+  },
   accounts: {
     get: 'accounts/{accountUid}',
     getCurrentUserPermissions:

src/request/index.js

   headers: {
     'Content-Type': 'application/json',
   },
-  withCredentials: true,
   paramsSerializer: (params) =>
     queryString.stringify(params, { arrayFormat: 'comma' }),
 });

src/request/scrappe.js

+import { getRequest, patchRequest, postRequest, replaceInUrl } from './index';
+import apiEndpoints from './apiEndpoints';
+
+export const getAllScrappes = () => getRequest(apiEndpoints.scrappe.getAll)
+export const getByIdScrappe = (id) => getRequest(replaceInUrl(apiEndpoints.scrappe.getById, { id }))
+export const createScrappes = (scrappe) => postRequest(apiEndpoints.scrappe.create, scrappe)
+export const executeScrappes = (id) => patchRequest(replaceInUrl(apiEndpoints.scrappe.execute, { id }))