Mongo database connected

app.js

@@ -1,26 +1,41 @@
+var config = require('config');
 const express = require('express');
-var cors = require('cors');
+const cors = require('cors');
+const MongoClient = require('mongodb').MongoClient;
+const ObjectID = require('mongodb').ObjectID;
 
+// express application
 const app = express();
-// app.use(bodyParser.urlencoded({ extended: false }));
+
 app.use(express.json());
 app.use(cors());
-// app.use(bodyParser.raw());
+
+// database setup
+var mongoUrl = config.get("mongo");
+var database = config.get("database");
+
+var client = undefined;
+MongoClient.connect(mongoUrl, function (err, db) {
+  if (err) throw err;
+  console.log("Database created!");
+  console.log(mongoUrl);
+  client = db;
+
+  var dbo = db.db(database);
+  dbo.createCollection("scrapes", function (err, res) {
+    if (err) {
+      console.log("Collection already created!");
+      return;
+    }
+    console.log("Collection created!");
+  });
+});
 
 // const cron = require('node-cron');
-// const {MongoClient} = require('mongodb');
-
-// const uri = "mongodb+srv://Nikola:Nikola@cluster0.jvqzh.mongodb.net/admin?authSource=admin&replicaSet=atlas-7exvp3-shard-0&w=majority&readPreference=primary&appname=MongoDB%20Compass&retryWrites=true&ssl=true";
-// const client = new MongoClient(uri);
-// client.connect();
-// createListing =  async function (client, data){
-//   const result = await client.db("scraper").collection("scraping").insertOne(data);
-//   console.log(`New scraping created with the following id: ${result.insertedId}`);
-// }
+
 // var apartments = require('./apartments.js');
 // var houses = require('./houses.js');
 
-
 // app.set('json spaces', 2);
 
 // const axios = require('axios');
@@ -82,54 +97,33 @@ app.use(cors());
 //   });
 // });
 
-app.get("/scrapes", (req, res) => {
-  res.json(
-    [
-      { 
-        id: 1,
-        location: "Chicago, IL",
-        count: 21,
-        estimate: Date.now(),
-        sourceUrl: "https://www.apartments.com",
-        filters: [
-          { name: "price", value: "1000"},
-          { name: "beds", value: "2"},
-        ],
-        status: "requested"
-      },
-      { 
-        id: 2,
-        location: "New York, NY",
-        count: 21,
-        estimate: Date.now(),
-        sourceUrl: "https://www.apartments.com",
-        filters: [
-          { name: "lifestyle", value: "2"},
-        ],
-        status: "pending"
-      },{ 
-        id: 3,
-        location: "Los Angeles, CA",
-        count: 21,
-        estimate: Date.now(),
-        sourceUrl: "https://www.apartments.com",
-        filters: [
-          { name: "type", value: "apartments"},
-        ],
-        status: "done"
-      }
-    ]
-  )
+app.get("/scrapes", async (req, res) => {
+  try {
+    const dbo = client.db(database);
+    let collection = dbo.collection('scrapes');
+    let data = await collection.find({}).toArray();
+    return res.json(data);
+  } catch (err) {
+    console.log(err);
+    return res.status(500).json();
+  }
 });
-app.get("/scrapes/:id", (req, res) => {
+app.get("/scrapes/:id", async (req, res) => {
   const id = req.params.id;
-  //todo: get data from mongo
-
-  res.json(id);
+  try {
+    const dbo = client.db(database);
+    let collection = dbo.collection('scrapes');
+
+    var o_id = new ObjectID(id);
+    let data = await collection.findOne({ _id: o_id});
+    return res.json(data);
+  } catch (err) {
+    console.log(err);
+    res.status(500).json();
+  }
 });
-app.post("/scrapes/", (req, res) => {
+app.post("/scrapes/", async (req, res) => {
   const location = req.body.location;
-  console.log(req.body)
   const price = req.body.price;
   const beds = req.body.beds;
   const type = req.body.type;
@@ -138,22 +132,45 @@ app.post("/scrapes/", (req, res) => {
   // query builder
 
   //todo: save data into the database
-  res.json({
-    id: 1,
-    location: location,
-    filters:[
-      { name: 'price', value: price },
-      { name: 'beds', value: beds },
-      { name: 'type', value: type },
-      { name: 'lifestyle', value: lifestyle },
-    ]
-  });
+  try {
+    const dbo = client.db(database);
+    let collection = dbo.collection('scrapes');
+
+    let res = await collection.insertOne({
+      count: 21,
+      estimate: Date.now(),
+      sourceUrl: "https://www.apartments.com",
+      location: location,
+      filters: [
+        { name: 'price', value: price },
+        { name: 'beds', value: beds },
+        { name: 'type', value: type },
+        { name: 'lifestyle', value: lifestyle },
+      ],
+      status: "requested"
+    });
+    console.log(res);
+  } catch (err) {
+    console.log(err);
+    return res.status(500).json();
+  }
+  return res.json();
 });
-app.patch("/scrapes/:id/execute", (req, res) => {
+app.patch("/scrapes/:id/execute", async (req, res) => {
   const id = req.params.id;
-  //todo: get scrape data from db
-  //todo: mark scrape for execution in the job queue
-  res.status(200).json(id);
+
+  try {
+    const dbo = client.db(database);
+    let collection = dbo.collection('scrapes');
+
+    var o_id = new ObjectID(id);
+    var newvalues = { $set: {status: "pending"} };
+    let data = await collection.updateOne({ _id: o_id}, newvalues);
+    return res.status(204).json();
+  } catch (err) {
+    console.log(err);
+    res.status(500).json();
+  }
 });
 
 const port = 3333;
@@ -166,7 +183,7 @@ const port = 3333;
 //     path: '/apartments/https://www.apartments.com/essex-on-the-park-chicago-il/begd58b/',
 //     method: 'GET'
 //   };
-  
+
 
 // task.start()
 
@@ -174,4 +191,7 @@ const port = 3333;
 
 app.listen(port, () => {
   console.log(`Example app listening at http://localhost:${port}`)
+});
+process.on('exit', function () {
+  client.close();
 });

config/default.json

@@ -0,0 +1,4 @@
+{
+  "mongo":"mongodb://localhost:27017/",
+  "database":"scraper"
+}

package-lock.json

@@ -391,6 +391,14 @@
       "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=",
       "dev": true
     },
+    "config": {
+      "version": "3.3.6",
+      "resolved": "https://registry.npmjs.org/config/-/config-3.3.6.tgz",
+      "integrity": "sha512-Hj5916C5HFawjYJat1epbyY2PlAgLpBtDUlr0MxGLgo3p5+7kylyvnRY18PqJHgnNWXcdd0eWDemT7eYWuFgwg==",
+      "requires": {
+        "json5": "^2.1.1"
+      }
+    },
     "configstore": {
       "version": "5.0.1",
       "resolved": "https://registry.npmjs.org/configstore/-/configstore-5.0.1.tgz",
@@ -1034,6 +1042,14 @@
       "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz",
       "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus="
     },
+    "json5": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.0.tgz",
+      "integrity": "sha512-f+8cldu7X/y7RAJurMEJmdoKXGB/X550w2Nr3tTbezL6RwEE/iMcm+tZnXeoZtKuOq6ft8+CqzEkrIgx1fPoQA==",
+      "requires": {
+        "minimist": "^1.2.5"
+      }
+    },
     "jsprim": {
       "version": "1.4.1",
       "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz",
@@ -1143,8 +1159,7 @@
     "minimist": {
       "version": "1.2.5",
       "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz",
-      "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==",
-      "dev": true
+      "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw=="
     },
     "moment": {
       "version": "2.29.1",

package.json

@@ -12,6 +12,7 @@
     "axios": "^0.21.1",
     "body-parser": "^1.19.0",
     "cheerio": "^1.0.0-rc.10",
+    "config": "^3.3.6",
     "cors": "^2.8.5",
     "express": "^4.17.1",
     "http": "0.0.1-security",