Update to use BlinkDB

docker-compose.yml

@@ -8,18 +8,7 @@ services:
       - '4050:3000'
     environment:
       DB_URL: 'mongodb://mongodb:27017'
-    depends_on:
+      REDIS_URL: 'redis://cache'
-      - mongodb
-    logging:
-      driver: local
-      options:
-        max-size: '10m'
-    
-  mongodb:
-    image: mongo:4.2
-    restart: unless-stopped
-    volumes:
-      - mongo-data:/data/db
     logging:
       driver: local
       options:

package-lock.json

@@ -9,10 +9,11 @@
       "version": "1.0.0",
       "license": "ISC",
       "dependencies": {
+        "async-lock": "^1.4.0",
+        "blinkdb": "^0.9.0",
         "body-parser": "^1.20.1",
         "cors": "^2.8.5",
         "date-fns": "^2.29.3",
-        "date-fns-tz": "^1.3.7",
         "dotenv": "^16.0.3",
         "express": "^4.18.2",
         "mongoose": "^6.8.4",
@@ -1218,6 +1219,11 @@
       "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
       "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg=="
     },
+    "node_modules/async-lock": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/async-lock/-/async-lock-1.4.0.tgz",
+      "integrity": "sha512-coglx5yIWuetakm3/1dsX9hxCNox22h7+V80RQOu2XUUMidtArxKoZoOtHUPuR84SycKTXzgGzAUR5hJxujyJQ=="
+    },
     "node_modules/base64-js": {
       "version": "1.5.1",
       "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
@@ -1237,6 +1243,17 @@
         }
       ]
     },
+    "node_modules/blinkdb": {
+      "version": "0.9.0",
+      "resolved": "https://registry.npmjs.org/blinkdb/-/blinkdb-0.9.0.tgz",
+      "integrity": "sha512-MVBs8epK67JDw1Lkv3838iCYb6DouH51GcPihT1adnqDpXWxVJOiaJazkMpRNBTHEoP2xYXz+o+ZyYhpw7tECA==",
+      "dependencies": {
+        "fast-deep-equal": "^3.1.3",
+        "hyperid": "^3.0.1",
+        "rfdc": "^1.3.0",
+        "sorted-btree": "^1.8.0"
+      }
+    },
     "node_modules/body-parser": {
       "version": "1.20.1",
       "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz",
@@ -1384,14 +1401,6 @@
         "url": "https://opencollective.com/date-fns"
       }
     },
-    "node_modules/date-fns-tz": {
-      "version": "1.3.7",
-      "resolved": "https://registry.npmjs.org/date-fns-tz/-/date-fns-tz-1.3.7.tgz",
-      "integrity": "sha512-1t1b8zyJo+UI8aR+g3iqr5fkUHWpd58VBx8J/ZSQ+w7YrGlw80Ag4sA86qkfCXRBLmMc4I2US+aPMd4uKvwj5g==",
-      "peerDependencies": {
-        "date-fns": ">=2.0.0"
-      }
-    },
     "node_modules/debug": {
       "version": "2.6.9",
       "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
@@ -1492,6 +1501,11 @@
         "node": ">= 0.10.0"
       }
     },
+    "node_modules/fast-deep-equal": {
+      "version": "3.1.3",
+      "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
+      "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="
+    },
     "node_modules/fast-xml-parser": {
       "version": "4.0.11",
       "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.0.11.tgz",
@@ -1629,6 +1643,15 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/hyperid": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/hyperid/-/hyperid-3.1.1.tgz",
+      "integrity": "sha512-RveV33kIksycSf7HLkq1sHB5wW0OwuX8ot8MYnY++gaaPXGFfKpBncHrAWxdpuEeRlazUMGWefwP1w6o6GaumA==",
+      "dependencies": {
+        "uuid": "^8.3.2",
+        "uuid-parse": "^1.1.0"
+      }
+    },
     "node_modules/iconv-lite": {
       "version": "0.4.24",
       "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
@@ -1978,6 +2001,11 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/rfdc": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.3.0.tgz",
+      "integrity": "sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA=="
+    },
     "node_modules/safe-buffer": {
       "version": "5.2.1",
       "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
@@ -2101,6 +2129,11 @@
         "npm": ">= 3.0.0"
       }
     },
+    "node_modules/sorted-btree": {
+      "version": "1.8.1",
+      "resolved": "https://registry.npmjs.org/sorted-btree/-/sorted-btree-1.8.1.tgz",
+      "integrity": "sha512-395+XIP+wqNn3USkFSrNz7G3Ss/MXlZEqesxvzCRFwL14h6e8LukDHdLBePn5pwbm5OQ9vGu8mDyz2lLDIqamQ=="
+    },
     "node_modules/sparse-bitfield": {
       "version": "3.0.3",
       "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz",
@@ -2194,11 +2227,15 @@
       "version": "8.3.2",
       "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
       "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==",
-      "optional": true,
       "bin": {
         "uuid": "dist/bin/uuid"
       }
     },
+    "node_modules/uuid-parse": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/uuid-parse/-/uuid-parse-1.1.0.tgz",
+      "integrity": "sha512-OdmXxA8rDsQ7YpNVbKSJkNzTw2I+S5WsbMDnCtIWSQaosNAcWtFuI/YK1TjzUI6nbkgiqEyh8gWngfcv8Asd9A=="
+    },
     "node_modules/vary": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz",
@@ -3264,11 +3301,27 @@
       "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
       "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg=="
     },
+    "async-lock": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/async-lock/-/async-lock-1.4.0.tgz",
+      "integrity": "sha512-coglx5yIWuetakm3/1dsX9hxCNox22h7+V80RQOu2XUUMidtArxKoZoOtHUPuR84SycKTXzgGzAUR5hJxujyJQ=="
+    },
     "base64-js": {
       "version": "1.5.1",
       "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
       "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA=="
     },
+    "blinkdb": {
+      "version": "0.9.0",
+      "resolved": "https://registry.npmjs.org/blinkdb/-/blinkdb-0.9.0.tgz",
+      "integrity": "sha512-MVBs8epK67JDw1Lkv3838iCYb6DouH51GcPihT1adnqDpXWxVJOiaJazkMpRNBTHEoP2xYXz+o+ZyYhpw7tECA==",
+      "requires": {
+        "fast-deep-equal": "^3.1.3",
+        "hyperid": "^3.0.1",
+        "rfdc": "^1.3.0",
+        "sorted-btree": "^1.8.0"
+      }
+    },
     "body-parser": {
       "version": "1.20.1",
       "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz",
@@ -3367,12 +3420,6 @@
       "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.29.3.tgz",
       "integrity": "sha512-dDCnyH2WnnKusqvZZ6+jA1O51Ibt8ZMRNkDZdyAyK4YfbDwa/cEmuztzG5pk6hqlp9aSBPYcjOlktquahGwGeA=="
     },
-    "date-fns-tz": {
-      "version": "1.3.7",
-      "resolved": "https://registry.npmjs.org/date-fns-tz/-/date-fns-tz-1.3.7.tgz",
-      "integrity": "sha512-1t1b8zyJo+UI8aR+g3iqr5fkUHWpd58VBx8J/ZSQ+w7YrGlw80Ag4sA86qkfCXRBLmMc4I2US+aPMd4uKvwj5g==",
-      "requires": {}
-    },
     "debug": {
       "version": "2.6.9",
       "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
@@ -3454,6 +3501,11 @@
         "vary": "~1.1.2"
       }
     },
+    "fast-deep-equal": {
+      "version": "3.1.3",
+      "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
+      "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="
+    },
     "fast-xml-parser": {
       "version": "4.0.11",
       "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.0.11.tgz",
@@ -3544,6 +3596,15 @@
         "toidentifier": "1.0.1"
       }
     },
+    "hyperid": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/hyperid/-/hyperid-3.1.1.tgz",
+      "integrity": "sha512-RveV33kIksycSf7HLkq1sHB5wW0OwuX8ot8MYnY++gaaPXGFfKpBncHrAWxdpuEeRlazUMGWefwP1w6o6GaumA==",
+      "requires": {
+        "uuid": "^8.3.2",
+        "uuid-parse": "^1.1.0"
+      }
+    },
     "iconv-lite": {
       "version": "0.4.24",
       "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
@@ -3777,6 +3838,11 @@
         "unpipe": "1.0.0"
       }
     },
+    "rfdc": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.3.0.tgz",
+      "integrity": "sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA=="
+    },
     "safe-buffer": {
       "version": "5.2.1",
       "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
@@ -3868,6 +3934,11 @@
         "smart-buffer": "^4.2.0"
       }
     },
+    "sorted-btree": {
+      "version": "1.8.1",
+      "resolved": "https://registry.npmjs.org/sorted-btree/-/sorted-btree-1.8.1.tgz",
+      "integrity": "sha512-395+XIP+wqNn3USkFSrNz7G3Ss/MXlZEqesxvzCRFwL14h6e8LukDHdLBePn5pwbm5OQ9vGu8mDyz2lLDIqamQ=="
+    },
     "sparse-bitfield": {
       "version": "3.0.3",
       "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz",
@@ -3935,8 +4006,12 @@
     "uuid": {
       "version": "8.3.2",
       "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
-      "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==",
+      "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg=="
-      "optional": true
+    },
+    "uuid-parse": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/uuid-parse/-/uuid-parse-1.1.0.tgz",
+      "integrity": "sha512-OdmXxA8rDsQ7YpNVbKSJkNzTw2I+S5WsbMDnCtIWSQaosNAcWtFuI/YK1TjzUI6nbkgiqEyh8gWngfcv8Asd9A=="
     },
     "vary": {
       "version": "1.1.2",

package.json

@@ -5,7 +5,8 @@
   "main": "index.js",
   "type": "module",
   "scripts": {
-    "start": "npx tsc && node build/index.js",
+    "start": "npx tsc && node --experimental-specifier-resolution=node build/index.js",
+    "start:dev": "export PORT=4500 npx tsc && node --experimental-specifier-resolution=node --trace-warnings build/index.js",
     "test": "echo \"Error: no test specified\" && exit 1"
   },
   "repository": {
@@ -15,10 +16,11 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
+    "async-lock": "^1.4.0",
+    "blinkdb": "^0.9.0",
     "body-parser": "^1.20.1",
     "cors": "^2.8.5",
     "date-fns": "^2.29.3",
-    "date-fns-tz": "^1.3.7",
     "dotenv": "^16.0.3",
     "express": "^4.18.2",
     "mongoose": "^6.8.4",

src/index.ts

@@ -1,32 +1,38 @@
 import bodyParser from 'body-parser';
 import express from 'express';
 import 'dotenv/config';
-import mongoose, { Schema } from 'mongoose';
 import fetch from 'node-fetch';
-import { subHours } from 'date-fns'
+import { subHours, subSeconds } from 'date-fns'
-import dateFnsTz from 'date-fns-tz';
 import cors from 'cors';
+import {
+  createDB,
+  createTable,
+  insert,
+  many,
+  removeWhere,
+  uuid,
+} from 'blinkdb';
 
 const { json } = bodyParser;
-const { formatInTimeZone } = dateFnsTz;
-
 
 const RAIN_GAUGE_ENDPOINT = process.env.RAIN_GAUGE_ENDPOINT;
 const SOIL_SENSOR_ENDPOINT = process.env.SOIL_SENSOR_ENDPOINT;
 const TEASENSE_ENDPOINT = process.env.TEASENSE_ENDPOINT;
 const PORT = process.env.PORT || 3000;
-const DATA_COLLECT_SLEEP = 100;
+const DATA_COLLECT_SLEEP = 1000;
 const PURGE_SLEEP = 60000;
+const PURGE_TEASENSE_SLEEP = 1000;
 
-mongoose.connect(process.env.DB_URL ?? '');
+interface DataPoint {
+  id: string;
+  name: string;
+  value: number;
+  timestamp: Date;
+};
 
-const DataSchema = new Schema({
+const db = createDB();
-  name: Schema.Types.String,
-  value: Schema.Types.Number,
-  timestamp: Schema.Types.Date,
-});
 
-const DataModel = mongoose.model('data', DataSchema);
+const dataPointsTable = createTable<DataPoint>(db, 'dataPoints')();
 
 (async () => {
   const app = express();
@@ -40,24 +46,13 @@ const DataModel = mongoose.model('data', DataSchema);
 
   app.get('/:name', async (req: any, res: any) => {
     const startTime = subHours(new Date(), 24);
-    DataModel.find({
+    const result = await many(dataPointsTable, {
+      where: {
         name: req.params.name,
-      timestamp: {
-        $gte: startTime,
       }
-    }, (err: any, arr: any) => {
+    });
-      if (arr) {
+
-        res.json(arr.map((item: any) => {
+    res.json(result);
-          const newItem = {
-            value: item.value,
-            timestamp: formatInTimeZone(item.timestamp, 'UTC', 'yyyy-MM-dd\'T\'HH:mm:ss'),
-          }
-          return newItem;
-        }));
-      } else {
-        res.status(500).send('There were complications.');
-      }
-    })
   });
 
   app.listen(PORT);
@@ -67,11 +62,12 @@ const DataModel = mongoose.model('data', DataSchema);
 let rainfallTimestamp = '';
 
 const saveDataPoint = async (name: string, value: number) => {
-  const tempData = new DataModel();
+  await insert(dataPointsTable, {
-  tempData.name = name;
+    id: uuid(),
-  tempData.value = value;
+    name,
-  tempData.timestamp = new Date();
+    value,
-  tempData.save();
+    timestamp: new Date(),
+  });
 }
 
 const collectData = async () => {
@@ -95,8 +91,7 @@ const collectData = async () => {
         signal: AbortSignal.timeout(5000),
       });
       const data: any = await response.json();
-      saveDataPoint('soil_temperature', data.temperature);
+      saveDataPoint('soil_voltage', data.moisture);
-      saveDataPoint('soil_capacitence', data.capacitive);
     } catch (e) {
       console.log(e);
     }
@@ -104,9 +99,7 @@ const collectData = async () => {
 
   if (TEASENSE_ENDPOINT && TEASENSE_ENDPOINT.trim() !== '') {
     try {
-      const response = await fetch(TEASENSE_ENDPOINT, {
+      const response = await fetch(TEASENSE_ENDPOINT);
-        signal: AbortSignal.timeout(5000),
-      });
       const data: any = await response.json();
       saveDataPoint('teasense', data.temperature);
     } catch (e) {
@@ -119,12 +112,30 @@ const collectData = async () => {
 
 const purger = async () => {
   const startTime = subHours(new Date(), 24);
-  await DataModel.deleteMany({
+  await removeWhere(dataPointsTable, {
+    where: {
       timestamp: {
-      $lt: startTime,
+        lt: startTime,
+      },
+    },
+  });
+};
+
+const purger5Sec = async () => {
+  const startTime = subSeconds(new Date(), 5);
+  await removeWhere(dataPointsTable, {
+    where: {
+      name: {
+        in: ['teasense', 'soil_voltage'],
+      },
+      timestamp: {
+        lt: startTime,
+      },
     },
   });
 };
 
 setTimeout(() => collectData(), DATA_COLLECT_SLEEP);
+
 setInterval(() => purger(), PURGE_SLEEP);
+setInterval(() => purger5Sec(), PURGE_TEASENSE_SLEEP);

tsconfig.json

@@ -48,7 +48,7 @@
     // "declarationMap": true,                           /* Create sourcemaps for d.ts files. */
     // "emitDeclarationOnly": true,                      /* Only output d.ts files and not JavaScript files. */
     // "sourceMap": true,                                /* Create source map files for emitted JavaScript files. */
-    // "outFile": "./",                                  /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */
+    // "outFile": "./build/index.js",                                  /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */
     "outDir": "./build",                                   /* Specify an output folder for all emitted files. */
     // "removeComments": true,                           /* Disable emitting comments. */
     // "noEmit": true,                                   /* Disable emitting files from a compilation. */