Update to use BlinkDB

2023-02-25 03:55:47 -06:00 · 2023-02-25 03:55:47 -06:00 · b1936e89bc
commit b1936e89bc
parent f6a509eefa
5 changed files with 155 additions and 78 deletions
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -8,18 +8,7 @@ services:
      - '4050:3000'
    environment:
      DB_URL: 'mongodb://mongodb:27017'
-    depends_on:
-      - mongodb
-    logging:
-      driver: local
-      options:
-        max-size: '10m'
-    
-  mongodb:
-    image: mongo:4.2
-    restart: unless-stopped
-    volumes:
-      - mongo-data:/data/db
+      REDIS_URL: 'redis://cache'
    logging:
      driver: local
      options:
--- a/package-lock.json
+++ b/package-lock.json
@ -9,10 +9,11 @@
      "version": "1.0.0",
      "license": "ISC",
      "dependencies": {
+        "async-lock": "^1.4.0",
+        "blinkdb": "^0.9.0",
        "body-parser": "^1.20.1",
        "cors": "^2.8.5",
        "date-fns": "^2.29.3",
-        "date-fns-tz": "^1.3.7",
        "dotenv": "^16.0.3",
        "express": "^4.18.2",
        "mongoose": "^6.8.4",
@ -1218,6 +1219,11 @@
      "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
      "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg=="
    },
+    "node_modules/async-lock": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/async-lock/-/async-lock-1.4.0.tgz",
+      "integrity": "sha512-coglx5yIWuetakm3/1dsX9hxCNox22h7+V80RQOu2XUUMidtArxKoZoOtHUPuR84SycKTXzgGzAUR5hJxujyJQ=="
+    },
    "node_modules/base64-js": {
      "version": "1.5.1",
      "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
@ -1237,6 +1243,17 @@
        }
      ]
    },
+    "node_modules/blinkdb": {
+      "version": "0.9.0",
+      "resolved": "https://registry.npmjs.org/blinkdb/-/blinkdb-0.9.0.tgz",
+      "integrity": "sha512-MVBs8epK67JDw1Lkv3838iCYb6DouH51GcPihT1adnqDpXWxVJOiaJazkMpRNBTHEoP2xYXz+o+ZyYhpw7tECA==",
+      "dependencies": {
+        "fast-deep-equal": "^3.1.3",
+        "hyperid": "^3.0.1",
+        "rfdc": "^1.3.0",
+        "sorted-btree": "^1.8.0"
+      }
+    },
    "node_modules/body-parser": {
      "version": "1.20.1",
      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz",
@ -1384,14 +1401,6 @@
        "url": "https://opencollective.com/date-fns"
      }
    },
-    "node_modules/date-fns-tz": {
-      "version": "1.3.7",
-      "resolved": "https://registry.npmjs.org/date-fns-tz/-/date-fns-tz-1.3.7.tgz",
-      "integrity": "sha512-1t1b8zyJo+UI8aR+g3iqr5fkUHWpd58VBx8J/ZSQ+w7YrGlw80Ag4sA86qkfCXRBLmMc4I2US+aPMd4uKvwj5g==",
-      "peerDependencies": {
-        "date-fns": ">=2.0.0"
-      }
-    },
    "node_modules/debug": {
      "version": "2.6.9",
      "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
@ -1492,6 +1501,11 @@
        "node": ">= 0.10.0"
      }
    },
+    "node_modules/fast-deep-equal": {
+      "version": "3.1.3",
+      "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
+      "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="
+    },
    "node_modules/fast-xml-parser": {
      "version": "4.0.11",
      "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.0.11.tgz",
@ -1629,6 +1643,15 @@
        "node": ">= 0.8"
      }
    },
+    "node_modules/hyperid": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/hyperid/-/hyperid-3.1.1.tgz",
+      "integrity": "sha512-RveV33kIksycSf7HLkq1sHB5wW0OwuX8ot8MYnY++gaaPXGFfKpBncHrAWxdpuEeRlazUMGWefwP1w6o6GaumA==",
+      "dependencies": {
+        "uuid": "^8.3.2",
+        "uuid-parse": "^1.1.0"
+      }
+    },
    "node_modules/iconv-lite": {
      "version": "0.4.24",
      "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
@ -1978,6 +2001,11 @@
        "node": ">= 0.8"
      }
    },
+    "node_modules/rfdc": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.3.0.tgz",
+      "integrity": "sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA=="
+    },
    "node_modules/safe-buffer": {
      "version": "5.2.1",
      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
@ -2101,6 +2129,11 @@
        "npm": ">= 3.0.0"
      }
    },
+    "node_modules/sorted-btree": {
+      "version": "1.8.1",
+      "resolved": "https://registry.npmjs.org/sorted-btree/-/sorted-btree-1.8.1.tgz",
+      "integrity": "sha512-395+XIP+wqNn3USkFSrNz7G3Ss/MXlZEqesxvzCRFwL14h6e8LukDHdLBePn5pwbm5OQ9vGu8mDyz2lLDIqamQ=="
+    },
    "node_modules/sparse-bitfield": {
      "version": "3.0.3",
      "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz",
@ -2194,11 +2227,15 @@
      "version": "8.3.2",
      "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
      "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==",
-      "optional": true,
      "bin": {
        "uuid": "dist/bin/uuid"
      }
    },
+    "node_modules/uuid-parse": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/uuid-parse/-/uuid-parse-1.1.0.tgz",
+      "integrity": "sha512-OdmXxA8rDsQ7YpNVbKSJkNzTw2I+S5WsbMDnCtIWSQaosNAcWtFuI/YK1TjzUI6nbkgiqEyh8gWngfcv8Asd9A=="
+    },
    "node_modules/vary": {
      "version": "1.1.2",
      "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz",
@ -3264,11 +3301,27 @@
      "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
      "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg=="
    },
+    "async-lock": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/async-lock/-/async-lock-1.4.0.tgz",
+      "integrity": "sha512-coglx5yIWuetakm3/1dsX9hxCNox22h7+V80RQOu2XUUMidtArxKoZoOtHUPuR84SycKTXzgGzAUR5hJxujyJQ=="
+    },
    "base64-js": {
      "version": "1.5.1",
      "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
      "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA=="
    },
+    "blinkdb": {
+      "version": "0.9.0",
+      "resolved": "https://registry.npmjs.org/blinkdb/-/blinkdb-0.9.0.tgz",
+      "integrity": "sha512-MVBs8epK67JDw1Lkv3838iCYb6DouH51GcPihT1adnqDpXWxVJOiaJazkMpRNBTHEoP2xYXz+o+ZyYhpw7tECA==",
+      "requires": {
+        "fast-deep-equal": "^3.1.3",
+        "hyperid": "^3.0.1",
+        "rfdc": "^1.3.0",
+        "sorted-btree": "^1.8.0"
+      }
+    },
    "body-parser": {
      "version": "1.20.1",
      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz",
@ -3367,12 +3420,6 @@
      "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.29.3.tgz",
      "integrity": "sha512-dDCnyH2WnnKusqvZZ6+jA1O51Ibt8ZMRNkDZdyAyK4YfbDwa/cEmuztzG5pk6hqlp9aSBPYcjOlktquahGwGeA=="
    },
-    "date-fns-tz": {
-      "version": "1.3.7",
-      "resolved": "https://registry.npmjs.org/date-fns-tz/-/date-fns-tz-1.3.7.tgz",
-      "integrity": "sha512-1t1b8zyJo+UI8aR+g3iqr5fkUHWpd58VBx8J/ZSQ+w7YrGlw80Ag4sA86qkfCXRBLmMc4I2US+aPMd4uKvwj5g==",
-      "requires": {}
-    },
    "debug": {
      "version": "2.6.9",
      "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
@ -3454,6 +3501,11 @@
        "vary": "~1.1.2"
      }
    },
+    "fast-deep-equal": {
+      "version": "3.1.3",
+      "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
+      "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="
+    },
    "fast-xml-parser": {
      "version": "4.0.11",
      "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.0.11.tgz",
@ -3544,6 +3596,15 @@
        "toidentifier": "1.0.1"
      }
    },
+    "hyperid": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/hyperid/-/hyperid-3.1.1.tgz",
+      "integrity": "sha512-RveV33kIksycSf7HLkq1sHB5wW0OwuX8ot8MYnY++gaaPXGFfKpBncHrAWxdpuEeRlazUMGWefwP1w6o6GaumA==",
+      "requires": {
+        "uuid": "^8.3.2",
+        "uuid-parse": "^1.1.0"
+      }
+    },
    "iconv-lite": {
      "version": "0.4.24",
      "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
@ -3777,6 +3838,11 @@
        "unpipe": "1.0.0"
      }
    },
+    "rfdc": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.3.0.tgz",
+      "integrity": "sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA=="
+    },
    "safe-buffer": {
      "version": "5.2.1",
      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
@ -3868,6 +3934,11 @@
        "smart-buffer": "^4.2.0"
      }
    },
+    "sorted-btree": {
+      "version": "1.8.1",
+      "resolved": "https://registry.npmjs.org/sorted-btree/-/sorted-btree-1.8.1.tgz",
+      "integrity": "sha512-395+XIP+wqNn3USkFSrNz7G3Ss/MXlZEqesxvzCRFwL14h6e8LukDHdLBePn5pwbm5OQ9vGu8mDyz2lLDIqamQ=="
+    },
    "sparse-bitfield": {
      "version": "3.0.3",
      "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz",
@ -3935,8 +4006,12 @@
    "uuid": {
      "version": "8.3.2",
      "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
-      "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==",
-      "optional": true
+      "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg=="
+    },
+    "uuid-parse": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/uuid-parse/-/uuid-parse-1.1.0.tgz",
+      "integrity": "sha512-OdmXxA8rDsQ7YpNVbKSJkNzTw2I+S5WsbMDnCtIWSQaosNAcWtFuI/YK1TjzUI6nbkgiqEyh8gWngfcv8Asd9A=="
    },
    "vary": {
      "version": "1.1.2",
--- a/package.json
+++ b/package.json
@ -5,7 +5,8 @@
  "main": "index.js",
  "type": "module",
  "scripts": {
-    "start": "npx tsc && node build/index.js",
+    "start": "npx tsc && node --experimental-specifier-resolution=node build/index.js",
+    "start:dev": "export PORT=4500 npx tsc && node --experimental-specifier-resolution=node --trace-warnings build/index.js",
    "test": "echo \"Error: no test specified\" && exit 1"
  },
  "repository": {
@ -15,10 +16,11 @@
  "author": "",
  "license": "ISC",
  "dependencies": {
+    "async-lock": "^1.4.0",
+    "blinkdb": "^0.9.0",
    "body-parser": "^1.20.1",
    "cors": "^2.8.5",
    "date-fns": "^2.29.3",
-    "date-fns-tz": "^1.3.7",
    "dotenv": "^16.0.3",
    "express": "^4.18.2",
    "mongoose": "^6.8.4",
--- a/src/index.ts
+++ b/src/index.ts
@ -1,32 +1,38 @@
 import bodyParser from 'body-parser';
 import express from 'express';
 import 'dotenv/config';
-import mongoose, { Schema } from 'mongoose';
 import fetch from 'node-fetch';
-import { subHours } from 'date-fns'
-import dateFnsTz from 'date-fns-tz';
+import { subHours, subSeconds } from 'date-fns'
 import cors from 'cors';
+import {
+  createDB,
+  createTable,
+  insert,
+  many,
+  removeWhere,
+  uuid,
+} from 'blinkdb';

 const { json } = bodyParser;
-const { formatInTimeZone } = dateFnsTz;
-

 const RAIN_GAUGE_ENDPOINT = process.env.RAIN_GAUGE_ENDPOINT;
 const SOIL_SENSOR_ENDPOINT = process.env.SOIL_SENSOR_ENDPOINT;
 const TEASENSE_ENDPOINT = process.env.TEASENSE_ENDPOINT;
 const PORT = process.env.PORT || 3000;
-const DATA_COLLECT_SLEEP = 100;
+const DATA_COLLECT_SLEEP = 1000;
 const PURGE_SLEEP = 60000;
+const PURGE_TEASENSE_SLEEP = 1000;

-mongoose.connect(process.env.DB_URL ?? '');
+interface DataPoint {
+  id: string;
+  name: string;
+  value: number;
+  timestamp: Date;
+};

-const DataSchema = new Schema({
-  name: Schema.Types.String,
-  value: Schema.Types.Number,
-  timestamp: Schema.Types.Date,
-});
+const db = createDB();

-const DataModel = mongoose.model('data', DataSchema);
+const dataPointsTable = createTable<DataPoint>(db, 'dataPoints')();

 (async () => {
  const app = express();
@ -40,24 +46,13 @@ const DataModel = mongoose.model('data', DataSchema);

  app.get('/:name', async (req: any, res: any) => {
    const startTime = subHours(new Date(), 24);
-    DataModel.find({
-      name: req.params.name,
-      timestamp: {
-        $gte: startTime,
+    const result = await many(dataPointsTable, {
+      where: {
+        name: req.params.name,
      }
-    }, (err: any, arr: any) => {
-      if (arr) {
-        res.json(arr.map((item: any) => {
-          const newItem = {
-            value: item.value,
-            timestamp: formatInTimeZone(item.timestamp, 'UTC', 'yyyy-MM-dd\'T\'HH:mm:ss'),
-          }
-          return newItem;
-        }));
-      } else {
-        res.status(500).send('There were complications.');
-      }
-    })
+    });
+
+    res.json(result);
  });

  app.listen(PORT);
@ -67,11 +62,12 @@ const DataModel = mongoose.model('data', DataSchema);
 let rainfallTimestamp = '';

 const saveDataPoint = async (name: string, value: number) => {
-  const tempData = new DataModel();
-  tempData.name = name;
-  tempData.value = value;
-  tempData.timestamp = new Date();
-  tempData.save();
+  await insert(dataPointsTable, {
+    id: uuid(),
+    name,
+    value,
+    timestamp: new Date(),
+  });
 }

 const collectData = async () => {
@ -95,8 +91,7 @@ const collectData = async () => {
        signal: AbortSignal.timeout(5000),
      });
      const data: any = await response.json();
-      saveDataPoint('soil_temperature', data.temperature);
-      saveDataPoint('soil_capacitence', data.capacitive);
+      saveDataPoint('soil_voltage', data.moisture);
    } catch (e) {
      console.log(e);
    }
@ -104,9 +99,7 @@ const collectData = async () => {

  if (TEASENSE_ENDPOINT && TEASENSE_ENDPOINT.trim() !== '') {
    try {
-      const response = await fetch(TEASENSE_ENDPOINT, {
-        signal: AbortSignal.timeout(5000),
-      });
+      const response = await fetch(TEASENSE_ENDPOINT);
      const data: any = await response.json();
      saveDataPoint('teasense', data.temperature);
    } catch (e) {
@ -119,12 +112,30 @@ const collectData = async () => {

 const purger = async () => {
  const startTime = subHours(new Date(), 24);
-  await DataModel.deleteMany({
-    timestamp: {
-      $lt: startTime,
+  await removeWhere(dataPointsTable, {
+    where: {
+      timestamp: {
+        lt: startTime,
+      },
+    },
+  });
+};
+
+const purger5Sec = async () => {
+  const startTime = subSeconds(new Date(), 5);
+  await removeWhere(dataPointsTable, {
+    where: {
+      name: {
+        in: ['teasense', 'soil_voltage'],
+      },
+      timestamp: {
+        lt: startTime,
+      },
    },
  });
 };

 setTimeout(() => collectData(), DATA_COLLECT_SLEEP);
+
 setInterval(() => purger(), PURGE_SLEEP);
+setInterval(() => purger5Sec(), PURGE_TEASENSE_SLEEP);
--- a/tsconfig.json
+++ b/tsconfig.json
@ -48,7 +48,7 @@
    // "declarationMap": true,                           /* Create sourcemaps for d.ts files. */
    // "emitDeclarationOnly": true,                      /* Only output d.ts files and not JavaScript files. */
    // "sourceMap": true,                                /* Create source map files for emitted JavaScript files. */
-    // "outFile": "./",                                  /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */
+    // "outFile": "./build/index.js",                                  /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */
    "outDir": "./build",                                   /* Specify an output folder for all emitted files. */
    // "removeComments": true,                           /* Disable emitting comments. */
    // "noEmit": true,                                   /* Disable emitting files from a compilation. */