From b1936e89bceb097c7408d89d808b20008a87db04 Mon Sep 17 00:00:00 2001 From: William Moore Date: Sat, 25 Feb 2023 03:55:47 -0600 Subject: [PATCH] Update to use BlinkDB --- docker-compose.yml | 15 +----- package-lock.json | 111 +++++++++++++++++++++++++++++++++++++-------- package.json | 6 ++- src/index.ts | 99 ++++++++++++++++++++++------------------ tsconfig.json | 2 +- 5 files changed, 155 insertions(+), 78 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index d027ecf..1e815fe 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,21 +8,10 @@ services: - '4050:3000' environment: DB_URL: 'mongodb://mongodb:27017' - depends_on: - - mongodb - logging: - driver: local - options: - max-size: '10m' - - mongodb: - image: mongo:4.2 - restart: unless-stopped - volumes: - - mongo-data:/data/db + REDIS_URL: 'redis://cache' logging: driver: local options: max-size: '10m' volumes: - mongo-data: \ No newline at end of file + mongo-data: diff --git a/package-lock.json b/package-lock.json index 084e055..355e1d1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,10 +9,11 @@ "version": "1.0.0", "license": "ISC", "dependencies": { + "async-lock": "^1.4.0", + "blinkdb": "^0.9.0", "body-parser": "^1.20.1", "cors": "^2.8.5", "date-fns": "^2.29.3", - "date-fns-tz": "^1.3.7", "dotenv": "^16.0.3", "express": "^4.18.2", "mongoose": "^6.8.4", @@ -1218,6 +1219,11 @@ "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==" }, + "node_modules/async-lock": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/async-lock/-/async-lock-1.4.0.tgz", + "integrity": "sha512-coglx5yIWuetakm3/1dsX9hxCNox22h7+V80RQOu2XUUMidtArxKoZoOtHUPuR84SycKTXzgGzAUR5hJxujyJQ==" + }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -1237,6 +1243,17 @@ } ] }, + "node_modules/blinkdb": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/blinkdb/-/blinkdb-0.9.0.tgz", + "integrity": "sha512-MVBs8epK67JDw1Lkv3838iCYb6DouH51GcPihT1adnqDpXWxVJOiaJazkMpRNBTHEoP2xYXz+o+ZyYhpw7tECA==", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "hyperid": "^3.0.1", + "rfdc": "^1.3.0", + "sorted-btree": "^1.8.0" + } + }, "node_modules/body-parser": { "version": "1.20.1", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz", @@ -1384,14 +1401,6 @@ "url": "https://opencollective.com/date-fns" } }, - "node_modules/date-fns-tz": { - "version": "1.3.7", - "resolved": "https://registry.npmjs.org/date-fns-tz/-/date-fns-tz-1.3.7.tgz", - "integrity": "sha512-1t1b8zyJo+UI8aR+g3iqr5fkUHWpd58VBx8J/ZSQ+w7YrGlw80Ag4sA86qkfCXRBLmMc4I2US+aPMd4uKvwj5g==", - "peerDependencies": { - "date-fns": ">=2.0.0" - } - }, "node_modules/debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", @@ -1492,6 +1501,11 @@ "node": ">= 0.10.0" } }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" + }, "node_modules/fast-xml-parser": { "version": "4.0.11", "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.0.11.tgz", @@ -1629,6 +1643,15 @@ "node": ">= 0.8" } }, + "node_modules/hyperid": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/hyperid/-/hyperid-3.1.1.tgz", + "integrity": "sha512-RveV33kIksycSf7HLkq1sHB5wW0OwuX8ot8MYnY++gaaPXGFfKpBncHrAWxdpuEeRlazUMGWefwP1w6o6GaumA==", + "dependencies": { + "uuid": "^8.3.2", + "uuid-parse": "^1.1.0" + } + }, "node_modules/iconv-lite": { "version": "0.4.24", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", @@ -1978,6 +2001,11 @@ "node": ">= 0.8" } }, + "node_modules/rfdc": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.3.0.tgz", + "integrity": "sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA==" + }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -2101,6 +2129,11 @@ "npm": ">= 3.0.0" } }, + "node_modules/sorted-btree": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/sorted-btree/-/sorted-btree-1.8.1.tgz", + "integrity": "sha512-395+XIP+wqNn3USkFSrNz7G3Ss/MXlZEqesxvzCRFwL14h6e8LukDHdLBePn5pwbm5OQ9vGu8mDyz2lLDIqamQ==" + }, "node_modules/sparse-bitfield": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz", @@ -2194,11 +2227,15 @@ "version": "8.3.2", "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", - "optional": true, "bin": { "uuid": "dist/bin/uuid" } }, + "node_modules/uuid-parse": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/uuid-parse/-/uuid-parse-1.1.0.tgz", + "integrity": "sha512-OdmXxA8rDsQ7YpNVbKSJkNzTw2I+S5WsbMDnCtIWSQaosNAcWtFuI/YK1TjzUI6nbkgiqEyh8gWngfcv8Asd9A==" + }, "node_modules/vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", @@ -3264,11 +3301,27 @@ "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==" }, + "async-lock": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/async-lock/-/async-lock-1.4.0.tgz", + "integrity": "sha512-coglx5yIWuetakm3/1dsX9hxCNox22h7+V80RQOu2XUUMidtArxKoZoOtHUPuR84SycKTXzgGzAUR5hJxujyJQ==" + }, "base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" }, + "blinkdb": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/blinkdb/-/blinkdb-0.9.0.tgz", + "integrity": "sha512-MVBs8epK67JDw1Lkv3838iCYb6DouH51GcPihT1adnqDpXWxVJOiaJazkMpRNBTHEoP2xYXz+o+ZyYhpw7tECA==", + "requires": { + "fast-deep-equal": "^3.1.3", + "hyperid": "^3.0.1", + "rfdc": "^1.3.0", + "sorted-btree": "^1.8.0" + } + }, "body-parser": { "version": "1.20.1", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz", @@ -3367,12 +3420,6 @@ "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.29.3.tgz", "integrity": "sha512-dDCnyH2WnnKusqvZZ6+jA1O51Ibt8ZMRNkDZdyAyK4YfbDwa/cEmuztzG5pk6hqlp9aSBPYcjOlktquahGwGeA==" }, - "date-fns-tz": { - "version": "1.3.7", - "resolved": "https://registry.npmjs.org/date-fns-tz/-/date-fns-tz-1.3.7.tgz", - "integrity": "sha512-1t1b8zyJo+UI8aR+g3iqr5fkUHWpd58VBx8J/ZSQ+w7YrGlw80Ag4sA86qkfCXRBLmMc4I2US+aPMd4uKvwj5g==", - "requires": {} - }, "debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", @@ -3454,6 +3501,11 @@ "vary": "~1.1.2" } }, + "fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" + }, "fast-xml-parser": { "version": "4.0.11", "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.0.11.tgz", @@ -3544,6 +3596,15 @@ "toidentifier": "1.0.1" } }, + "hyperid": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/hyperid/-/hyperid-3.1.1.tgz", + "integrity": "sha512-RveV33kIksycSf7HLkq1sHB5wW0OwuX8ot8MYnY++gaaPXGFfKpBncHrAWxdpuEeRlazUMGWefwP1w6o6GaumA==", + "requires": { + "uuid": "^8.3.2", + "uuid-parse": "^1.1.0" + } + }, "iconv-lite": { "version": "0.4.24", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", @@ -3777,6 +3838,11 @@ "unpipe": "1.0.0" } }, + "rfdc": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.3.0.tgz", + "integrity": "sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA==" + }, "safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -3868,6 +3934,11 @@ "smart-buffer": "^4.2.0" } }, + "sorted-btree": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/sorted-btree/-/sorted-btree-1.8.1.tgz", + "integrity": "sha512-395+XIP+wqNn3USkFSrNz7G3Ss/MXlZEqesxvzCRFwL14h6e8LukDHdLBePn5pwbm5OQ9vGu8mDyz2lLDIqamQ==" + }, "sparse-bitfield": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz", @@ -3935,8 +4006,12 @@ "uuid": { "version": "8.3.2", "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", - "optional": true + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==" + }, + "uuid-parse": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/uuid-parse/-/uuid-parse-1.1.0.tgz", + "integrity": "sha512-OdmXxA8rDsQ7YpNVbKSJkNzTw2I+S5WsbMDnCtIWSQaosNAcWtFuI/YK1TjzUI6nbkgiqEyh8gWngfcv8Asd9A==" }, "vary": { "version": "1.1.2", diff --git a/package.json b/package.json index e6ebf47..cdab7e3 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,8 @@ "main": "index.js", "type": "module", "scripts": { - "start": "npx tsc && node build/index.js", + "start": "npx tsc && node --experimental-specifier-resolution=node build/index.js", + "start:dev": "export PORT=4500 npx tsc && node --experimental-specifier-resolution=node --trace-warnings build/index.js", "test": "echo \"Error: no test specified\" && exit 1" }, "repository": { @@ -15,10 +16,11 @@ "author": "", "license": "ISC", "dependencies": { + "async-lock": "^1.4.0", + "blinkdb": "^0.9.0", "body-parser": "^1.20.1", "cors": "^2.8.5", "date-fns": "^2.29.3", - "date-fns-tz": "^1.3.7", "dotenv": "^16.0.3", "express": "^4.18.2", "mongoose": "^6.8.4", diff --git a/src/index.ts b/src/index.ts index 632fa2b..42aac5d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,32 +1,38 @@ import bodyParser from 'body-parser'; import express from 'express'; import 'dotenv/config'; -import mongoose, { Schema } from 'mongoose'; import fetch from 'node-fetch'; -import { subHours } from 'date-fns' -import dateFnsTz from 'date-fns-tz'; +import { subHours, subSeconds } from 'date-fns' import cors from 'cors'; +import { + createDB, + createTable, + insert, + many, + removeWhere, + uuid, +} from 'blinkdb'; const { json } = bodyParser; -const { formatInTimeZone } = dateFnsTz; - const RAIN_GAUGE_ENDPOINT = process.env.RAIN_GAUGE_ENDPOINT; const SOIL_SENSOR_ENDPOINT = process.env.SOIL_SENSOR_ENDPOINT; const TEASENSE_ENDPOINT = process.env.TEASENSE_ENDPOINT; const PORT = process.env.PORT || 3000; -const DATA_COLLECT_SLEEP = 100; +const DATA_COLLECT_SLEEP = 1000; const PURGE_SLEEP = 60000; +const PURGE_TEASENSE_SLEEP = 1000; -mongoose.connect(process.env.DB_URL ?? ''); +interface DataPoint { + id: string; + name: string; + value: number; + timestamp: Date; +}; -const DataSchema = new Schema({ - name: Schema.Types.String, - value: Schema.Types.Number, - timestamp: Schema.Types.Date, -}); +const db = createDB(); -const DataModel = mongoose.model('data', DataSchema); +const dataPointsTable = createTable(db, 'dataPoints')(); (async () => { const app = express(); @@ -40,24 +46,13 @@ const DataModel = mongoose.model('data', DataSchema); app.get('/:name', async (req: any, res: any) => { const startTime = subHours(new Date(), 24); - DataModel.find({ - name: req.params.name, - timestamp: { - $gte: startTime, + const result = await many(dataPointsTable, { + where: { + name: req.params.name, } - }, (err: any, arr: any) => { - if (arr) { - res.json(arr.map((item: any) => { - const newItem = { - value: item.value, - timestamp: formatInTimeZone(item.timestamp, 'UTC', 'yyyy-MM-dd\'T\'HH:mm:ss'), - } - return newItem; - })); - } else { - res.status(500).send('There were complications.'); - } - }) + }); + + res.json(result); }); app.listen(PORT); @@ -67,11 +62,12 @@ const DataModel = mongoose.model('data', DataSchema); let rainfallTimestamp = ''; const saveDataPoint = async (name: string, value: number) => { - const tempData = new DataModel(); - tempData.name = name; - tempData.value = value; - tempData.timestamp = new Date(); - tempData.save(); + await insert(dataPointsTable, { + id: uuid(), + name, + value, + timestamp: new Date(), + }); } const collectData = async () => { @@ -95,8 +91,7 @@ const collectData = async () => { signal: AbortSignal.timeout(5000), }); const data: any = await response.json(); - saveDataPoint('soil_temperature', data.temperature); - saveDataPoint('soil_capacitence', data.capacitive); + saveDataPoint('soil_voltage', data.moisture); } catch (e) { console.log(e); } @@ -104,9 +99,7 @@ const collectData = async () => { if (TEASENSE_ENDPOINT && TEASENSE_ENDPOINT.trim() !== '') { try { - const response = await fetch(TEASENSE_ENDPOINT, { - signal: AbortSignal.timeout(5000), - }); + const response = await fetch(TEASENSE_ENDPOINT); const data: any = await response.json(); saveDataPoint('teasense', data.temperature); } catch (e) { @@ -119,12 +112,30 @@ const collectData = async () => { const purger = async () => { const startTime = subHours(new Date(), 24); - await DataModel.deleteMany({ - timestamp: { - $lt: startTime, + await removeWhere(dataPointsTable, { + where: { + timestamp: { + lt: startTime, + }, + }, + }); +}; + +const purger5Sec = async () => { + const startTime = subSeconds(new Date(), 5); + await removeWhere(dataPointsTable, { + where: { + name: { + in: ['teasense', 'soil_voltage'], + }, + timestamp: { + lt: startTime, + }, }, }); }; setTimeout(() => collectData(), DATA_COLLECT_SLEEP); -setInterval(() => purger(), PURGE_SLEEP); \ No newline at end of file + +setInterval(() => purger(), PURGE_SLEEP); +setInterval(() => purger5Sec(), PURGE_TEASENSE_SLEEP); \ No newline at end of file diff --git a/tsconfig.json b/tsconfig.json index 99f0ed5..5ff8960 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -48,7 +48,7 @@ // "declarationMap": true, /* Create sourcemaps for d.ts files. */ // "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */ // "sourceMap": true, /* Create source map files for emitted JavaScript files. */ - // "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */ + // "outFile": "./build/index.js", /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */ "outDir": "./build", /* Specify an output folder for all emitted files. */ // "removeComments": true, /* Disable emitting comments. */ // "noEmit": true, /* Disable emitting files from a compilation. */