commit bedd69436b5f0b09a58d407f53c85bb27bdaf573 Author: grey Date: Sun Mar 6 18:36:36 2022 +0100 Inital commit diff --git a/.eslintignore b/.eslintignore new file mode 100644 index 0000000..eaf16d6 --- /dev/null +++ b/.eslintignore @@ -0,0 +1,3 @@ +node_modules +dist +main.js \ No newline at end of file diff --git a/.eslintrc.json b/.eslintrc.json new file mode 100644 index 0000000..86e16e2 --- /dev/null +++ b/.eslintrc.json @@ -0,0 +1,17 @@ +{ + "root": true, + + "plugins": [ + "@typescript-eslint" + ], + "extends": [ + "eslint:recommended", + "plugin:@typescript-eslint/eslint-recommended", + "plugin:@typescript-eslint/recommended" + ], + "rules": { + "@typescript-eslint/no-var-requires": 0, + "no-control-regex": 0 + } + + } \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4f530f7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +node_modules +package-lock.json +config/default.json +/main.js +static/css/tailwindInclude.css +FLAG +*/db_table_config.sql \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..c9287dd --- /dev/null +++ b/LICENSE @@ -0,0 +1,71 @@ +GNU LESSER GENERAL PUBLIC LICENSE +Version 3, 29 June 2007 + +Copyright (C) 2007 Free Software Foundation, Inc. + +Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. + +This version of the GNU Lesser General Public License incorporates the terms and conditions of version 3 of the GNU General Public License, supplemented by the additional permissions listed below. + +0. Additional Definitions. + +As used herein, "this License" refers to version 3 of the GNU Lesser General Public License, and the "GNU GPL" refers to version 3 of the GNU General Public License. + +"The Library" refers to a covered work governed by this License, other than an Application or a Combined Work as defined below. + +An "Application" is any work that makes use of an interface provided by the Library, but which is not otherwise based on the Library. Defining a subclass of a class defined by the Library is deemed a mode of using an interface provided by the Library. + +A "Combined Work" is a work produced by combining or linking an Application with the Library. The particular version of the Library with which the Combined Work was made is also called the "Linked Version". + +The "Minimal Corresponding Source" for a Combined Work means the Corresponding Source for the Combined Work, excluding any source code for portions of the Combined Work that, considered in isolation, are based on the Application, and not on the Linked Version. + +The "Corresponding Application Code" for a Combined Work means the object code and/or source code for the Application, including any data and utility programs needed for reproducing the Combined Work from the Application, but excluding the System Libraries of the Combined Work. + +1. Exception to Section 3 of the GNU GPL. +You may convey a covered work under sections 3 and 4 of this License without being bound by section 3 of the GNU GPL. + +2. Conveying Modified Versions. +If you modify a copy of the Library, and, in your modifications, a facility refers to a function or data to be supplied by an Application that uses the facility (other than as an argument passed when the facility is invoked), then you may convey a copy of the modified version: + + a) under this License, provided that you make a good faith effort to ensure that, in the event an Application does not supply the function or data, the facility still operates, and performs whatever part of its purpose remains meaningful, or + + b) under the GNU GPL, with none of the additional permissions of this License applicable to that copy. + +3. Object Code Incorporating Material from Library Header Files. +The object code form of an Application may incorporate material from a header file that is part of the Library. You may convey such object code under terms of your choice, provided that, if the incorporated material is not limited to numerical parameters, data structure layouts and accessors, or small macros, inline functions and templates (ten or fewer lines in length), you do both of the following: + + a) Give prominent notice with each copy of the object code that the Library is used in it and that the Library and its use are covered by this License. + + b) Accompany the object code with a copy of the GNU GPL and this license document. + +4. Combined Works. +You may convey a Combined Work under terms of your choice that, taken together, effectively do not restrict modification of the portions of the Library contained in the Combined Work and reverse engineering for debugging such modifications, if you also do each of the following: + + a) Give prominent notice with each copy of the Combined Work that the Library is used in it and that the Library and its use are covered by this License. + + b) Accompany the Combined Work with a copy of the GNU GPL and this license document. + + c) For a Combined Work that displays copyright notices during execution, include the copyright notice for the Library among these notices, as well as a reference directing the user to the copies of the GNU GPL and this license document. + + d) Do one of the following: + + 0) Convey the Minimal Corresponding Source under the terms of this License, and the Corresponding Application Code in a form suitable for, and under terms that permit, the user to recombine or relink the Application with a modified version of the Linked Version to produce a modified Combined Work, in the manner specified by section 6 of the GNU GPL for conveying Corresponding Source. + + 1) Use a suitable shared library mechanism for linking with the Library. A suitable mechanism is one that (a) uses at run time a copy of the Library already present on the user's computer system, and (b) will operate properly with a modified version of the Library that is interface-compatible with the Linked Version. + + e) Provide Installation Information, but only if you would otherwise be required to provide such information under section 6 of the GNU GPL, and only to the extent that such information is necessary to install and execute a modified version of the Combined Work produced by recombining or relinking the Application with a modified version of the Linked Version. (If you use option 4d0, the Installation Information must accompany the Minimal Corresponding Source and Corresponding Application Code. If you use option 4d1, you must provide the Installation Information in the manner specified by section 6 of the GNU GPL for conveying Corresponding Source.) + +5. Combined Libraries. +You may place library facilities that are a work based on the Library side by side in a single library together with other library facilities that are not Applications and are not covered by this License, and convey such a combined library under terms of your choice, if you do both of the following: + + a) Accompany the combined library with a copy of the same work based on the Library, uncombined with any other library facilities, conveyed under the terms of this License. + + b) Give prominent notice with the combined library that part of it is a work based on the Library, and explaining where to find the accompanying uncombined form of the same work. + +6. Revised Versions of the GNU Lesser General Public License. +The Free Software Foundation may publish revised and/or new versions of the GNU Lesser General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library as you received it specifies that a certain numbered version of the GNU Lesser General Public License "or any later version" applies to it, you have the option of following the terms and conditions either of that published version or of any later version published by the Free Software Foundation. If the Library as you received it does not specify a version number of the GNU Lesser General Public License, you may choose any version of the GNU Lesser General Public License ever published by the Free Software Foundation. + +If the Library as you received it specifies that a proxy can decide whether future versions of the GNU Lesser General Public License shall +apply, that proxy's public statement of acceptance of any version is permanent authorization for you to choose that version for the Library. diff --git a/README.MD b/README.MD new file mode 100644 index 0000000..1b9195b --- /dev/null +++ b/README.MD @@ -0,0 +1,82 @@ + +# Pointsight +Pointsight is a centrailzed geolocatlized api aggreagtor. + +# To be done before feature lock +- [X] Feature to report points +- [ ] API usage tracking (MySQL done; influx done; limiting tbd) +- [X] Proper error messages, on page crash loading failure, lost connection +- [X] Shareable points, open by link +- [X] Offline version +- [X] Filter sidebar + + +# Configuration +This is the configuration with all it's defaults. +``` +let jsonConfigGlobal = { + fontAwesome: undefined, + mapboxAccessToken: undefined, + cookieSecret: undefined, + mapquest: undefined, + here: undefined, + sentryDsn: undefined, + database: { + host: "127.0.0.1", + user: "pointsight", + password: "CHANGE_ME", + database: "pointsight", + customSettings: { + wait_timout: 28800 + } + }, + env: "PROD", + maint: true, + betaMode: false, + port: 3000, + adress: '127.0.0.1', + taxonomyCacheInterval: 5, + metrics: { + influx: { + enable: false, + host: "127.0.0.1", + database: "pointsight", + user: "pointsight", + password: "CHANGE_ME", + }, + writeMetricsToMySQL: true + } +} +``` + +# How to deploy + +1. Make sure Node.js is installed +2. Clone repo +3. Run the `npm install` command +4. Create the `config/default.json` file with proper settings + 1. Make sure `env` is set to `PROD` + 2. Make sure `maint` is set to `false` + 3. Make sure the database credentials are set +5. Start the server with `npm start`, this starts the server on port 3000 + +## Troubleshooting +**The map stays empty** + +Cause: The data cannot be loaded because the API key was not imported. + +Solution: Import the `b03f8aaf-1f32-4d9e-914a-9a50f904833d` into the `apikeys` table. After importing it you may use the apikey manager to change it's options. + +**The map is not being displayed** + +Cause: The maps tiles cannot be loaded as the API key is missing. + +Solution: Make sure the config contains the correct apikey. + +# For developers +## Adding a new module +1. Copy the example module folder +2. Change the `main.js` to your needs. + 1. Update the modules meta data in the `getModuleMeta()` funcion + 2. Make sure to use the right country for your module + diff --git a/apiHandler/airspydirectory/db_table_config.sql b/apiHandler/airspydirectory/db_table_config.sql new file mode 100644 index 0000000..0ff96e8 --- /dev/null +++ b/apiHandler/airspydirectory/db_table_config.sql @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS `airspy` +CREATE TABLE IF NOT EXISTS `airspy` ( `id` BIGINT unsigned NOT NULL AUTO_INCREMENT, `name` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, `description` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, `source_id` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, `loc` POINT NOT NULL, `protocol_id` SMALLINT unsigned NOT NULL, `link` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, UNIQUE (id)); \ No newline at end of file diff --git a/apiHandler/airspydirectory/db_table_config.sql.template b/apiHandler/airspydirectory/db_table_config.sql.template new file mode 100644 index 0000000..cf197e6 --- /dev/null +++ b/apiHandler/airspydirectory/db_table_config.sql.template @@ -0,0 +1,13 @@ +DROP TABLE IF EXISTS `airspy` +#NL +CREATE TABLE IF NOT EXISTS `airspy` ( + `id` BIGINT unsigned NOT NULL AUTO_INCREMENT, + `name` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + `description` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + `source_id` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + `loc` POINT NOT NULL, + `protocol_id` SMALLINT unsigned NOT NULL, + `link` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + UNIQUE (id) +); +#NL diff --git a/apiHandler/airspydirectory/main.js b/apiHandler/airspydirectory/main.js new file mode 100644 index 0000000..bcfc621 --- /dev/null +++ b/apiHandler/airspydirectory/main.js @@ -0,0 +1,157 @@ +const fs = require("fs"); +const bent = require("bent"); +const mysql = require("mysql"); +const tools = require("../../functions"); + +let con = undefined; + +function initialize(conection) { + con = conection; + // Prepare all of the tables needed + const array = fs + .readFileSync("./apiHandler/airspydirectory/db_table_config.sql") + .toString() + .split("\n"); + + for (i in array) { + con.query(array[i], function (err, result) { + tools.handleMysqlErrors(err, getModuleMeta().title); + console.log("Table created for AirspyDirectory"); + }); + } + + const airspyDirectoryApiUrl = "https://airspy.com/directory/status.json"; + const mysqlQuery = + "INSERT INTO airspy (name, description, loc, source_id, protocol_id, link) VALUES ( ?, ?, POINT(?,?), ?, ?, ?)"; + + // Request API and save into table + const request = bent(airspyDirectoryApiUrl, "GET", "json", 200); + request().then(function(body){ + console.log("[AirSpyDirectory] Request done"); + for (let c = 0; c < body.servers.length; c++) { + const elm = body.servers[c]; + con.query( + mysqlQuery, + [ + elm.deviceType, + elm.generalDescription + + "
Adress: " + + elm.streamingHost + + ":" + + elm.streamingPort + + "
" + + "Owned by: " + + elm.ownerName, + elm.antennaLocation.lat, + elm.antennaLocation.long, + 0, + 0, + "https://airspy.com/directory/status.json", + ], + function (err, result) { + tools.handleMysqlErrors(err, getModuleMeta().title); + } + ); + } + console.log("[AirSpyDirectory] Insert into Database done!"); + }, + function (err) { + console.warn( + "[" + + getModuleMeta().title + + "] Was unable to resolve request with error: " + + err + ); + }); +} + +function queryData(boundNorthEastLatLng, boundSouthWestLatLng, minimal) { + // Create a new promise to return, DBs are slow. + return new Promise(function (resolve, reject) { + // The Promise constructor should catch any errors thrown on + // this tick. Alternately, try/catch and reject(err) on catch. + let elementsToQuery = "*"; + if (minimal) { + elementsToQuery = "id,loc,source_id"; + } + // This is taken from https://stackoverflow.com/questions/21208697/select-all-geospatial-points-inside-a-bounding-box + const sqlQuery = " \ + SELECT " + elementsToQuery + " \ + FROM airspy \ + WHERE MBRContains( \ + GeomFromText( 'LINESTRING(? ?,? ?)' ), \ + loc)"; + + con.query(sqlQuery, [boundNorthEastLatLng.lat, boundNorthEastLatLng.lng, boundSouthWestLatLng.lat, boundSouthWestLatLng.lng], function (err, rows, fields) { + // Call reject on error states, + // call resolve with results + if (err) { + return reject(err); // Probably bad for PROD + } + + const results = []; + for (let e = 0; e < rows.length; e++) { + const currentRow = rows[e]; + let element = { + type: "general-poi", + source: "AirSpyDirectory", + titel: currentRow.name, + description: currentRow.description, + url: currentRow.link, + uid: "airspydirectory_" + currentRow.id, + location: currentRow.loc, + icon: "signalIcon", + }; + if (minimal) { + const stripableElements = ["title", "description", "url", "type"]; + for (h in stripableElements) { + delete element[stripableElements[h]]; + } + } + results.push(element); + } + resolve(results); + }); + }); +} + +function queryItem(uid) { + const uidParts = uid.split("_"); + const sqlQuery = "SELECT * FROM airspy WHERE id=?"; + return new Promise(function (resolve, reject) { + con.query(sqlQuery, [parseInt(uidParts[1])], function (err, rows, fields) { + if (err) { + return reject(err); + } + const results = []; + for (let e = 0; e < rows.length; e++) { + const currentRow = rows[e]; + const element = { + type: "general-poi", + source: "AirSpyDirectory", + titel: currentRow.name, + description: currentRow.description, + url: currentRow.link, + uid: "airspydirectory_" + currentRow.id, + location: currentRow.loc, + icon: "signalIcon", + }; + results.push(element); + } + resolve(results); + }); + }); +} + +function getModuleMeta() { + return { + title: "AirSpyDirectory", + exactName: "airspydirectory", + country: ["*"], + tags: ["RF"], + features: ["tags"], + limited: false, + }; +} + +module.exports = { initialize, queryData, getModuleMeta, queryItem }; diff --git a/apiHandler/autobahn/db_table_config.sql b/apiHandler/autobahn/db_table_config.sql new file mode 100644 index 0000000..d0c25a0 --- /dev/null +++ b/apiHandler/autobahn/db_table_config.sql @@ -0,0 +1,4 @@ +DROP TABLE IF EXISTS `autobahn` +DROP TABLE IF EXISTS `source` +CREATE TABLE IF NOT EXISTS `source` ( `source_id` INT unsigned NOT NULL AUTO_INCREMENT, `source_name` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, `source_live` BIT NOT NULL, `source_link` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, `source_type` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, `source_area` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, UNIQUE (source_id)); +CREATE TABLE IF NOT EXISTS `autobahn` ( `id` BIGINT unsigned NOT NULL AUTO_INCREMENT, `name` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, `source_id` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, `loc` POINT NOT NULL, `protocol_id` SMALLINT unsigned NOT NULL, `link` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, UNIQUE (id)); \ No newline at end of file diff --git a/apiHandler/autobahn/db_table_config.sql.template b/apiHandler/autobahn/db_table_config.sql.template new file mode 100644 index 0000000..17334f9 --- /dev/null +++ b/apiHandler/autobahn/db_table_config.sql.template @@ -0,0 +1,24 @@ +DROP TABLE IF EXISTS `autobahn` +#NL +DROP TABLE IF EXISTS `source` +#NL +CREATE TABLE IF NOT EXISTS `source` ( + `source_id` INT unsigned NOT NULL AUTO_INCREMENT, + `source_name` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + `source_live` BIT NOT NULL, + `source_link` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + `source_type` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + `source_area` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + UNIQUE (source_id) +); +#NL +CREATE TABLE IF NOT EXISTS `autobahn` ( + `id` BIGINT unsigned NOT NULL AUTO_INCREMENT, + `name` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + `source_id` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + `loc` POINT NOT NULL, + `protocol_id` SMALLINT unsigned NOT NULL, + `link` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + UNIQUE (id) +); +#NL diff --git a/apiHandler/autobahn/main.js b/apiHandler/autobahn/main.js new file mode 100644 index 0000000..448fe4a --- /dev/null +++ b/apiHandler/autobahn/main.js @@ -0,0 +1,188 @@ +var fs = require("fs"); +const bent = require("bent"); +const mysql = require("mysql"); +const tools = require("../../functions"); + +let con = undefined; + +function initialize(connection) { + con = connection; + var array = fs + .readFileSync("./apiHandler/autobahn/db_table_config.sql") + .toString() + .split("\n"); + + for (i in array) { + con.query(array[i], function (err, result) { + tools.handleMysqlErrors(err, getModuleMeta().title); + console.log("Table created for autobahn"); + }); + } + + const roadURL = "https://verkehr.autobahn.de/o/autobahn/"; + const baseURL = "https://verkehr.autobahn.de/o/autobahn/#/services/webcam"; + + var hws = []; + var index_count_cams = 0; + + console.log( + "[" + + getModuleMeta().title + + "] Starting requests" + ); + + const roadRequest = bent(roadURL, "GET", "json", 200); + roadRequest().then( + function (body) { + hws = body.roads; + + for (h in hws) { + if (!hws[h].includes("/")) { + let camRequest = bent( + baseURL.replace("#", hws[h]), + "GET", + "json", + 200 + ); + camRequest().then( + function (body) { + for (var i in body.webcam) { + var q = + "INSERT INTO autobahn (name, source_id, loc, protocol_id, link) VALUES (?, ?, POINT(?, ?), 0, ?)" + if (body.webcam[i].linkurl != "") { + index_count_cams += 1; + con.query(q, [body.webcam[i].title, getModuleMeta().exactName, body.webcam[i].coordinate.lat, body.webcam[i].coordinate.long, body.webcam[i].linkurl], function (err, result) { + tools.handleMysqlErrors(err, getModuleMeta().title); + }); + } + } + }, + function (err) { + console.warn( + "[" + + getModuleMeta().title + + "] Was unable to resolve request with error: " + + err + ); + } + ); + } + } + }, + function (err) { + console.warn( + "[" + + getModuleMeta().title + + "] Was unable to resolve request with error: " + + err + ); + } + ); + console.log( + "[" + + getModuleMeta().title + + "] Done" + ); +} + +function queryData(boundNorthEastLatLng, boundSouthWestLatLng, minimal) { + /* + radius in km + + */ + + return new Promise(function (resolve, reject) { + // The Promise constructor should catch any errors thrown on + // this tick. Alternately, try/catch and reject(err) on catch. + var connection = con; + + let elementsToQuery = "*"; + if (minimal) { + elementsToQuery = "id,loc,source_id"; + } + + const sqlQuery = " \ + SELECT " + elementsToQuery + " \ +FROM autobahn \ +WHERE MBRContains( \ + GeomFromText( 'LINESTRING(? ?,? ?)' ), \ + loc)"; + + connection.query( + sqlQuery, + [boundNorthEastLatLng.lat, boundNorthEastLatLng.lng, boundSouthWestLatLng.lat, boundSouthWestLatLng.lng], + function (err, rows, fields) { + // Call reject on error states, + // call resolve with results + if (err) { + return reject(err); + } + let results = []; + for (let e = 0; e < rows.length; e++) { + const currentRow = rows[e]; + element = { + type: "webcam-iframe", + source: "Autobahn", + titel: currentRow.name, + description: "An example webcam with a url", + url: currentRow.link, + uid: currentRow.source_id + "_" + currentRow.id, + location: currentRow.loc, + icon: "cctvIcon", + }; + + if (minimal) { + const stripableElements = ["title", "description", "url", "type"]; + for (h in stripableElements) { + delete element[stripableElements[h]]; + } + } + + results.push(element); + } + resolve(results); + } + ); + }); +} + +function queryItem(uid) { + const uidParts = uid.split("_"); + + const sqlQuery = "SELECT * FROM " + getModuleMeta().exactName + " WHERE id=?"; + return new Promise(function (resolve, reject) { + con.query(sqlQuery, [parseInt(uidParts[1])], function (err, rows, fields) { + if (err) { + return reject(err); + } + const results = []; + for (let e = 0; e < rows.length; e++) { + const currentRow = rows[e]; + const element = { + type: "webcam-iframe", + source: "Autobahn", + titel: currentRow.name, + description: "An example webcam with a url", + url: currentRow.link, + uid: currentRow.source_id + "_" + currentRow.id, + location: currentRow.loc, + icon: "cctvIcon", + }; + results.push(element); + } + resolve(results); + }); + }); +} + +function getModuleMeta() { + return { + title: "Autobahn", + exactName: "autobahn", + country: ["DEU"], + tags: ["webcam"], + limited: false, + }; +} + +module.exports = { initialize, queryData, getModuleMeta, queryItem }; diff --git a/apiHandler/customWebcams/db_table_config.sql b/apiHandler/customWebcams/db_table_config.sql new file mode 100644 index 0000000..b1e93b2 --- /dev/null +++ b/apiHandler/customWebcams/db_table_config.sql @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS `customWebcams` +CREATE TABLE IF NOT EXISTS `customWebcams` ( `id` BIGINT unsigned NOT NULL AUTO_INCREMENT, `name` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, `source_id` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, `loc` POINT NOT NULL, `protocol_id` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, `link` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, UNIQUE (id)); \ No newline at end of file diff --git a/apiHandler/customWebcams/db_table_config.sql.template b/apiHandler/customWebcams/db_table_config.sql.template new file mode 100644 index 0000000..ed4b3b6 --- /dev/null +++ b/apiHandler/customWebcams/db_table_config.sql.template @@ -0,0 +1,12 @@ +DROP TABLE IF EXISTS `customWebcams` +#NL +CREATE TABLE IF NOT EXISTS `customWebcams` ( + `id` BIGINT unsigned NOT NULL AUTO_INCREMENT, + `name` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + `source_id` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + `loc` POINT NOT NULL, + `protocol_id` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + `link` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + UNIQUE (id) +); +#NL diff --git a/apiHandler/customWebcams/main.js b/apiHandler/customWebcams/main.js new file mode 100644 index 0000000..f2d9d38 --- /dev/null +++ b/apiHandler/customWebcams/main.js @@ -0,0 +1,145 @@ +var fs = require("fs"); +const bent = require("bent"); +const mysql = require("mysql"); +const tools = require("../../functions"); + +let con = undefined; + +function initialize(connection) { + con = connection; + var array = fs + .readFileSync("./apiHandler/customWebcams/db_table_config.sql") + .toString() + .split("\n"); + + for (i in array) { + con.query(array[i], function (err, result) { + tools.handleMysqlErrors(err, getModuleMeta().title); + console.log("Table created for customWebcam"); + }); + } + + console.log("[" + getModuleMeta().title + "] Starting requests"); + + const data = fs.readFileSync("./apiHandler/customWebcams/webcams.json"); + const webcams = JSON.parse(data); + + for (var i in webcams.cams) { + var q = + "INSERT INTO customWebcams (name, source_id, loc, protocol_id, link) VALUES (?, ?, POINT(?,?), ?, ?)" + con.query(q, [webcams.cams[i].title, getModuleMeta().exactName, webcams.cams[i].loc.lat, webcams.cams[i].loc.lng, webcams.cams[i].embedType, webcams.cams[i].url], function (err, result) { + tools.handleMysqlErrors(err, getModuleMeta().title); + }); + } + + console.log("[" + getModuleMeta().title + "] Done"); +} + +function queryData(boundNorthEastLatLng, boundSouthWestLatLng, minimal) { + return new Promise(function (resolve, reject) { + // The Promise constructor should catch any errors thrown on + // this tick. Alternately, try/catch and reject(err) on catch. + var connection = con; + + let elementsToQuery = "*"; + if (minimal) { + elementsToQuery = "id,loc,source_id"; + } + + const sqlQuery = " \ + SELECT " + elementsToQuery + " \ +FROM customWebcams \ +WHERE MBRContains( \ + GeomFromText( 'LINESTRING(? ?,? ?)' ), \ + loc)"; + + con.query(sqlQuery, [boundNorthEastLatLng.lat, boundNorthEastLatLng.lng, boundSouthWestLatLng.lat, boundSouthWestLatLng.lng], function (err, rows, fields) { + // Call reject on error states, + // call resolve with results + if (err) { + return reject(err); + } + let results = []; + for (let e = 0; e < rows.length; e++) { + const currentRow = rows[e]; + let ty = ""; + if (currentRow.protocol_id == "IFRAME") { + ty = "webcam-iframe"; + } else if (currentRow.protocol_id == "IMAGE") { + ty = "webcam-image"; + } + element = { + type: ty, + source: "customWebcam", + titel: currentRow.name, + description: "An example webcam with a url", + url: currentRow.link, + uid: currentRow.source_id + "_" + currentRow.id, + location: currentRow.loc, + icon: "cctvIcon", + }; + + if (minimal) { + const stripableElements = ["title", "description", "url", "type"]; + for (h in stripableElements) { + delete element[stripableElements[h]]; + } + } + + results.push(element); + } + resolve(results); + } + ); + }); +} + +function queryItem(uid) { + const uidParts = uid.split("_"); + + const sqlQuery = "SELECT * FROM " + getModuleMeta().exactName + " WHERE id=?"; + return new Promise(function (resolve, reject) { + con.query(sqlQuery, [parseInt(uidParts[1])], function (err, rows, fields) { + if (err) { + return reject(err); + } + const results = []; + for (let e = 0; e < rows.length; e++) { + const currentRow = rows[e]; + let ty = ""; + if (currentRow.protocol_id == "IFRAME") { + ty = "webcam-iframe"; + } else if (currentRow.protocol_id == "IMAGE") { + ty = "webcam-image"; + } + element = { + type: ty, + source: "customWebcam", + titel: currentRow.name, + description: "A web cam read from a file", + url: currentRow.link, + uid: currentRow.source_id + "_" + currentRow.id, + location: { + lat: currentRow.loc_lat, + lng: currentRow.loc_lng, + }, + icon: "cctvIcon", + }; + results.push(element); + } + resolve(results); + }); + }); +} + +function getModuleMeta() { + return { + title: "custom webcams", + exactName: "customWebcams", + country: ["*"], + tags: ["webcam"], + limited: false, + }; +} + +module.exports = { initialize, queryData, getModuleMeta, queryItem }; diff --git a/apiHandler/customWebcams/webcams.json b/apiHandler/customWebcams/webcams.json new file mode 100644 index 0000000..073e5bc --- /dev/null +++ b/apiHandler/customWebcams/webcams.json @@ -0,0 +1,11 @@ +{"cams": [ + { + "title": "Blick auf den Großenbroder Südstrand", + "loc": { + "lat": 54.35780113081485, + "lng": 11.087519716842067 + }, + "url":"https://www.grossenbrode.de/webcamImage.php?secretmagickey=HEdqH9cwfqt2q35uuj75j5qdDdx3F7", + "embedType": "IMAGE" + } +]} \ No newline at end of file diff --git a/apiHandler/example/db_table_config.sql b/apiHandler/example/db_table_config.sql new file mode 100644 index 0000000..e69de29 diff --git a/apiHandler/example/db_table_config.sql.template b/apiHandler/example/db_table_config.sql.template new file mode 100644 index 0000000..e69de29 diff --git a/apiHandler/example/main.js b/apiHandler/example/main.js new file mode 100644 index 0000000..337feb6 --- /dev/null +++ b/apiHandler/example/main.js @@ -0,0 +1,114 @@ +const fs = require("fs"); +const request = require("request"); +const mysql = require("mysql"); +const tools = require("../../functions"); + +let con = undefined; + +function initialize(conection) { + console.error("!! THE EXAMPLE MODULE HAS BEEN ENABLED, NEVER ENABLE TO EXAMPLE MODULE !!") + con = conection; + const filePath = "./apiHandler/" + getModuleMeta().exactName + "/db_table_config.sql" + const array = fs + .readFileSync(filePath) + .toString() + .split("\n"); + + for (i in array) { + con.query(array[i], function (err, result) { + if (err) throw err; + console.log("Table created for " + getModuleMeta().title); + }); + } + + const requestURL = "https://example.com/api.json"; + const mysqlQuery = "INSERT INTO " + getModuleMeta().exactName + " (" + getModuleMeta().exactName + "_id, " + getModuleMeta().exactName + "_name, " + getModuleMeta().exactName + "_description, source_id, " + getModuleMeta().exactName + "_loc_lat, " + getModuleMeta().exactName + "_loc_lng, " + getModuleMeta().exactName + "_protocol_id, " + getModuleMeta().exactName + "_link) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"; + + request(requestURL, { json: true }, (err, res2, body) => { + if (err) { + throw err; + } + console.log("[" + getModuleMeta().title + "] Request done") + for (let c = 0; c < body.length; c++) { + + const elm = body[c]; + con.query(mysqlQuery, [c, elm.name, "", 0, elm.currentLocation.coordinates[1], elm.currentLocation.coordinates[0], 0, "https://api.opensensemap.org/boxes/" + elm._id], function (err, result) { + if (err) { throw (err) } + }) + } + console.log("[" + getModuleMeta().title + "] Insert into Database done!") + }); +} + +function queryData(lat, lng, radius, minimal) { + /* + radius in km + + */ + + return new Promise(function (resolve, reject) { + // The Promise constructor should catch any errors thrown on + // this tick. Alternately, try/catch and reject(err) on catch. + + let elementsToQuery = "*"; + if (minimal) { + elementsToQuery = "id,loc_lat,loc_lng,source_id"; + } + + const sqlQuery = + "SELECT \ + " + + elementsToQuery + + ", \ + ( 6371 \ + * acos( cos( radians(?) ) \ + * cos( radians( " + getModuleMeta().exactName + "_loc_lat ) ) \ + * cos( radians( " + getModuleMeta().exactName + "_loc_lng ) - radians(?) ) \ + + sin( radians(?) ) \ + * sin( radians( " + getModuleMeta().exactName + "_loc_lat ) ) \ + ) \ + ) \ + AS distance \ +FROM " + getModuleMeta().exactName + " \ +HAVING distance < ? \ +ORDER BY distance"; + + con.query(sqlQuery, [lat, lng, lat, radius], function (err, rows, fields) { + // Call reject on error states, + // call resolve with results + if (err) { + return reject(err); + } + + const results = []; + for (let e = 0; e < rows.length; e++) { + const currentRow = rows[e] + const element = { + "type": "request-info", + "source": getModuleMeta().title, + "titel": currentRow.example_name, + "description": "", + "url": currentRow.example_link, + "distance": currentRow.distance, + "location": { + "lat": currentRow.example_loc_lat, + "lng": currentRow.example_loc_lng + } + } + results.push(element) + } + resolve(results); + }); + }); +} + +function getModuleMeta() { + return { + title: "Example", + exactName: "example", + country: ["*"], + limited: false, + }; +} + +module.exports = { initialize, queryData, getModuleMeta }; diff --git a/apiHandler/freifunk/db_table_config.sql b/apiHandler/freifunk/db_table_config.sql new file mode 100644 index 0000000..33cbbfa --- /dev/null +++ b/apiHandler/freifunk/db_table_config.sql @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS `freifunk` +CREATE TABLE IF NOT EXISTS `freifunk` ( `id` BIGINT unsigned NOT NULL AUTO_INCREMENT, `name` TEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, `description` TEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, `source_id` TEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, `loc` POINT NOT NULL, `protocol_id` SMALLINT unsigned NOT NULL, `link` TEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, UNIQUE (id)); \ No newline at end of file diff --git a/apiHandler/freifunk/db_table_config.sql.template b/apiHandler/freifunk/db_table_config.sql.template new file mode 100644 index 0000000..f2a76da --- /dev/null +++ b/apiHandler/freifunk/db_table_config.sql.template @@ -0,0 +1,13 @@ +DROP TABLE IF EXISTS `freifunk` +#NL +CREATE TABLE IF NOT EXISTS `freifunk` ( + `id` BIGINT unsigned NOT NULL AUTO_INCREMENT, + `name` TEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + `description` TEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + `source_id` TEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + `loc` POINT NOT NULL, + `protocol_id` SMALLINT unsigned NOT NULL, + `link` TEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + UNIQUE (id) +); +#NL diff --git a/apiHandler/freifunk/main.js b/apiHandler/freifunk/main.js new file mode 100644 index 0000000..4b9290d --- /dev/null +++ b/apiHandler/freifunk/main.js @@ -0,0 +1,146 @@ +const fs = require("fs"); +const bent = require('bent') +const mysql = require("mysql"); +const tools = require("../../functions"); + +let con = undefined; + +function initialize(connection) { + con = connection; + const array = fs + .readFileSync("./apiHandler/freifunk/db_table_config.sql") + .toString() + .split("\n"); + + for (i in array) { + con.query(array[i], function (err, result) { + tools.handleMysqlErrors(err, getModuleMeta().title); + console.log("Table created for freifunk"); + }); + } + const freifunkDataUrl = "https://www.freifunk-karte.de/data.php"; + const mysqlQuery = + "INSERT INTO freifunk (name, description, source_id, loc, protocol_id, link) VALUES (?, ?, ?, POINT(?, ?), ?, ?)"; + const resulter = bent(freifunkDataUrl, 'GET', 'json', 200); + resulter().then(function(body){ + console.log("[Freifunk] Request done"); + for (let c = 0; c < body.allTheRouters.length; c++) { + const elm = body.allTheRouters[c]; + con.query( + mysqlQuery, + [ + elm.name, + elm.community, + getModuleMeta().exactName, + elm.lat, + elm.long, + 0, + "https://www.freifunk-karte.de/", + ], + function (err, result) { + tools.handleMysqlErrors(err, getModuleMeta().title); + } + ); + } + console.log("[Freifunk] Insert into Database done!"); + }, function(err){ + console.warn("[" + getModuleMeta().title + "] Was unable to resolve request with error: " + err) + }); +} + +function queryData(boundNorthEastLatLng, boundSouthWestLatLng, minimal) { + return new Promise(function (resolve, reject) { + // The Promise constructor should catch any errors thrown on + // this tick. Alternately, try/catch and reject(err) on catch. + let elementsToQuery = "*"; + if (minimal) { + elementsToQuery = "id,loc,source_id"; + } + const sqlQuery = " \ + SELECT " + elementsToQuery + " \ +FROM freifunk \ +WHERE MBRContains( \ + GeomFromText( 'LINESTRING(? ?,? ?)' ), \ + loc)"; + + con.query(sqlQuery, [boundNorthEastLatLng.lat, boundNorthEastLatLng.lng, boundSouthWestLatLng.lat, boundSouthWestLatLng.lng], function (err, rows, fields) { + // Call reject on error states, + // call resolve with results + if (err) { + return reject(err); + } + + const results = []; + for (let e = 0; e < rows.length; e++) { + const currentRow = rows[e]; + const element = { + type: "general-poi", + source: "Freifunk", + titel: currentRow.name, + description: + "This is an AP by the " + + currentRow.description + + " Freifunk Comunity", + url: currentRow.link, + uid: "freifunk_" + currentRow.id, + location: currentRow.loc, + icon: "wifiIcon" + }; + + if (minimal) { + const stripableElements = ["title", "description", "url", "type"]; + for (h in stripableElements) { + delete element[stripableElements[h]]; + } + } + + results.push(element); + } + resolve(results); + }); + }); +} + +function queryItem(uid){ + const uidParts = uid.split("_"); + + const sqlQuery = "SELECT * FROM " + getModuleMeta().exactName + " WHERE id=?" + return new Promise(function (resolve, reject) { + con.query(sqlQuery, [parseInt(uidParts[1])], function (err, rows, fields) { + if (err) { + return reject(err); + } + const results = []; + for (let e = 0; e < rows.length; e++) { + const currentRow = rows[e]; + const element = { + type: "general-poi", + source: "Freifunk", + titel: currentRow.name, + description: + "This is an AP by the " + + currentRow.description + + " Freifunk Comunity", + url: currentRow.link, + uid: "freifunk_" + currentRow.id, + location: currentRow.loc, + icon: "wifiIcon" + }; + results.push(element); + } + resolve(results) + }); +}); +} + +function getModuleMeta() { + return { + title: "Freifunk", + exactName: "freifunk", + country: ["*"], + tags: ["AP"], + limited: false, + }; +} + +module.exports = { initialize, queryData, getModuleMeta, queryItem }; diff --git a/apiHandler/opensensemap/db_table_config.sql b/apiHandler/opensensemap/db_table_config.sql new file mode 100644 index 0000000..7f83f37 --- /dev/null +++ b/apiHandler/opensensemap/db_table_config.sql @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS `opensensemap` +CREATE TABLE IF NOT EXISTS `opensensemap` ( `id` BIGINT unsigned NOT NULL AUTO_INCREMENT, `name` TEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, `description` TEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, `source_id` TEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, `loc` POINT NOT NULL, `protocol_id` SMALLINT unsigned NOT NULL, `link` TEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, UNIQUE (id)); \ No newline at end of file diff --git a/apiHandler/opensensemap/db_table_config.sql.template b/apiHandler/opensensemap/db_table_config.sql.template new file mode 100644 index 0000000..1f6b12d --- /dev/null +++ b/apiHandler/opensensemap/db_table_config.sql.template @@ -0,0 +1,13 @@ +DROP TABLE IF EXISTS `opensensemap` +#NL +CREATE TABLE IF NOT EXISTS `opensensemap` ( + `id` BIGINT unsigned NOT NULL AUTO_INCREMENT, + `name` TEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + `description` TEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + `source_id` TEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + `loc` POINT NOT NULL, + `protocol_id` SMALLINT unsigned NOT NULL, + `link` TEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + UNIQUE (id) +); +#NL diff --git a/apiHandler/opensensemap/main.js b/apiHandler/opensensemap/main.js new file mode 100644 index 0000000..db32eba --- /dev/null +++ b/apiHandler/opensensemap/main.js @@ -0,0 +1,145 @@ +const fs = require("fs"); +const mysql = require("mysql"); +const bent = require('bent') +const tools = require("../../functions"); + +let con = undefined; + +function initialize(conection) { + con = conection; + const filePath = + "./apiHandler/" + getModuleMeta().exactName + "/db_table_config.sql"; + const array = fs.readFileSync(filePath).toString().split("\n"); + + for (i in array) { + con.query(array[i], function (err, result) { + if (err) throw err; + console.log("Table created for " + getModuleMeta().title); + }); + } + + const requestURL = "https://api.opensensemap.org/boxes"; + const mysqlQuery = + "INSERT INTO opensensemap (name, description, source_id, loc, protocol_id, link) VALUES (?, ?, ?, POINT(?, ?), ?, ?)"; + const res = bent(requestURL, 'GET', 'json', 200); + res().then(function(response){ + body = response + console.log("[" + getModuleMeta().title + "] Request done"); + for (let c = 0; c < body.length; c++) { + const elm = body[c]; + con.query( + mysqlQuery, + [ + elm.name, + "", + getModuleMeta().exactName, + elm.currentLocation.coordinates[1], + elm.currentLocation.coordinates[0], + 0, + "https://sensebox.github.io/opensensemap-widget/iframe.html?senseboxId=" + elm._id, + ], + function (err, result) { + tools.handleMysqlErrors(err, getModuleMeta().title); + } + ); + } + console.log("[" + getModuleMeta().title + "] Insert into Database done!"); + }, function(err){ + console.warn("[" + getModuleMeta().title + "] Was unable to resolve request with error: " + err) + }); +} + +function queryData(boundNorthEastLatLng, boundSouthWestLatLng, minimal) { + return new Promise(function (resolve, reject) { + // The Promise constructor should catch any errors thrown on + // this tick. Alternately, try/catch and reject(err) on catch. + let elementsToQuery = "*"; + if (minimal) { + elementsToQuery = "id,loc,source_id"; + } + // This is taken from https://stackoverflow.com/questions/21208697/select-all-geospatial-points-inside-a-bounding-box + const sqlQuery = " \ + SELECT " + elementsToQuery + " \ + FROM opensensemap \ + WHERE MBRContains( \ + GeomFromText( 'LINESTRING(? ?,? ?)' ), \ + loc)"; + + con.query(sqlQuery, [boundNorthEastLatLng.lat, boundNorthEastLatLng.lng, boundSouthWestLatLng.lat, boundSouthWestLatLng.lng], function (err, rows, fields) { + // Call reject on error states, + // call resolve with results + if (err) { + return reject(err); + } + + const results = []; + for (let e = 0; e < rows.length; e++) { + const currentRow = rows[e]; + const element = { + type: "iframe", + source: getModuleMeta().exactName, + titel: currentRow.name, + description: "", + url: currentRow.link, + uid: "opensensemap_" + currentRow.id, + location: currentRow.loc, + icon: "temperatureIcon" + }; + if (minimal) { + const stripableElements = ["title", "description", "url", "type"]; + for (h in stripableElements) { + delete element[stripableElements[h]]; + } + } + + results.push(element); + } + resolve(results); + }); + }); +} + +function queryItem(uid){ + const uidParts = uid.split("_"); + + const sqlQuery = "SELECT * FROM `" + getModuleMeta().exactName + "` WHERE id=?" + return new Promise(function (resolve, reject) { + con.query(sqlQuery, [parseInt(uidParts[1])], function (err, rows, fields) { + if (err) { + return reject(err); + } + console.log() + const results = []; + for (let e = 0; e < rows.length; e++) { + const currentRow = rows[e]; + const element = { + type: "iframe", + source: getModuleMeta().exactName, + titel: currentRow.name, + description: "", + url: currentRow.link, + uid: "opensensemap_" + currentRow.id, + distance: tools.roundOff(currentRow.distance, 4), + location: currentRow.loc, + icon: "temperatureIcon" + }; + results.push(element); + } + resolve(results) + }); +}); +} + + + +function getModuleMeta() { + return { + title: "OpenSensemap", + exactName: "opensensemap", + tags: ["temperature", "envoriment"], + country: ["*"], + limited: false, + }; +} + +module.exports = { initialize, queryData, getModuleMeta, queryItem }; diff --git a/apiHandler/opentopia/db_table_config.sql b/apiHandler/opentopia/db_table_config.sql new file mode 100644 index 0000000..4a23a89 --- /dev/null +++ b/apiHandler/opentopia/db_table_config.sql @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS `opentopia` +CREATE TABLE IF NOT EXISTS `opentopia` ( `id` BIGINT unsigned NOT NULL AUTO_INCREMENT, `name` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, `source_id` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, `loc` POINT NOT NULL, `protocol_id` SMALLINT unsigned NOT NULL, `link` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, UNIQUE (id), UNIQUE (`link`)); \ No newline at end of file diff --git a/apiHandler/opentopia/db_table_config.sql.template b/apiHandler/opentopia/db_table_config.sql.template new file mode 100644 index 0000000..70926ad --- /dev/null +++ b/apiHandler/opentopia/db_table_config.sql.template @@ -0,0 +1,13 @@ +DROP TABLE IF EXISTS `opentopia` +#NL +CREATE TABLE IF NOT EXISTS `opentopia` ( + `id` BIGINT unsigned NOT NULL AUTO_INCREMENT, + `name` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + `source_id` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + `loc` POINT NOT NULL, + `protocol_id` SMALLINT unsigned NOT NULL, + `link` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + UNIQUE (id), + UNIQUE (`link`) +); +#NL \ No newline at end of file diff --git a/apiHandler/opentopia/main.js b/apiHandler/opentopia/main.js new file mode 100644 index 0000000..6ecd8ed --- /dev/null +++ b/apiHandler/opentopia/main.js @@ -0,0 +1,317 @@ +var fs = require("fs"); +const bent = require("bent"); +const mysql = require("mysql"); +const tools = require("../../functions"); +const _ = require("underscore"); + +let con = undefined; +let config; + +function requestWithPromise(n) { + const listURL = + "http://www.opentopia.com/hiddencam.php?showmode=standard&country=%2A&seewhat=highlyrated&p="; + return new Promise(function (resolve, reject) { + const request = bent(listURL + n, "GET", 200); + request().then( + function (body) { + body.text().then(function (text) { + resolve(text); + }); + }, + function (err) { + console.warn( + "[" + + getModuleMeta().title + + "] Was unable to resolve request with error: " + + err + ); + } + ); + }); +} + +function initialize(connection, configL, replaceOld = false) { + con = connection; + config = configL; + if(!fs.existsSync("./apiHandler/opentopia/FLAG")){ + fs.writeFileSync("./apiHandler/opentopia/FLAG", "") + replaceOld = true + } + if (replaceOld) { // Just do it once, calling that API ain't cheap + var array = fs + .readFileSync("./apiHandler/opentopia/db_table_config.sql") + .toString() + .split("\n"); + + for (i in array) { + con.query(array[i], function (err, result) { + tools.handleMysqlErrors(err, getModuleMeta().title + "L32"); + console.log("Table created for " + getModuleMeta().title); + }); + } + + let allProms = []; + + for (let n = 0; n < 26; n++) { + allProms.push(requestWithPromise(n)); + } + + const mysqlQuery = + "INSERT INTO opentopia (name, source_id, loc, protocol_id, link) VALUES (?, ?, POINT(?, ?), ?, ?)"; + + Promise.all(allProms).then(function (res) { + for (l in res) { + const baseurl = "http://www.opentopia.com/webcam/"; + let allResults = []; + // Process the pages + let part = res[l].split("")[1]; + part = part.split("/webcam/"); + part.shift(); + for (h in part) { + part[h] = part[h].replace("\t", ""); + part[h] = part[h] + .split('target="_self">')[0] + .replace('"', "") + .split(">")[0]; + } + + pages = part; + camProms = []; + for (let pI = 0; pI < pages.length; pI++) { + camProms.push( + new Promise(function (resolve, reject) { + const url = baseurl + pages[pI] + "?viewmode=livevideo"; + _.throttle(function () { + setTimeout(function () { + const request = bent(url, "GET", 200); + request().then( + function (body) { + body.text().then(function (text) { + let pre; + let result = { + name: "", + url: "", + location: { lat: 0, lng: 0 }, + }; + try { + const body = text; + + let part = body.split("")[1]; + camPart = part.split('
')[1]; + camPart = camPart.split('')[1]; + pre = part; + + result.name = part.split("
")[0]; + result.name = result.name + .replaceAll("\t", "") + .replaceAll(" ", ""); + part = part.split('")[0]; + } catch (error) { + reqPart1 = pre + .split('region">')[1] + .split("")[0]; + } + reqPart2 = pre + .split('country-name">')[1] + .split("")[0]; + const geoCoderReq = bent( + "https://open.mapquestapi.com/geocoding/v1/address?key=" + + config.mapquest + + "&location=" + + reqPart1 + + "," + + reqPart2, + "GET", + "json", + 200 + ); + geoCoderReq().then(function (body) { + result.location.lat = + body.results[0].locations[0].latLng.lat; + result.location.lng = + body.results[0].locations[0].latLng.lng; + resolve(result); + }); + console.log(reqPart1 + "," + reqPart2); + } + } + }); + }, + function (err) { + console.warn( + "[" + + getModuleMeta().title + + "] Was unable to resolve request with error: " + + err + ); + } + ); + }, Math.floor(Math.random() * 500)); + }, 200)(); + }) + ); + let globalI = 0; + Promise.all(camProms).then(function (result) { + // console.log(result); + for (elm in result) { + const row = result[elm]; + globalI++; + try { + con.query( + mysqlQuery, + [ + row.name, + getModuleMeta().exactName, + row.location.lat, + row.location.lng, + 0, + row.url, + ], + function (err, result) { + if (err) { + if (err.code == "ER_DUP_ENTRY") { + //console.warn("[Opentopia] Skipped duplicate webcam"); + } else { + tools.handleMysqlErrors( + err, + getModuleMeta().title + "L128" + ); + } + } + } + ); + } catch (error) { + /*console.warn( + "[Opentopia] Experienced an issue adding element to DB, error was: " + + error + );*/ + } + } + }); + } + } + }); + } +} + +function queryData(boundNorthEastLatLng, boundSouthWestLatLng, minimal) { + return new Promise(function (resolve, reject) { + // The Promise constructor should catch any errors thrown on + // this tick. Alternately, try/catch and reject(err) on catch. + var connection = con; + + let elementsToQuery = "*"; + if (minimal) { + elementsToQuery = "id,loc,source_id"; + } + + const sqlQuery = " \ + SELECT " + elementsToQuery + " \ +FROM opentopia \ +WHERE MBRContains( \ + GeomFromText( 'LINESTRING(? ?,? ?)' ), \ + loc)"; + + + con.query(sqlQuery, [boundNorthEastLatLng.lat, boundNorthEastLatLng.lng, boundSouthWestLatLng.lat, boundSouthWestLatLng.lng], function (err, rows, fields) { + // Call reject on error states, + // call resolve with results + if (err) { + tools.handleMysqlErrors(err, getModuleMeta().title + "-L187"); + return reject(err); + } + let results = []; + for (let e = 0; e < rows.length; e++) { + const currentRow = rows[e]; + element = { + type: "webcam-iframe", + source: "Opentopia", + titel: currentRow.name, + description: "An example webcam with a url", + url: currentRow.link, + uid: "opentopia_" + currentRow.id, + location: currentRow.loc, + icon: "cctvIcon", + }; + + if (minimal) { + const stripableElements = ["title", "description", "url", "type"]; + for (h in stripableElements) { + delete element[stripableElements[h]]; + } + } + + results.push(element); + } + resolve(results); + } + ); + }); +} + +function queryItem(uid) { + const uidParts = uid.split("_"); + + const sqlQuery = "SELECT * FROM " + getModuleMeta().exactName + " WHERE id=?"; + return new Promise(function (resolve, reject) { + con.query(sqlQuery, [parseInt(uidParts[1])], function (err, rows, fields) { + if (err) { + tools.handleMysqlErrors(err, getModuleMeta().title + "L230"); + return reject(err); + } + const results = []; + for (let e = 0; e < rows.length; e++) { + const currentRow = rows[e]; + const element = { + type: "webcam-iframe", + source: "Opentopia", + titel: currentRow.name, + description: "An example webcam with a url", + url: currentRow.link, + uid: "opentopia_" + currentRow.id, + location: currentRow.loc, + icon: "cctvIcon", + }; + results.push(element); + } + resolve(results); + }); + }); +} + +function getModuleMeta() { + return { + title: "Opentopia", + exactName: "opentopia", + country: ["*"], + tags: ["webcam"], + limited: false, + }; +} + +module.exports = { initialize, queryData, getModuleMeta, queryItem }; diff --git a/apiHandler/stub/db_table_config.sql b/apiHandler/stub/db_table_config.sql new file mode 100644 index 0000000..e69de29 diff --git a/apiHandler/stub/db_table_config.sql.template b/apiHandler/stub/db_table_config.sql.template new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/apiHandler/stub/db_table_config.sql.template @@ -0,0 +1 @@ + diff --git a/apiHandler/stub/main.js b/apiHandler/stub/main.js new file mode 100644 index 0000000..fa2fcec --- /dev/null +++ b/apiHandler/stub/main.js @@ -0,0 +1,28 @@ +let con = undefined; + +function initialize(connection) { + con = connection; + // Do nothing and die. +} + +function queryData(lat, lng, radius, minimal) { + // Does nothing. + return new Promise(function (resolve, reject) {resolve([]);}); +} + +function queryItem(uid) { + // Does nothing. + return new Promise(function (resolve, reject) {resolve([]);}); +} + +function getModuleMeta() { + return { + title: "Stub", + exactName: "stub", + country: [""], + tags: [""], + limited: true, + }; +} + +module.exports = { initialize, queryData, getModuleMeta, queryItem }; diff --git a/apiHandler/ttnantennas/db_table_config.sql b/apiHandler/ttnantennas/db_table_config.sql new file mode 100644 index 0000000..bdd7e5c --- /dev/null +++ b/apiHandler/ttnantennas/db_table_config.sql @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS `ttnantenna` +CREATE TABLE IF NOT EXISTS `ttnantenna` ( `id` BIGINT unsigned NOT NULL AUTO_INCREMENT, `name` TEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, `description` TEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, `source_id` TEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, `loc` POINT NOT NULL, `protocol_id` SMALLINT unsigned NOT NULL, `link` TEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, UNIQUE (id)); \ No newline at end of file diff --git a/apiHandler/ttnantennas/db_table_config.sql.template b/apiHandler/ttnantennas/db_table_config.sql.template new file mode 100644 index 0000000..163d614 --- /dev/null +++ b/apiHandler/ttnantennas/db_table_config.sql.template @@ -0,0 +1,13 @@ +DROP TABLE IF EXISTS `ttnantenna` +#NL +CREATE TABLE IF NOT EXISTS `ttnantenna` ( + `id` BIGINT unsigned NOT NULL AUTO_INCREMENT, + `name` TEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + `description` TEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + `source_id` TEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + `loc` POINT NOT NULL, + `protocol_id` SMALLINT unsigned NOT NULL, + `link` TEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + UNIQUE (id) +); +#NL diff --git a/apiHandler/ttnantennas/main.js b/apiHandler/ttnantennas/main.js new file mode 100644 index 0000000..0c86bf8 --- /dev/null +++ b/apiHandler/ttnantennas/main.js @@ -0,0 +1,175 @@ +const fs = require("fs"); +const bent = require("bent"); +const mysql = require("mysql"); +const tools = require("../../functions"); + +let con = undefined; + +function initialize(connection) { + con = connection; + const array = fs + .readFileSync("./apiHandler/ttnantennas/db_table_config.sql") + .toString() + .split("\n"); + + for (i in array) { + con.query(array[i], function (err, result) { + if (err) throw err; + console.log("Table created for ttnantenna"); + }); + } + const ttnGateways = + "https://www.thethingsnetwork.org/gateway-data/"; + const mysqlQuery = + "INSERT INTO ttnantenna (name, description, source_id, loc, protocol_id, link) VALUES (?, ?, ?, POINT(?, ?), ?, ?)"; + const ttnGateRequest = bent(ttnGateways, "GET", "json", 200); + ttnGateRequest().then(function(body){ + console.log("[TheThingsnetwork Gateways] Request done"); + let i = 0; + for (const key in body) { + const elm = body[key]; + if (elm.location != undefined) { + if(elm.attributes == undefined){ + elm.attributes = {frequency_plan: "Unknown", placement: "Unknown"} + }else{ + if(elm.attributes.frequency_plan == undefined){ + elm.attributes.frequency_plan = "Unknown" + } + if(elm.attributes.placement == undefined){ + elm.attributes.placement = "Unknown" + } + } + if ( + elm.location.latitude != undefined && + elm.location.longitude != undefined + ) { + con.query( + mysqlQuery, + [ + + key, + elm.description + + "
Frequency plan: " + + elm.attributes.frequency_plan + + "
Altitude: " + + elm.location.altitude + + "
Placement: " + elm.attributes.placement, + getModuleMeta().exactName, + elm.location.latitude, + elm.location.longitude, + 0, + "", + ], + function (err, result) { + tools.handleMysqlErrors(err, getModuleMeta().title); + } + ); + i++ + } + } + } + console.log("[TheThingsnetwork Gateways] Insert into Database done!"); + }, + function (err) { + console.warn( + "[" + + getModuleMeta().title + + "] Was unable to resolve request with error: " + + err + ); + }); +} + +function queryData(boundNorthEastLatLng, boundSouthWestLatLng, minimal) { + return new Promise(function (resolve, reject) { + // The Promise constructor should catch any errors thrown on + // this tick. Alternately, try/catch and reject(err) on catch. + var connection = con; + + let elementsToQuery = "*"; + if (minimal) { + elementsToQuery = "id,loc,source_id"; + } + + const sqlQuery = " \ + SELECT " + elementsToQuery + " \ +FROM ttnantenna \ +WHERE MBRContains( \ + GeomFromText( 'LINESTRING(? ?,? ?)' ), \ + loc)"; + + + con.query(sqlQuery, [boundNorthEastLatLng.lat, boundNorthEastLatLng.lng, boundSouthWestLatLng.lat, boundSouthWestLatLng.lng], function (err, rows, fields) { + // Call reject on error states, + // call resolve with results + if (err) { + return reject(err); + } + + const results = []; + for (let e = 0; e < rows.length; e++) { + const currentRow = rows[e]; + const element = { + type: "general-poi", + source: "ttnantennas", + titel: currentRow.name, + description: currentRow.description, + url: currentRow.link, + uid: "ttnantennas_" + currentRow.id, + location: currentRow.loc, + icon: "signalIcon" + }; + if (minimal) { + const stripableElements = ["title", "description", "url", "type"]; + for (h in stripableElements) { + delete element[stripableElements[h]]; + } + } + results.push(element); + } + resolve(results); + }); + }); +} + +function queryItem(uid) { + const uidParts = uid.split("_"); + + const sqlQuery = "SELECT * FROM ttnantenna WHERE id=?"; + return new Promise(function (resolve, reject) { + con.query(sqlQuery, [parseInt(uidParts[1])], function (err, rows, fields) { + if (err) { + return reject(err); + } + const results = []; + for (let e = 0; e < rows.length; e++) { + const currentRow = rows[e]; + const element = { + type: "general-poi", + source: "ttnantennas", + titel: currentRow.name, + description: currentRow.description, + url: currentRow.link, + uid: "ttnantennas_" + currentRow.id, + location: { + lat: currentRow.loc_lat, + lng: currentRow.loc_lng, + }, + }; + results.push(element); + } + resolve(results); + }); + }); +} + +function getModuleMeta() { + return { + title: "TheThingsnetwork Gateways", + exactName: "ttnantennas", + country: ["*"], + limited: false, + }; +} + +module.exports = { initialize, queryData, getModuleMeta, queryItem }; diff --git a/apiHandler/ttncomms/db_table_config.sql b/apiHandler/ttncomms/db_table_config.sql new file mode 100644 index 0000000..8e9e627 --- /dev/null +++ b/apiHandler/ttncomms/db_table_config.sql @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS `ttncomms` +CREATE TABLE IF NOT EXISTS `ttncomms` ( `id` BIGINT unsigned NOT NULL AUTO_INCREMENT, `name` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, `description` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, `source_id` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, `loc` POINT NOT NULL, `protocol_id` SMALLINT unsigned NOT NULL, `link` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, UNIQUE (id)); \ No newline at end of file diff --git a/apiHandler/ttncomms/db_table_config.sql.template b/apiHandler/ttncomms/db_table_config.sql.template new file mode 100644 index 0000000..02a03b2 --- /dev/null +++ b/apiHandler/ttncomms/db_table_config.sql.template @@ -0,0 +1,13 @@ +DROP TABLE IF EXISTS `ttncomms` +#NL +CREATE TABLE IF NOT EXISTS `ttncomms` ( + `id` BIGINT unsigned NOT NULL AUTO_INCREMENT, + `name` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + `description` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + `source_id` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + `loc` POINT NOT NULL, + `protocol_id` SMALLINT unsigned NOT NULL, + `link` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + UNIQUE (id) +); +#NL diff --git a/apiHandler/ttncomms/main.js b/apiHandler/ttncomms/main.js new file mode 100644 index 0000000..ad7a0d3 --- /dev/null +++ b/apiHandler/ttncomms/main.js @@ -0,0 +1,165 @@ +const fs = require("fs"); +const bent = require('bent') +const tools = require("../../functions"); + +let con = undefined; + +function initialize(connection) { + con = connection; + const array = fs + .readFileSync("./apiHandler/ttncomms/db_table_config.sql") + .toString() + .split("\n"); + + for (i in array) { + con.query(array[i], function (err, result) { + if (err) throw err; + console.log("Table created for ttncomms"); + }); + } + const ttnCommUrl = + "https://www.thethingsnetwork.org/community.json"; + const mysqlQuery = + "INSERT INTO ttncomms (name, description, source_id, loc, protocol_id, link) VALUES (?, ?, ?, POINT(?, ?), ?, ?)"; + const res = bent(ttnCommUrl, 'GET', 'json', 200); + + res().then(function(body){ + console.log("[TheThingsnetwork Communities] Request done"); + for (let c = 0; c < body.length; c++) { + const elm = body[c]; + if (elm.lat != null && elm.lon != null) { + con.query( + mysqlQuery, + [ + "TTN Community " + elm.title, + "", + getModuleMeta().exactName, + elm.lat, + elm.lon, + 0, + "https://www.thethingsnetwork.org" + elm.path, + ], + function (err, result) { + if (err) { + throw err; + } + } + ); + } + } + console.log("[TheThingsnetwork Communities] Insert into Database done!"); + }, function(err){ + console.warn("[" + getModuleMeta().title + "] Was unable to resolve request with error: " + err) + }); +} + +function queryData(boundNorthEastLatLng, boundSouthWestLatLng, minimal) { + // Create a new promise to return, DBs are slow. + return new Promise(function (resolve, reject) { + // The Promise constructor should catch any errors thrown on + // this tick. Alternately, try/catch and reject(err) on catch. + let elementsToQuery = "*"; + if (minimal) { + elementsToQuery = "id,loc,source_id"; + } + // This is taken from https://stackoverflow.com/questions/21208697/select-all-geospatial-points-inside-a-bounding-box + const sqlQuery = " \ + SELECT " + elementsToQuery + " \ + FROM airspy \ + WHERE MBRContains( \ + GeomFromText( 'LINESTRING(? ?,? ?)' ), \ + loc)"; + + con.query(sqlQuery, [boundNorthEastLatLng.lat, boundNorthEastLatLng.lng, boundSouthWestLatLng.lat, boundSouthWestLatLng.lng], function (err, rows, fields) { + // Call reject on error states, + // call resolve with results + if (err) { + return reject(err); + } + + const results = []; + for (let e = 0; e < rows.length; e++) { + const currentRow = rows[e]; + const element = { + type: "general-poi", + source: "ttncomms", + titel: currentRow.name, + description: "This is a TheThingsnetwork Comunity. URL: " + currentRow.link, + url: currentRow.link, + uid: "ttncomms_" + currentRow.id, + location: currentRow.loc, + }; + if (minimal) { + const stripableElements = ["title", "description", "url", "type"]; + for (h in stripableElements) { + delete element[stripableElements[h]]; + } + } + results.push(element); + } + resolve(results); + }); + }); +} + +function queryItem(uid){ + const uidParts = uid.split("_"); + console.log("[ttncomms] Query item with uid: " + uid + " using id: " + parseInt(uidParts[1])); + + const sqlQuery = "SELECT * FROM " + getModuleMeta().exactName + " WHERE id=?" + return new Promise(function (resolve, reject) { + con.query(sqlQuery, [parseInt(uidParts[1])], function (err, rows, fields) { + if (err) { + return reject(err); + } + console.log("[ttncomms] Query done, result: " + rows); + let results = []; + if(rows.length > 0){ + + for (let e = 0; e < rows.length; e++) { + const currentRow = rows[e]; + const element = { + type: "general-poi", + source: "ttncomms", + titel: currentRow.name, + description: "This is a TheThingsnetwork Comunity. URL: " + currentRow.link, + url: currentRow.link, + uid: "ttncomms_" + currentRow.id, + location: { + lat: currentRow.loc_lat, + lng: currentRow.loc_lng, + }, + }; + results.push(element); + } + + }/*else{ + results = [{ + type: "general-poi", + source: "ttncomms", + titel: "Invalid", + description: "This is broken. Something is broken.", + url: "INVALID", + uid: uid, + location: { + lat: 0, + lng: 0, + }, + }]; + }*/ + + resolve(results) + }); +}); +} + +function getModuleMeta() { + return { + title: "TheThingsnetwork Communities", + exactName: "ttncomms", + country: ["*"], + limited: false, + }; +} + +module.exports = { initialize, queryData, getModuleMeta, queryItem }; diff --git a/config/default.json.save b/config/default.json.save new file mode 100644 index 0000000..696010a --- /dev/null +++ b/config/default.json.save @@ -0,0 +1,15 @@ +{ + "fontAwesome": "d7b80a780b", + "mapboxAccessToken": "pk.eyJ1IjoiZ3JleWRpYW1vbmQiLCJhIjoiY2p5NXJyZmZtMDlzYjNibGV2cnZ1MTZxbyJ9.7kB9q9Hijmx9MewGM0F0MQ", + "cookieSecret": "fdgdfgdgsfg156516ds561156dsf10g65d1f0g651fd651/", + "database": { + "host": "localhost", + "user": "knowMap", + "password": "uwCG5rVc51qv4Tb8", + "database": "knowMapexit", + "multipleStatements": true, + "charset": "utf8mb4" + }, + "env": "DEV" + +} diff --git a/config/default.json.template b/config/default.json.template new file mode 100644 index 0000000..7e3cd3d --- /dev/null +++ b/config/default.json.template @@ -0,0 +1,16 @@ +{ + "fontAwesome": "", + "mapboxAccessToken": "", + "cookieSecret": "", + "mapquest": "", + "database": { + "host": "localhost", + "user": "", + "password": "", + "database": "" + }, + "env": "DEV", + "maint": false, + "betaMode": true, + "port": 3000 +} \ No newline at end of file diff --git a/functions.js b/functions.js new file mode 100644 index 0000000..36c6609 --- /dev/null +++ b/functions.js @@ -0,0 +1,25 @@ +// Rounds a number to ˋplacesˋ amount of digits +// Example roundOff(5.2434234234, 3) => 5.243 +let roundOff = (num, places) => { + const x = Math.pow(10, places); + return Math.round(num * x) / x; +}; + +function handleMysqlErrors(err, title) { + if (err) { + if (err.code == "ECONNREFUSED") { + console.log( + "⚠ Connection to database failed. Is the database server running? ⚠" + ); + process.exit(5); + } else if (err.code == "ECONNRESET") { + console.log("⚠ Module " + title + " experienced a connection reset ⚠"); + process.exit(5); + } else { + console.warn("!!!!!!!!!!!!!!!! NOW THROWING " + err.code + " !!!!!!!!!!!!!!!!") + throw err; + } + } +} + +module.exports = { roundOff, handleMysqlErrors }; diff --git a/libs/jsonHand.lib.js b/libs/jsonHand.lib.js new file mode 100644 index 0000000..0e911d9 --- /dev/null +++ b/libs/jsonHand.lib.js @@ -0,0 +1,10 @@ +function IsJsonString(str) { + try { + JSON.parse(str); + } catch (e) { + return false; + } + return true; +} + +module.exports = { IsJsonString } \ No newline at end of file diff --git a/libs/metaHandler.lib.js b/libs/metaHandler.lib.js new file mode 100644 index 0000000..94df09b --- /dev/null +++ b/libs/metaHandler.lib.js @@ -0,0 +1,26 @@ +function returnTags(module){ + const meta = module.getModuleMeta() + if(meta.features != undefined){ + if(meta.features.includes("tags")){ + return(meta.tags) + } + }else if(meta.tags != undefined){ + return(meta.tags) + }else{ + return([]) // Assume the module does not support tags + } +} + +function returnCountries(module){ + const meta = module.getModuleMeta() + if(meta.features != undefined){ + if(meta.features.includes("countries")){ + return(meta.country) + } + }if(meta.country != undefined){ + return(meta.country) + }else{ + return(["*"]) // Assume the module does not support tags + } +} +module.exports = { returnTags, returnCountries } \ No newline at end of file diff --git a/license-report-config.json b/license-report-config.json new file mode 100644 index 0000000..668a2e0 --- /dev/null +++ b/license-report-config.json @@ -0,0 +1,10 @@ +{ + "fields": [ + "name", + "licenseType", + "link", + "comment", + "installedVersion", + "author" + ] + } \ No newline at end of file diff --git a/main.ts b/main.ts new file mode 100644 index 0000000..5583324 --- /dev/null +++ b/main.ts @@ -0,0 +1,370 @@ +/*** + * Copyright © 2021 Sören W. Oesterwind (TheGreydiamond). All rights reserved. + */ + +// Requires (Imports) +const express = require("express"); +const fs = require("fs"); +const mysql = require('mysql'); +const autobahnAPI = require("./apiHandler/autobahn/main.js"); +/*const ffhAPI = require("./apiHandler/ffh/main.js"); +const stadtkoelnAPI = require("./apiHandler/ffh/main.js");*/ +const customWebcamAPI = require("./apiHandler/customWebcams/main.js"); +const airspyAPI = require("./apiHandler/airspydirectory/main"); +const ttncommsAPI = require("./apiHandler/ttncomms/main"); +const freifunkAPI = require("./apiHandler/freifunk/main"); +const senseboxAPI = require("./apiHandler/opensensemap/main.js"); +const opentopiaAPI = require("./apiHandler/opentopia/main.js"); +const ttnGateway = require("./apiHandler/ttnantennas/main"); +const stubMod = require("./apiHandler/stub/main"); +const _ = require("underscore"); +const chalk = require('chalk'); +const func = require("./functions.js") +const minify = require('express-minify'); +const compression = require('compression'); +const cookieParser = require('cookie-parser') +const bodyParser = require('body-parser'); +const Sentry = require('@sentry/node'); +const Tracing = require('@sentry/tracing'); +const bent = require("bent"); +const Influx = require("influx"); + + +function padLeadingZeros(num, size) { + let s = num + ""; + while (s.length < size) s = "0" + s; + return s; +} + +const enabledModules = [airspyAPI, autobahnAPI, customWebcamAPI, freifunkAPI, senseboxAPI, opentopiaAPI, ttnGateway, ttncommsAPI]; + +// Webserver init +const app = express(); + + +const allTags = {}; +const metaGlobals = { "desc": "Pointsight is a centralized geolocalized API aggragator.", "titlePrefx": "Pointsight - " } + +// Skeleton Variables +let jsonConfigGlobal = { + fontAwesome: undefined, + mapboxAccessToken: undefined, + cookieSecret: undefined, + mapquest: undefined, + here: undefined, + sentryDsn: undefined, + database: { + host: "127.0.0.1", + user: "pointsight", + password: "", + database: "pointsight", + customSettings: { + wait_timout: 28800 + } + }, + env: "PROD", + maint: true, + betaMode: false, + port: 3000, + adress: '127.0.0.1', + taxonomyCacheInterval: 5, + metrics: { + influx: { + enable: false, + host: "127.0.0.1", + database: "pointsight", + user: "pointsight", + password: "" + }, + writeMetricsToMySQL: true + } +} + +// Load config +try { + const data = fs.readFileSync("config/default.json", "utf8"); + jsonConfigGlobal = _.extend(jsonConfigGlobal, JSON.parse(data)); +} catch (error) { + console.error( + "While reading the config an error occured. The error was: " + error + ); +} +console.log(jsonConfigGlobal.metrics.influx) + +Sentry.init({ + dsn: jsonConfigGlobal.sentryDsn, + integrations: [ + // enable HTTP calls tracing + new Sentry.Integrations.Http({ tracing: true }), + // enable Express.js middleware tracing + new Tracing.Integrations.Express({ app }), + new Tracing.Integrations.Mysql({ useMysql: true }) + ], + // Set tracesSampleRate to 1.0 to capture 100% + // of transactions for performance monitoring. + // We recommend adjusting this value in production + tracesSampleRate: 0.8, +}); +app.use(Sentry.Handlers.requestHandler()); +app.use(Sentry.Handlers.tracingHandler()); + +const con = mysql.createConnection(jsonConfigGlobal.database); +const connectionPool = mysql.createPool(jsonConfigGlobal.database); +const allPoolConnections = []; +let influxD = undefined; +let isInfluxReady = false; +let lastHttpRequest = Math.floor(Date.now() / 1000) + +if(jsonConfigGlobal.metrics.influx.enable){ + console.log("[Influx] Connecting to InfluxDB..."); + influxD = new Influx.InfluxDB({ + host: jsonConfigGlobal.metrics.influx.host, + database: jsonConfigGlobal.metrics.influx.database, + user: jsonConfigGlobal.metrics.influx.user, + password: jsonConfigGlobal.metrics.influx.password, + schema: [ + { + measurement: 'apitaxonomy', + fields: { calls: Influx.FieldType.INTEGER }, + tags: [] + } + ] + }); + console.log("[Influx] Checking if database exists..."); + influxD.getDatabaseNames() + .then(names => { + if (!names.includes('pointsight')) { + return influxD.createDatabase('pointsight'); + } + }) + .then(() => { + console.log("[Influx] Database ready."); + isInfluxReady = true; + }) + .catch(error => console.log({ error })); +} + +connectionPool.getConnection((err, conn) => { + conn.on('error', function (err) { + console.log("----[MYSQL ERROR]----") + console.log(err); // 'ER_BAD_DB_ERROR' + console.log("[MYSQL ERROR] Connection pool error"); + console.log("----]MYSQL ERROR[----") + }); + + func.handleMysqlErrors(err) + //Create system tabels + const queries = ["CREATE TABLE IF NOT EXISTS `apikeys` ( `id` BIGINT NOT NULL AUTO_INCREMENT , `apikey` VARCHAR(64) NOT NULL , `owner` VARCHAR(64) NOT NULL , `hosts` JSON NOT NULL , `created` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP , `expire` DATETIME NOT NULL , PRIMARY KEY (`id`));", "CREATE TABLE IF NOT EXISTS `betatokens` ( `id` BIGINT NOT NULL AUTO_INCREMENT , `token` VARCHAR(64) NOT NULL , `owner` VARCHAR(64) NOT NULL, `created` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP , `expire` DATETIME NOT NULL , PRIMARY KEY (`id`));", + "CREATE TABLE IF NOT EXISTS `reports` ( `id` INT NOT NULL AUTO_INCREMENT , `point_id` VARCHAR(512) NOT NULL , `mail` VARCHAR(512) NOT NULL , `comment` VARCHAR(2048) NOT NULL , PRIMARY KEY (`id`)) ENGINE = InnoDB;", + "CREATE TABLE IF NOT EXISTS `apitaxonomy` ( `id` INT NOT NULL AUTO_INCREMENT , `apiKey` VARCHAR(255) NOT NULL , `date` DATE NOT NULL DEFAULT CURRENT_TIMESTAMP , `calls` INT NOT NULL DEFAULT '0' , PRIMARY KEY (`id`)) ENGINE = InnoDB;"] + queries.forEach(element => { + conn.query(element, function (err) { + if (err) throw err; + console.log("Table created for Main system"); + }); + }); + conn.release() +}) + +for (let i = 0; i < enabledModules.length; i++) { + connectionPool.getConnection((err, conn) => { + if (err) { + if (err.code == "ECONNREFUSED") { + console.log(chalk.red("⚠ Connection to database failed. Is the database server running? ⚠")) + server.close() + process.exit(5) + } else { + throw err; + } + } + + allPoolConnections.push(conn); + conn.on('error', function (err) { + console.log("----[MYSQL ERROR]----") + console.log("[MYSQL ERROR] Module connection error"); + console.log(err); // 'ER_BAD_DB_ERROR' + if (err.code == "ECONNRESET") { + connectionPool.getConnection((err, connP) => { + if (err) throw err; + conn = connP // Trying to mitigate ECONNRESET issues, even tho this didn't fix it lol + }) + } + console.log("----]MYSQL ERROR[----") + }); + _.each(allPoolConnections, function (arg) { setInterval(function () { arg.query("SELECT 1") }, jsonConfigGlobal.database.customSettings.wait_timout - 50) }) + try { + enabledModules[i].initialize(conn, jsonConfigGlobal) + const currTags = enabledModules[i].getModuleMeta().tags + for (const tagy in currTags) { + const tag = currTags[tagy] + if (_.isFinite(allTags[tag])) { + allTags[tag] = allTags[tag] + 1; + } else { + allTags[tag] = 1; + } + } + } catch (error) { + console.warn(chalk.red("⚠ Module " + enabledModules[i].getModuleMeta().title + " failed to initialize() ⚠" + "\n Error: " + error + "\n Stacktrace: " + error.stack)) + enabledModules[i] = stubMod; + } + }); + + setInterval(function () { + for (const conyT in allPoolConnections) { + allPoolConnections[conyT].query("SELECT 1;", function (err) { + if (err) throw err; + console.log("Querrying heartbeat for connection pool number: " + conyT); + }); + } + }, jsonConfigGlobal.database.customSettings.wait_timout * 1000) +} + + +app.use(express.static("static")); + +// A crappy logging middleware, please don't look at this +app.use(function (req, res, next) { + lastHttpRequest = Math.floor(Date.now() / 1000) + const date_ob = new Date(); + const date = ("0" + date_ob.getDate()).slice(-2); + // current month + const month = ("0" + (date_ob.getMonth() + 1)).slice(-2); + // current year + const year = date_ob.getFullYear(); + // current hours + const hours = padLeadingZeros(date_ob.getHours(), 2) + // current minutes + const minutes = padLeadingZeros(date_ob.getMinutes(), 2) + // current seconds + const seconds = padLeadingZeros(date_ob.getSeconds(), 2) + // prints date & time in YYYY-MM-DD HH:MM:SS format + console.log("[" + year + "-" + month + "-" + date + " " + hours + ":" + minutes + ":" + seconds + "] " + req.method + " " + req.path); + next(); +}); +app.use(bodyParser.urlencoded({ extended: true })); +app.use(cookieParser(jsonConfigGlobal.cookieSecret)); +if (jsonConfigGlobal.betaMode) { + require('./middleware/beta.middleware')(app, con, jsonConfigGlobal); +} + +// API Taxonomy +const apiTaxonomyCache = {}; +function flushApiTaxonomyCache():void { + if(jsonConfigGlobal.metrics.writeMetricsToMySQL){ + const updateSql = "UPDATE apitaxonomy SET calls = calls+? WHERE apikey LIKE ? AND DATE(date) = CURDATE()"; + const insertSql = "INSERT INTO apitaxonomy (apikey, calls) VALUES(?, ?);" + console.log( Object.entries(apiTaxonomyCache).length + " entries in apiTaxonomyCache") + for (const [key, value] of Object.entries(apiTaxonomyCache)) { + con.query(updateSql, [value, key], function (err, result) { + if (err) { + console.error(err); + } + if(!jsonConfigGlobal.metrics.influx.enable){ + apiTaxonomyCache[key] = 0; + } + + if (result.affectedRows == 0) { + con.query(insertSql, [key, value], function (err, result) { + if (err) { + console.error(err); + } + }); + } + }); + } + }else{ + if(!jsonConfigGlobal.metrics.influx.enable){ + console.log(chalk.red("⚠ MySQL Metrics writing has been disabled, but Influx is not setup! ⚠"));} + } + if(jsonConfigGlobal.metrics.influx.enable){ + const toWritePoints = []; + console.log( Object.entries(apiTaxonomyCache).length + " entries in apiTaxonomyCache in Influx") + for (const [key, value] of Object.entries(apiTaxonomyCache)) { + const val = apiTaxonomyCache[key] + toWritePoints.push({ + measurement: 'apitaxonomy', + tags: {}, + fields: { calls: val }, + timestamp: new Date, + }) + apiTaxonomyCache[key] = 0; + } + influxD.writePoints(toWritePoints, { + database: jsonConfigGlobal.metrics.influx.database, + precision: 's', + }) + .catch(error => { + console.error(`[InfluxDB] Error saving data to InfluxDB! ${error.stack}`) + }); + } +} + +if(jsonConfigGlobal.env == "DEV"){ + jsonConfigGlobal.taxonomyCacheInterval = 0.5; +} + +setInterval(function () { + flushApiTaxonomyCache() + console.log("[API TAXONOMY] Wrote taxonomy to database"); +}, jsonConfigGlobal.taxonomyCacheInterval * 60 * 1000); // Every `taxonomyCacheInterval` minutes + + +connectionPool.getConnection((err, conn) => { + conn.on('error', function (err) { + console.log("----[MYSQL ERROR]----") + console.log("[MYSQL ERROR] System connection error"); + console.log(err); + console.log("----]MYSQL ERROR[----") + }); + + setInterval(function () { + conn.query("SELECT 1;", function (err) { + if (err) throw err; + console.log("Querrying heartbeat for system connection"); + }); + }, jsonConfigGlobal.database.customSettings.wait_timout * 1000) // Every 20 Minutes + func.handleMysqlErrors(err) + require('./middleware/maint.middleware')(app, jsonConfigGlobal, metaGlobals); + require('./middleware/apitoken.middleware')(app, con, apiTaxonomyCache); + require('./routes/api.route.ts')(app, enabledModules, conn, { tags: allTags }); + require('./routes/index.route.js')(app, metaGlobals, jsonConfigGlobal.fontAwesome, jsonConfigGlobal.mapboxAccessToken, jsonConfigGlobal); + app.use(Sentry.Handlers.errorHandler()); + require('./routes/error.route.js')(app, jsonConfigGlobal, metaGlobals); // Make sure this is always last + app.use(minify()); + app.use(compression()); +}) + +let server = undefined; + +setInterval(function () { + if (Math.floor(Date.now() / 1000) - lastHttpRequest >= 60 * 60) { // One hour after the last http request + const request = bent("https://pointsight.project-name-here.de/api/getPOI?boundingEast=55.21649013168979;10.696563720703127&boundingWest=54.59354980818523;9.825897216796877&key=b03f8aaf-1f32-4d9e-914a-9a50f904833d", "GET", "json", 200); + request().then(function (body) { + console.log("Tried to pull POI from pointsight.project-name-here.de") + }) + } +}, 60000); + +process.on('SIGINT', function () { + console.log("Caught interrupt signal and shutting down gracefully"); + flushApiTaxonomyCache(); // Save the api taxonomy cache before quitting + server.close(); // Make th express server stop + _.each(allPoolConnections, function (arg) { arg.release(); }) // Let go off all mysql pool connections + con.end(); // Close the system's mysql connection + Sentry.close(2000).then(function () { // Wait for sentry to quit + console.log(chalk.blue("👋 Goodbye! 👋")) + process.exit(); // Quit the application + }); +}); + +// Start server +server = app.listen(jsonConfigGlobal.port, jsonConfigGlobal.adress, () => { + console.log(`The Pointsight is running at http://localhost:${jsonConfigGlobal.port}`) + setTimeout(function () { + if (jsonConfigGlobal.env == "DEV") { + console.log(allTags); + } + + }, 1000) +}) + diff --git a/middleware/apitoken.middleware.js b/middleware/apitoken.middleware.js new file mode 100644 index 0000000..519535d --- /dev/null +++ b/middleware/apitoken.middleware.js @@ -0,0 +1,86 @@ +module.exports = function (app, con, apiTaxonomyCache) { + const _ = require("underscore"); + function isFutureDate(value) { + const d_now = new Date(); + const d_inp = new Date(value); + return d_now.getTime() <= d_inp.getTime(); + } + app.use(function (req, res, next) { + // API key handling middleware + if (req.path.includes("/api/")) { + if (req.query.key != undefined) { + const sql = "SELECT * FROM apikeys WHERE apikey LIKE ? LIMIT 1"; + con.query(sql, [req.query.key], function (err, result) { + if (err) { + throw err; + } + if (result.length == 0) { + // There is atleast one result + res.status(401); + res.setHeader("Content-Type", "application/json"); + res.send( + JSON.stringify({ state: "Failed", message: "Invalid API key" }) + ); + } else { + // console.log(req.headers); + if ( + JSON.parse(result[0].hosts).includes(req.hostname) || + JSON.parse(result[0].hosts).includes("*") + ) { + // Is the key even allowed for this host? + if (isFutureDate(result[0].expire)) { // Has the key expired? + next(); // Allow it to pass + if(!_.isFinite(apiTaxonomyCache[req.query.key])){ + apiTaxonomyCache[req.query.key] = 0; + } + apiTaxonomyCache[req.query.key] = apiTaxonomyCache[req.query.key]+1; + // console.log(apiTaxonomyCache) + /*const updateSql = "UPDATE apitaxonomy SET calls = calls+1 WHERE apikey LIKE ? AND DATE(date) = CURDATE()"; + const insertSql = "INSERT INTO apitaxonomy (apikey, calls) VALUES(?, ?);" + con.query(updateSql, [req.query.key], function (err, result) { + if (err) { + console.error(err); + } + if(result.affectedRows == 0) { + con.query(insertSql, [req.query.key, 1], function (err, result) { + if (err) { + console.error(err); + } + }); + } + });*/ + } else { + // Yes? Then no passing! + res.status(401); + res.setHeader("Content-Type", "application/json"); + res.send( + JSON.stringify({ + state: "Failed", + message: "Expired API key", + }) + ); + } + } else { + res.status(401); + res.setHeader("Content-Type", "application/json"); + res.send( + JSON.stringify({ + state: "Failed", + message: "Invalid Hostname for API key", + }) + ); + } + } + }); + } else { + res.status(401); + res.setHeader("Content-Type", "application/json"); + res.send( + JSON.stringify({ state: "Failed", message: "Missing API key" }) + ); + } + } else { + next(); + } + }); +}; diff --git a/middleware/beta.middleware.js b/middleware/beta.middleware.js new file mode 100644 index 0000000..2016ad4 --- /dev/null +++ b/middleware/beta.middleware.js @@ -0,0 +1,142 @@ +module.exports = function (app, con, config) { + const uuid = require("uuid"); + const Eta = require("eta"); + const fs = require("fs"); + + function isFutureDate(value) { + const d_now = new Date(); + const d_inp = new Date(value); + return d_now.getTime() <= d_inp.getTime(); + } + + app.use(function (req, res, next) { + res.locals.valid = true; + if ( + req.path.includes("/api/") || + req.path.includes("favicon") || + req.path.includes("/betaLogin") || + req.path.includes("/redirectUrl") || + req.path.includes("/beta/Invite") + ) { + next(); + } else { + if (config.env == "DEV") { + console.log("[beta.middleware.js]", req.path); + } + if (uuid.validate(req.signedCookies.betaToken)) { + next(); + } else { + // res.cookie('betaToken', uuid.v4(), { signed: true }) + const data = fs.readFileSync("templates/redr.eta.html", "utf8"); + res.locals.valid = false; + res.send( + Eta.render(data, { + siteTitel: "Pointsight - BetaLogin", + redirectUrl: "/betaLogin", + }) + ); + } + } + }); + + app.post("/betaLogin", function (req, res) { + const sql = "SELECT * FROM betatokens WHERE token LIKE ? LIMIT 1"; + + con.query(sql, [req.body.betatoken], function (err, result) { + if (err) { + throw err; + } + if (result.length == 0) { + // There is atleast one result + res.status(401); + res.setHeader("Content-Type", "application/json"); + res.send( + JSON.stringify({ state: "Failed", message: "Invalid API key" }) + ); + } else { + if (isFutureDate(result[0].expire)) { + // Has the key expired? + res.status(200); + res.cookie("betaToken", uuid.v4(), { signed: true }); + res.redirect("/"); + } else { + // Yes? Then no passing! + res.status(401); + res.setHeader("Content-Type", "application/json"); + res.send( + JSON.stringify({ state: "Failed", message: "Expires API key" }) + ); + } + } + }); + + console.log(req.body); + }); + app.get("/beta/Invite", function (req, res) { + const myInv = req.query.invite; // Invite code + let invtee = req.query.invitee; // Used for personalized invites + if(invtee == undefined){ + invtee = "Someone"; // Fallback if none is given + } + if (myInv == undefined) { + const data = fs.readFileSync("templates/redr.eta.html", "utf8"); + res.locals.valid = false; + res.send( + Eta.render(data, { + siteTitel: "Pointsight", + redirectUrl: "/", + }) + ); + } else { + const data = fs.readFileSync("templates/beta/invitePage.eta.html", "utf8"); + res.send( + Eta.render(data, { + siteTitel: "Pointsight - BetaToken - Invite", + invite: myInv, + invitee: invtee + }) + ); + } + }); + app.get("/betaLogin", function (req, res) { + if (req.query.betaKey != undefined) { + console.log(req.query.betaKey); + const sql = "SELECT * FROM betatokens WHERE token LIKE ? LIMIT 1"; + + con.query(sql, [req.query.betaKey], function (err, result) { + if (err) { + throw err; + } + if (result.length == 0) { + // There is atleast one result + res.status(401); + res.setHeader("Content-Type", "application/json"); + res.send( + JSON.stringify({ state: "Failed", message: "Invalid API key" }) + ); + } else { + if (isFutureDate(result[0].expire)) { + // Has the key expired? + res.status(200); + res.cookie("betaToken", uuid.v4(), { signed: true }); + res.redirect("/"); + } else { + // Yes? Then no passing! + res.status(401); + res.setHeader("Content-Type", "application/json"); + res.send( + JSON.stringify({ state: "Failed", message: "Expires API key" }) + ); + } + } + }); + } else { + const data = fs.readFileSync("templates/betaLogin.eta.html", "utf8"); + res.send( + Eta.render(data, { + siteTitel: "Pointsight - BetaLogin", + }) + ); + } + }); +}; diff --git a/middleware/maint.middleware.js b/middleware/maint.middleware.js new file mode 100644 index 0000000..9d07667 --- /dev/null +++ b/middleware/maint.middleware.js @@ -0,0 +1,19 @@ +module.exports = function (app, jsonConfigGlobal, metaGlobals) { + const fs = require("fs"); + const Eta = require("eta"); + app.use(function (req, res, next) { + if(jsonConfigGlobal.maint){ + const data = fs.readFileSync("templates/error/maint.eta", "utf8"); + res.send( + Eta.render(data, { + desc: metaGlobals.desc, + siteTitel: metaGlobals.titlePrefx + "Maintanance work", + debug: {}, + fontawesomeKey: jsonConfigGlobal.fontAwesome + }) + ); + }else{ + next() + } + }); +} \ No newline at end of file diff --git a/mysqlTables.sql b/mysqlTables.sql new file mode 100644 index 0000000..2a5f40d --- /dev/null +++ b/mysqlTables.sql @@ -0,0 +1,2 @@ +CREATE TABLE `apikeys` ( `id` BIGINT NOT NULL AUTO_INCREMENT , `apikey` VARCHAR(64) NOT NULL , `owner` VARCHAR(64) NOT NULL , `hosts` JSON NOT NULL , `created` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP , `expire` DATETIME NOT NULL , PRIMARY KEY (`id`)); +CREATE TABLE `betatokens` ( `id` BIGINT NOT NULL AUTO_INCREMENT , `token` VARCHAR(64) NOT NULL , `owner` VARCHAR(64) NOT NULL, `created` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP , `expire` DATETIME NOT NULL , PRIMARY KEY (`id`)); \ No newline at end of file diff --git a/package.json b/package.json new file mode 100644 index 0000000..52e5c2a --- /dev/null +++ b/package.json @@ -0,0 +1,64 @@ +{ + "name": "pointsight", + "version": "1.1.1", + "description": "The ultimate knowledge map", + "main": "main.js", + "scripts": { + "makeJS": "tsc main.ts", + "lint": "eslint ./*.js && eslint ./*.ts", + "makeJSwatch": "tsc -w main.ts", + "start": "node tools/sqlMinifyer.js && tsc main.ts && node main.js", + "startDev": "node tools/sqlMinifyer.js && tsc main.ts && node --inspect main.js", + "preChecks": "npm outdated && npm audit", + "startBeforeMerge": "eslint . --ext .ts && tsc main.ts && node main.js", + "nodemon": "nodemon main.js", + "tailwind:css": "npx tailwindcss -i ./static/css/tailwindTemp.css -o ./static/css/tailwindInclude.css", + "checkLicense": "license-report --output=table --only=prod --config license-report-config.json" + }, + "author": "[Project-Name-Here]", + "license": "LGPL-3.0", + "dependencies": { + "@sentry/node": "^6.16.0", + "@sentry/tracing": "^6.16.0", + "@themesberg/flowbite": "^1.2.0", + "autoprefixer": "^10.4.1", + "bent": "^7.3.12", + "body-parser": "^1.19.0", + "chalk": "^4.1.2", + "cli-progress": "^3.9.1", + "compression": "^1.7.4", + "cookie-parser": "^1.4.6", + "deep-email-validator": "^0.1.18", + "eta": "^1.12.3", + "express": "^4.17.2", + "express-hcaptcha": "^0.0.3", + "express-minify": "^1.0.0", + "express-winston": "^4.2.0", + "influx": "^5.9.2", + "license-report": "^4.5.0", + "log-symbols": "^5.1.0", + "mysql": "^2.18.1", + "postcss": "^8.4.5", + "postcss-cli": "^9.1.0", + "pp-which-country": "^1.0.2", + "safe-stable-stringify": "^2.3.1", + "tailwindcss": "^3.0.11", + "underscore": "^1.13.2", + "winston": "^3.3.3" + }, + "devDependencies": { + "@types/node": "^16.11.19", + "@typescript-eslint/eslint-plugin": "^5.9.0", + "@typescript-eslint/parser": "^5.9.0", + "chai": "^4.3.4", + "chai-json": "^1.0.0", + "eslint": "^8.6.0", + "eslint-config-strongloop": "^2.1.0", + "inquirer": "^8.2.0", + "ora": "^6.0.1", + "prompts": "^2.4.2", + "supertest": "^6.1.6", + "typescript": "^4.5.4", + "uuid": "^8.3.2" + } +} diff --git a/postcss.config.js b/postcss.config.js new file mode 100644 index 0000000..283a097 --- /dev/null +++ b/postcss.config.js @@ -0,0 +1,6 @@ +module.exports = { + plugins: [ + require('tailwindcss'), + require('autoprefixer'), + ] + } \ No newline at end of file diff --git a/proto/filterTest.js b/proto/filterTest.js new file mode 100644 index 0000000..e084dfe --- /dev/null +++ b/proto/filterTest.js @@ -0,0 +1,76 @@ +const sensemap = require("../apiHandler/opensensemap/main"); +const ttnantennas = require("../apiHandler/ttnantennas/main"); +const autobahn = require("../apiHandler/autobahn/main"); +const libs = require("../libs/metaHandler.lib.js"); + +const mods = [sensemap, ttnantennas, autobahn]; // Provided by main.ts in a real env. + +const jsonData = { + filters: [ + ["tag", "webcam"], + ["country", "DEU"], + ], + conjunction: "AND", +}; // Provided by request in a real env + +const uMods = mods.filter(function (elm) { + // Loops through all elements in a list and lets you decide if weather to keep 'em or not + let amount = 0; + for (fId in jsonData.filters) { + // Do this is each provided filter + const fi = jsonData.filters[fId]; // Gets the current filters name + switch (fi[0]) { + case "tag": + if (jsonData.conjunction == "OR") { + if (libs.returnTags(elm).includes(fi[1])) { + return elm; + } else { + break; + } + } else { + if (libs.returnTags(elm).includes(fi[1])) { + amount++; + } + break; + } + case "country": + if (jsonData.conjunction == "OR") { + if (libs.returnCountries(elm).includes(fi[1])) { + return elm; + } else { + break; + } + } else { + if (libs.returnCountries(elm).includes(fi[1])) { + amount++; + } + break; + } + } + } + if (jsonData.conjunction == "AND") { + if (amount == jsonData.filters.length) { + return elm; + } + } +}); + +console.log(uMods); + +/** + * This things is sorting all given modules by a JSON array. + * You can supply the ˋfilterˋ array with a list of filters + * They are made up of ["metaData", "metaValue"] + * Another argument is ˋconjunctionˋ it can be either AND or OR + * OR means that any of the given filters should apply + * AND means that all filter HAVE to apply + * + * Current state + * This things can already handle OR requests, AND requests + * are not supported yet. I also use the /libs/metaHandler lib, + * at least I finnaly found a usecase for that waste of space. + * + * This is a small prototype which will be added into /routes/api.route.js + * at some point. But what I was doing before was almost production testing. + * Not good. I will need to test this further and throw some edge cases at it. + */ diff --git a/proto/scrapeDemo.js b/proto/scrapeDemo.js new file mode 100644 index 0000000..98d2c5d --- /dev/null +++ b/proto/scrapeDemo.js @@ -0,0 +1,58 @@ +// Trying to scrape https://www.opentopia.com/hiddencam.php?p=n for data +// THIS IS A PROTOTYPE - DO NOT USE +// [Project-Name-Here] 2021 +// Current state: Able to get cams from page, get location, unable to get live feed +process.exit(1) +const fs = require("fs"); +const request = require("request"); + + +function requestCamList(n){ + request("http://www.opentopia.com/hiddencam.php?showmode=standard&country=%2A&seewhat=highlyrated&p="+n, { json: false }, (err, res2, body) => { + if (err) { + throw err; + } + let part = body.split("")[1] + // part = part.split('
')[0] + //part = part.split('
    ')[1] + part = part.split("/webcam/") + part.shift() + for(h in part){ + part[h] = part[h].replace("\t", "") + part[h] = part[h].split('target="_self">')[0].replace('"', "").split(">")[0] + console.log(part[h]) + } + return(part) + }); +} + +function getMoreInfo(id){ + const baseurl = "http://www.opentopia.com/webcam/" + const url = baseurl + id + "?viewmode=livevideo" + request(url, { json: false }, (err, res2, body) => { + if (err) { + throw err; + } + let result = {name: "", url:"", location:{lat:0,lng:0}} + let part = body.split("")[1] + camPart = part.split("
    ")[1] + camPart = camPart.split('')[1] + + result.name = part.split("
    ")[0] + result.name = result.name.replaceAll("\t", "").replaceAll(" ", "") + part = part.split("