Inital commit
This commit is contained in:
2
apiHandler/opentopia/db_table_config.sql
Normal file
2
apiHandler/opentopia/db_table_config.sql
Normal file
@ -0,0 +1,2 @@
|
||||
DROP TABLE IF EXISTS `opentopia`
|
||||
CREATE TABLE IF NOT EXISTS `opentopia` ( `id` BIGINT unsigned NOT NULL AUTO_INCREMENT, `name` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, `source_id` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, `loc` POINT NOT NULL, `protocol_id` SMALLINT unsigned NOT NULL, `link` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, UNIQUE (id), UNIQUE (`link`));
|
||||
13
apiHandler/opentopia/db_table_config.sql.template
Normal file
13
apiHandler/opentopia/db_table_config.sql.template
Normal file
@ -0,0 +1,13 @@
|
||||
DROP TABLE IF EXISTS `opentopia`
|
||||
#NL
|
||||
CREATE TABLE IF NOT EXISTS `opentopia` (
|
||||
`id` BIGINT unsigned NOT NULL AUTO_INCREMENT,
|
||||
`name` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL,
|
||||
`source_id` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL,
|
||||
`loc` POINT NOT NULL,
|
||||
`protocol_id` SMALLINT unsigned NOT NULL,
|
||||
`link` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL,
|
||||
UNIQUE (id),
|
||||
UNIQUE (`link`)
|
||||
);
|
||||
#NL
|
||||
317
apiHandler/opentopia/main.js
Normal file
317
apiHandler/opentopia/main.js
Normal file
@ -0,0 +1,317 @@
|
||||
var fs = require("fs");
|
||||
const bent = require("bent");
|
||||
const mysql = require("mysql");
|
||||
const tools = require("../../functions");
|
||||
const _ = require("underscore");
|
||||
|
||||
let con = undefined;
|
||||
let config;
|
||||
|
||||
function requestWithPromise(n) {
|
||||
const listURL =
|
||||
"http://www.opentopia.com/hiddencam.php?showmode=standard&country=%2A&seewhat=highlyrated&p=";
|
||||
return new Promise(function (resolve, reject) {
|
||||
const request = bent(listURL + n, "GET", 200);
|
||||
request().then(
|
||||
function (body) {
|
||||
body.text().then(function (text) {
|
||||
resolve(text);
|
||||
});
|
||||
},
|
||||
function (err) {
|
||||
console.warn(
|
||||
"[" +
|
||||
getModuleMeta().title +
|
||||
"] Was unable to resolve request with error: " +
|
||||
err
|
||||
);
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function initialize(connection, configL, replaceOld = false) {
|
||||
con = connection;
|
||||
config = configL;
|
||||
if(!fs.existsSync("./apiHandler/opentopia/FLAG")){
|
||||
fs.writeFileSync("./apiHandler/opentopia/FLAG", "")
|
||||
replaceOld = true
|
||||
}
|
||||
if (replaceOld) { // Just do it once, calling that API ain't cheap
|
||||
var array = fs
|
||||
.readFileSync("./apiHandler/opentopia/db_table_config.sql")
|
||||
.toString()
|
||||
.split("\n");
|
||||
|
||||
for (i in array) {
|
||||
con.query(array[i], function (err, result) {
|
||||
tools.handleMysqlErrors(err, getModuleMeta().title + "L32");
|
||||
console.log("Table created for " + getModuleMeta().title);
|
||||
});
|
||||
}
|
||||
|
||||
let allProms = [];
|
||||
|
||||
for (let n = 0; n < 26; n++) {
|
||||
allProms.push(requestWithPromise(n));
|
||||
}
|
||||
|
||||
const mysqlQuery =
|
||||
"INSERT INTO opentopia (name, source_id, loc, protocol_id, link) VALUES (?, ?, POINT(?, ?), ?, ?)";
|
||||
|
||||
Promise.all(allProms).then(function (res) {
|
||||
for (l in res) {
|
||||
const baseurl = "http://www.opentopia.com/webcam/";
|
||||
let allResults = [];
|
||||
// Process the pages
|
||||
let part = res[l].split("<body>")[1];
|
||||
part = part.split("/webcam/");
|
||||
part.shift();
|
||||
for (h in part) {
|
||||
part[h] = part[h].replace("\t", "");
|
||||
part[h] = part[h]
|
||||
.split('target="_self">')[0]
|
||||
.replace('"', "")
|
||||
.split(">")[0];
|
||||
}
|
||||
|
||||
pages = part;
|
||||
camProms = [];
|
||||
for (let pI = 0; pI < pages.length; pI++) {
|
||||
camProms.push(
|
||||
new Promise(function (resolve, reject) {
|
||||
const url = baseurl + pages[pI] + "?viewmode=livevideo";
|
||||
_.throttle(function () {
|
||||
setTimeout(function () {
|
||||
const request = bent(url, "GET", 200);
|
||||
request().then(
|
||||
function (body) {
|
||||
body.text().then(function (text) {
|
||||
let pre;
|
||||
let result = {
|
||||
name: "",
|
||||
url: "",
|
||||
location: { lat: 0, lng: 0 },
|
||||
};
|
||||
try {
|
||||
const body = text;
|
||||
|
||||
let part = body.split("<body>")[1];
|
||||
camPart = part.split('<div class="big">')[1];
|
||||
camPart = camPart.split('<img src="')[1];
|
||||
camPart = camPart.split('"')[0];
|
||||
result.url = camPart;
|
||||
part = part.split("Facility")[1];
|
||||
part = part.split("Rating")[0];
|
||||
part = part.split('<label class="right">')[1];
|
||||
pre = part;
|
||||
|
||||
result.name = part.split("</label></div>")[0];
|
||||
result.name = result.name
|
||||
.replaceAll("\t", "")
|
||||
.replaceAll(" ", "");
|
||||
part = part.split('<label class="right geo">')[1];
|
||||
|
||||
part = part.split('<span class="latitude">')[1];
|
||||
result.location.lat = parseFloat(
|
||||
part.split("</span>")[0]
|
||||
);
|
||||
part = part.split('<span class="longitude">')[1];
|
||||
result.location.lng = parseFloat(
|
||||
part.split("</span>")[0]
|
||||
);
|
||||
resolve(result);
|
||||
} catch (ex) {
|
||||
if (pre == undefined) {
|
||||
resolve([]);
|
||||
} else {
|
||||
let reqPart1;
|
||||
let reqPart2;
|
||||
try {
|
||||
reqPart1 = pre
|
||||
.split('locality">')[1]
|
||||
.split("</label>")[0];
|
||||
} catch (error) {
|
||||
reqPart1 = pre
|
||||
.split('region">')[1]
|
||||
.split("</label>")[0];
|
||||
}
|
||||
reqPart2 = pre
|
||||
.split('country-name">')[1]
|
||||
.split("</label>")[0];
|
||||
const geoCoderReq = bent(
|
||||
"https://open.mapquestapi.com/geocoding/v1/address?key=" +
|
||||
config.mapquest +
|
||||
"&location=" +
|
||||
reqPart1 +
|
||||
"," +
|
||||
reqPart2,
|
||||
"GET",
|
||||
"json",
|
||||
200
|
||||
);
|
||||
geoCoderReq().then(function (body) {
|
||||
result.location.lat =
|
||||
body.results[0].locations[0].latLng.lat;
|
||||
result.location.lng =
|
||||
body.results[0].locations[0].latLng.lng;
|
||||
resolve(result);
|
||||
});
|
||||
console.log(reqPart1 + "," + reqPart2);
|
||||
}
|
||||
}
|
||||
});
|
||||
},
|
||||
function (err) {
|
||||
console.warn(
|
||||
"[" +
|
||||
getModuleMeta().title +
|
||||
"] Was unable to resolve request with error: " +
|
||||
err
|
||||
);
|
||||
}
|
||||
);
|
||||
}, Math.floor(Math.random() * 500));
|
||||
}, 200)();
|
||||
})
|
||||
);
|
||||
let globalI = 0;
|
||||
Promise.all(camProms).then(function (result) {
|
||||
// console.log(result);
|
||||
for (elm in result) {
|
||||
const row = result[elm];
|
||||
globalI++;
|
||||
try {
|
||||
con.query(
|
||||
mysqlQuery,
|
||||
[
|
||||
row.name,
|
||||
getModuleMeta().exactName,
|
||||
row.location.lat,
|
||||
row.location.lng,
|
||||
0,
|
||||
row.url,
|
||||
],
|
||||
function (err, result) {
|
||||
if (err) {
|
||||
if (err.code == "ER_DUP_ENTRY") {
|
||||
//console.warn("[Opentopia] Skipped duplicate webcam");
|
||||
} else {
|
||||
tools.handleMysqlErrors(
|
||||
err,
|
||||
getModuleMeta().title + "L128"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
);
|
||||
} catch (error) {
|
||||
/*console.warn(
|
||||
"[Opentopia] Experienced an issue adding element to DB, error was: " +
|
||||
error
|
||||
);*/
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function queryData(boundNorthEastLatLng, boundSouthWestLatLng, minimal) {
|
||||
return new Promise(function (resolve, reject) {
|
||||
// The Promise constructor should catch any errors thrown on
|
||||
// this tick. Alternately, try/catch and reject(err) on catch.
|
||||
var connection = con;
|
||||
|
||||
let elementsToQuery = "*";
|
||||
if (minimal) {
|
||||
elementsToQuery = "id,loc,source_id";
|
||||
}
|
||||
|
||||
const sqlQuery = " \
|
||||
SELECT " + elementsToQuery + " \
|
||||
FROM opentopia \
|
||||
WHERE MBRContains( \
|
||||
GeomFromText( 'LINESTRING(? ?,? ?)' ), \
|
||||
loc)";
|
||||
|
||||
|
||||
con.query(sqlQuery, [boundNorthEastLatLng.lat, boundNorthEastLatLng.lng, boundSouthWestLatLng.lat, boundSouthWestLatLng.lng], function (err, rows, fields) {
|
||||
// Call reject on error states,
|
||||
// call resolve with results
|
||||
if (err) {
|
||||
tools.handleMysqlErrors(err, getModuleMeta().title + "-L187");
|
||||
return reject(err);
|
||||
}
|
||||
let results = [];
|
||||
for (let e = 0; e < rows.length; e++) {
|
||||
const currentRow = rows[e];
|
||||
element = {
|
||||
type: "webcam-iframe",
|
||||
source: "Opentopia",
|
||||
titel: currentRow.name,
|
||||
description: "An example webcam with a url",
|
||||
url: currentRow.link,
|
||||
uid: "opentopia_" + currentRow.id,
|
||||
location: currentRow.loc,
|
||||
icon: "cctvIcon",
|
||||
};
|
||||
|
||||
if (minimal) {
|
||||
const stripableElements = ["title", "description", "url", "type"];
|
||||
for (h in stripableElements) {
|
||||
delete element[stripableElements[h]];
|
||||
}
|
||||
}
|
||||
|
||||
results.push(element);
|
||||
}
|
||||
resolve(results);
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function queryItem(uid) {
|
||||
const uidParts = uid.split("_");
|
||||
|
||||
const sqlQuery = "SELECT * FROM " + getModuleMeta().exactName + " WHERE id=?";
|
||||
return new Promise(function (resolve, reject) {
|
||||
con.query(sqlQuery, [parseInt(uidParts[1])], function (err, rows, fields) {
|
||||
if (err) {
|
||||
tools.handleMysqlErrors(err, getModuleMeta().title + "L230");
|
||||
return reject(err);
|
||||
}
|
||||
const results = [];
|
||||
for (let e = 0; e < rows.length; e++) {
|
||||
const currentRow = rows[e];
|
||||
const element = {
|
||||
type: "webcam-iframe",
|
||||
source: "Opentopia",
|
||||
titel: currentRow.name,
|
||||
description: "An example webcam with a url",
|
||||
url: currentRow.link,
|
||||
uid: "opentopia_" + currentRow.id,
|
||||
location: currentRow.loc,
|
||||
icon: "cctvIcon",
|
||||
};
|
||||
results.push(element);
|
||||
}
|
||||
resolve(results);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function getModuleMeta() {
|
||||
return {
|
||||
title: "Opentopia",
|
||||
exactName: "opentopia",
|
||||
country: ["*"],
|
||||
tags: ["webcam"],
|
||||
limited: false,
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = { initialize, queryData, getModuleMeta, queryItem };
|
||||
Reference in New Issue
Block a user