318 lines
10 KiB
JavaScript
318 lines
10 KiB
JavaScript
|
var fs = require("fs");
|
||
|
|
const bent = require("bent");
|
||
|
|
const mysql = require("mysql");
|
||
|
|
const tools = require("../../functions");
|
||
|
|
const _ = require("underscore");
|
||
|
|
|
||
|
|
let con = undefined;
|
||
|
|
let config;
|
||
|
|
|
||
|
|
function requestWithPromise(n) {
|
||
|
|
const listURL =
|
||
|
|
"http://www.opentopia.com/hiddencam.php?showmode=standard&country=%2A&seewhat=highlyrated&p=";
|
||
|
|
return new Promise(function (resolve, reject) {
|
||
|
|
const request = bent(listURL + n, "GET", 200);
|
||
|
|
request().then(
|
||
|
|
function (body) {
|
||
|
|
body.text().then(function (text) {
|
||
|
|
resolve(text);
|
||
|
|
});
|
||
|
|
},
|
||
|
|
function (err) {
|
||
|
|
console.warn(
|
||
|
|
"[" +
|
||
|
|
getModuleMeta().title +
|
||
|
|
"] Was unable to resolve request with error: " +
|
||
|
|
err
|
||
|
|
);
|
||
|
|
}
|
||
|
|
);
|
||
|
|
});
|
||
|
|
}
|
||
|
|
|
||
|
|
function initialize(connection, configL, replaceOld = false) {
|
||
|
|
con = connection;
|
||
|
|
config = configL;
|
||
|
|
if(!fs.existsSync("./apiHandler/opentopia/FLAG")){
|
||
|
|
fs.writeFileSync("./apiHandler/opentopia/FLAG", "")
|
||
|
|
replaceOld = true
|
||
|
|
}
|
||
|
|
if (replaceOld) { // Just do it once, calling that API ain't cheap
|
||
|
|
var array = fs
|
||
|
|
.readFileSync("./apiHandler/opentopia/db_table_config.sql")
|
||
|
|
.toString()
|
||
|
|
.split("\n");
|
||
|
|
|
||
|
|
for (i in array) {
|
||
|
|
con.query(array[i], function (err, result) {
|
||
|
|
tools.handleMysqlErrors(err, getModuleMeta().title + "L32");
|
||
|
|
console.log("Table created for " + getModuleMeta().title);
|
||
|
|
});
|
||
|
|
}
|
||
|
|
|
||
|
|
let allProms = [];
|
||
|
|
|
||
|
|
for (let n = 0; n < 26; n++) {
|
||
|
|
allProms.push(requestWithPromise(n));
|
||
|
|
}
|
||
|
|
|
||
|
|
const mysqlQuery =
|
||
|
|
"INSERT INTO opentopia (name, source_id, loc, protocol_id, link) VALUES (?, ?, POINT(?, ?), ?, ?)";
|
||
|
|
|
||
|
|
Promise.all(allProms).then(function (res) {
|
||
|
|
for (l in res) {
|
||
|
|
const baseurl = "http://www.opentopia.com/webcam/";
|
||
|
|
let allResults = [];
|
||
|
|
// Process the pages
|
||
|
|
let part = res[l].split("<body>")[1];
|
||
|
|
part = part.split("/webcam/");
|
||
|
|
part.shift();
|
||
|
|
for (h in part) {
|
||
|
|
part[h] = part[h].replace("\t", "");
|
||
|
|
part[h] = part[h]
|
||
|
|
.split('target="_self">')[0]
|
||
|
|
.replace('"', "")
|
||
|
|
.split(">")[0];
|
||
|
|
}
|
||
|
|
|
||
|
|
pages = part;
|
||
|
|
camProms = [];
|
||
|
|
for (let pI = 0; pI < pages.length; pI++) {
|
||
|
|
camProms.push(
|
||
|
|
new Promise(function (resolve, reject) {
|
||
|
|
const url = baseurl + pages[pI] + "?viewmode=livevideo";
|
||
|
|
_.throttle(function () {
|
||
|
|
setTimeout(function () {
|
||
|
|
const request = bent(url, "GET", 200);
|
||
|
|
request().then(
|
||
|
|
function (body) {
|
||
|
|
body.text().then(function (text) {
|
||
|
|
let pre;
|
||
|
|
let result = {
|
||
|
|
name: "",
|
||
|
|
url: "",
|
||
|
|
location: { lat: 0, lng: 0 },
|
||
|
|
};
|
||
|
|
try {
|
||
|
|
const body = text;
|
||
|
|
|
||
|
|
let part = body.split("<body>")[1];
|
||
|
|
camPart = part.split('<div class="big">')[1];
|
||
|
|
camPart = camPart.split('<img src="')[1];
|
||
|
|
camPart = camPart.split('"')[0];
|
||
|
|
result.url = camPart;
|
||
|
|
part = part.split("Facility")[1];
|
||
|
|
part = part.split("Rating")[0];
|
||
|
|
part = part.split('<label class="right">')[1];
|
||
|
|
pre = part;
|
||
|
|
|
||
|
|
result.name = part.split("</label></div>")[0];
|
||
|
|
result.name = result.name
|
||
|
|
.replaceAll("\t", "")
|
||
|
|
.replaceAll(" ", "");
|
||
|
|
part = part.split('<label class="right geo">')[1];
|
||
|
|
|
||
|
|
part = part.split('<span class="latitude">')[1];
|
||
|
|
result.location.lat = parseFloat(
|
||
|
|
part.split("</span>")[0]
|
||
|
|
);
|
||
|
|
part = part.split('<span class="longitude">')[1];
|
||
|
|
result.location.lng = parseFloat(
|
||
|
|
part.split("</span>")[0]
|
||
|
|
);
|
||
|
|
resolve(result);
|
||
|
|
} catch (ex) {
|
||
|
|
if (pre == undefined) {
|
||
|
|
resolve([]);
|
||
|
|
} else {
|
||
|
|
let reqPart1;
|
||
|
|
let reqPart2;
|
||
|
|
try {
|
||
|
|
reqPart1 = pre
|
||
|
|
.split('locality">')[1]
|
||
|
|
.split("</label>")[0];
|
||
|
|
} catch (error) {
|
||
|
|
reqPart1 = pre
|
||
|
|
.split('region">')[1]
|
||
|
|
.split("</label>")[0];
|
||
|
|
}
|
||
|
|
reqPart2 = pre
|
||
|
|
.split('country-name">')[1]
|
||
|
|
.split("</label>")[0];
|
||
|
|
const geoCoderReq = bent(
|
||
|
|
"https://open.mapquestapi.com/geocoding/v1/address?key=" +
|
||
|
|
config.mapquest +
|
||
|
|
"&location=" +
|
||
|
|
reqPart1 +
|
||
|
|
"," +
|
||
|
|
reqPart2,
|
||
|
|
"GET",
|
||
|
|
"json",
|
||
|
|
200
|
||
|
|
);
|
||
|
|
geoCoderReq().then(function (body) {
|
||
|
|
result.location.lat =
|
||
|
|
body.results[0].locations[0].latLng.lat;
|
||
|
|
result.location.lng =
|
||
|
|
body.results[0].locations[0].latLng.lng;
|
||
|
|
resolve(result);
|
||
|
|
});
|
||
|
|
console.log(reqPart1 + "," + reqPart2);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
});
|
||
|
|
},
|
||
|
|
function (err) {
|
||
|
|
console.warn(
|
||
|
|
"[" +
|
||
|
|
getModuleMeta().title +
|
||
|
|
"] Was unable to resolve request with error: " +
|
||
|
|
err
|
||
|
|
);
|
||
|
|
}
|
||
|
|
);
|
||
|
|
}, Math.floor(Math.random() * 500));
|
||
|
|
}, 200)();
|
||
|
|
})
|
||
|
|
);
|
||
|
|
let globalI = 0;
|
||
|
|
Promise.all(camProms).then(function (result) {
|
||
|
|
// console.log(result);
|
||
|
|
for (elm in result) {
|
||
|
|
const row = result[elm];
|
||
|
|
globalI++;
|
||
|
|
try {
|
||
|
|
con.query(
|
||
|
|
mysqlQuery,
|
||
|
|
[
|
||
|
|
row.name,
|
||
|
|
getModuleMeta().exactName,
|
||
|
|
row.location.lat,
|
||
|
|
row.location.lng,
|
||
|
|
0,
|
||
|
|
row.url,
|
||
|
|
],
|
||
|
|
function (err, result) {
|
||
|
|
if (err) {
|
||
|
|
if (err.code == "ER_DUP_ENTRY") {
|
||
|
|
//console.warn("[Opentopia] Skipped duplicate webcam");
|
||
|
|
} else {
|
||
|
|
tools.handleMysqlErrors(
|
||
|
|
err,
|
||
|
|
getModuleMeta().title + "L128"
|
||
|
|
);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
);
|
||
|
|
} catch (error) {
|
||
|
|
/*console.warn(
|
||
|
|
"[Opentopia] Experienced an issue adding element to DB, error was: " +
|
||
|
|
error
|
||
|
|
);*/
|
||
|
|
}
|
||
|
|
}
|
||
|
|
});
|
||
|
|
}
|
||
|
|
}
|
||
|
|
});
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
function queryData(boundNorthEastLatLng, boundSouthWestLatLng, minimal) {
|
||
|
|
return new Promise(function (resolve, reject) {
|
||
|
|
// The Promise constructor should catch any errors thrown on
|
||
|
|
// this tick. Alternately, try/catch and reject(err) on catch.
|
||
|
|
var connection = con;
|
||
|
|
|
||
|
|
let elementsToQuery = "*";
|
||
|
|
if (minimal) {
|
||
|
|
elementsToQuery = "id,loc,source_id";
|
||
|
|
}
|
||
|
|
|
||
|
|
const sqlQuery = " \
|
||
|
|
SELECT " + elementsToQuery + " \
|
||
|
|
FROM opentopia \
|
||
|
|
WHERE MBRContains( \
|
||
|
|
GeomFromText( 'LINESTRING(? ?,? ?)' ), \
|
||
|
|
loc)";
|
||
|
|
|
||
|
|
|
||
|
|
con.query(sqlQuery, [boundNorthEastLatLng.lat, boundNorthEastLatLng.lng, boundSouthWestLatLng.lat, boundSouthWestLatLng.lng], function (err, rows, fields) {
|
||
|
|
// Call reject on error states,
|
||
|
|
// call resolve with results
|
||
|
|
if (err) {
|
||
|
|
tools.handleMysqlErrors(err, getModuleMeta().title + "-L187");
|
||
|
|
return reject(err);
|
||
|
|
}
|
||
|
|
let results = [];
|
||
|
|
for (let e = 0; e < rows.length; e++) {
|
||
|
|
const currentRow = rows[e];
|
||
|
|
element = {
|
||
|
|
type: "webcam-iframe",
|
||
|
|
source: "Opentopia",
|
||
|
|
titel: currentRow.name,
|
||
|
|
description: "An example webcam with a url",
|
||
|
|
url: currentRow.link,
|
||
|
|
uid: "opentopia_" + currentRow.id,
|
||
|
|
location: currentRow.loc,
|
||
|
|
icon: "cctvIcon",
|
||
|
|
};
|
||
|
|
|
||
|
|
if (minimal) {
|
||
|
|
const stripableElements = ["title", "description", "url", "type"];
|
||
|
|
for (h in stripableElements) {
|
||
|
|
delete element[stripableElements[h]];
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
results.push(element);
|
||
|
|
}
|
||
|
|
resolve(results);
|
||
|
|
}
|
||
|
|
);
|
||
|
|
});
|
||
|
|
}
|
||
|
|
|
||
|
|
function queryItem(uid) {
|
||
|
|
const uidParts = uid.split("_");
|
||
|
|
|
||
|
|
const sqlQuery = "SELECT * FROM " + getModuleMeta().exactName + " WHERE id=?";
|
||
|
|
return new Promise(function (resolve, reject) {
|
||
|
|
con.query(sqlQuery, [parseInt(uidParts[1])], function (err, rows, fields) {
|
||
|
|
if (err) {
|
||
|
|
tools.handleMysqlErrors(err, getModuleMeta().title + "L230");
|
||
|
|
return reject(err);
|
||
|
|
}
|
||
|
|
const results = [];
|
||
|
|
for (let e = 0; e < rows.length; e++) {
|
||
|
|
const currentRow = rows[e];
|
||
|
|
const element = {
|
||
|
|
type: "webcam-iframe",
|
||
|
|
source: "Opentopia",
|
||
|
|
titel: currentRow.name,
|
||
|
|
description: "An example webcam with a url",
|
||
|
|
url: currentRow.link,
|
||
|
|
uid: "opentopia_" + currentRow.id,
|
||
|
|
location: currentRow.loc,
|
||
|
|
icon: "cctvIcon",
|
||
|
|
};
|
||
|
|
results.push(element);
|
||
|
|
}
|
||
|
|
resolve(results);
|
||
|
|
});
|
||
|
|
});
|
||
|
|
}
|
||
|
|
|
||
|
|
function getModuleMeta() {
|
||
|
|
return {
|
||
|
|
title: "Opentopia",
|
||
|
|
exactName: "opentopia",
|
||
|
|
country: ["*"],
|
||
|
|
tags: ["webcam"],
|
||
|
|
limited: false,
|
||
|
|
};
|
||
|
|
}
|
||
|
|
|
||
|
|
module.exports = { initialize, queryData, getModuleMeta, queryItem };
|