coder-xiaomo/nuclear_data_spider
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity GitHub Gitee

提交代码

Browse Source
This commit is contained in:
程序员小墨 2023-08-27 22:30:32 +08:00
commit 1c76c2cd8c
9 changed files with 1402 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored Normal file
View File

@ -0,0 +1 @@
node_modules

40
README.md Normal file
View File

@ -0,0 +1,40 @@
# 辐射环境历史数据查询
> 可查询自 2023-08-25 起的国家核安全局辐射环境监测数据历史数据
数据来源：[国家核安全局](https://nnsa.mee.gov.cn/) - [辐射环境监测数据](https://data.rmtc.org.cn/gis)
视频版说明(B站)：[<链接>]()
## 目录结构
- 后端服务：backend
- 前端页面：html
- 数据采集：spider
- 数据库表结构：sql
## 项目环境
数据库使用 MySQL 8.0，逻辑上讲 MySQL 5.7 版本应该也能使用，但我没有测试过。
后端和数据采集端需要在 node 环境下运行，初次使用需要 `npm i` 安装依赖。
## 一些说明
这个项目代码写的比较烂，毕竟只是一个数据采集工具，除非官方网站更新，否则后期应该不会改动太多。大家如果要看代码，可以带着审视的眼光来看，相信大家能够写出比我的更优雅的代码。
## 免责声明
本站仅记录自 2023-08-25 起的辐射环境监测数据历史值，不对数据来源真实性做保证，数据使用权请查阅上方数据来源自行获取，仅通过技术层面保存监测数据历史记录，方便学术研究使用。

208
backend/server.js Normal file
View File

@ -0,0 +1,208 @@
const express = require('express');
const mysql = require('mysql');
const app = express();
// 创建连接池
const pool = mysql.createPool({
connectionLimit: 5, // 设置连接池的大小
host: 'localhost',
user: 'root',
password: '123456',
database: 'open_data'
});
// 设置跨域访问
app.all('*', function (req, res, next) {
res.header('Access-Control-Allow-Origin', '*');
res.header('Access-Control-Allow-Headers', 'Content-Type');
res.header('Access-Control-Allow-Methods', '*');
next();
});
app.use(express.json())
app.use(express.urlencoded({ extended: false }))
// app.get('/nuclear_data/options', function (req, res) {
// // 从连接池中获取一个连接
// pool.getConnection(function (err, connection) {
// if (err) {
// console.error(err);
// res.status(500).send('服务器错误');
// } else {
// // 查询数据库，获取所有的名称和位置，去重
// // let sql = `SELECT DISTINCT name, location FROM nuclear_data`;
// let sql = `SELECT DISTINCT name FROM nuclear_data`;
// connection.query(sql, function (err, result) {
// if (err) {
// console.error(err);
// res.status(500).send('服务器错误');
// } else {
// // 返回查询结果
// res.json(result);
// }
// // 释放连接
// connection.release();
// });
// }
// });
// });
app.post('/nuclear_data/options/names', function (req, res) {
// 获取请求参数
let type = req.body?.type;
// 从连接池中获取一个连接
pool.getConnection(function (err, connection) {
if (err) {
console.error(err);
res.status(500).send('服务器错误');
} else {
// 查询数据库，获取指定名称下的所有位置，去重
let sql = `SELECT DISTINCT name FROM nuclear_data WHERE type = ? ORDER BY name`;
connection.query(sql, [type], function (err, result) {
if (err) {
console.error(err);
res.status(500).send('服务器错误');
} else {
// 返回查询结果
res.json(result);
}
// 释放连接
connection.release();
});
}
});
});
app.post('/nuclear_data/options/locations', function (req, res) {
// 获取请求参数
let type = req.body?.type;
let name = req.body?.name;
// 从连接池中获取一个连接
pool.getConnection(function (err, connection) {
if (err) {
console.error(err);
res.status(500).send('服务器错误');
} else {
// 查询数据库，获取指定名称下的所有位置，去重
let sql = `SELECT DISTINCT location FROM nuclear_data WHERE type = ? AND name = ? ORDER BY location`;
connection.query(sql, [type, name], function (err, result) {
if (err) {
console.error(err);
res.status(500).send('服务器错误');
} else {
// 返回查询结果
res.json(result);
}
// 释放连接
connection.release();
});
}
});
});
app.post('/nuclear_data', function (req, res) {
// 获取请求参数
let name = req.body?.name;
let location = req.body?.location;
let date = req.body?.date;
// console.log('param', name, location, date)
// 从连接池中获取一个连接
pool.getConnection(function (err, connection) {
if (err) {
console.error(err);
res.status(500).send('服务器错误');
} else {
// 查询数据库
// let sql = `SELECT * FROM nuclear_data WHERE name = ? AND location = ? AND date = ?`;
let sql = `SELECT * FROM nuclear_data WHERE name = ? AND location = ? AND date BETWEEN DATE_SUB(?, INTERVAL 3 DAY) AND ? ORDER BY date`;
connection.query(sql, [name, location, date, date], function (err, result) {
if (err) {
console.error(err);
res.status(500).send('服务器错误');
} else {
// 返回查询结果
res.json(result);
}
// 释放连接
connection.release();
});
}
});
});
// 开放接口
app.post('/api/nuclear_data/export', function (req, res) {
// 获取请求参数
let year = req.body?.year;
let month = req.body?.month;
// let day = req.body?.day;
if (!year || !month || year.indexOf('.') != -1 || month.indexOf('.') != -1) {
res.status(400).send('参数错误');
}
year = Number(year);
month = Number(month);
if (isNaN(year) || isNaN(month) || year < 2023 || year > 3000 || month < 1 || month > 12) {
res.status(400).send('参数错误');
return
}
// if (day) {
// if (day.indexOf('.') != -1) {
// res.status(400).send('参数错误');
// }
// day = Number(day);
// if (isNaN(day) || day < 1 || day > 31) {
// res.status(400).send('参数错误');
// return
// }
// }
// console.log('param', name, location, date)
// 从连接池中获取一个连接
pool.getConnection(function (err, connection) {
if (err) {
console.error(err);
res.status(500).send('服务器错误');
} else {
// 查询数据库
let sql = `SELECT date,type,name,location,value FROM nuclear_data WHERE YEAR(date) = ? AND MONTH(date) = ? ORDER BY date,type,name,location`;
connection.query(sql, [year, month], function (err, result) {
if (err) {
console.error(err);
res.status(500).send('服务器错误');
} else {
// 返回查询结果
res.json(result.map(i => {
if (i.date) {
let date = new Date(i.date)
let year = date.getFullYear();
let month = date.getMonth() + 1;
if (month < 10) month = `0${month}`
let day = date.getDate();
if (day < 10) day = `0${day}`
i.date = year + "-" + month + "-" + day;
// i.date2 = new Date(new Date(i.date).getTime() + 8 * 3600 * 1000).toISOString().substring(0, 10)
}
return i
}));
}
// 释放连接
connection.release();
});
}
});
});
// 启动服务器
app.listen(3000, function () {
console.log('Server is running on port 3000');
});

204
html/index.html Normal file
View File

@ -0,0 +1,204 @@
<html>
<head>
<meta charset="utf-8">
<title>辐射环境历史数据查询</title>
</head>
<body>
<h1>辐射环境历史数据查询</h1>
<p>可查询自 2023-08-25 起的国家核安全局辐射环境监测数据历史数据</p>
<hr>
<form id="form">
<label for="type">监测点：</label>
<select id="type" name="type" required>
<option value="0">环境质量监测点</option>
<option value="1">核电厂监测点</option>
</select><br>
<label for="name">名称：</label>
<select id="name" name="name" required></select><br>
<label for="location">位置：</label>
<select id="location" name="location" required></select><br>
<label for="date">日期：</label>
<input type="date" id="date" name="date" required min="2023-08-25"><br>
<button type="submit" id="submit-btn">查询</button>
</form>
<hr>
<div id="result"></div>
<div class="footer">
<p>
开放接口：按月批量查询所有点位数据接口<br>
备注：目前数据较少。
</p>
<p>
开源项目：本项目由程序员小墨开发并开源，仅作学习研究使用。<br>
<a target="_blank" href="#">GitHub</a>
<a target="_blank" href="#">Gitee</a>
<a target="_blank" href="#">自建仓库</a>
</p>
<p>
特别说明：请勿使用爬虫爬取本站接口，感谢！如需全部历史数据，请<a href="https://www.only4.work/about/feedback/"
target="_blank">点击留言</a>，我会尽量在当日将数据打包发送回您的邮箱。
</p>
<p>
</p>
<p>
数据来源：<a href="https://nnsa.mee.gov.cn/" target="_blank">国家核安全局</a>
-
<a href="https://data.rmtc.org.cn/gis" target="_blank">辐射环境监测数据</a>
</p>
<p>
免责声明：本站仅记录自 2023-08-25 起的辐射环境监测数据历史值，不对数据来源真实性做保证，数据使用权请查阅上方数据来源自行获取，仅通过技术层面保存监测数据历史记录，方便学术研究使用。
</p>
</div>
<script src="./jquery.min.js"></script>
<!-- <script src="https://cdn.bootcdn.net/ajax/libs/jquery/3.6.0/jquery.min.js"></script> -->
<script>
function setDefalutValue() {
// 日期当前值 2天前
var today = new Date(Date.now() - 2 * 24 * 3600 * 1000); // 获取当前的日期对象
var day = today.getDate(); // 获取当前的日
var month = today.getMonth() + 1; // 获取当前的月，需要加1
var year = today.getFullYear(); // 获取当前的年
if (day < 10) { // 如果日小于10，前面补0
day = "0" + day;
}
if (month < 10) { // 如果月小于10，前面补0
month = "0" + month;
}
var date = year + "-" + month + "-" + day; // 拼接成 yyyy-MM-dd 格式的字符串
document.getElementById("date").value = date; // 将字符串赋值给日期输入框
// 日期最大值 1天前
var today = new Date(Date.now() - 24 * 3600 * 1000); // 获取当前的日期对象
var day = today.getDate(); // 获取当前的日
var month = today.getMonth() + 1; // 获取当前的月，需要加1
var year = today.getFullYear(); // 获取当前的年
if (day < 10) { // 如果日小于10，前面补0
day = "0" + day;
}
if (month < 10) { // 如果月小于10，前面补0
month = "0" + month;
}
var date = year + "-" + month + "-" + day; // 拼接成 yyyy-MM-dd 格式的字符串
document.getElementById("date").max = date; // 将字符串赋值给日期输入框
}
// // 在页面加载时发送get请求，获取所有的名称和位置，并填充到下拉框中
// $(document).ready(function () {
// $.get('http://localhost:3000/nuclear_data/options', function (res) {
// // 遍历返回的数组，将每个名称和位置添加到对应的下拉框中
// for (let item of res) {
// $('#name').append(`<option value="${item.name}">${item.name}</option>`);
// $('#location').append(`<option value="${item.location}">${item.location}</option>`);
// }
// });
// });
// // 在页面加载时发送get请求，获取所有的名称，并填充到名称的下拉框中
// $(document).ready(function () {
// $.get('http://localhost:3000/nuclear_data/options', function (res) {
// // 遍历返回的数组，将每个名称添加到名称的下拉框中
// for (let item of res) {
// $('#name').append(`<option value="${item.name}">${item.name}</option>`);
// }
// // 触发名称的下拉框的change事件，以获取对应的位置
// $('#name').trigger('change');
// });
// });
// 在页面加载时发送get请求，获取所有的名称，并填充到名称的下拉框中
$(document).ready(function () {
setDefalutValue()
// 触发名称的下拉框的change事件，以获取对应的位置
$('#type').trigger('change');
});
// 绑定名称的下拉框的change事件，当选择某个名称时，发送post请求，获取对应的名称，并填充到名称的下拉框中
$('#type').on('change', function () {
// 获取选择的名称
let type = $(this).val();
// 发送post请求
$.post('http://localhost:3000/nuclear_data/options/names', { type: type }, function (res) {
// 清空位置的下拉框
$('#name').empty();
// 遍历返回的数组，将每个位置添加到位置的下拉框中
for (let item of res) {
$('#name').append(`<option value="${item.name}">${item.name}</option>`);
}
// 触发名称的下拉框的change事件，以获取对应的位置
$('#name').trigger('change');
});
});
// 绑定名称的下拉框的change事件，当选择某个名称时，发送post请求，获取对应的位置，并填充到位置的下拉框中
$('#name').on('change', function () {
// 获取选择的名称
let type = $('#type').val();
let name = $(this).val();
// 发送post请求
$.post('http://localhost:3000/nuclear_data/options/locations', { type: type, name: name }, function (res) {
// 清空位置的下拉框
$('#location').empty();
// 遍历返回的数组，将每个位置添加到位置的下拉框中
for (let item of res) {
$('#location').append(`<option value="${item.location}">${item.location}</option>`);
}
});
});
$('#location').on('change', function () {
$('#submit-btn').trigger('click');
});
$('#date').on('change', function () {
$('#submit-btn').trigger('click');
});
function rtime(timeStr) {
let date = new Date(timeStr)
// return new Date(date.getTime() + 8 * 3600 * 1000).toISOString().replace('T', ' ').replace('Z', '')
return new Date(date.getTime() + 8 * 3600 * 1000).toISOString().substring(0, 10)
}
// 绑定表单提交事件
$('#form').on('submit', function (e) {
e.preventDefault(); // 阻止默认行为
// 获取表单数据
let data = $(this).serialize();
// 发送post请求
$.post('http://localhost:3000/nuclear_data', data, function (res) {
// 清空结果区域
$('#result').empty();
// 判断是否有数据返回
if (res.length > 0) {
// 遍历返回的数组，显示每条数据的信息
for (let item of res) {
$('#result').append(`<p>名称：${item.name}</p>`);
$('#result').append(`<p>位置：${item.location}</p>`);
$('#result').append(`<p>值：${item.value}</p>`);
$('#result').append(`<p>日期：${rtime(item.date)} <span style="color: grey; font-size: small;">(${item.date})</span></p>`);
$('#result').append(`<p>类型：${item.type}</p>`);
$('#result').append(`<p>数据更新日期：${item.time || '暂无'}</p>`);
$('#result').append(`<hr>`); // 分隔线
}
} else {
// 没有数据返回，显示提示信息
$('#result').text('没有找到符合条件的数据');
}
});
});
</script>
</body>
</html>

2
html/jquery.min.js vendored Normal file
View File

File diff suppressed because one or more lines are too long

669
package-lock.json generated Normal file
View File

@ -0,0 +1,669 @@
{
"name": "数据爬取",
"version": "1.0.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"version": "1.0.0",
"license": "ISC",
"dependencies": {
"express": "^4.18.2",
"mysql": "^2.18.1",
"xmldom": "^0.6.0",
"xpath": "^0.0.33"
}
},
"node_modules/accepts": {
"version": "1.3.8",
"resolved": "https://registry.npmmirror.com/accepts/-/accepts-1.3.8.tgz",
"integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==",
"dependencies": {
"mime-types": "~2.1.34",
"negotiator": "0.6.3"
},
"engines": {
"node": ">= 0.6"
}
},
"node_modules/array-flatten": {
"version": "1.1.1",
"resolved": "https://registry.npmmirror.com/array-flatten/-/array-flatten-1.1.1.tgz",
"integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg=="
},
"node_modules/bignumber.js": {
"version": "9.0.0",
"resolved": "https://registry.npmmirror.com/bignumber.js/-/bignumber.js-9.0.0.tgz",
"integrity": "sha512-t/OYhhJ2SD+YGBQcjY8GzzDHEk9f3nerxjtfa6tlMXfe7frs/WozhvCNoGvpM0P3bNf3Gq5ZRMlGr5f3r4/N8A==",
"engines": {
"node": "*"
}
},
"node_modules/body-parser": {
"version": "1.20.1",
"resolved": "https://registry.npmmirror.com/body-parser/-/body-parser-1.20.1.tgz",
"integrity": "sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==",
"dependencies": {
"bytes": "3.1.2",
"content-type": "~1.0.4",
"debug": "2.6.9",
"depd": "2.0.0",
"destroy": "1.2.0",
"http-errors": "2.0.0",
"iconv-lite": "0.4.24",
"on-finished": "2.4.1",
"qs": "6.11.0",
"raw-body": "2.5.1",
"type-is": "~1.6.18",
"unpipe": "1.0.0"
},
"engines": {
"node": ">= 0.8",
"npm": "1.2.8000 || >= 1.4.16"
}
},
"node_modules/bytes": {
"version": "3.1.2",
"resolved": "https://registry.npmmirror.com/bytes/-/bytes-3.1.2.tgz",
"integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==",
"engines": {
"node": ">= 0.8"
}
},
"node_modules/call-bind": {
"version": "1.0.2",
"resolved": "https://registry.npmmirror.com/call-bind/-/call-bind-1.0.2.tgz",
"integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==",
"dependencies": {
"function-bind": "^1.1.1",
"get-intrinsic": "^1.0.2"
}
},
"node_modules/content-disposition": {
"version": "0.5.4",
"resolved": "https://registry.npmmirror.com/content-disposition/-/content-disposition-0.5.4.tgz",
"integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==",
"dependencies": {
"safe-buffer": "5.2.1"
},
"engines": {
"node": ">= 0.6"
}
},
"node_modules/content-disposition/node_modules/safe-buffer": {
"version": "5.2.1",
"resolved": "https://registry.npmmirror.com/safe-buffer/-/safe-buffer-5.2.1.tgz",
"integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="
},
"node_modules/content-type": {
"version": "1.0.5",
"resolved": "https://registry.npmmirror.com/content-type/-/content-type-1.0.5.tgz",
"integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==",
"engines": {
"node": ">= 0.6"
}
},
"node_modules/cookie": {
"version": "0.5.0",
"resolved": "https://registry.npmmirror.com/cookie/-/cookie-0.5.0.tgz",
"integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==",
"engines": {
"node": ">= 0.6"
}
},
"node_modules/cookie-signature": {
"version": "1.0.6",
"resolved": "https://registry.npmmirror.com/cookie-signature/-/cookie-signature-1.0.6.tgz",
"integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ=="
},
"node_modules/core-util-is": {
"version": "1.0.3",
"resolved": "https://registry.npmmirror.com/core-util-is/-/core-util-is-1.0.3.tgz",
"integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ=="
},
"node_modules/debug": {
"version": "2.6.9",
"resolved": "https://registry.npmmirror.com/debug/-/debug-2.6.9.tgz",
"integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
"dependencies": {
"ms": "2.0.0"
}
},
"node_modules/depd": {
"version": "2.0.0",
"resolved": "https://registry.npmmirror.com/depd/-/depd-2.0.0.tgz",
"integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==",
"engines": {
"node": ">= 0.8"
}
},
"node_modules/destroy": {
"version": "1.2.0",
"resolved": "https://registry.npmmirror.com/destroy/-/destroy-1.2.0.tgz",
"integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==",
"engines": {
"node": ">= 0.8",
"npm": "1.2.8000 || >= 1.4.16"
}
},
"node_modules/ee-first": {
"version": "1.1.1",
"resolved": "https://registry.npmmirror.com/ee-first/-/ee-first-1.1.1.tgz",
"integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow=="
},
"node_modules/encodeurl": {
"version": "1.0.2",
"resolved": "https://registry.npmmirror.com/encodeurl/-/encodeurl-1.0.2.tgz",
"integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==",
"engines": {
"node": ">= 0.8"
}
},
"node_modules/escape-html": {
"version": "1.0.3",
"resolved": "https://registry.npmmirror.com/escape-html/-/escape-html-1.0.3.tgz",
"integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow=="
},
"node_modules/etag": {
"version": "1.8.1",
"resolved": "https://registry.npmmirror.com/etag/-/etag-1.8.1.tgz",
"integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==",
"engines": {
"node": ">= 0.6"
}
},
"node_modules/express": {
"version": "4.18.2",
"resolved": "https://registry.npmmirror.com/express/-/express-4.18.2.tgz",
"integrity": "sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==",
"dependencies": {
"accepts": "~1.3.8",
"array-flatten": "1.1.1",
"body-parser": "1.20.1",
"content-disposition": "0.5.4",
"content-type": "~1.0.4",
"cookie": "0.5.0",
"cookie-signature": "1.0.6",
"debug": "2.6.9",
"depd": "2.0.0",
"encodeurl": "~1.0.2",
"escape-html": "~1.0.3",
"etag": "~1.8.1",
"finalhandler": "1.2.0",
"fresh": "0.5.2",
"http-errors": "2.0.0",
"merge-descriptors": "1.0.1",
"methods": "~1.1.2",
"on-finished": "2.4.1",
"parseurl": "~1.3.3",
"path-to-regexp": "0.1.7",
"proxy-addr": "~2.0.7",
"qs": "6.11.0",
"range-parser": "~1.2.1",
"safe-buffer": "5.2.1",
"send": "0.18.0",
"serve-static": "1.15.0",
"setprototypeof": "1.2.0",
"statuses": "2.0.1",
"type-is": "~1.6.18",
"utils-merge": "1.0.1",
"vary": "~1.1.2"
},
"engines": {
"node": ">= 0.10.0"
}
},
"node_modules/express/node_modules/safe-buffer": {
"version": "5.2.1",
"resolved": "https://registry.npmmirror.com/safe-buffer/-/safe-buffer-5.2.1.tgz",
"integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="
},
"node_modules/finalhandler": {
"version": "1.2.0",
"resolved": "https://registry.npmmirror.com/finalhandler/-/finalhandler-1.2.0.tgz",
"integrity": "sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==",
"dependencies": {
"debug": "2.6.9",
"encodeurl": "~1.0.2",
"escape-html": "~1.0.3",
"on-finished": "2.4.1",
"parseurl": "~1.3.3",
"statuses": "2.0.1",
"unpipe": "~1.0.0"
},
"engines": {
"node": ">= 0.8"
}
},
"node_modules/forwarded": {
"version": "0.2.0",
"resolved": "https://registry.npmmirror.com/forwarded/-/forwarded-0.2.0.tgz",
"integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==",
"engines": {
"node": ">= 0.6"
}
},
"node_modules/fresh": {
"version": "0.5.2",
"resolved": "https://registry.npmmirror.com/fresh/-/fresh-0.5.2.tgz",
"integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==",
"engines": {
"node": ">= 0.6"
}
},
"node_modules/function-bind": {
"version": "1.1.1",
"resolved": "https://registry.npmmirror.com/function-bind/-/function-bind-1.1.1.tgz",
"integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A=="
},
"node_modules/get-intrinsic": {
"version": "1.2.1",
"resolved": "https://registry.npmmirror.com/get-intrinsic/-/get-intrinsic-1.2.1.tgz",
"integrity": "sha512-2DcsyfABl+gVHEfCOaTrWgyt+tb6MSEGmKq+kI5HwLbIYgjgmMcV8KQ41uaKz1xxUcn9tJtgFbQUEVcEbd0FYw==",
"dependencies": {
"function-bind": "^1.1.1",
"has": "^1.0.3",
"has-proto": "^1.0.1",
"has-symbols": "^1.0.3"
}
},
"node_modules/has": {
"version": "1.0.3",
"resolved": "https://registry.npmmirror.com/has/-/has-1.0.3.tgz",
"integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==",
"dependencies": {
"function-bind": "^1.1.1"
},
"engines": {
"node": ">= 0.4.0"
}
},
"node_modules/has-proto": {
"version": "1.0.1",
"resolved": "https://registry.npmmirror.com/has-proto/-/has-proto-1.0.1.tgz",
"integrity": "sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==",
"engines": {
"node": ">= 0.4"
}
},
"node_modules/has-symbols": {
"version": "1.0.3",
"resolved": "https://registry.npmmirror.com/has-symbols/-/has-symbols-1.0.3.tgz",
"integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==",
"engines": {
"node": ">= 0.4"
}
},
"node_modules/http-errors": {
"version": "2.0.0",
"resolved": "https://registry.npmmirror.com/http-errors/-/http-errors-2.0.0.tgz",
"integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==",
"dependencies": {
"depd": "2.0.0",
"inherits": "2.0.4",
"setprototypeof": "1.2.0",
"statuses": "2.0.1",
"toidentifier": "1.0.1"
},
"engines": {
"node": ">= 0.8"
}
},
"node_modules/iconv-lite": {
"version": "0.4.24",
"resolved": "https://registry.npmmirror.com/iconv-lite/-/iconv-lite-0.4.24.tgz",
"integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
"dependencies": {
"safer-buffer": ">= 2.1.2 < 3"
},
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/inherits": {
"version": "2.0.4",
"resolved": "https://registry.npmmirror.com/inherits/-/inherits-2.0.4.tgz",
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
},
"node_modules/ipaddr.js": {
"version": "1.9.1",
"resolved": "https://registry.npmmirror.com/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
"integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==",
"engines": {
"node": ">= 0.10"
}
},
"node_modules/isarray": {
"version": "1.0.0",
"resolved": "https://registry.npmmirror.com/isarray/-/isarray-1.0.0.tgz",
"integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ=="
},
"node_modules/media-typer": {
"version": "0.3.0",
"resolved": "https://registry.npmmirror.com/media-typer/-/media-typer-0.3.0.tgz",
"integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==",
"engines": {
"node": ">= 0.6"
}
},
"node_modules/merge-descriptors": {
"version": "1.0.1",
"resolved": "https://registry.npmmirror.com/merge-descriptors/-/merge-descriptors-1.0.1.tgz",
"integrity": "sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w=="
},
"node_modules/methods": {
"version": "1.1.2",
"resolved": "https://registry.npmmirror.com/methods/-/methods-1.1.2.tgz",
"integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==",
"engines": {
"node": ">= 0.6"
}
},
"node_modules/mime": {
"version": "1.6.0",
"resolved": "https://registry.npmmirror.com/mime/-/mime-1.6.0.tgz",
"integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==",
"bin": {
"mime": "cli.js"
},
"engines": {
"node": ">=4"
}
},
"node_modules/mime-db": {
"version": "1.52.0",
"resolved": "https://registry.npmmirror.com/mime-db/-/mime-db-1.52.0.tgz",
"integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
"engines": {
"node": ">= 0.6"
}
},
"node_modules/mime-types": {
"version": "2.1.35",
"resolved": "https://registry.npmmirror.com/mime-types/-/mime-types-2.1.35.tgz",
"integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
"dependencies": {
"mime-db": "1.52.0"
},
"engines": {
"node": ">= 0.6"
}
},
"node_modules/ms": {
"version": "2.0.0",
"resolved": "https://registry.npmmirror.com/ms/-/ms-2.0.0.tgz",
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="
},
"node_modules/mysql": {
"version": "2.18.1",
"resolved": "https://registry.npmmirror.com/mysql/-/mysql-2.18.1.tgz",
"integrity": "sha512-Bca+gk2YWmqp2Uf6k5NFEurwY/0td0cpebAucFpY/3jhrwrVGuxU2uQFCHjU19SJfje0yQvi+rVWdq78hR5lig==",
"dependencies": {
"bignumber.js": "9.0.0",
"readable-stream": "2.3.7",
"safe-buffer": "5.1.2",
"sqlstring": "2.3.1"
},
"engines": {
"node": ">= 0.6"
}
},
"node_modules/negotiator": {
"version": "0.6.3",
"resolved": "https://registry.npmmirror.com/negotiator/-/negotiator-0.6.3.tgz",
"integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==",
"engines": {
"node": ">= 0.6"
}
},
"node_modules/object-inspect": {
"version": "1.12.3",
"resolved": "https://registry.npmmirror.com/object-inspect/-/object-inspect-1.12.3.tgz",
"integrity": "sha512-geUvdk7c+eizMNUDkRpW1wJwgfOiOeHbxBR/hLXK1aT6zmVSO0jsQcs7fj6MGw89jC/cjGfLcNOrtMYtGqm81g=="
},
"node_modules/on-finished": {
"version": "2.4.1",
"resolved": "https://registry.npmmirror.com/on-finished/-/on-finished-2.4.1.tgz",
"integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==",
"dependencies": {
"ee-first": "1.1.1"
},
"engines": {
"node": ">= 0.8"
}
},
"node_modules/parseurl": {
"version": "1.3.3",
"resolved": "https://registry.npmmirror.com/parseurl/-/parseurl-1.3.3.tgz",
"integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==",
"engines": {
"node": ">= 0.8"
}
},
"node_modules/path-to-regexp": {
"version": "0.1.7",
"resolved": "https://registry.npmmirror.com/path-to-regexp/-/path-to-regexp-0.1.7.tgz",
"integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ=="
},
"node_modules/process-nextick-args": {
"version": "2.0.1",
"resolved": "https://registry.npmmirror.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
},
"node_modules/proxy-addr": {
"version": "2.0.7",
"resolved": "https://registry.npmmirror.com/proxy-addr/-/proxy-addr-2.0.7.tgz",
"integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==",
"dependencies": {
"forwarded": "0.2.0",
"ipaddr.js": "1.9.1"
},
"engines": {
"node": ">= 0.10"
}
},
"node_modules/qs": {
"version": "6.11.0",
"resolved": "https://registry.npmmirror.com/qs/-/qs-6.11.0.tgz",
"integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==",
"dependencies": {
"side-channel": "^1.0.4"
},
"engines": {
"node": ">=0.6"
}
},
"node_modules/range-parser": {
"version": "1.2.1",
"resolved": "https://registry.npmmirror.com/range-parser/-/range-parser-1.2.1.tgz",
"integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==",
"engines": {
"node": ">= 0.6"
}
},
"node_modules/raw-body": {
"version": "2.5.1",
"resolved": "https://registry.npmmirror.com/raw-body/-/raw-body-2.5.1.tgz",
"integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==",
"dependencies": {
"bytes": "3.1.2",
"http-errors": "2.0.0",
"iconv-lite": "0.4.24",
"unpipe": "1.0.0"
},
"engines": {
"node": ">= 0.8"
}
},
"node_modules/readable-stream": {
"version": "2.3.7",
"resolved": "https://registry.npmmirror.com/readable-stream/-/readable-stream-2.3.7.tgz",
"integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
"dependencies": {
"core-util-is": "~1.0.0",
"inherits": "~2.0.3",
"isarray": "~1.0.0",
"process-nextick-args": "~2.0.0",
"safe-buffer": "~5.1.1",
"string_decoder": "~1.1.1",
"util-deprecate": "~1.0.1"
}
},
"node_modules/safe-buffer": {
"version": "5.1.2",
"resolved": "https://registry.npmmirror.com/safe-buffer/-/safe-buffer-5.1.2.tgz",
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
},
"node_modules/safer-buffer": {
"version": "2.1.2",
"resolved": "https://registry.npmmirror.com/safer-buffer/-/safer-buffer-2.1.2.tgz",
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
},
"node_modules/send": {
"version": "0.18.0",
"resolved": "https://registry.npmmirror.com/send/-/send-0.18.0.tgz",
"integrity": "sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==",
"dependencies": {
"debug": "2.6.9",
"depd": "2.0.0",
"destroy": "1.2.0",
"encodeurl": "~1.0.2",
"escape-html": "~1.0.3",
"etag": "~1.8.1",
"fresh": "0.5.2",
"http-errors": "2.0.0",
"mime": "1.6.0",
"ms": "2.1.3",
"on-finished": "2.4.1",
"range-parser": "~1.2.1",
"statuses": "2.0.1"
},
"engines": {
"node": ">= 0.8.0"
}
},
"node_modules/send/node_modules/ms": {
"version": "2.1.3",
"resolved": "https://registry.npmmirror.com/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
},
"node_modules/serve-static": {
"version": "1.15.0",
"resolved": "https://registry.npmmirror.com/serve-static/-/serve-static-1.15.0.tgz",
"integrity": "sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g==",
"dependencies": {
"encodeurl": "~1.0.2",
"escape-html": "~1.0.3",
"parseurl": "~1.3.3",
"send": "0.18.0"
},
"engines": {
"node": ">= 0.8.0"
}
},
"node_modules/setprototypeof": {
"version": "1.2.0",
"resolved": "https://registry.npmmirror.com/setprototypeof/-/setprototypeof-1.2.0.tgz",
"integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw=="
},
"node_modules/side-channel": {
"version": "1.0.4",
"resolved": "https://registry.npmmirror.com/side-channel/-/side-channel-1.0.4.tgz",
"integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==",
"dependencies": {
"call-bind": "^1.0.0",
"get-intrinsic": "^1.0.2",
"object-inspect": "^1.9.0"
}
},
"node_modules/sqlstring": {
"version": "2.3.1",
"resolved": "https://registry.npmmirror.com/sqlstring/-/sqlstring-2.3.1.tgz",
"integrity": "sha512-ooAzh/7dxIG5+uDik1z/Rd1vli0+38izZhGzSa34FwR7IbelPWCCKSNIl8jlL/F7ERvy8CB2jNeM1E9i9mXMAQ==",
"engines": {
"node": ">= 0.6"
}
},
"node_modules/statuses": {
"version": "2.0.1",
"resolved": "https://registry.npmmirror.com/statuses/-/statuses-2.0.1.tgz",
"integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==",
"engines": {
"node": ">= 0.8"
}
},
"node_modules/string_decoder": {
"version": "1.1.1",
"resolved": "https://registry.npmmirror.com/string_decoder/-/string_decoder-1.1.1.tgz",
"integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
"dependencies": {
"safe-buffer": "~5.1.0"
}
},
"node_modules/toidentifier": {
"version": "1.0.1",
"resolved": "https://registry.npmmirror.com/toidentifier/-/toidentifier-1.0.1.tgz",
"integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==",
"engines": {
"node": ">=0.6"
}
},
"node_modules/type-is": {
"version": "1.6.18",
"resolved": "https://registry.npmmirror.com/type-is/-/type-is-1.6.18.tgz",
"integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==",
"dependencies": {
"media-typer": "0.3.0",
"mime-types": "~2.1.24"
},
"engines": {
"node": ">= 0.6"
}
},
"node_modules/unpipe": {
"version": "1.0.0",
"resolved": "https://registry.npmmirror.com/unpipe/-/unpipe-1.0.0.tgz",
"integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==",
"engines": {
"node": ">= 0.8"
}
},
"node_modules/util-deprecate": {
"version": "1.0.2",
"resolved": "https://registry.npmmirror.com/util-deprecate/-/util-deprecate-1.0.2.tgz",
"integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw=="
},
"node_modules/utils-merge": {
"version": "1.0.1",
"resolved": "https://registry.npmmirror.com/utils-merge/-/utils-merge-1.0.1.tgz",
"integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==",
"engines": {
"node": ">= 0.4.0"
}
},
"node_modules/vary": {
"version": "1.1.2",
"resolved": "https://registry.npmmirror.com/vary/-/vary-1.1.2.tgz",
"integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==",
"engines": {
"node": ">= 0.8"
}
},
"node_modules/xmldom": {
"version": "0.6.0",
"resolved": "https://registry.npmmirror.com/xmldom/-/xmldom-0.6.0.tgz",
"integrity": "sha512-iAcin401y58LckRZ0TkI4k0VSM1Qg0KGSc3i8rU+xrxe19A/BN1zHyVSJY7uoutVlaTSzYyk/v5AmkewAP7jtg==",
"engines": {
"node": ">=10.0.0"
}
},
"node_modules/xpath": {
"version": "0.0.33",
"resolved": "https://registry.npmmirror.com/xpath/-/xpath-0.0.33.tgz",
"integrity": "sha512-NNXnzrkDrAzalLhIUc01jO2mOzXGXh1JwPgkihcLLzw98c0WgYDmmjSh1Kl3wzaxSVWMuA+fe0WTWOBDWCBmNA==",
"engines": {
"node": ">=0.6.0"
}
}
}
}

17
package.json Normal file
View File

@ -0,0 +1,17 @@
{
"name": "",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"author": "",
"license": "ISC",
"dependencies": {
"express": "^4.18.2",
"mysql": "^2.18.1",
"xmldom": "^0.6.0",
"xpath": "^0.0.33"
}
}

245
spider/spider.js Normal file
View File

@ -0,0 +1,245 @@
// 导入需要的模块
const request = require('request');
const fs = require('fs');
const xpath = require('xpath');
const dom = require('xmldom').DOMParser;
const mysql = require('mysql');
// 要爬取的网页地址
// https://data.rmtc.org.cn/gis/PubIndexM.html?type=0
// https://data.rmtc.org.cn/gis/PubIndexM.html?type=1
const mysqlConfig = {
host: 'localhost', // 数据库地址
user: 'root', // 数据库用户名
password: '123456', // 数据库密码
database: 'open_data' // 数据库名称
}
main()
async function main() {
await doFetch('0')
await doFetch('1')
console.log('爬取完成')
}
//
async function doFetch(type) {
const url = 'https://data.rmtc.org.cn/gis/PubIndexM.html?type=' + type
await new Promise((resolve) => {
// 发送请求，获取网页内容
request(url, async (error, response, body) => {
if (error) {
console.error(error);
return
}
// fs.writeFileSync('output/html.json', body, 'utf-8');
// 使用dom模块解析body内容，生成文档对象
const doc = new dom().parseFromString(body);
// 修改了这个xpath表达式，用li标签而不是text()，因为text()会匹配所有文本节点，包括空白和换行
const lis = xpath.select("//ul[@data-role='listview']/li", doc);
// 定义一个空数组，用于存储转换后的数据
const data = [];
const childrenPage = [];
// 遍历提取到的数据，将其转换为对象，并存入数组中
for (let i = 0; i < lis.length; i++) {
const li = lis[i]
/*
<li><a data-ajax="false" href="PubStationlistM.html?type=1&id=2410283902&">
<h2><span class="epfy">红沿河核电厂</span> (<span class="epfy">复州城</span>)</h2>
<p>
<span class="span-count">83 nGy/h</span>
<span class="showtime">2023-08-25</span> &nbsp; </p>
</a>
</li>
*/
// 使用nodeValue属性获取文本内容
const name = xpath.select1(".//h2/span[@class='epfy'][1]/text()", li).nodeValue.trim();
const location = xpath.select1(".//h2/span[@class='epfy'][2]/text()", li).nodeValue.trim();
const value = xpath.select1(".//p/span[@class='span-count']/text()", li).nodeValue.trim();
const date = xpath.select1(".//p/span[@class='showtime']/text()", li).nodeValue.trim();
// 使用正则表达式匹配type和id的值
const href = xpath.select1("./a/@href", li).nodeValue.trim();
const regex = /type=(\d+)&id=(\d+)/;
const match = regex.exec(href);
const type = match[1];
const id = match[2];
// 创建一个对象，存储每一条数据
const item = {
name,
location,
value,
date,
parent_id: null,
type,
id
};
// 将对象推入数组中
data.push(item);
childrenPage.push({ type, id, href, name })
}
// 打印转换后的数据
// console.log(data);
// console.log('主表数据完成', JSON.stringify(data));
console.log('主表数据完成');
saveToDb(data)
// fs.writeFileSync('output/data.json', JSON.stringify(data, null, 4), 'utf-8');
console.log('childrenPage', childrenPage)
for (let page of childrenPage) {
console.log('#######################################################')
console.log('page', page)
await doFetch2(page.type, page.id, page.name, page.href)
console.log('#######################################################')
await new Promise((resolve) => {
setTimeout(resolve, 1000)
})
}
resolve()
});
})
}
async function doFetch2(parentType, parentId, parentName, relativeUrl) {
const url = 'https://data.rmtc.org.cn/gis/' + relativeUrl
// 发送请求，获取网页内容
let data = await new Promise((resolve) => {
request(url, (error, response, body) => {
if (error) {
console.error(error);
return
}
const doc = new dom().parseFromString(body);
const lis = xpath.select("//ul[@data-role='listview']/li", doc);
const data = [];
// 遍历提取到的数据，将其转换为对象，并存入数组中
for (let i = 0; i < lis.length; i++) {
const li = lis[i]
/*
<li><a stid="42974" time="2023-08-26 00:00:00" itemkey="43061" itemcode="0102060301" itemname="辐射剂量率" class="showboxlink" href="#drawdialog">
<h2>老渔窝</h2>
<p>
<span class="span-count">71 nGy/h</span>
<span class="showtime">2023-08-25</span> &nbsp; </p>
</a>
</li>
*/
// 使用nodeValue属性获取文本内容
const time = xpath.select1(".//a/@time", li).nodeValue.trim();
const itemkey = xpath.select1(".//a/@itemkey", li).nodeValue.trim();
const itemcode = xpath.select1(".//a/@itemcode", li).nodeValue.trim();
const itemname = xpath.select1(".//a/@itemname", li).nodeValue.trim();
const location = xpath.select1(".//h2/text()", li).nodeValue.trim();
const value = xpath.select1(".//p/span[@class='span-count']/text()", li).nodeValue.trim();
const date = xpath.select1(".//p/span[@class='showtime']/text()", li).nodeValue.trim();
// 创建一个对象，存储每一条数据
const item = {
name: parentName,
location,
value,
date,
parent_id: parentId,
type: parentType,
id: null,
itemkey: itemkey,
itemcode: itemcode,
itemname: itemname,
time,
};
// 将对象推入数组中
data.push(item);
}
// 打印转换后的数据
// console.log(data);
// console.log('二级分类数据完成', parentType, parentId, JSON.stringify(data));
console.log('二级分类数据完成', parentType, parentId);
resolve(data)
// fs.writeFileSync('output/data.json', JSON.stringify(data, null, 4), 'utf-8');
});
})
saveToDb(data)
}
function saveToDb(data) {
if (!data || data.length == 0) {
console.error("没有数据需要保存")
return
}
// 创建一个数据库连接对象
const connection = mysql.createConnection(mysqlConfig);
// 连接数据库
connection.connect((err) => {
if (err) {
console.error(err);
} else {
console.log('数据库连接成功');
const sql = `INSERT IGNORE INTO nuclear_data (${Object.keys(data[0]).join(', ')}) VALUES ?`;
// `REPLACE INTO nuclear_data (name, location, value, date, parent_id, type, id) VALUES ?`;
// `INSERT IGNORE INTO nuclear_data (name, location, value, date, parent_id, type, id) VALUES ?`;
// 将数据转换为二维数组，方便插入
const values = data.map(item => Object.values(item));
// let data = [
// {
// name: '红沿河核电厂',
// location: '复州城',
// value: '83 nGy/h',
// date: '2023-08-25',
// parent_id: '1',
// type: '1',
// id: '2410283902'
// },
// ];
// const values = [
// [
// "红沿河核电厂",
// "复州城",
// "83 nGy/h",
// "2023-08-25",
// "1",
// "1",
// "2410283902"
// ]
// ]
// 执行插入或替换数据的语句，传入values作为参数
connection.query(sql, [values], (err, result) => {
if (err) {
console.error(err);
} else {
console.log('成功');
// 关闭数据库连接
connection.end();
}
});
}
});
}

16
sql/create_table.sql Normal file
View File

@ -0,0 +1,16 @@
CREATE TABLE `nuclear_data` (
`nuclear_data_id` int(11) NOT NULL AUTO_INCREMENT,
`name` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL,
`location` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL,
`value` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL,
`date` date DEFAULT NULL,
`parent_id` varchar(15) DEFAULT NULL,
`type` varchar(5) DEFAULT NULL,
`id` varchar(20) DEFAULT NULL,
`itemkey` varchar(20) DEFAULT NULL,
`itemcode` varchar(30) DEFAULT NULL,
`itemname` varchar(30) DEFAULT NULL,
`time` varchar(30) DEFAULT NULL,
PRIMARY KEY (`nuclear_data_id`),
UNIQUE KEY `uniquekey` (`name`, `location`, `date`) USING BTREE
) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_0900_ai_ci;