coder-xiaomo/nuclear_data_spider
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity GitHub Gitee

添加定时任务，部署相关优化

Browse Source
This commit is contained in:
程序员小墨 2023-08-27 23:51:47 +08:00
parent 70e4c8adee
commit b75ec7d6e7
7 changed files with 131 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
backend/server.js
View File

@ -2,6 +2,8 @@ const express = require('express');
const mysql = require('mysql');
const app = express();
const port = 35590
// 创建连接池
const pool = mysql.createPool({
connectionLimit: 5, // 设置连接池的大小
@ -202,7 +204,14 @@ app.post('/api/nuclear_data/export', function (req, res) {
});
});
// 启动服务器
app.listen(3000, function () {
console.log('Server is running on port 3000');
});
function serve() {
// 启动服务器
app.listen(port, function () {
console.log('Server is running on port ' + port);
});
}
module.exports = {
serve,
}

15
html/index.html
View File

@ -61,6 +61,11 @@
<script src="./jquery.min.js"></script>
<!-- <script src="https://cdn.bootcdn.net/ajax/libs/jquery/3.6.0/jquery.min.js"></script> -->
<script>
const host = location.host.includes('localhost') || location.host.includes('127.0.0.1')
? 'http://localhost:35590/'
: '/api/'
console.log('host', host)
function setDefalutValue() {
// 日期当前值 2天前
var today = new Date(Date.now() - 2 * 24 * 3600 * 1000); // 获取当前的日期对象
@ -93,7 +98,7 @@
// // 在页面加载时发送get请求，获取所有的名称和位置，并填充到下拉框中
// $(document).ready(function () {
// $.get('http://localhost:3000/nuclear_data/options', function (res) {
// $.get(host + 'nuclear_data/options', function (res) {
// // 遍历返回的数组，将每个名称和位置添加到对应的下拉框中
// for (let item of res) {
// $('#name').append(`<option value="${item.name}">${item.name}</option>`);
@ -104,7 +109,7 @@
// // 在页面加载时发送get请求，获取所有的名称，并填充到名称的下拉框中
// $(document).ready(function () {
// $.get('http://localhost:3000/nuclear_data/options', function (res) {
// $.get(host + 'nuclear_data/options', function (res) {
// // 遍历返回的数组，将每个名称添加到名称的下拉框中
// for (let item of res) {
// $('#name').append(`<option value="${item.name}">${item.name}</option>`);
@ -128,7 +133,7 @@
// 获取选择的名称
let type = $(this).val();
// 发送post请求
$.post('http://localhost:3000/nuclear_data/options/names', { type: type }, function (res) {
$.post(host + 'nuclear_data/options/names', { type: type }, function (res) {
// 清空位置的下拉框
$('#name').empty();
// 遍历返回的数组，将每个位置添加到位置的下拉框中
@ -146,7 +151,7 @@
let type = $('#type').val();
let name = $(this).val();
// 发送post请求
$.post('http://localhost:3000/nuclear_data/options/locations', { type: type, name: name }, function (res) {
$.post(host + 'nuclear_data/options/locations', { type: type, name: name }, function (res) {
// 清空位置的下拉框
$('#location').empty();
// 遍历返回的数组，将每个位置添加到位置的下拉框中
@ -177,7 +182,7 @@
// 获取表单数据
let data = $(this).serialize();
// 发送post请求
$.post('http://localhost:3000/nuclear_data', data, function (res) {
$.post(host + 'nuclear_data', data, function (res) {
// 清空结果区域
$('#result').empty();
// 判断是否有数据返回

11
index.js Normal file
View File

@ -0,0 +1,11 @@
const schedule = require('node-schedule')
const { run } = require('./spider/spider')
const { serve } = require('./backend/server')
// 每日 11:11 23:11 执行采集
schedule.scheduleJob('11 11 11,23 * * *', run)
// 启动后端服务
serve()
console.log('项目后端及定时采集已启动')

48
package-lock.json generated
View File

@ -1,15 +1,17 @@
{
"name": "数据爬取",
"name": "nuclear_data_spider",
"version": "1.0.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "nuclear_data_spider",
"version": "1.0.0",
"license": "ISC",
"license": "MIT",
"dependencies": {
"express": "^4.18.2",
"mysql": "^2.18.1",
"node-schedule": "^2.1.1",
"request": "^2.88.2",
"xmldom": "^0.6.0",
"xpath": "^0.0.33"
@ -191,6 +193,17 @@
"resolved": "https://registry.npmmirror.com/core-util-is/-/core-util-is-1.0.3.tgz",
"integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ=="
},
"node_modules/cron-parser": {
"version": "4.9.0",
"resolved": "https://registry.npmmirror.com/cron-parser/-/cron-parser-4.9.0.tgz",
"integrity": "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==",
"dependencies": {
"luxon": "^3.2.1"
},
"engines": {
"node": ">=12.0.0"
}
},
"node_modules/dashdash": {
"version": "1.14.1",
"resolved": "https://registry.npmmirror.com/dashdash/-/dashdash-1.14.1.tgz",
@ -567,6 +580,19 @@
"node": ">=0.6.0"
}
},
"node_modules/long-timeout": {
"version": "0.1.1",
"resolved": "https://registry.npmmirror.com/long-timeout/-/long-timeout-0.1.1.tgz",
"integrity": "sha512-BFRuQUqc7x2NWxfJBCyUrN8iYUYznzL9JROmRz1gZ6KlOIgmoD+njPVbb+VNn2nGMKggMsK79iUNErillsrx7w=="
},
"node_modules/luxon": {
"version": "3.4.2",
"resolved": "https://registry.npmmirror.com/luxon/-/luxon-3.4.2.tgz",
"integrity": "sha512-uBoAVCVcajsrqy3pv7eo5jEUz1oeLmCcnMv8n4AJpT5hbpN9lUssAXibNElpbLce3Mhm9dyBzwYLs9zctM/0tA==",
"engines": {
"node": ">=12"
}
},
"node_modules/media-typer": {
"version": "0.3.0",
"resolved": "https://registry.npmmirror.com/media-typer/-/media-typer-0.3.0.tgz",
@ -645,6 +671,19 @@
"node": ">= 0.6"
}
},
"node_modules/node-schedule": {
"version": "2.1.1",
"resolved": "https://registry.npmmirror.com/node-schedule/-/node-schedule-2.1.1.tgz",
"integrity": "sha512-OXdegQq03OmXEjt2hZP33W2YPs/E5BcFQks46+G2gAxs4gHOIVD1u7EqlYLYSKsaIpyKCK9Gbk0ta1/gjRSMRQ==",
"dependencies": {
"cron-parser": "^4.2.0",
"long-timeout": "0.1.1",
"sorted-array-functions": "^1.3.0"
},
"engines": {
"node": ">=6"
}
},
"node_modules/oauth-sign": {
"version": "0.9.0",
"resolved": "https://registry.npmmirror.com/oauth-sign/-/oauth-sign-0.9.0.tgz",
@ -870,6 +909,11 @@
"object-inspect": "^1.9.0"
}
},
"node_modules/sorted-array-functions": {
"version": "1.3.0",
"resolved": "https://registry.npmmirror.com/sorted-array-functions/-/sorted-array-functions-1.3.0.tgz",
"integrity": "sha512-2sqgzeFlid6N4Z2fUQ1cvFmTOLRi/sEDzSQ0OKYchqgoPmQBVyM3959qYx3fpS6Esef80KjmpgPeEr028dP3OA=="
},
"node_modules/sqlstring": {
"version": "2.3.1",
"resolved": "https://registry.npmmirror.com/sqlstring/-/sqlstring-2.3.1.tgz",

11
package.json
View File

@ -1,18 +1,19 @@
{
"name": "",
"name": "nuclear_data_spider",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
"serve": "node index.js"
},
"author": "",
"license": "ISC",
"author": "程序员小墨",
"license": "MIT",
"dependencies": {
"express": "^4.18.2",
"mysql": "^2.18.1",
"node-schedule": "^2.1.1",
"request": "^2.88.2",
"xmldom": "^0.6.0",
"xpath": "^0.0.33"
}
}
}

17
pm2.sh Normal file
View File

@ -0,0 +1,17 @@
# npm i -g pm2
# npm install pm2-windows-startup -g
# 启动服务 并设置开机启动
pm2 start index.js --name nuclear-data-spider
pm2 save
pm2-startup install
# # 查看当前运行列表
# pm2 list
# pm2 status
# # 停止
# pm2 stop nuclear-data-spider
# # 重启
# pm2 restart nuclear-data-spider

34
spider/spider.js
View File

@ -16,23 +16,41 @@ const mysqlConfig = {
database: 'open_data' // 数据库名称
}
main()
// main()
async function main() {
await doFetch('0')
await doFetch('1')
try {
await doFetch('0')
} catch (err) {
console.error('type=0 爬取时出现错误，正在重试...')
try {
await doFetch('0')
} catch (err) {
console.error('type=0 爬取时出现错误，爬取失败')
}
}
try {
await doFetch('1')
} catch (err) {
console.error('type=1 爬取时出现错误，正在重试...')
try {
await doFetch('1')
} catch (err) {
console.error('type=1 爬取时出现错误，爬取失败')
}
}
console.log('爬取完成')
}
//
async function doFetch(type) {
const url = 'https://data.rmtc.org.cn/gis/PubIndexM.html?type=' + type
await new Promise((resolve) => {
await new Promise((resolve, reject) => {
// 发送请求，获取网页内容
request(url, async (error, response, body) => {
if (error) {
console.error(error);
return
// return
reject()
}
// fs.writeFileSync('output/html.json', body, 'utf-8');
@ -242,4 +260,8 @@ function saveToDb(data) {
});
}
});
}
module.exports = {
run: main,
}