添加定时任务，部署相关优化

backend/server.js

@@ -2,6 +2,8 @@ const express = require('express');
 const mysql = require('mysql');
 const app = express();
 
+const port = 35590
+
 // 创建连接池
 const pool = mysql.createPool({
     connectionLimit: 5, // 设置连接池的大小
@@ -202,7 +204,14 @@ app.post('/api/nuclear_data/export', function (req, res) {
     });
 });
 
+
+function serve() {
     // 启动服务器
-app.listen(3000, function () {
+    app.listen(port, function () {
-    console.log('Server is running on port 3000');
+        console.log('Server is running on port ' + port);
     });
+}
+
+module.exports = {
+    serve,
+}

html/index.html

@@ -61,6 +61,11 @@
     <script src="./jquery.min.js"></script>
     <!-- <script src="https://cdn.bootcdn.net/ajax/libs/jquery/3.6.0/jquery.min.js"></script> -->
     <script>
+        const host = location.host.includes('localhost') || location.host.includes('127.0.0.1')
+            ? 'http://localhost:35590/'
+            : '/api/'
+        console.log('host', host)
+
         function setDefalutValue() {
             // 日期当前值 2天前
             var today = new Date(Date.now() - 2 * 24 * 3600 * 1000); // 获取当前的日期对象
@@ -93,7 +98,7 @@
 
         // // 在页面加载时发送get请求，获取所有的名称和位置，并填充到下拉框中
         // $(document).ready(function () {
-        //     $.get('http://localhost:3000/nuclear_data/options', function (res) {
+        //     $.get(host + 'nuclear_data/options', function (res) {
         //         // 遍历返回的数组，将每个名称和位置添加到对应的下拉框中
         //         for (let item of res) {
         //             $('#name').append(`<option value="${item.name}">${item.name}</option>`);
@@ -104,7 +109,7 @@
 
         // // 在页面加载时发送get请求，获取所有的名称，并填充到名称的下拉框中
         // $(document).ready(function () {
-        //     $.get('http://localhost:3000/nuclear_data/options', function (res) {
+        //     $.get(host + 'nuclear_data/options', function (res) {
         //         // 遍历返回的数组，将每个名称添加到名称的下拉框中
         //         for (let item of res) {
         //             $('#name').append(`<option value="${item.name}">${item.name}</option>`);
@@ -128,7 +133,7 @@
             // 获取选择的名称
             let type = $(this).val();
             // 发送post请求
-            $.post('http://localhost:3000/nuclear_data/options/names', { type: type }, function (res) {
+            $.post(host + 'nuclear_data/options/names', { type: type }, function (res) {
                 // 清空位置的下拉框
                 $('#name').empty();
                 // 遍历返回的数组，将每个位置添加到位置的下拉框中
@@ -146,7 +151,7 @@
             let type = $('#type').val();
             let name = $(this).val();
             // 发送post请求
-            $.post('http://localhost:3000/nuclear_data/options/locations', { type: type, name: name }, function (res) {
+            $.post(host + 'nuclear_data/options/locations', { type: type, name: name }, function (res) {
                 // 清空位置的下拉框
                 $('#location').empty();
                 // 遍历返回的数组，将每个位置添加到位置的下拉框中
@@ -177,7 +182,7 @@
             // 获取表单数据
             let data = $(this).serialize();
             // 发送post请求
-            $.post('http://localhost:3000/nuclear_data', data, function (res) {
+            $.post(host + 'nuclear_data', data, function (res) {
                 // 清空结果区域
                 $('#result').empty();
                 // 判断是否有数据返回

index.js

@@ -0,0 +1,11 @@
+const schedule = require('node-schedule')
+const { run } = require('./spider/spider')
+const { serve } = require('./backend/server')
+
+// 每日 11:11 23:11 执行采集
+schedule.scheduleJob('11 11 11,23 * * *', run)
+
+// 启动后端服务
+serve()
+
+console.log('项目后端及定时采集已启动')

package-lock.json

@@ -1,15 +1,17 @@
 {
-  "name": "数据爬取",
+  "name": "nuclear_data_spider",
   "version": "1.0.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
+      "name": "nuclear_data_spider",
       "version": "1.0.0",
-      "license": "ISC",
+      "license": "MIT",
       "dependencies": {
         "express": "^4.18.2",
         "mysql": "^2.18.1",
+        "node-schedule": "^2.1.1",
         "request": "^2.88.2",
         "xmldom": "^0.6.0",
         "xpath": "^0.0.33"
@@ -191,6 +193,17 @@
       "resolved": "https://registry.npmmirror.com/core-util-is/-/core-util-is-1.0.3.tgz",
       "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ=="
     },
+    "node_modules/cron-parser": {
+      "version": "4.9.0",
+      "resolved": "https://registry.npmmirror.com/cron-parser/-/cron-parser-4.9.0.tgz",
+      "integrity": "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==",
+      "dependencies": {
+        "luxon": "^3.2.1"
+      },
+      "engines": {
+        "node": ">=12.0.0"
+      }
+    },
     "node_modules/dashdash": {
       "version": "1.14.1",
       "resolved": "https://registry.npmmirror.com/dashdash/-/dashdash-1.14.1.tgz",
@@ -567,6 +580,19 @@
         "node": ">=0.6.0"
       }
     },
+    "node_modules/long-timeout": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmmirror.com/long-timeout/-/long-timeout-0.1.1.tgz",
+      "integrity": "sha512-BFRuQUqc7x2NWxfJBCyUrN8iYUYznzL9JROmRz1gZ6KlOIgmoD+njPVbb+VNn2nGMKggMsK79iUNErillsrx7w=="
+    },
+    "node_modules/luxon": {
+      "version": "3.4.2",
+      "resolved": "https://registry.npmmirror.com/luxon/-/luxon-3.4.2.tgz",
+      "integrity": "sha512-uBoAVCVcajsrqy3pv7eo5jEUz1oeLmCcnMv8n4AJpT5hbpN9lUssAXibNElpbLce3Mhm9dyBzwYLs9zctM/0tA==",
+      "engines": {
+        "node": ">=12"
+      }
+    },
     "node_modules/media-typer": {
       "version": "0.3.0",
       "resolved": "https://registry.npmmirror.com/media-typer/-/media-typer-0.3.0.tgz",
@@ -645,6 +671,19 @@
         "node": ">= 0.6"
       }
     },
+    "node_modules/node-schedule": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmmirror.com/node-schedule/-/node-schedule-2.1.1.tgz",
+      "integrity": "sha512-OXdegQq03OmXEjt2hZP33W2YPs/E5BcFQks46+G2gAxs4gHOIVD1u7EqlYLYSKsaIpyKCK9Gbk0ta1/gjRSMRQ==",
+      "dependencies": {
+        "cron-parser": "^4.2.0",
+        "long-timeout": "0.1.1",
+        "sorted-array-functions": "^1.3.0"
+      },
+      "engines": {
+        "node": ">=6"
+      }
+    },
     "node_modules/oauth-sign": {
       "version": "0.9.0",
       "resolved": "https://registry.npmmirror.com/oauth-sign/-/oauth-sign-0.9.0.tgz",
@@ -870,6 +909,11 @@
         "object-inspect": "^1.9.0"
       }
     },
+    "node_modules/sorted-array-functions": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmmirror.com/sorted-array-functions/-/sorted-array-functions-1.3.0.tgz",
+      "integrity": "sha512-2sqgzeFlid6N4Z2fUQ1cvFmTOLRi/sEDzSQ0OKYchqgoPmQBVyM3959qYx3fpS6Esef80KjmpgPeEr028dP3OA=="
+    },
     "node_modules/sqlstring": {
       "version": "2.3.1",
       "resolved": "https://registry.npmmirror.com/sqlstring/-/sqlstring-2.3.1.tgz",

package.json

@@ -1,16 +1,17 @@
 {
-  "name": "",
+  "name": "nuclear_data_spider",
   "version": "1.0.0",
   "description": "",
   "main": "index.js",
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "serve": "node index.js"
   },
-  "author": "",
+  "author": "程序员小墨",
-  "license": "ISC",
+  "license": "MIT",
   "dependencies": {
     "express": "^4.18.2",
     "mysql": "^2.18.1",
+    "node-schedule": "^2.1.1",
     "request": "^2.88.2",
     "xmldom": "^0.6.0",
     "xpath": "^0.0.33"

pm2.sh

@@ -0,0 +1,17 @@
+# npm i -g pm2
+# npm install pm2-windows-startup -g
+
+# 启动服务 并设置开机启动
+pm2 start index.js --name nuclear-data-spider
+pm2 save
+pm2-startup install
+
+# # 查看当前运行列表
+# pm2 list
+# pm2 status
+
+# # 停止
+# pm2 stop nuclear-data-spider
+
+# # 重启
+# pm2 restart nuclear-data-spider

spider/spider.js

@@ -16,23 +16,41 @@ const mysqlConfig = {
     database: 'open_data' // 数据库名称
 }
 
-main()
+// main()
-
 async function main() {
+    try {
         await doFetch('0')
+    } catch (err) {
+        console.error('type=0 爬取时出现错误，正在重试...')
+        try {
+            await doFetch('0')
+        } catch (err) {
+            console.error('type=0 爬取时出现错误，爬取失败')
+        }
+    }
+    try {
         await doFetch('1')
+    } catch (err) {
+        console.error('type=1 爬取时出现错误，正在重试...')
+        try {
+            await doFetch('1')
+        } catch (err) {
+            console.error('type=1 爬取时出现错误，爬取失败')
+        }
+    }
     console.log('爬取完成')
 }
 
 //
 async function doFetch(type) {
     const url = 'https://data.rmtc.org.cn/gis/PubIndexM.html?type=' + type
-    await new Promise((resolve) => {
+    await new Promise((resolve, reject) => {
         // 发送请求，获取网页内容
         request(url, async (error, response, body) => {
             if (error) {
                 console.error(error);
-                return
+                // return
+                reject()
             }
 
             // fs.writeFileSync('output/html.json', body, 'utf-8');
@@ -243,3 +261,7 @@ function saveToDb(data) {
         }
     });
 }
+
+module.exports = {
+    run: main,
+}