diff --git a/.env b/.env index f6d7059..494a7a5 100644 --- a/.env +++ b/.env @@ -6,3 +6,7 @@ DEBUG_MODE=1 # 目录开头与结尾的 [./] [/] [\] [\\] 均可带可不带 # 默认为 data 文件夹 DATA_FOLDER=data + +# 是否在程序刚一启动时就抓取一次数据 +# 1为是 +EXECUTE_AT_STARTUP=1 \ No newline at end of file diff --git a/index.js b/index.js index a486d45..211d3c6 100644 --- a/index.js +++ b/index.js @@ -1,15 +1,18 @@ 'use strict'; +const dotenv = require('dotenv'); +const schedule = require('node-schedule'); + /** * 环境变量 */ -const dotenv = require('dotenv'); process.env = {}; // 清除系统自带的环境变量 dotenv.config('./.env'); // 导入 .env 文件中的环境变量 // console.log(process.env); // 调试模式 -const DEBUG_MODE = !!process.env.DEBUG_MODE; +const DEBUG_MODE = process.env.DEBUG_MODE == true; +const EXECUTE_AT_STARTUP = process.env.EXECUTE_AT_STARTUP == true; if (DEBUG_MODE) { console.log('DEBUG_MODE is on'); @@ -27,5 +30,19 @@ const get_hotband = require('./src/get_hotband'); */ console.log("Start running ..."); -// 爬取热搜数据 -get_hotband.main(); \ No newline at end of file +// 程序主函数 +function start() { + // 爬取热搜数据 + get_hotband.main(); +} + +// 调试模式下,程序一启动就首先运行一次 +if (EXECUTE_AT_STARTUP) { + process.stdout.write("程序启动时,立即运行一次\t"); + start(); +} + +// 每分钟的第 5 秒执行一次 +// 这里指定第 5 秒是为了稍微与微博服务器热榜更新时间错开,避免因为微秒级误差造成拉取两次相同的热榜数据 +// refer: https://www.npmjs.com/package/node-schedule +const scheduleJob = schedule.scheduleJob('05 * * * * *', start); diff --git a/src/get_hotband.js b/src/get_hotband.js index 0346237..abceba1 100644 --- a/src/get_hotband.js +++ b/src/get_hotband.js @@ -6,6 +6,7 @@ const { dirname } = require('path'); const path = require('path'); const DATA_FOLDER = path.join(dirname(__dirname), process.env.DATA_FOLDER ?? 'data'); +console.log("DATA_FOLDER", DATA_FOLDER); // 请求微博热搜 APi 接口 async function getApiResult(url) { @@ -70,8 +71,6 @@ function saveJSON({ now, fileNameSuffix, object, compress = true, uncompress = t } async function main() { - console.log("DATA_FOLDER", DATA_FOLDER); - let requestTimestamp = Date.now(); let now = new Date(requestTimestamp + 8 * 3600 * 1000).toISOString(); @@ -79,9 +78,9 @@ async function main() { // console.log("result", result); if (result.ok != 1) { - console.log("请求成功,但服务器处理失败。"); + console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), "请求成功,但服务器处理失败。"); } else { - console.log("请求成功。"); + console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), "请求成功"); /** * 保存原始数据 @@ -188,7 +187,8 @@ async function main() { fileNameSuffix: `final`, object: convert, compress: true, - uncompress: true + // uncompress: true, + uncompress: false, }); @@ -230,8 +230,10 @@ async function main() { now: now, fileNameSuffix: `simplify`, object: data, - compress: true, - uncompress: true + // compress: true, + // uncompress: true, + compress: false, + uncompress: false, });