From cf28f475e310d4dd7186414dc60591883aca4f5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E5=B0=8F=E5=A2=A8?= <2291200076@qq.com> Date: Thu, 28 Jul 2022 22:35:07 +0800 Subject: [PATCH] =?UTF-8?q?=E6=8A=BD=E7=A6=BB=E5=87=BA=E5=85=AC=E5=85=B1?= =?UTF-8?q?=E6=96=B9=E6=B3=95=EF=BC=9B=E6=B7=BB=E5=8A=A0B=E7=AB=99?= =?UTF-8?q?=E7=83=AD=E6=90=9C=E7=88=AC=E5=8F=96=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- html/weibo.html | 2 +- index.js | 2 + src/get_bilibili_hotband.js | 114 ++++++++++++++++++++++++++++++++++++ src/get_weibo_hotband.js | 72 ++++++----------------- src/utils/utils.js | 49 ++++++++++++++++ 5 files changed, 184 insertions(+), 55 deletions(-) create mode 100644 src/get_bilibili_hotband.js create mode 100644 src/utils/utils.js diff --git a/html/weibo.html b/html/weibo.html index 7e45e89..be75868 100644 --- a/html/weibo.html +++ b/html/weibo.html @@ -227,7 +227,7 @@ function getData() { var xhr = new XMLHttpRequest(); - xhr.open("GET", "../data/latest.json?t=" + Date.now(), true); + xhr.open("GET", "../data/weibo/latest.json?t=" + Date.now(), true); xhr.send(); xhr.onreadystatechange = function () { if (xhr.readyState !== 4) return; diff --git a/index.js b/index.js index c10bffe..f786500 100644 --- a/index.js +++ b/index.js @@ -37,6 +37,7 @@ if (DEBUG_MODE) { * 引入模块 */ const get_weibo_hotband = require('./src/get_weibo_hotband'); +const get_bilibili_hotband = require('./src/get_bilibili_hotband'); const execute_command = require('./src/execute_command'); @@ -52,6 +53,7 @@ console.log("Start running ..."); async function start() { // 爬取热搜数据 await get_weibo_hotband.main(); + await get_bilibili_hotband.main(); // 调试模式下 if (DEBUG_MODE) { diff --git a/src/get_bilibili_hotband.js b/src/get_bilibili_hotband.js new file mode 100644 index 0000000..9d29daa --- /dev/null +++ b/src/get_bilibili_hotband.js @@ -0,0 +1,114 @@ +'use strict'; + +const request = require('request'); +const fs = require('fs'); +const path = require('path'); + +const utils = require('./utils/utils'); + +const DATA_FOLDER = path.join(path.dirname(__dirname), process.env.DATA_FOLDER ?? 'data', 'bilibili'); +console.log("DATA_FOLDER", DATA_FOLDER); +utils.createFolder(DATA_FOLDER); // 程序运行就保证 data 目录存在 + +// 请求微博热搜 APi 接口 +async function getApiResult(url) { + var return_data = await new Promise((resolve) => { + request({ + method: 'GET', + url: url, + json: true, + }, (error, response, result) => { + if (!error && (response.statusCode == 200)) { + // 请求成功 + resolve(result); + } else { + // 请求失败 + console.log(`error is ${error}`); + resolve("error"); + } + }); + }); + // console.log(`return_data is ${JSON.stringify(return_data)}`); + return return_data; +} + +async function main() { + let requestTimestamp = Date.now(); + let now = new Date(requestTimestamp + 8 * 3600 * 1000).toISOString(); + + let result = await getApiResult("https://app.bilibili.com/x/v2/search/trending/ranking"); + if (result.code != 0) { + console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), "bilibili", "请求成功，但服务器处理失败，正在重试。"); + result = await getApiResult("https://app.bilibili.com/x/v2/search/trending/ranking"); + if (result.ok != 1) { + console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), "bilibili", "请求成功，但服务器处理失败，保存失败信息。"); + // ok 不为 1，那么久直接保存便于后续分析，不进行后续处理 + utils.saveJSON({ + saveFolder: DATA_FOLDER, + now: now, + fileNameSuffix: `origin-error`, + object: result, + compress: true, + uncompress: false + }); + return; + } + } + + console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), "bilibili", "请求成功"); + // console.log("result", result); + + let data = result.data; + + // 去除 trackid + delete data["trackid"]; + // console.log(data); + + /** + * 保存原始数据 + */ + utils.saveJSON({ + saveFolder: DATA_FOLDER, + now: now, + fileNameSuffix: `origin`, + object: result, + compress: true, + uncompress: false + }); + + + /** + * 获取需要的数据，进行转换 + */ + let convert = []; + data.list.forEach(item => { + // { + // "position": 1, + // "keyword": "关键词", + // "show_name": "热搜名称", + // "word_type": 8, + // "icon": "热搜的图标，也可能没有", + // "hot_id": 7399 // 热搜id + // } + convert.push(item); + }); + utils.saveJSON({ + saveFolder: DATA_FOLDER, + now: now, + fileNameSuffix: `final`, + object: convert, + compress: true, + uncompress: false, + }); + + /** + * 更新最新的 + */ + fs.writeFileSync(`${DATA_FOLDER}/latest.json`, JSON.stringify({ + update_time: requestTimestamp, + update_time_friendly: now.substring(0, 19).replace(/T/g, " "), + data: data + })); +} + +exports.main = main; diff --git a/src/get_weibo_hotband.js b/src/get_weibo_hotband.js index 1eec48c..e5a11c2 100644 --- a/src/get_weibo_hotband.js +++ b/src/get_weibo_hotband.js @@ -4,11 +4,11 @@ const request = require('request'); const fs = require('fs'); const path = require('path'); -const DATA_FOLDER = path.join(path.dirname(__dirname), process.env.DATA_FOLDER ?? 'data'); -console.log("DATA_FOLDER", DATA_FOLDER); -createFolder(DATA_FOLDER); // 程序运行就保证 data 目录存在 +const utils = require('./utils/utils'); -const LATEST_DATA_ONLY = process.env.LATEST_DATA_ONLY == true; +const DATA_FOLDER = path.join(path.dirname(__dirname), process.env.DATA_FOLDER ?? 'data', 'weibo'); +console.log("DATA_FOLDER", DATA_FOLDER); +utils.createFolder(DATA_FOLDER); // 程序运行就保证 data 目录存在 // 请求微博热搜 APi 接口 async function getApiResult(url) { @@ -32,59 +32,19 @@ async function getApiResult(url) { return return_data; } -// 创建目录 -async function createFolder(folderToCreate) { - let currentFolder = folderToCreate.replace(/\\/g, '/'); - let parentFolder = currentFolder.substring(0, currentFolder.lastIndexOf('/')); - if (!fs.existsSync(currentFolder)) { - // 文件夹不存在，创建文件夹 - createFolder(parentFolder); // 保证父级文件夹存在 - fs.mkdirSync(currentFolder); // 创建当前级文件夹 - } else { - // 否则就什么也不做 - } -} - -// 保存 JSON -function saveJSON({ now, fileNameSuffix, object, compress = true, uncompress = true }) { - if (LATEST_DATA_ONLY) return; - - let year = now.substring(0, 4); - let month = now.substring(5, 7); - let day = now.substring(8, 10); - let hour = now.substring(11, 13); - let minute = now.substring(14, 16); - // console.log(now); - // console.log( "year, month, day, hour, minute: " + year + ", " + month + ", " + day + ", " + hour + ", " + minute); - - // 创建当前文件夹 - let folder = `${DATA_FOLDER}/${fileNameSuffix}/${year}/${month}/${day}`; - createFolder(folder); - let fileName = `${folder}/${year}${month}${day}_${hour}${minute}`; - - // 生成文件名 - // '2022-07-23T10:11:38.650Z' => '20220723_1011' - // let fileName = now.replace(/T/, '_').replace(/:\d{2}.\d{3}Z/, '').replace(/[-:]/g, ''); - // console.log(`fileName is ${fileName}`); - - if (compress) - fs.writeFileSync(`${fileName}.min.json`, JSON.stringify(object)); - if (uncompress) - fs.writeFileSync(`${fileName}.json`, JSON.stringify(object, "", "\t")); -} - async function main() { let requestTimestamp = Date.now(); let now = new Date(requestTimestamp + 8 * 3600 * 1000).toISOString(); let result = await getApiResult("https://weibo.com/ajax/statuses/hot_band"); if (result.ok != 1) { - console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), "请求成功，但服务器处理失败，正在重试。"); + console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), "weibo", "请求成功，但服务器处理失败，正在重试。"); result = await getApiResult("https://weibo.com/ajax/statuses/hot_band"); if (result.ok != 1) { - console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), "请求成功，但服务器处理失败，保存失败信息。"); + console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), "weibo", "请求成功，但服务器处理失败，保存失败信息。"); // ok 不为 1，那么久直接保存便于后续分析，不进行后续处理 - saveJSON({ + utils.saveJSON({ + saveFolder: DATA_FOLDER, now: now, fileNameSuffix: `origin-error`, object: result, @@ -95,13 +55,14 @@ async function main() { } } - console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), "请求成功"); + console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), "weibo", "请求成功"); // console.log("result", result); /** * 保存原始数据 */ - saveJSON({ + utils.saveJSON({ + saveFolder: DATA_FOLDER, now: now, fileNameSuffix: `origin`, object: result, @@ -198,7 +159,8 @@ async function main() { }, }); }); - saveJSON({ + utils.saveJSON({ + saveFolder: DATA_FOLDER, now: now, fileNameSuffix: `final`, object: convert, @@ -222,7 +184,8 @@ async function main() { `原始:${item.raw_hot} 显示:${item.num} 调控: ${item.num - item.raw_hot}` ]); }); - saveJSON({ + utils.saveJSON({ + saveFolder: DATA_FOLDER, now: now, fileNameSuffix: `regulation`, object: { @@ -242,7 +205,8 @@ async function main() { data.band_list.forEach(function (item) { delete item["mblog"]; }); - saveJSON({ + utils.saveJSON({ + saveFolder: DATA_FOLDER, now: now, fileNameSuffix: `simplify`, object: data, @@ -264,4 +228,4 @@ async function main() { })); } -exports.main = main; \ No newline at end of file +exports.main = main; diff --git a/src/utils/utils.js b/src/utils/utils.js new file mode 100644 index 0000000..82aa0aa --- /dev/null +++ b/src/utils/utils.js @@ -0,0 +1,49 @@ +const fs = require('fs'); + +const LATEST_DATA_ONLY = process.env.LATEST_DATA_ONLY == true; + +// 创建目录 +async function createFolder(folderToCreate) { + let currentFolder = folderToCreate.replace(/\\/g, '/'); + let parentFolder = currentFolder.substring(0, currentFolder.lastIndexOf('/')); + if (!fs.existsSync(currentFolder)) { + // 文件夹不存在，创建文件夹 + createFolder(parentFolder); // 保证父级文件夹存在 + fs.mkdirSync(currentFolder); // 创建当前级文件夹 + } else { + // 否则就什么也不做 + } +} + +// 保存 JSON +function saveJSON({ saveFolder, now, fileNameSuffix, object, compress = true, uncompress = true }) { + if (LATEST_DATA_ONLY) return; + + let year = now.substring(0, 4); + let month = now.substring(5, 7); + let day = now.substring(8, 10); + let hour = now.substring(11, 13); + let minute = now.substring(14, 16); + // console.log(now); + // console.log( "year, month, day, hour, minute: " + year + ", " + month + ", " + day + ", " + hour + ", " + minute); + + // 创建当前文件夹 + let folder = `${saveFolder}/${fileNameSuffix}/${year}/${month}/${day}`; + createFolder(folder); + let fileName = `${folder}/${year}${month}${day}_${hour}${minute}`; + + // 生成文件名 + // '2022-07-23T10:11:38.650Z' => '20220723_1011' + // let fileName = now.replace(/T/, '_').replace(/:\d{2}.\d{3}Z/, '').replace(/[-:]/g, ''); + // console.log(`fileName is ${fileName}`); + + if (compress) + fs.writeFileSync(`${fileName}.min.json`, JSON.stringify(object)); + if (uncompress) + fs.writeFileSync(`${fileName}.json`, JSON.stringify(object, "", "\t")); +} + +module.exports = { + createFolder, + saveJSON, +} \ No newline at end of file