From 05ea84f47e5b140f95121befdcdaf9e22720f9b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E5=B0=8F=E5=A2=A8?= <2291200076@qq.com> Date: Fri, 29 Jul 2022 15:47:38 +0800 Subject: [PATCH] =?UTF-8?q?=E8=AF=B7=E6=B1=82API=E6=8A=BD=E7=A6=BB?= =?UTF-8?q?=E5=87=BA=E7=8B=AC=E7=AB=8B=E6=96=B9=E6=B3=95=20requestUtils?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/get_bilibili_hotband.js | 38 ++++++------------------- src/get_weibo_hotband.js | 42 +++++++--------------------- src/utils/{utils.js => fileUtils.js} | 0 src/utils/requestUtils.js | 27 ++++++++++++++++++ 4 files changed, 45 insertions(+), 62 deletions(-) rename src/utils/{utils.js => fileUtils.js} (100%) create mode 100644 src/utils/requestUtils.js diff --git a/src/get_bilibili_hotband.js b/src/get_bilibili_hotband.js index 3de3a4a..d39c5f1 100644 --- a/src/get_bilibili_hotband.js +++ b/src/get_bilibili_hotband.js @@ -1,49 +1,27 @@ 'use strict'; -const request = require('request'); const fs = require('fs'); const path = require('path'); -const utils = require('./utils/utils'); +const fileUtils = require('./utils/fileUtils'); +const requestUtils = require('./utils/requestUtils'); const DATA_FOLDER = path.join(path.dirname(__dirname), process.env.DATA_FOLDER ?? 'data', 'bilibili'); console.log("DATA_FOLDER", DATA_FOLDER); -utils.createFolder(DATA_FOLDER); // 程序运行就保证 data 目录存在 - -// 请求微博热搜 APi 接口 -async function getApiResult(url) { - var return_data = await new Promise((resolve) => { - request({ - method: 'GET', - url: url, - json: true, - }, (error, response, result) => { - if (!error && (response.statusCode == 200)) { - // 请求成功 - resolve(result); - } else { - // 请求失败 - console.log(`error is ${error}`); - resolve("error"); - } - }); - }); - // console.log(`return_data is ${JSON.stringify(return_data)}`); - return return_data; -} +fileUtils.createFolder(DATA_FOLDER); // 程序运行就保证 data 目录存在 async function main() { let requestTimestamp = Date.now(); let now = new Date(requestTimestamp + 8 * 3600 * 1000).toISOString(); - let result = await getApiResult("https://app.bilibili.com/x/v2/search/trending/ranking"); + let result = await requestUtils.getApiResult("https://app.bilibili.com/x/v2/search/trending/ranking"); if (result.code != 0) { console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), "bilibili", "请求成功，但服务器处理失败，正在重试。"); - result = await getApiResult("https://app.bilibili.com/x/v2/search/trending/ranking"); + result = await requestUtils.getApiResult("https://app.bilibili.com/x/v2/search/trending/ranking"); if (result.ok != 1) { console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), "bilibili", "请求成功，但服务器处理失败，保存失败信息。"); // ok 不为 1，那么久直接保存便于后续分析，不进行后续处理 - utils.saveJSON({ + fileUtils.saveJSON({ saveFolder: DATA_FOLDER, now: now, fileNameSuffix: `origin-error`, @@ -67,7 +45,7 @@ async function main() { /** * 保存原始数据 */ - utils.saveJSON({ + fileUtils.saveJSON({ saveFolder: DATA_FOLDER, now: now, fileNameSuffix: `origin`, @@ -92,7 +70,7 @@ async function main() { // } convert.push(item); }); - utils.saveJSON({ + fileUtils.saveJSON({ saveFolder: DATA_FOLDER, now: now, fileNameSuffix: `final`, diff --git a/src/get_weibo_hotband.js b/src/get_weibo_hotband.js index e5a11c2..d482d81 100644 --- a/src/get_weibo_hotband.js +++ b/src/get_weibo_hotband.js @@ -1,49 +1,27 @@ 'use strict'; -const request = require('request'); const fs = require('fs'); const path = require('path'); -const utils = require('./utils/utils'); +const fileUtils = require('./utils/fileUtils'); +const requestUtils = require('./utils/requestUtils'); const DATA_FOLDER = path.join(path.dirname(__dirname), process.env.DATA_FOLDER ?? 'data', 'weibo'); console.log("DATA_FOLDER", DATA_FOLDER); -utils.createFolder(DATA_FOLDER); // 程序运行就保证 data 目录存在 - -// 请求微博热搜 APi 接口 -async function getApiResult(url) { - var return_data = await new Promise((resolve) => { - request({ - method: 'GET', - url: url, - json: true, - }, (error, response, result) => { - if (!error && (response.statusCode == 200)) { - // 请求成功 - resolve(result); - } else { - // 请求失败 - console.log(`error is ${error}`); - resolve("error"); - } - }); - }); - // console.log(`return_data is ${JSON.stringify(return_data)}`); - return return_data; -} +fileUtils.createFolder(DATA_FOLDER); // 程序运行就保证 data 目录存在 async function main() { let requestTimestamp = Date.now(); let now = new Date(requestTimestamp + 8 * 3600 * 1000).toISOString(); - let result = await getApiResult("https://weibo.com/ajax/statuses/hot_band"); + let result = await requestUtils.getApiResult("https://weibo.com/ajax/statuses/hot_band"); if (result.ok != 1) { console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), "weibo", "请求成功，但服务器处理失败，正在重试。"); - result = await getApiResult("https://weibo.com/ajax/statuses/hot_band"); + result = await requestUtils.getApiResult("https://weibo.com/ajax/statuses/hot_band"); if (result.ok != 1) { console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), "weibo", "请求成功，但服务器处理失败，保存失败信息。"); // ok 不为 1，那么久直接保存便于后续分析，不进行后续处理 - utils.saveJSON({ + fileUtils.saveJSON({ saveFolder: DATA_FOLDER, now: now, fileNameSuffix: `origin-error`, @@ -61,7 +39,7 @@ async function main() { /** * 保存原始数据 */ - utils.saveJSON({ + fileUtils.saveJSON({ saveFolder: DATA_FOLDER, now: now, fileNameSuffix: `origin`, @@ -159,7 +137,7 @@ async function main() { }, }); }); - utils.saveJSON({ + fileUtils.saveJSON({ saveFolder: DATA_FOLDER, now: now, fileNameSuffix: `final`, @@ -184,7 +162,7 @@ async function main() { `原始:${item.raw_hot} 显示:${item.num} 调控: ${item.num - item.raw_hot}` ]); }); - utils.saveJSON({ + fileUtils.saveJSON({ saveFolder: DATA_FOLDER, now: now, fileNameSuffix: `regulation`, @@ -205,7 +183,7 @@ async function main() { data.band_list.forEach(function (item) { delete item["mblog"]; }); - utils.saveJSON({ + fileUtils.saveJSON({ saveFolder: DATA_FOLDER, now: now, fileNameSuffix: `simplify`, diff --git a/src/utils/utils.js b/src/utils/fileUtils.js similarity index 100% rename from src/utils/utils.js rename to src/utils/fileUtils.js diff --git a/src/utils/requestUtils.js b/src/utils/requestUtils.js new file mode 100644 index 0000000..ef37af4 --- /dev/null +++ b/src/utils/requestUtils.js @@ -0,0 +1,27 @@ +const request = require('request'); + +// 请求 APi 接口 +async function getApiResult(url) { + var return_data = await new Promise((resolve) => { + request({ + method: 'GET', + url: url, + json: true, + }, (error, response, result) => { + if (!error && (response.statusCode == 200)) { + // 请求成功 + resolve(result); + } else { + // 请求失败 + console.log(`error is ${error}`); + resolve("error"); + } + }); + }); + // console.log(`return_data is ${JSON.stringify(return_data)}`); + return return_data; +} + +module.exports = { + getApiResult, +} \ No newline at end of file