From 40173ce54d267fc334c2909579403b4db13ec559 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E5=B0=8F=E5=A2=A8?= <2291200076@qq.com> Date: Sat, 23 Jul 2022 21:12:48 +0800 Subject: [PATCH 01/50] first commit --- .gitignore | 7 + data/.gitkeep | 0 index.js | 247 +++++++++++ package-lock.json | 1011 +++++++++++++++++++++++++++++++++++++++++++++ package.json | 15 + 5 files changed, 1280 insertions(+) create mode 100644 .gitignore create mode 100644 data/.gitkeep create mode 100644 index.js create mode 100644 package-lock.json create mode 100644 package.json diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e43125c --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +.DS_Store +test.js + +node_modules + +data/* +!data/.gitkeep \ No newline at end of file diff --git a/data/.gitkeep b/data/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/index.js b/index.js new file mode 100644 index 0000000..1ccb8a0 --- /dev/null +++ b/index.js @@ -0,0 +1,247 @@ +'use strict'; + +const request = require('request'); +const fs = require('fs'); + +console.log("Start running ..."); + +// 请求微博热搜 APi 接口 +async function getApiResult(url) { + var return_data = await new Promise((resolve) => { + request({ + method: 'GET', + url: url, + json: true, + }, (error, response, result) => { + if (!error && (response.statusCode == 200)) { + // 请求成功 + resolve(result); + } else { + // 请求失败 + console.log(`error is ${error}`); + resolve("error"); + } + }); + }); + // console.log(`return_data is ${JSON.stringify(return_data)}`); + return return_data; +} + +// 创建目录 +async function createFolder(folderToCreate) { + let currentFolder = folderToCreate.replace(/\\/g, '/'); + let parentFolder = currentFolder.substring(0, currentFolder.lastIndexOf('/')); + if (!fs.existsSync(currentFolder)) { + // 文件夹不存在，创建文件夹 + createFolder(parentFolder); // 保证父级文件夹存在 + fs.mkdirSync(currentFolder); // 创建当前级文件夹 + } else { + // 否则就什么也不做 + } +} + +// 保存 JSON +function saveJSON({ now, fileNameSuffix, object, compress = true, uncompress = true }) { + + let year = now.substring(0, 4); + let month = now.substring(5, 7); + let day = now.substring(8, 10); + let hour = now.substring(11, 13); + let minute = now.substring(14, 16); + // console.log(now); + // console.log( "year, month, day, hour, minute: " + year + ", " + month + ", " + day + ", " + hour + ", " + minute); + + // 创建当前文件夹 + let folder = `./data/${year}/${month}/${day}`; + createFolder(folder); + let fileName = `${folder}/${year}${month}${day}_${hour}${minute}_${fileNameSuffix}`; + + // 生成文件名 + // '2022-07-23T10:11:38.650Z' => '20220723_1011' + // let fileName = now.replace(/T/, '_').replace(/:\d{2}.\d{3}Z/, '').replace(/[-:]/g, ''); + // console.log(`fileName is ${fileName}`); + + if (compress) + fs.writeFileSync(`${fileName}.min.json`, JSON.stringify(object)); + if (uncompress) + fs.writeFileSync(`${fileName}.json`, JSON.stringify(object, "", "\t")); +} + +async function main() { + let requestTimestamp = Date.now(); + let now = new Date(requestTimestamp + 8 * 3600 * 1000).toISOString(); + + let result = await getApiResult("https://weibo.com/ajax/statuses/hot_band"); + // console.log("result", result); + + if (result.ok != 1) { + console.log("请求成功，但服务器处理失败。"); + } else { + console.log("请求成功。"); + + /** + * 保存原始数据 + */ + saveJSON({ + now: now, + fileNameSuffix: `origin`, + object: result, + compress: true, + uncompress: false + }); + + let data = JSON.parse(JSON.stringify(result.data)); + + + /** + * 过滤掉不需要的数据 + */ + // hotgov + delete data.hotgov["mblog"]; + // 重复字段只保留一个 + delete data.hotgov["note"]; // note word + delete data.hotgov["small_icon_desc"]; // icon_desc small_icon_desc + delete data.hotgov["small_icon_desc_color"]; // icon_desc_color small_icon_desc_color + + // band_list + for (let i = 0; i < data.band_list.length; i++) { + const item = data.band_list[i]; + + // 过滤广告 + if (item.is_ad) { + data.band_list.splice(i, 1); + i--; + } + + // 过滤空字段 + delete item["ad_info"]; + + // 重复字段只保留一个 + delete item["note"]; // note word + delete item["icon_desc"]; delete item["small_icon_desc"]; // label_name icon_desc small_icon_desc + delete item["small_icon_desc_color"]; // icon_desc_color small_icon_desc_color + delete item["flag_desc"]; // flag_desc subject_label 这两个有值的时候相同，没有值的时候，前一个为 undefined，后一个为 "" + } + + + /** + * 获取需要的数据，进行转换 + */ + let convert = []; + data.band_list.forEach(item => { + let detail = ""; + let pic_ids = []; + if (item.mblog) { // 有些热搜没有 mblog + var regex = /(<([^>]+)>)/ig + detail = item.mblog.text.replace(regex, ""); + if (item.mblog.pics) { + pic_ids = item.mblog.pics.map(pic => `${pic}`); + } + } + convert.push({ + // 热搜排行顺序 + rank: item.rank, + realpos: item.realpos, + + // 热搜信息 + word: item.word, // 热搜标题 + word_scheme: item.word_scheme, // 热搜话题 "#热搜标题#" + emoticon: item.emoticon, // 热搜小表情，如 "[泪]" + label_name: item.label_name, // 热搜标签，如 "爆" "热" "新" "" + onboard_time: item.onboard_time, // 热搜上线时间，秒级时间戳，如 1658565575 + + /** + * 热搜数据 + * + * 大部分的 num 和 raw_hot 是相同的，页面上显示的是 num，可能是人工调控的热搜 + * + * 两者差值通过观测似乎最大是 1250000 + * 例如 【爆】唐山打架事件8名违法嫌疑人已到案 这条热搜一开始 delta 首先不断增大，最大达到 1250000 + * 然后热搜数量增加到 12600000 左右的时候，delta 逐渐减小到 1040000 左右 + * 所有热搜的 detla（带正负） 加起来就是基本上在100000-230000之间 + */ + num: item.num, + raw_hot: item.raw_hot, + detla: item.num - item.raw_hot, // 计算值 + + url: `https://s.weibo.com/weibo?q=${encodeURIComponent(item.word_scheme)}`, // 热搜话题链接 + + // 分类 + category: item.category ? item.category.split(',') : "", + subject_label: item.subject_label, + + // 其他 + more: { + is_new: item.is_new, + subject_querys: item.subject_querys, + mid: item.mid, + icon_desc_color: item.icon_desc_color, + detail: detail, + }, + }); + }); + saveJSON({ + now: now, + fileNameSuffix: `final`, + object: convert, + compress: true, + uncompress: true + }); + + + /** + * 只统计微博调控信息 + */ + let convert2 = []; + let total = 0; + data.band_list.forEach(item => { + total += item.num; + total -= item.raw_hot; + if (item.num - item.raw_hot == 0) return; + convert2.push([ + `[${item.realpos}] ${item.word}【${item.label_name}】`, + `原始:${item.raw_hot} 显示:${item.num} 调控: ${item.num - item.raw_hot}` + ]); + }); + saveJSON({ + now: now, + fileNameSuffix: `regulation`, + object: { + total_delta: total, // 所有调控值之和 + data: convert2 + }, + compress: false, + uncompress: true + }); + + + /** + * 保存预处理后数据 + */ + // 过滤掉不需要的数据 + // band_list + data.band_list.forEach(function (item) { + delete item["mblog"]; + }); + saveJSON({ + now: now, + fileNameSuffix: `simplify`, + object: data, + compress: true, + uncompress: true + }); + + + /** + * 更新最新的 + */ + fs.writeFileSync(`./data/latest.json`, JSON.stringify({ + update_time: requestTimestamp, + update_time_friendly: now.substring(0, 19).replace(/T/g, " "), + regulation: convert2, + data: convert + })); + } +} + +main(); \ No newline at end of file diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..e509250 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,1011 @@ +{ + "name": "weibo-hot-band", + "version": "0.0.0", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "weibo-hot-band", + "version": "0.0.0", + "dependencies": { + "dotenv": "^16.0.1", + "fs": "^0.0.1-security", + "node-schedule": "^2.1.0", + "request": "^2.88.2" + } + }, + "node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmmirror.com/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "node_modules/asn1": { + "version": "0.2.6", + "resolved": "https://registry.npmmirror.com/asn1/-/asn1-0.2.6.tgz", + "integrity": "sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==", + "dependencies": { + "safer-buffer": "~2.1.0" + } + }, + "node_modules/assert-plus": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha512-NfJ4UzBCcQGLDlQq7nHxH+tv3kyZ0hHQqF5BO6J7tNJeP5do1llPr8dZ8zHonfhAu0PHAdMkSo+8o0wxg9lZWw==", + "engines": { + "node": ">=0.8" + } + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmmirror.com/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" + }, + "node_modules/aws-sign2": { + "version": "0.7.0", + "resolved": "https://registry.npmmirror.com/aws-sign2/-/aws-sign2-0.7.0.tgz", + "integrity": "sha512-08kcGqnYf/YmjoRhfxyu+CLxBjUtHLXLXX/vUfx9l2LYzG3c1m61nrpyFUZI6zeS+Li/wWMMidD9KgrqtGq3mA==", + "engines": { + "node": "*" + } + }, + "node_modules/aws4": { + "version": "1.11.0", + "resolved": "https://registry.npmmirror.com/aws4/-/aws4-1.11.0.tgz", + "integrity": "sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA==" + }, + "node_modules/bcrypt-pbkdf": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", + "integrity": "sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==", + "dependencies": { + "tweetnacl": "^0.14.3" + } + }, + "node_modules/call-bind": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/call-bind/-/call-bind-1.0.2.tgz", + "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", + "dependencies": { + "function-bind": "^1.1.1", + "get-intrinsic": "^1.0.2" + } + }, + "node_modules/caseless": { + "version": "0.12.0", + "resolved": "https://registry.npmmirror.com/caseless/-/caseless-0.12.0.tgz", + "integrity": "sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==" + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmmirror.com/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/core-util-is": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ==" + }, + "node_modules/cron-parser": { + "version": "3.5.0", + "resolved": "https://registry.npmmirror.com/cron-parser/-/cron-parser-3.5.0.tgz", + "integrity": "sha512-wyVZtbRs6qDfFd8ap457w3XVntdvqcwBGxBoTvJQH9KGVKL/fB+h2k3C8AqiVxvUQKN1Ps/Ns46CNViOpVDhfQ==", + "dependencies": { + "is-nan": "^1.3.2", + "luxon": "^1.26.0" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/dashdash": { + "version": "1.14.1", + "resolved": "https://registry.npmmirror.com/dashdash/-/dashdash-1.14.1.tgz", + "integrity": "sha512-jRFi8UDGo6j+odZiEpjazZaWqEal3w/basFjQHQEwVtZJGDpxbH1MeYluwCS8Xq5wmLJooDlMgvVarmWfGM44g==", + "dependencies": { + "assert-plus": "^1.0.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/define-properties": { + "version": "1.1.4", + "resolved": "https://registry.npmmirror.com/define-properties/-/define-properties-1.1.4.tgz", + "integrity": "sha512-uckOqKcfaVvtBdsVkdPv3XjveQJsNQqmhXgRi8uhvWWuPYZCNlzT8qAyblUgNoXdHdjMTzAqeGjAoli8f+bzPA==", + "dependencies": { + "has-property-descriptors": "^1.0.0", + "object-keys": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/dotenv": { + "version": "16.0.1", + "resolved": "https://registry.npmmirror.com/dotenv/-/dotenv-16.0.1.tgz", + "integrity": "sha512-1K6hR6wtk2FviQ4kEiSjFiH5rpzEVi8WW0x96aztHVMhEspNpc4DVOUTEHtEva5VThQ8IaBX1Pe4gSzpVVUsKQ==", + "engines": { + "node": ">=12" + } + }, + "node_modules/ecc-jsbn": { + "version": "0.1.2", + "resolved": "https://registry.npmmirror.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", + "integrity": "sha512-eh9O+hwRHNbG4BLTjEl3nw044CkGm5X6LoaCf7LPp7UU8Qrt47JYNi6nPX8xjW97TKGKm1ouctg0QSpZe9qrnw==", + "dependencies": { + "jsbn": "~0.1.0", + "safer-buffer": "^2.1.0" + } + }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmmirror.com/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" + }, + "node_modules/extsprintf": { + "version": "1.3.0", + "resolved": "https://registry.npmmirror.com/extsprintf/-/extsprintf-1.3.0.tgz", + "integrity": "sha512-11Ndz7Nv+mvAC1j0ktTa7fAb0vLyGGX+rMHNBYQviQDGU0Hw7lhctJANqbPhu9nV9/izT/IntTgZ7Im/9LJs9g==", + "engines": [ + "node >=0.6.0" + ] + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmmirror.com/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" + }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==" + }, + "node_modules/forever-agent": { + "version": "0.6.1", + "resolved": "https://registry.npmmirror.com/forever-agent/-/forever-agent-0.6.1.tgz", + "integrity": "sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw==", + "engines": { + "node": "*" + } + }, + "node_modules/form-data": { + "version": "2.3.3", + "resolved": "https://registry.npmmirror.com/form-data/-/form-data-2.3.3.tgz", + "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.6", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 0.12" + } + }, + "node_modules/fs": { + "version": "0.0.1-security", + "resolved": "https://registry.npmmirror.com/fs/-/fs-0.0.1-security.tgz", + "integrity": "sha512-3XY9e1pP0CVEUCdj5BmfIZxRBTSDycnbqhIOGec9QYtmVH2fbLpj86CFWkrNOkt/Fvty4KZG5lTglL9j/gJ87w==" + }, + "node_modules/function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmmirror.com/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" + }, + "node_modules/get-intrinsic": { + "version": "1.1.2", + "resolved": "https://registry.npmmirror.com/get-intrinsic/-/get-intrinsic-1.1.2.tgz", + "integrity": "sha512-Jfm3OyCxHh9DJyc28qGk+JmfkpO41A4XkneDSujN9MDXrm4oDKdHvndhZ2dN94+ERNfkYJWDclW6k2L/ZGHjXA==", + "dependencies": { + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.3" + } + }, + "node_modules/getpass": { + "version": "0.1.7", + "resolved": "https://registry.npmmirror.com/getpass/-/getpass-0.1.7.tgz", + "integrity": "sha512-0fzj9JxOLfJ+XGLhR8ze3unN0KZCgZwiSSDz168VERjK8Wl8kVSdcu2kspd4s4wtAa1y/qrVRiAA0WclVsu0ng==", + "dependencies": { + "assert-plus": "^1.0.0" + } + }, + "node_modules/har-schema": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/har-schema/-/har-schema-2.0.0.tgz", + "integrity": "sha512-Oqluz6zhGX8cyRaTQlFMPw80bSJVG2x/cFb8ZPhUILGgHka9SsokCCOQgpveePerqidZOrT14ipqfJb7ILcW5Q==", + "engines": { + "node": ">=4" + } + }, + "node_modules/har-validator": { + "version": "5.1.5", + "resolved": "https://registry.npmmirror.com/har-validator/-/har-validator-5.1.5.tgz", + "integrity": "sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w==", + "deprecated": "this library is no longer supported", + "dependencies": { + "ajv": "^6.12.3", + "har-schema": "^2.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/has": { + "version": "1.0.3", + "resolved": "https://registry.npmmirror.com/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "dependencies": { + "function-bind": "^1.1.1" + }, + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/has-property-descriptors": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/has-property-descriptors/-/has-property-descriptors-1.0.0.tgz", + "integrity": "sha512-62DVLZGoiEBDHQyqG4w9xCuZ7eJEwNmJRWw2VY84Oedb7WFcA27fiEVe8oUQx9hAUJ4ekurquucTGwsyO1XGdQ==", + "dependencies": { + "get-intrinsic": "^1.1.1" + } + }, + "node_modules/has-symbols": { + "version": "1.0.3", + "resolved": "https://registry.npmmirror.com/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/http-signature": { + "version": "1.2.0", + "resolved": "https://registry.npmmirror.com/http-signature/-/http-signature-1.2.0.tgz", + "integrity": "sha512-CAbnr6Rz4CYQkLYUtSNXxQPUH2gK8f3iWexVlsnMeD+GjlsQ0Xsy1cOX+mN3dtxYomRy21CiOzU8Uhw6OwncEQ==", + "dependencies": { + "assert-plus": "^1.0.0", + "jsprim": "^1.2.2", + "sshpk": "^1.7.0" + }, + "engines": { + "node": ">=0.8", + "npm": ">=1.3.7" + } + }, + "node_modules/is-nan": { + "version": "1.3.2", + "resolved": "https://registry.npmmirror.com/is-nan/-/is-nan-1.3.2.tgz", + "integrity": "sha512-E+zBKpQ2t6MEo1VsonYmluk9NxGrbzpeeLC2xIViuO2EjU2xsXsBPwTr3Ykv9l08UYEVEdWeRZNouaZqF6RN0w==", + "dependencies": { + "call-bind": "^1.0.0", + "define-properties": "^1.1.3" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/is-typedarray": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/is-typedarray/-/is-typedarray-1.0.0.tgz", + "integrity": "sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA==" + }, + "node_modules/isstream": { + "version": "0.1.2", + "resolved": "https://registry.npmmirror.com/isstream/-/isstream-0.1.2.tgz", + "integrity": "sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g==" + }, + "node_modules/jsbn": { + "version": "0.1.1", + "resolved": "https://registry.npmmirror.com/jsbn/-/jsbn-0.1.1.tgz", + "integrity": "sha512-UVU9dibq2JcFWxQPA6KCqj5O42VOmAY3zQUfEKxU0KpTGXwNoCjkX1e13eHNvw/xPynt6pU0rZ1htjWTNTSXsg==" + }, + "node_modules/json-schema": { + "version": "0.4.0", + "resolved": "https://registry.npmmirror.com/json-schema/-/json-schema-0.4.0.tgz", + "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==" + }, + "node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmmirror.com/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" + }, + "node_modules/json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmmirror.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==" + }, + "node_modules/jsprim": { + "version": "1.4.2", + "resolved": "https://registry.npmmirror.com/jsprim/-/jsprim-1.4.2.tgz", + "integrity": "sha512-P2bSOMAc/ciLz6DzgjVlGJP9+BrJWu5UDGK70C2iweC5QBIeFf0ZXRvGjEj2uYgrY2MkAAhsSWHDWlFtEroZWw==", + "dependencies": { + "assert-plus": "1.0.0", + "extsprintf": "1.3.0", + "json-schema": "0.4.0", + "verror": "1.10.0" + }, + "engines": { + "node": ">=0.6.0" + } + }, + "node_modules/long-timeout": { + "version": "0.1.1", + "resolved": "https://registry.npmmirror.com/long-timeout/-/long-timeout-0.1.1.tgz", + "integrity": "sha512-BFRuQUqc7x2NWxfJBCyUrN8iYUYznzL9JROmRz1gZ6KlOIgmoD+njPVbb+VNn2nGMKggMsK79iUNErillsrx7w==" + }, + "node_modules/luxon": { + "version": "1.28.0", + "resolved": "https://registry.npmmirror.com/luxon/-/luxon-1.28.0.tgz", + "integrity": "sha512-TfTiyvZhwBYM/7QdAVDh+7dBTBA29v4ik0Ce9zda3Mnf8on1S5KJI8P2jKFZ8+5C0jhmr0KwJEO/Wdpm0VeWJQ==", + "engines": { + "node": "*" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmmirror.com/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmmirror.com/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/node-schedule": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/node-schedule/-/node-schedule-2.1.0.tgz", + "integrity": "sha512-nl4JTiZ7ZQDc97MmpTq9BQjYhq7gOtoh7SiPH069gBFBj0PzD8HI7zyFs6rzqL8Y5tTiEEYLxgtbx034YPrbyQ==", + "dependencies": { + "cron-parser": "^3.5.0", + "long-timeout": "0.1.1", + "sorted-array-functions": "^1.3.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/oauth-sign": { + "version": "0.9.0", + "resolved": "https://registry.npmmirror.com/oauth-sign/-/oauth-sign-0.9.0.tgz", + "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==", + "engines": { + "node": "*" + } + }, + "node_modules/object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmmirror.com/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/performance-now": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/performance-now/-/performance-now-2.1.0.tgz", + "integrity": "sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==" + }, + "node_modules/psl": { + "version": "1.9.0", + "resolved": "https://registry.npmmirror.com/psl/-/psl-1.9.0.tgz", + "integrity": "sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==" + }, + "node_modules/punycode": { + "version": "2.1.1", + "resolved": "https://registry.npmmirror.com/punycode/-/punycode-2.1.1.tgz", + "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", + "engines": { + "node": ">=6" + } + }, + "node_modules/qs": { + "version": "6.5.3", + "resolved": "https://registry.npmmirror.com/qs/-/qs-6.5.3.tgz", + "integrity": "sha512-qxXIEh4pCGfHICj1mAJQ2/2XVZkjCDTcEgfoSQxc/fYivUZxTkk7L3bDBJSoNrEzXI17oUO5Dp07ktqE5KzczA==", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/request": { + "version": "2.88.2", + "resolved": "https://registry.npmmirror.com/request/-/request-2.88.2.tgz", + "integrity": "sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==", + "deprecated": "request has been deprecated, see https://github.com/request/request/issues/3142", + "dependencies": { + "aws-sign2": "~0.7.0", + "aws4": "^1.8.0", + "caseless": "~0.12.0", + "combined-stream": "~1.0.6", + "extend": "~3.0.2", + "forever-agent": "~0.6.1", + "form-data": "~2.3.2", + "har-validator": "~5.1.3", + "http-signature": "~1.2.0", + "is-typedarray": "~1.0.0", + "isstream": "~0.1.2", + "json-stringify-safe": "~5.0.1", + "mime-types": "~2.1.19", + "oauth-sign": "~0.9.0", + "performance-now": "^2.1.0", + "qs": "~6.5.2", + "safe-buffer": "^5.1.2", + "tough-cookie": "~2.5.0", + "tunnel-agent": "^0.6.0", + "uuid": "^3.3.2" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmmirror.com/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, + "node_modules/sorted-array-functions": { + "version": "1.3.0", + "resolved": "https://registry.npmmirror.com/sorted-array-functions/-/sorted-array-functions-1.3.0.tgz", + "integrity": "sha512-2sqgzeFlid6N4Z2fUQ1cvFmTOLRi/sEDzSQ0OKYchqgoPmQBVyM3959qYx3fpS6Esef80KjmpgPeEr028dP3OA==" + }, + "node_modules/sshpk": { + "version": "1.17.0", + "resolved": "https://registry.npmmirror.com/sshpk/-/sshpk-1.17.0.tgz", + "integrity": "sha512-/9HIEs1ZXGhSPE8X6Ccm7Nam1z8KcoCqPdI7ecm1N33EzAetWahvQWVqLZtaZQ+IDKX4IyA2o0gBzqIMkAagHQ==", + "dependencies": { + "asn1": "~0.2.3", + "assert-plus": "^1.0.0", + "bcrypt-pbkdf": "^1.0.0", + "dashdash": "^1.12.0", + "ecc-jsbn": "~0.1.1", + "getpass": "^0.1.1", + "jsbn": "~0.1.0", + "safer-buffer": "^2.0.2", + "tweetnacl": "~0.14.0" + }, + "bin": { + "sshpk-conv": "bin/sshpk-conv", + "sshpk-sign": "bin/sshpk-sign", + "sshpk-verify": "bin/sshpk-verify" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/tough-cookie": { + "version": "2.5.0", + "resolved": "https://registry.npmmirror.com/tough-cookie/-/tough-cookie-2.5.0.tgz", + "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", + "dependencies": { + "psl": "^1.1.28", + "punycode": "^2.1.1" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmmirror.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", + "dependencies": { + "safe-buffer": "^5.0.1" + }, + "engines": { + "node": "*" + } + }, + "node_modules/tweetnacl": { + "version": "0.14.5", + "resolved": "https://registry.npmmirror.com/tweetnacl/-/tweetnacl-0.14.5.tgz", + "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==" + }, + "node_modules/uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmmirror.com/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/uuid": { + "version": "3.4.0", + "resolved": "https://registry.npmmirror.com/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==", + "deprecated": "Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details.", + "bin": { + "uuid": "bin/uuid" + } + }, + "node_modules/verror": { + "version": "1.10.0", + "resolved": "https://registry.npmmirror.com/verror/-/verror-1.10.0.tgz", + "integrity": "sha512-ZZKSmDAEFOijERBLkmYfJ+vmk3w+7hOLYDNkRCuRuMJGEmqYNCNLyBBFwWKVMhfwaEF3WOd0Zlw86U/WC/+nYw==", + "engines": [ + "node >=0.6.0" + ], + "dependencies": { + "assert-plus": "^1.0.0", + "core-util-is": "1.0.2", + "extsprintf": "^1.2.0" + } + } + }, + "dependencies": { + "ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmmirror.com/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "requires": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "asn1": { + "version": "0.2.6", + "resolved": "https://registry.npmmirror.com/asn1/-/asn1-0.2.6.tgz", + "integrity": "sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==", + "requires": { + "safer-buffer": "~2.1.0" + } + }, + "assert-plus": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha512-NfJ4UzBCcQGLDlQq7nHxH+tv3kyZ0hHQqF5BO6J7tNJeP5do1llPr8dZ8zHonfhAu0PHAdMkSo+8o0wxg9lZWw==" + }, + "asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmmirror.com/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" + }, + "aws-sign2": { + "version": "0.7.0", + "resolved": "https://registry.npmmirror.com/aws-sign2/-/aws-sign2-0.7.0.tgz", + "integrity": "sha512-08kcGqnYf/YmjoRhfxyu+CLxBjUtHLXLXX/vUfx9l2LYzG3c1m61nrpyFUZI6zeS+Li/wWMMidD9KgrqtGq3mA==" + }, + "aws4": { + "version": "1.11.0", + "resolved": "https://registry.npmmirror.com/aws4/-/aws4-1.11.0.tgz", + "integrity": "sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA==" + }, + "bcrypt-pbkdf": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", + "integrity": "sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==", + "requires": { + "tweetnacl": "^0.14.3" + } + }, + "call-bind": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/call-bind/-/call-bind-1.0.2.tgz", + "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", + "requires": { + "function-bind": "^1.1.1", + "get-intrinsic": "^1.0.2" + } + }, + "caseless": { + "version": "0.12.0", + "resolved": "https://registry.npmmirror.com/caseless/-/caseless-0.12.0.tgz", + "integrity": "sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==" + }, + "combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmmirror.com/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "requires": { + "delayed-stream": "~1.0.0" + } + }, + "core-util-is": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ==" + }, + "cron-parser": { + "version": "3.5.0", + "resolved": "https://registry.npmmirror.com/cron-parser/-/cron-parser-3.5.0.tgz", + "integrity": "sha512-wyVZtbRs6qDfFd8ap457w3XVntdvqcwBGxBoTvJQH9KGVKL/fB+h2k3C8AqiVxvUQKN1Ps/Ns46CNViOpVDhfQ==", + "requires": { + "is-nan": "^1.3.2", + "luxon": "^1.26.0" + } + }, + "dashdash": { + "version": "1.14.1", + "resolved": "https://registry.npmmirror.com/dashdash/-/dashdash-1.14.1.tgz", + "integrity": "sha512-jRFi8UDGo6j+odZiEpjazZaWqEal3w/basFjQHQEwVtZJGDpxbH1MeYluwCS8Xq5wmLJooDlMgvVarmWfGM44g==", + "requires": { + "assert-plus": "^1.0.0" + } + }, + "define-properties": { + "version": "1.1.4", + "resolved": "https://registry.npmmirror.com/define-properties/-/define-properties-1.1.4.tgz", + "integrity": "sha512-uckOqKcfaVvtBdsVkdPv3XjveQJsNQqmhXgRi8uhvWWuPYZCNlzT8qAyblUgNoXdHdjMTzAqeGjAoli8f+bzPA==", + "requires": { + "has-property-descriptors": "^1.0.0", + "object-keys": "^1.1.1" + } + }, + "delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==" + }, + "dotenv": { + "version": "16.0.1", + "resolved": "https://registry.npmmirror.com/dotenv/-/dotenv-16.0.1.tgz", + "integrity": "sha512-1K6hR6wtk2FviQ4kEiSjFiH5rpzEVi8WW0x96aztHVMhEspNpc4DVOUTEHtEva5VThQ8IaBX1Pe4gSzpVVUsKQ==" + }, + "ecc-jsbn": { + "version": "0.1.2", + "resolved": "https://registry.npmmirror.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", + "integrity": "sha512-eh9O+hwRHNbG4BLTjEl3nw044CkGm5X6LoaCf7LPp7UU8Qrt47JYNi6nPX8xjW97TKGKm1ouctg0QSpZe9qrnw==", + "requires": { + "jsbn": "~0.1.0", + "safer-buffer": "^2.1.0" + } + }, + "extend": { + "version": "3.0.2", + "resolved": "https://registry.npmmirror.com/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" + }, + "extsprintf": { + "version": "1.3.0", + "resolved": "https://registry.npmmirror.com/extsprintf/-/extsprintf-1.3.0.tgz", + "integrity": "sha512-11Ndz7Nv+mvAC1j0ktTa7fAb0vLyGGX+rMHNBYQviQDGU0Hw7lhctJANqbPhu9nV9/izT/IntTgZ7Im/9LJs9g==" + }, + "fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmmirror.com/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" + }, + "fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==" + }, + "forever-agent": { + "version": "0.6.1", + "resolved": "https://registry.npmmirror.com/forever-agent/-/forever-agent-0.6.1.tgz", + "integrity": "sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw==" + }, + "form-data": { + "version": "2.3.3", + "resolved": "https://registry.npmmirror.com/form-data/-/form-data-2.3.3.tgz", + "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.6", + "mime-types": "^2.1.12" + } + }, + "fs": { + "version": "0.0.1-security", + "resolved": "https://registry.npmmirror.com/fs/-/fs-0.0.1-security.tgz", + "integrity": "sha512-3XY9e1pP0CVEUCdj5BmfIZxRBTSDycnbqhIOGec9QYtmVH2fbLpj86CFWkrNOkt/Fvty4KZG5lTglL9j/gJ87w==" + }, + "function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmmirror.com/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" + }, + "get-intrinsic": { + "version": "1.1.2", + "resolved": "https://registry.npmmirror.com/get-intrinsic/-/get-intrinsic-1.1.2.tgz", + "integrity": "sha512-Jfm3OyCxHh9DJyc28qGk+JmfkpO41A4XkneDSujN9MDXrm4oDKdHvndhZ2dN94+ERNfkYJWDclW6k2L/ZGHjXA==", + "requires": { + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.3" + } + }, + "getpass": { + "version": "0.1.7", + "resolved": "https://registry.npmmirror.com/getpass/-/getpass-0.1.7.tgz", + "integrity": "sha512-0fzj9JxOLfJ+XGLhR8ze3unN0KZCgZwiSSDz168VERjK8Wl8kVSdcu2kspd4s4wtAa1y/qrVRiAA0WclVsu0ng==", + "requires": { + "assert-plus": "^1.0.0" + } + }, + "har-schema": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/har-schema/-/har-schema-2.0.0.tgz", + "integrity": "sha512-Oqluz6zhGX8cyRaTQlFMPw80bSJVG2x/cFb8ZPhUILGgHka9SsokCCOQgpveePerqidZOrT14ipqfJb7ILcW5Q==" + }, + "har-validator": { + "version": "5.1.5", + "resolved": "https://registry.npmmirror.com/har-validator/-/har-validator-5.1.5.tgz", + "integrity": "sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w==", + "requires": { + "ajv": "^6.12.3", + "har-schema": "^2.0.0" + } + }, + "has": { + "version": "1.0.3", + "resolved": "https://registry.npmmirror.com/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "requires": { + "function-bind": "^1.1.1" + } + }, + "has-property-descriptors": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/has-property-descriptors/-/has-property-descriptors-1.0.0.tgz", + "integrity": "sha512-62DVLZGoiEBDHQyqG4w9xCuZ7eJEwNmJRWw2VY84Oedb7WFcA27fiEVe8oUQx9hAUJ4ekurquucTGwsyO1XGdQ==", + "requires": { + "get-intrinsic": "^1.1.1" + } + }, + "has-symbols": { + "version": "1.0.3", + "resolved": "https://registry.npmmirror.com/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==" + }, + "http-signature": { + "version": "1.2.0", + "resolved": "https://registry.npmmirror.com/http-signature/-/http-signature-1.2.0.tgz", + "integrity": "sha512-CAbnr6Rz4CYQkLYUtSNXxQPUH2gK8f3iWexVlsnMeD+GjlsQ0Xsy1cOX+mN3dtxYomRy21CiOzU8Uhw6OwncEQ==", + "requires": { + "assert-plus": "^1.0.0", + "jsprim": "^1.2.2", + "sshpk": "^1.7.0" + } + }, + "is-nan": { + "version": "1.3.2", + "resolved": "https://registry.npmmirror.com/is-nan/-/is-nan-1.3.2.tgz", + "integrity": "sha512-E+zBKpQ2t6MEo1VsonYmluk9NxGrbzpeeLC2xIViuO2EjU2xsXsBPwTr3Ykv9l08UYEVEdWeRZNouaZqF6RN0w==", + "requires": { + "call-bind": "^1.0.0", + "define-properties": "^1.1.3" + } + }, + "is-typedarray": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/is-typedarray/-/is-typedarray-1.0.0.tgz", + "integrity": "sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA==" + }, + "isstream": { + "version": "0.1.2", + "resolved": "https://registry.npmmirror.com/isstream/-/isstream-0.1.2.tgz", + "integrity": "sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g==" + }, + "jsbn": { + "version": "0.1.1", + "resolved": "https://registry.npmmirror.com/jsbn/-/jsbn-0.1.1.tgz", + "integrity": "sha512-UVU9dibq2JcFWxQPA6KCqj5O42VOmAY3zQUfEKxU0KpTGXwNoCjkX1e13eHNvw/xPynt6pU0rZ1htjWTNTSXsg==" + }, + "json-schema": { + "version": "0.4.0", + "resolved": "https://registry.npmmirror.com/json-schema/-/json-schema-0.4.0.tgz", + "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==" + }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmmirror.com/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" + }, + "json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmmirror.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==" + }, + "jsprim": { + "version": "1.4.2", + "resolved": "https://registry.npmmirror.com/jsprim/-/jsprim-1.4.2.tgz", + "integrity": "sha512-P2bSOMAc/ciLz6DzgjVlGJP9+BrJWu5UDGK70C2iweC5QBIeFf0ZXRvGjEj2uYgrY2MkAAhsSWHDWlFtEroZWw==", + "requires": { + "assert-plus": "1.0.0", + "extsprintf": "1.3.0", + "json-schema": "0.4.0", + "verror": "1.10.0" + } + }, + "long-timeout": { + "version": "0.1.1", + "resolved": "https://registry.npmmirror.com/long-timeout/-/long-timeout-0.1.1.tgz", + "integrity": "sha512-BFRuQUqc7x2NWxfJBCyUrN8iYUYznzL9JROmRz1gZ6KlOIgmoD+njPVbb+VNn2nGMKggMsK79iUNErillsrx7w==" + }, + "luxon": { + "version": "1.28.0", + "resolved": "https://registry.npmmirror.com/luxon/-/luxon-1.28.0.tgz", + "integrity": "sha512-TfTiyvZhwBYM/7QdAVDh+7dBTBA29v4ik0Ce9zda3Mnf8on1S5KJI8P2jKFZ8+5C0jhmr0KwJEO/Wdpm0VeWJQ==" + }, + "mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmmirror.com/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==" + }, + "mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmmirror.com/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "requires": { + "mime-db": "1.52.0" + } + }, + "node-schedule": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/node-schedule/-/node-schedule-2.1.0.tgz", + "integrity": "sha512-nl4JTiZ7ZQDc97MmpTq9BQjYhq7gOtoh7SiPH069gBFBj0PzD8HI7zyFs6rzqL8Y5tTiEEYLxgtbx034YPrbyQ==", + "requires": { + "cron-parser": "^3.5.0", + "long-timeout": "0.1.1", + "sorted-array-functions": "^1.3.0" + } + }, + "oauth-sign": { + "version": "0.9.0", + "resolved": "https://registry.npmmirror.com/oauth-sign/-/oauth-sign-0.9.0.tgz", + "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==" + }, + "object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmmirror.com/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==" + }, + "performance-now": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/performance-now/-/performance-now-2.1.0.tgz", + "integrity": "sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==" + }, + "psl": { + "version": "1.9.0", + "resolved": "https://registry.npmmirror.com/psl/-/psl-1.9.0.tgz", + "integrity": "sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==" + }, + "punycode": { + "version": "2.1.1", + "resolved": "https://registry.npmmirror.com/punycode/-/punycode-2.1.1.tgz", + "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" + }, + "qs": { + "version": "6.5.3", + "resolved": "https://registry.npmmirror.com/qs/-/qs-6.5.3.tgz", + "integrity": "sha512-qxXIEh4pCGfHICj1mAJQ2/2XVZkjCDTcEgfoSQxc/fYivUZxTkk7L3bDBJSoNrEzXI17oUO5Dp07ktqE5KzczA==" + }, + "request": { + "version": "2.88.2", + "resolved": "https://registry.npmmirror.com/request/-/request-2.88.2.tgz", + "integrity": "sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==", + "requires": { + "aws-sign2": "~0.7.0", + "aws4": "^1.8.0", + "caseless": "~0.12.0", + "combined-stream": "~1.0.6", + "extend": "~3.0.2", + "forever-agent": "~0.6.1", + "form-data": "~2.3.2", + "har-validator": "~5.1.3", + "http-signature": "~1.2.0", + "is-typedarray": "~1.0.0", + "isstream": "~0.1.2", + "json-stringify-safe": "~5.0.1", + "mime-types": "~2.1.19", + "oauth-sign": "~0.9.0", + "performance-now": "^2.1.0", + "qs": "~6.5.2", + "safe-buffer": "^5.1.2", + "tough-cookie": "~2.5.0", + "tunnel-agent": "^0.6.0", + "uuid": "^3.3.2" + } + }, + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmmirror.com/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + }, + "safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, + "sorted-array-functions": { + "version": "1.3.0", + "resolved": "https://registry.npmmirror.com/sorted-array-functions/-/sorted-array-functions-1.3.0.tgz", + "integrity": "sha512-2sqgzeFlid6N4Z2fUQ1cvFmTOLRi/sEDzSQ0OKYchqgoPmQBVyM3959qYx3fpS6Esef80KjmpgPeEr028dP3OA==" + }, + "sshpk": { + "version": "1.17.0", + "resolved": "https://registry.npmmirror.com/sshpk/-/sshpk-1.17.0.tgz", + "integrity": "sha512-/9HIEs1ZXGhSPE8X6Ccm7Nam1z8KcoCqPdI7ecm1N33EzAetWahvQWVqLZtaZQ+IDKX4IyA2o0gBzqIMkAagHQ==", + "requires": { + "asn1": "~0.2.3", + "assert-plus": "^1.0.0", + "bcrypt-pbkdf": "^1.0.0", + "dashdash": "^1.12.0", + "ecc-jsbn": "~0.1.1", + "getpass": "^0.1.1", + "jsbn": "~0.1.0", + "safer-buffer": "^2.0.2", + "tweetnacl": "~0.14.0" + } + }, + "tough-cookie": { + "version": "2.5.0", + "resolved": "https://registry.npmmirror.com/tough-cookie/-/tough-cookie-2.5.0.tgz", + "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", + "requires": { + "psl": "^1.1.28", + "punycode": "^2.1.1" + } + }, + "tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmmirror.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", + "requires": { + "safe-buffer": "^5.0.1" + } + }, + "tweetnacl": { + "version": "0.14.5", + "resolved": "https://registry.npmmirror.com/tweetnacl/-/tweetnacl-0.14.5.tgz", + "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==" + }, + "uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmmirror.com/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "requires": { + "punycode": "^2.1.0" + } + }, + "uuid": { + "version": "3.4.0", + "resolved": "https://registry.npmmirror.com/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" + }, + "verror": { + "version": "1.10.0", + "resolved": "https://registry.npmmirror.com/verror/-/verror-1.10.0.tgz", + "integrity": "sha512-ZZKSmDAEFOijERBLkmYfJ+vmk3w+7hOLYDNkRCuRuMJGEmqYNCNLyBBFwWKVMhfwaEF3WOd0Zlw86U/WC/+nYw==", + "requires": { + "assert-plus": "^1.0.0", + "core-util-is": "1.0.2", + "extsprintf": "^1.2.0" + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..bcf7de7 --- /dev/null +++ b/package.json @@ -0,0 +1,15 @@ +{ + "name": "weibo-hot-band", + "version": "0.0.0", + "description": "weibo-hot-band", + "main": "server.js", + "author": { + "name": "" + }, + "dependencies": { + "dotenv": "^16.0.1", + "fs": "^0.0.1-security", + "node-schedule": "^2.1.0", + "request": "^2.88.2" + } +} \ No newline at end of file From 8067df5ae275b0e19862a076afb5d3fdb39099fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E5=B0=8F=E5=A2=A8?= <2291200076@qq.com> Date: Sat, 23 Jul 2022 23:19:13 +0800 Subject: [PATCH 02/50] =?UTF-8?q?=E6=B7=BB=E5=8A=A0node=E6=89=A7=E8=A1=8C?= =?UTF-8?q?=E5=91=BD=E4=BB=A4=E8=A1=8C=E7=9B=B8=E5=85=B3=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=EF=BC=9B=E4=BB=A3=E7=A0=81=E7=BB=93=E6=9E=84=E8=B0=83=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + index.js | 248 +---------------------------------------- src/execute_command.js | 20 ++++ src/get_hotband.js | 247 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 270 insertions(+), 246 deletions(-) create mode 100644 src/execute_command.js create mode 100644 src/get_hotband.js diff --git a/.gitignore b/.gitignore index e43125c..f72892b 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ test.js node_modules +.VSCodeCounter data/* !data/.gitkeep \ No newline at end of file diff --git a/index.js b/index.js index 1ccb8a0..4dd6cf8 100644 --- a/index.js +++ b/index.js @@ -1,247 +1,3 @@ -'use strict'; +const get_hotband = require('./src/get_hotband'); -const request = require('request'); -const fs = require('fs'); - -console.log("Start running ..."); - -// 请求微博热搜 APi 接口 -async function getApiResult(url) { - var return_data = await new Promise((resolve) => { - request({ - method: 'GET', - url: url, - json: true, - }, (error, response, result) => { - if (!error && (response.statusCode == 200)) { - // 请求成功 - resolve(result); - } else { - // 请求失败 - console.log(`error is ${error}`); - resolve("error"); - } - }); - }); - // console.log(`return_data is ${JSON.stringify(return_data)}`); - return return_data; -} - -// 创建目录 -async function createFolder(folderToCreate) { - let currentFolder = folderToCreate.replace(/\\/g, '/'); - let parentFolder = currentFolder.substring(0, currentFolder.lastIndexOf('/')); - if (!fs.existsSync(currentFolder)) { - // 文件夹不存在，创建文件夹 - createFolder(parentFolder); // 保证父级文件夹存在 - fs.mkdirSync(currentFolder); // 创建当前级文件夹 - } else { - // 否则就什么也不做 - } -} - -// 保存 JSON -function saveJSON({ now, fileNameSuffix, object, compress = true, uncompress = true }) { - - let year = now.substring(0, 4); - let month = now.substring(5, 7); - let day = now.substring(8, 10); - let hour = now.substring(11, 13); - let minute = now.substring(14, 16); - // console.log(now); - // console.log( "year, month, day, hour, minute: " + year + ", " + month + ", " + day + ", " + hour + ", " + minute); - - // 创建当前文件夹 - let folder = `./data/${year}/${month}/${day}`; - createFolder(folder); - let fileName = `${folder}/${year}${month}${day}_${hour}${minute}_${fileNameSuffix}`; - - // 生成文件名 - // '2022-07-23T10:11:38.650Z' => '20220723_1011' - // let fileName = now.replace(/T/, '_').replace(/:\d{2}.\d{3}Z/, '').replace(/[-:]/g, ''); - // console.log(`fileName is ${fileName}`); - - if (compress) - fs.writeFileSync(`${fileName}.min.json`, JSON.stringify(object)); - if (uncompress) - fs.writeFileSync(`${fileName}.json`, JSON.stringify(object, "", "\t")); -} - -async function main() { - let requestTimestamp = Date.now(); - let now = new Date(requestTimestamp + 8 * 3600 * 1000).toISOString(); - - let result = await getApiResult("https://weibo.com/ajax/statuses/hot_band"); - // console.log("result", result); - - if (result.ok != 1) { - console.log("请求成功，但服务器处理失败。"); - } else { - console.log("请求成功。"); - - /** - * 保存原始数据 - */ - saveJSON({ - now: now, - fileNameSuffix: `origin`, - object: result, - compress: true, - uncompress: false - }); - - let data = JSON.parse(JSON.stringify(result.data)); - - - /** - * 过滤掉不需要的数据 - */ - // hotgov - delete data.hotgov["mblog"]; - // 重复字段只保留一个 - delete data.hotgov["note"]; // note word - delete data.hotgov["small_icon_desc"]; // icon_desc small_icon_desc - delete data.hotgov["small_icon_desc_color"]; // icon_desc_color small_icon_desc_color - - // band_list - for (let i = 0; i < data.band_list.length; i++) { - const item = data.band_list[i]; - - // 过滤广告 - if (item.is_ad) { - data.band_list.splice(i, 1); - i--; - } - - // 过滤空字段 - delete item["ad_info"]; - - // 重复字段只保留一个 - delete item["note"]; // note word - delete item["icon_desc"]; delete item["small_icon_desc"]; // label_name icon_desc small_icon_desc - delete item["small_icon_desc_color"]; // icon_desc_color small_icon_desc_color - delete item["flag_desc"]; // flag_desc subject_label 这两个有值的时候相同，没有值的时候，前一个为 undefined，后一个为 "" - } - - - /** - * 获取需要的数据，进行转换 - */ - let convert = []; - data.band_list.forEach(item => { - let detail = ""; - let pic_ids = []; - if (item.mblog) { // 有些热搜没有 mblog - var regex = /(<([^>]+)>)/ig - detail = item.mblog.text.replace(regex, ""); - if (item.mblog.pics) { - pic_ids = item.mblog.pics.map(pic => `${pic}`); - } - } - convert.push({ - // 热搜排行顺序 - rank: item.rank, - realpos: item.realpos, - - // 热搜信息 - word: item.word, // 热搜标题 - word_scheme: item.word_scheme, // 热搜话题 "#热搜标题#" - emoticon: item.emoticon, // 热搜小表情，如 "[泪]" - label_name: item.label_name, // 热搜标签，如 "爆" "热" "新" "" - onboard_time: item.onboard_time, // 热搜上线时间，秒级时间戳，如 1658565575 - - /** - * 热搜数据 - * - * 大部分的 num 和 raw_hot 是相同的，页面上显示的是 num，可能是人工调控的热搜 - * - * 两者差值通过观测似乎最大是 1250000 - * 例如 【爆】唐山打架事件8名违法嫌疑人已到案 这条热搜一开始 delta 首先不断增大，最大达到 1250000 - * 然后热搜数量增加到 12600000 左右的时候，delta 逐渐减小到 1040000 左右 - * 所有热搜的 detla（带正负） 加起来就是基本上在100000-230000之间 - */ - num: item.num, - raw_hot: item.raw_hot, - detla: item.num - item.raw_hot, // 计算值 - - url: `https://s.weibo.com/weibo?q=${encodeURIComponent(item.word_scheme)}`, // 热搜话题链接 - - // 分类 - category: item.category ? item.category.split(',') : "", - subject_label: item.subject_label, - - // 其他 - more: { - is_new: item.is_new, - subject_querys: item.subject_querys, - mid: item.mid, - icon_desc_color: item.icon_desc_color, - detail: detail, - }, - }); - }); - saveJSON({ - now: now, - fileNameSuffix: `final`, - object: convert, - compress: true, - uncompress: true - }); - - - /** - * 只统计微博调控信息 - */ - let convert2 = []; - let total = 0; - data.band_list.forEach(item => { - total += item.num; - total -= item.raw_hot; - if (item.num - item.raw_hot == 0) return; - convert2.push([ - `[${item.realpos}] ${item.word}【${item.label_name}】`, - `原始:${item.raw_hot} 显示:${item.num} 调控: ${item.num - item.raw_hot}` - ]); - }); - saveJSON({ - now: now, - fileNameSuffix: `regulation`, - object: { - total_delta: total, // 所有调控值之和 - data: convert2 - }, - compress: false, - uncompress: true - }); - - - /** - * 保存预处理后数据 - */ - // 过滤掉不需要的数据 - // band_list - data.band_list.forEach(function (item) { - delete item["mblog"]; - }); - saveJSON({ - now: now, - fileNameSuffix: `simplify`, - object: data, - compress: true, - uncompress: true - }); - - - /** - * 更新最新的 - */ - fs.writeFileSync(`./data/latest.json`, JSON.stringify({ - update_time: requestTimestamp, - update_time_friendly: now.substring(0, 19).replace(/T/g, " "), - regulation: convert2, - data: convert - })); - } -} - -main(); \ No newline at end of file +get_hotband.main(); \ No newline at end of file diff --git a/src/execute_command.js b/src/execute_command.js new file mode 100644 index 0000000..76372e3 --- /dev/null +++ b/src/execute_command.js @@ -0,0 +1,20 @@ +var child_process = require('child_process'); +var cmds = [ + 'git status', + 'dir', +]; + +cmds.forEach(async function (cmd) { + let result = await new Promise(function (resolve) { + child_process.exec(cmd, function (error, stdout, stderr) { + // 获取命令执行的输出 + resolve({ + cmd: cmd, + error: error, + stdout: stdout, + stderr: stderr, + }); + }); + }); + console.log(result); +}); \ No newline at end of file diff --git a/src/get_hotband.js b/src/get_hotband.js new file mode 100644 index 0000000..5159a32 --- /dev/null +++ b/src/get_hotband.js @@ -0,0 +1,247 @@ +'use strict'; + +const request = require('request'); +const fs = require('fs'); + +console.log("Start running ..."); + +// 请求微博热搜 APi 接口 +async function getApiResult(url) { + var return_data = await new Promise((resolve) => { + request({ + method: 'GET', + url: url, + json: true, + }, (error, response, result) => { + if (!error && (response.statusCode == 200)) { + // 请求成功 + resolve(result); + } else { + // 请求失败 + console.log(`error is ${error}`); + resolve("error"); + } + }); + }); + // console.log(`return_data is ${JSON.stringify(return_data)}`); + return return_data; +} + +// 创建目录 +async function createFolder(folderToCreate) { + let currentFolder = folderToCreate.replace(/\\/g, '/'); + let parentFolder = currentFolder.substring(0, currentFolder.lastIndexOf('/')); + if (!fs.existsSync(currentFolder)) { + // 文件夹不存在，创建文件夹 + createFolder(parentFolder); // 保证父级文件夹存在 + fs.mkdirSync(currentFolder); // 创建当前级文件夹 + } else { + // 否则就什么也不做 + } +} + +// 保存 JSON +function saveJSON({ now, fileNameSuffix, object, compress = true, uncompress = true }) { + + let year = now.substring(0, 4); + let month = now.substring(5, 7); + let day = now.substring(8, 10); + let hour = now.substring(11, 13); + let minute = now.substring(14, 16); + // console.log(now); + // console.log( "year, month, day, hour, minute: " + year + ", " + month + ", " + day + ", " + hour + ", " + minute); + + // 创建当前文件夹 + let folder = `./data/${year}/${month}/${day}`; + createFolder(folder); + let fileName = `${folder}/${year}${month}${day}_${hour}${minute}_${fileNameSuffix}`; + + // 生成文件名 + // '2022-07-23T10:11:38.650Z' => '20220723_1011' + // let fileName = now.replace(/T/, '_').replace(/:\d{2}.\d{3}Z/, '').replace(/[-:]/g, ''); + // console.log(`fileName is ${fileName}`); + + if (compress) + fs.writeFileSync(`${fileName}.min.json`, JSON.stringify(object)); + if (uncompress) + fs.writeFileSync(`${fileName}.json`, JSON.stringify(object, "", "\t")); +} + +async function main() { + let requestTimestamp = Date.now(); + let now = new Date(requestTimestamp + 8 * 3600 * 1000).toISOString(); + + let result = await getApiResult("https://weibo.com/ajax/statuses/hot_band"); + // console.log("result", result); + + if (result.ok != 1) { + console.log("请求成功，但服务器处理失败。"); + } else { + console.log("请求成功。"); + + /** + * 保存原始数据 + */ + saveJSON({ + now: now, + fileNameSuffix: `origin`, + object: result, + compress: true, + uncompress: false + }); + + let data = JSON.parse(JSON.stringify(result.data)); + + + /** + * 过滤掉不需要的数据 + */ + // hotgov + delete data.hotgov["mblog"]; + // 重复字段只保留一个 + delete data.hotgov["note"]; // note word + delete data.hotgov["small_icon_desc"]; // icon_desc small_icon_desc + delete data.hotgov["small_icon_desc_color"]; // icon_desc_color small_icon_desc_color + + // band_list + for (let i = 0; i < data.band_list.length; i++) { + const item = data.band_list[i]; + + // 过滤广告 + if (item.is_ad) { + data.band_list.splice(i, 1); + i--; + } + + // 过滤空字段 + delete item["ad_info"]; + + // 重复字段只保留一个 + delete item["note"]; // note word + delete item["icon_desc"]; delete item["small_icon_desc"]; // label_name icon_desc small_icon_desc + delete item["small_icon_desc_color"]; // icon_desc_color small_icon_desc_color + delete item["flag_desc"]; // flag_desc subject_label 这两个有值的时候相同，没有值的时候，前一个为 undefined，后一个为 "" + } + + + /** + * 获取需要的数据，进行转换 + */ + let convert = []; + data.band_list.forEach(item => { + let detail = ""; + let pic_ids = []; + if (item.mblog) { // 有些热搜没有 mblog + var regex = /(<([^>]+)>)/ig + detail = item.mblog.text.replace(regex, ""); + if (item.mblog.pics) { + pic_ids = item.mblog.pics.map(pic => `${pic}`); + } + } + convert.push({ + // 热搜排行顺序 + rank: item.rank, + realpos: item.realpos, + + // 热搜信息 + word: item.word, // 热搜标题 + word_scheme: item.word_scheme, // 热搜话题 "#热搜标题#" + emoticon: item.emoticon, // 热搜小表情，如 "[泪]" + label_name: item.label_name, // 热搜标签，如 "爆" "热" "新" "" + onboard_time: item.onboard_time, // 热搜上线时间，秒级时间戳，如 1658565575 + + /** + * 热搜数据 + * + * 大部分的 num 和 raw_hot 是相同的，页面上显示的是 num，可能是人工调控的热搜 + * + * 两者差值通过观测似乎最大是 1250000 + * 例如 【爆】唐山打架事件8名违法嫌疑人已到案 这条热搜一开始 delta 首先不断增大，最大达到 1250000 + * 然后热搜数量增加到 12600000 左右的时候，delta 逐渐减小到 1040000 左右 + * 所有热搜的 detla（带正负） 加起来就是基本上在100000-230000之间 + */ + num: item.num, + raw_hot: item.raw_hot, + detla: item.num - item.raw_hot, // 计算值 + + url: `https://s.weibo.com/weibo?q=${encodeURIComponent(item.word_scheme)}`, // 热搜话题链接 + + // 分类 + category: item.category ? item.category.split(',') : "", + subject_label: item.subject_label, + + // 其他 + more: { + is_new: item.is_new, + subject_querys: item.subject_querys, + mid: item.mid, + icon_desc_color: item.icon_desc_color, + detail: detail, + }, + }); + }); + saveJSON({ + now: now, + fileNameSuffix: `final`, + object: convert, + compress: true, + uncompress: true + }); + + + /** + * 只统计微博调控信息 + */ + let convert2 = []; + let total = 0; + data.band_list.forEach(item => { + total += item.num; + total -= item.raw_hot; + if (item.num - item.raw_hot == 0) return; + convert2.push([ + `[${item.realpos}] ${item.word}【${item.label_name}】`, + `原始:${item.raw_hot} 显示:${item.num} 调控: ${item.num - item.raw_hot}` + ]); + }); + saveJSON({ + now: now, + fileNameSuffix: `regulation`, + object: { + total_delta: total, // 所有调控值之和 + data: convert2 + }, + compress: false, + uncompress: true + }); + + + /** + * 保存预处理后数据 + */ + // 过滤掉不需要的数据 + // band_list + data.band_list.forEach(function (item) { + delete item["mblog"]; + }); + saveJSON({ + now: now, + fileNameSuffix: `simplify`, + object: data, + compress: true, + uncompress: true + }); + + + /** + * 更新最新的 + */ + fs.writeFileSync(`./data/latest.json`, JSON.stringify({ + update_time: requestTimestamp, + update_time_friendly: now.substring(0, 19).replace(/T/g, " "), + regulation: convert2, + data: convert + })); + } +} + +exports.main = main; \ No newline at end of file From bbe9753ad37ca40327a4f81fa1bc3ad9560f3480 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E5=B0=8F=E5=A2=A8?= <2291200076@qq.com> Date: Sat, 23 Jul 2022 23:48:55 +0800 Subject: [PATCH 03/50] =?UTF-8?q?=E5=BC=95=E5=85=A5=E7=8E=AF=E5=A2=83?= =?UTF-8?q?=E5=8F=98=E9=87=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env | 8 ++++++++ index.js | 28 ++++++++++++++++++++++++++++ src/execute_command.js | 1 + src/get_hotband.js | 11 +++++++---- 4 files changed, 44 insertions(+), 4 deletions(-) create mode 100644 .env diff --git a/.env b/.env new file mode 100644 index 0000000..f6d7059 --- /dev/null +++ b/.env @@ -0,0 +1,8 @@ +# 调试模式 +# 1为开启调试 +DEBUG_MODE=1 + +# 爬取数据保存的文件夹 +# 目录开头与结尾的 [./] [/] [\] [\\] 均可带可不带 +# 默认为 data 文件夹 +DATA_FOLDER=data diff --git a/index.js b/index.js index 4dd6cf8..a486d45 100644 --- a/index.js +++ b/index.js @@ -1,3 +1,31 @@ +'use strict'; + +/** + * 环境变量 + */ +const dotenv = require('dotenv'); +process.env = {}; // 清除系统自带的环境变量 +dotenv.config('./.env'); // 导入 .env 文件中的环境变量 +// console.log(process.env); + +// 调试模式 +const DEBUG_MODE = !!process.env.DEBUG_MODE; + +if (DEBUG_MODE) { + console.log('DEBUG_MODE is on'); + console.log('Environment variables: ', process.env); +} + +/** + * 引入模块 + */ const get_hotband = require('./src/get_hotband'); + +/** + * 开始运行 + */ +console.log("Start running ..."); + +// 爬取热搜数据 get_hotband.main(); \ No newline at end of file diff --git a/src/execute_command.js b/src/execute_command.js index 76372e3..557016f 100644 --- a/src/execute_command.js +++ b/src/execute_command.js @@ -1,3 +1,4 @@ +// refer: https://www.webhek.com/post/execute-a-command-line-binary-with-node-js/ var child_process = require('child_process'); var cmds = [ 'git status', diff --git a/src/get_hotband.js b/src/get_hotband.js index 5159a32..0346237 100644 --- a/src/get_hotband.js +++ b/src/get_hotband.js @@ -2,8 +2,10 @@ const request = require('request'); const fs = require('fs'); +const { dirname } = require('path'); +const path = require('path'); -console.log("Start running ..."); +const DATA_FOLDER = path.join(dirname(__dirname), process.env.DATA_FOLDER ?? 'data'); // 请求微博热搜 APi 接口 async function getApiResult(url) { @@ -52,7 +54,7 @@ function saveJSON({ now, fileNameSuffix, object, compress = true, uncompress = t // console.log( "year, month, day, hour, minute: " + year + ", " + month + ", " + day + ", " + hour + ", " + minute); // 创建当前文件夹 - let folder = `./data/${year}/${month}/${day}`; + let folder = `${DATA_FOLDER}/${year}/${month}/${day}`; createFolder(folder); let fileName = `${folder}/${year}${month}${day}_${hour}${minute}_${fileNameSuffix}`; @@ -68,6 +70,8 @@ function saveJSON({ now, fileNameSuffix, object, compress = true, uncompress = t } async function main() { + console.log("DATA_FOLDER", DATA_FOLDER); + let requestTimestamp = Date.now(); let now = new Date(requestTimestamp + 8 * 3600 * 1000).toISOString(); @@ -158,7 +162,6 @@ async function main() { * 两者差值通过观测似乎最大是 1250000 * 例如 【爆】唐山打架事件8名违法嫌疑人已到案 这条热搜一开始 delta 首先不断增大，最大达到 1250000 * 然后热搜数量增加到 12600000 左右的时候，delta 逐渐减小到 1040000 左右 - * 所有热搜的 detla（带正负） 加起来就是基本上在100000-230000之间 */ num: item.num, raw_hot: item.raw_hot, @@ -235,7 +238,7 @@ async function main() { /** * 更新最新的 */ - fs.writeFileSync(`./data/latest.json`, JSON.stringify({ + fs.writeFileSync(`${DATA_FOLDER}/latest.json`, JSON.stringify({ update_time: requestTimestamp, update_time_friendly: now.substring(0, 19).replace(/T/g, " "), regulation: convert2, From 766658ea1123eef24c187aab18f2182f081aa0d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E5=B0=8F=E5=A2=A8?= <2291200076@qq.com> Date: Sun, 24 Jul 2022 00:11:13 +0800 Subject: [PATCH 04/50] =?UTF-8?q?=E5=BC=95=E5=85=A5=E5=AE=9A=E6=97=B6?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env | 4 ++++ index.js | 25 +++++++++++++++++++++---- src/get_hotband.js | 16 +++++++++------- 3 files changed, 34 insertions(+), 11 deletions(-) diff --git a/.env b/.env index f6d7059..494a7a5 100644 --- a/.env +++ b/.env @@ -6,3 +6,7 @@ DEBUG_MODE=1 # 目录开头与结尾的 [./] [/] [\] [\\] 均可带可不带 # 默认为 data 文件夹 DATA_FOLDER=data + +# 是否在程序刚一启动时就抓取一次数据 +# 1为是 +EXECUTE_AT_STARTUP=1 \ No newline at end of file diff --git a/index.js b/index.js index a486d45..211d3c6 100644 --- a/index.js +++ b/index.js @@ -1,15 +1,18 @@ 'use strict'; +const dotenv = require('dotenv'); +const schedule = require('node-schedule'); + /** * 环境变量 */ -const dotenv = require('dotenv'); process.env = {}; // 清除系统自带的环境变量 dotenv.config('./.env'); // 导入 .env 文件中的环境变量 // console.log(process.env); // 调试模式 -const DEBUG_MODE = !!process.env.DEBUG_MODE; +const DEBUG_MODE = process.env.DEBUG_MODE == true; +const EXECUTE_AT_STARTUP = process.env.EXECUTE_AT_STARTUP == true; if (DEBUG_MODE) { console.log('DEBUG_MODE is on'); @@ -27,5 +30,19 @@ const get_hotband = require('./src/get_hotband'); */ console.log("Start running ..."); -// 爬取热搜数据 -get_hotband.main(); \ No newline at end of file +// 程序主函数 +function start() { + // 爬取热搜数据 + get_hotband.main(); +} + +// 调试模式下，程序一启动就首先运行一次 +if (EXECUTE_AT_STARTUP) { + process.stdout.write("程序启动时，立即运行一次\t"); + start(); +} + +// 每分钟的第 5 秒执行一次 +// 这里指定第 5 秒是为了稍微与微博服务器热榜更新时间错开，避免因为微秒级误差造成拉取两次相同的热榜数据 +// refer: https://www.npmjs.com/package/node-schedule +const scheduleJob = schedule.scheduleJob('05 * * * * *', start); diff --git a/src/get_hotband.js b/src/get_hotband.js index 0346237..abceba1 100644 --- a/src/get_hotband.js +++ b/src/get_hotband.js @@ -6,6 +6,7 @@ const { dirname } = require('path'); const path = require('path'); const DATA_FOLDER = path.join(dirname(__dirname), process.env.DATA_FOLDER ?? 'data'); +console.log("DATA_FOLDER", DATA_FOLDER); // 请求微博热搜 APi 接口 async function getApiResult(url) { @@ -70,8 +71,6 @@ function saveJSON({ now, fileNameSuffix, object, compress = true, uncompress = t } async function main() { - console.log("DATA_FOLDER", DATA_FOLDER); - let requestTimestamp = Date.now(); let now = new Date(requestTimestamp + 8 * 3600 * 1000).toISOString(); @@ -79,9 +78,9 @@ async function main() { // console.log("result", result); if (result.ok != 1) { - console.log("请求成功，但服务器处理失败。"); + console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), "请求成功，但服务器处理失败。"); } else { - console.log("请求成功。"); + console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), "请求成功"); /** * 保存原始数据 @@ -188,7 +187,8 @@ async function main() { fileNameSuffix: `final`, object: convert, compress: true, - uncompress: true + // uncompress: true, + uncompress: false, }); @@ -230,8 +230,10 @@ async function main() { now: now, fileNameSuffix: `simplify`, object: data, - compress: true, - uncompress: true + // compress: true, + // uncompress: true, + compress: false, + uncompress: false, }); From 63c54911014c059eb99cb0647969126258e21e16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E5=B0=8F=E5=A2=A8?= <2291200076@qq.com> Date: Sun, 24 Jul 2022 00:35:31 +0800 Subject: [PATCH 05/50] =?UTF-8?q?=E6=89=A7=E8=A1=8C=E5=91=BD=E4=BB=A4?= =?UTF-8?q?=E8=A1=8C=E4=BB=A3=E7=A0=81=E9=80=BB=E8=BE=91=E4=BC=98=E5=8C=96?= =?UTF-8?q?=EF=BC=9B=E8=A7=A3=E5=86=B3=E4=B8=AD=E6=96=87=E4=B9=B1=E7=A0=81?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- data/.gitkeep | 0 package-lock.json | 20 ++++++++++++++++ package.json | 3 ++- src/execute_command.js | 53 ++++++++++++++++++++++++++++++------------ src/get_hotband.js | 3 +-- 5 files changed, 61 insertions(+), 18 deletions(-) delete mode 100644 data/.gitkeep diff --git a/data/.gitkeep b/data/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/package-lock.json b/package-lock.json index e509250..44ef4cf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,7 @@ "dependencies": { "dotenv": "^16.0.1", "fs": "^0.0.1-security", + "iconv-lite": "^0.6.3", "node-schedule": "^2.1.0", "request": "^2.88.2" } @@ -291,6 +292,17 @@ "npm": ">=1.3.7" } }, + "node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmmirror.com/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/is-nan": { "version": "1.3.2", "resolved": "https://registry.npmmirror.com/is-nan/-/is-nan-1.3.2.tgz", @@ -791,6 +803,14 @@ "sshpk": "^1.7.0" } }, + "iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmmirror.com/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "requires": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + } + }, "is-nan": { "version": "1.3.2", "resolved": "https://registry.npmmirror.com/is-nan/-/is-nan-1.3.2.tgz", diff --git a/package.json b/package.json index bcf7de7..af61168 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,8 @@ "dependencies": { "dotenv": "^16.0.1", "fs": "^0.0.1-security", + "iconv-lite": "^0.6.3", "node-schedule": "^2.1.0", "request": "^2.88.2" } -} \ No newline at end of file +} diff --git a/src/execute_command.js b/src/execute_command.js index 557016f..23fec0e 100644 --- a/src/execute_command.js +++ b/src/execute_command.js @@ -1,21 +1,44 @@ -// refer: https://www.webhek.com/post/execute-a-command-line-binary-with-node-js/ -var child_process = require('child_process'); +'use strict'; + +const path = require('path'); +const child_process = require('child_process'); +const iconv = require("iconv-lite"); + +const encoding = "cp936"; +const bufferEncoding = "binary"; +const DATA_FOLDER = path.join(path.dirname(__dirname), process.env.DATA_FOLDER ?? 'data'); + var cmds = [ - 'git status', 'dir', + 'git status', ]; -cmds.forEach(async function (cmd) { - let result = await new Promise(function (resolve) { - child_process.exec(cmd, function (error, stdout, stderr) { - // 获取命令执行的输出 - resolve({ - cmd: cmd, - error: error, - stdout: stdout, - stderr: stderr, +async function main() { + for (let cmd of cmds) { + let result = await new Promise(function (resolve) { + // refer: https://www.webhek.com/post/execute-a-command-line-binary-with-node-js/ + child_process.exec(cmd, { + cwd: DATA_FOLDER, // 脚本执行目录 + encoding: bufferEncoding + }, function (err, stdout, stderr) { + if (err) { + resolve({ + cmd: cmd, + err: err, + // err_stack: iconv.decode(Buffer.from(err.stack, bufferEncoding), encoding), + // err_message: iconv.decode(Buffer.from(err.message, bufferEncoding), encoding), + }); + } else { + // 获取命令执行的输出 + resolve({ + cmd: cmd, + stdout: iconv.decode(Buffer.from(stdout, bufferEncoding), encoding), + stderr: iconv.decode(Buffer.from(stderr, bufferEncoding), encoding), + }); + } }); }); - }); - console.log(result); -}); \ No newline at end of file + console.log(result); + } +} +main(); \ No newline at end of file diff --git a/src/get_hotband.js b/src/get_hotband.js index abceba1..8b5d31d 100644 --- a/src/get_hotband.js +++ b/src/get_hotband.js @@ -2,10 +2,9 @@ const request = require('request'); const fs = require('fs'); -const { dirname } = require('path'); const path = require('path'); -const DATA_FOLDER = path.join(dirname(__dirname), process.env.DATA_FOLDER ?? 'data'); +const DATA_FOLDER = path.join(path.dirname(__dirname), process.env.DATA_FOLDER ?? 'data'); console.log("DATA_FOLDER", DATA_FOLDER); // 请求微博热搜 APi 接口 From da446224eb3c8c9972712b924311b5aef3d66dd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E5=B0=8F=E5=A2=A8?= <2291200076@qq.com> Date: Sun, 24 Jul 2022 00:36:18 +0800 Subject: [PATCH 06/50] =?UTF-8?q?nodemon=20=E5=9C=A8=20json=20=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E5=8F=98=E5=8C=96=E6=97=B6=E4=B8=8D=E9=87=8D=E5=90=AF?= =?UTF-8?q?=E7=A8=8B=E5=BA=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 - nodemon.json | 8 ++++++++ 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 nodemon.json diff --git a/.gitignore b/.gitignore index f72892b..5d77e60 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,3 @@ node_modules .VSCodeCounter data/* -!data/.gitkeep \ No newline at end of file diff --git a/nodemon.json b/nodemon.json new file mode 100644 index 0000000..4878d76 --- /dev/null +++ b/nodemon.json @@ -0,0 +1,8 @@ +{ + "ignore": [ + ".git", + ".svn", + "node_modules/**/node_modules" + ], + "ext": "js" +} \ No newline at end of file From a2af3be807279b334ff981124d28f579ee590954 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E5=B0=8F=E5=A2=A8?= <2291200076@qq.com> Date: Sun, 24 Jul 2022 01:01:23 +0800 Subject: [PATCH 07/50] =?UTF-8?q?=E6=AF=8F=E5=B0=8F=E6=97=B6=E5=B0=86data?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E5=A4=B9=E4=B8=AD=E7=9A=84=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E6=8E=A8=E9=80=81=E5=88=B0git=E4=BB=93=E5=BA=93=E4=B8=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- index.js | 41 ++++++++++++++++++++++++++++++++++++++--- src/execute_command.js | 18 +++++++----------- 2 files changed, 45 insertions(+), 14 deletions(-) diff --git a/index.js b/index.js index 211d3c6..c64729d 100644 --- a/index.js +++ b/index.js @@ -2,6 +2,8 @@ const dotenv = require('dotenv'); const schedule = require('node-schedule'); +const path = require('path'); +const os = require('os'); /** * 环境变量 @@ -10,10 +12,15 @@ process.env = {}; // 清除系统自带的环境变量 dotenv.config('./.env'); // 导入 .env 文件中的环境变量 // console.log(process.env); -// 调试模式 const DEBUG_MODE = process.env.DEBUG_MODE == true; const EXECUTE_AT_STARTUP = process.env.EXECUTE_AT_STARTUP == true; +const ROOT_PATH = path.join(__dirname, process.env.DATA_FOLDER ?? 'data'); + + +/** + * 调试模式 + */ if (DEBUG_MODE) { console.log('DEBUG_MODE is on'); console.log('Environment variables: ', process.env); @@ -23,6 +30,7 @@ if (DEBUG_MODE) { * 引入模块 */ const get_hotband = require('./src/get_hotband'); +const execute_command = require('./src/execute_command'); /** @@ -31,9 +39,9 @@ const get_hotband = require('./src/get_hotband'); console.log("Start running ..."); // 程序主函数 -function start() { +async function start() { // 爬取热搜数据 - get_hotband.main(); + await get_hotband.main(); } // 调试模式下，程序一启动就首先运行一次 @@ -46,3 +54,30 @@ if (EXECUTE_AT_STARTUP) { // 这里指定第 5 秒是为了稍微与微博服务器热榜更新时间错开，避免因为微秒级误差造成拉取两次相同的热榜数据 // refer: https://www.npmjs.com/package/node-schedule const scheduleJob = schedule.scheduleJob('05 * * * * *', start); + +// 定时推送 +async function pushToGitRepo() { + let commands = [ + 'git status', + 'git add .', + `git commit -m "${new Date(Date.now() + 8 * 3600 * 1000).toISOString().substring(0, 19).replace('T', ' ')} update"`, + `git push origin master`, + 'git status', + ]; + switch (os.type()) { + case 'Windows_NT': // Windows + commands.unshift('dir'); + break; + + case 'Darwin': // Mac OS X + case 'Linux': // Linux + default: + commands.unshift('pwd'); + break; + } + let outputs = await execute_command.execute(ROOT_PATH, commands); + console.log(commands, outputs); +} + +// 每个小时同步一次 +schedule.scheduleJob('0 0 * * * *', pushToGitRepo); diff --git a/src/execute_command.js b/src/execute_command.js index 23fec0e..df21067 100644 --- a/src/execute_command.js +++ b/src/execute_command.js @@ -1,24 +1,18 @@ 'use strict'; -const path = require('path'); const child_process = require('child_process'); const iconv = require("iconv-lite"); const encoding = "cp936"; const bufferEncoding = "binary"; -const DATA_FOLDER = path.join(path.dirname(__dirname), process.env.DATA_FOLDER ?? 'data'); -var cmds = [ - 'dir', - 'git status', -]; - -async function main() { +async function execute(rootPath, cmds) { + let outputs = []; for (let cmd of cmds) { let result = await new Promise(function (resolve) { // refer: https://www.webhek.com/post/execute-a-command-line-binary-with-node-js/ child_process.exec(cmd, { - cwd: DATA_FOLDER, // 脚本执行目录 + cwd: rootPath, // 脚本执行目录 encoding: bufferEncoding }, function (err, stdout, stderr) { if (err) { @@ -38,7 +32,9 @@ async function main() { } }); }); - console.log(result); + outputs.push(result); } + return outputs; } -main(); \ No newline at end of file + +exports.execute = execute; From e588e463d5c4b4df2d89fe4918b98b04c0bb73ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E5=B0=8F=E5=A2=A8?= <2291200076@qq.com> Date: Sun, 24 Jul 2022 01:54:44 +0800 Subject: [PATCH 08/50] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E7=9B=AE=E5=BD=95=E7=BB=93=E6=9E=84=EF=BC=9B=E5=B0=8F=E6=94=B9?= =?UTF-8?q?=E5=8A=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- index.js | 11 +++++++++-- src/get_hotband.js | 8 ++++---- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/index.js b/index.js index c64729d..7692bc9 100644 --- a/index.js +++ b/index.js @@ -38,7 +38,10 @@ const execute_command = require('./src/execute_command'); */ console.log("Start running ..."); -// 程序主函数 + +/** + * 程序主函数 + */ async function start() { // 爬取热搜数据 await get_hotband.main(); @@ -55,10 +58,14 @@ if (EXECUTE_AT_STARTUP) { // refer: https://www.npmjs.com/package/node-schedule const scheduleJob = schedule.scheduleJob('05 * * * * *', start); -// 定时推送 + +/** + * 定时将热搜数据推送到 Git 仓库 + */ async function pushToGitRepo() { let commands = [ 'git status', + 'git pull', 'git add .', `git commit -m "${new Date(Date.now() + 8 * 3600 * 1000).toISOString().substring(0, 19).replace('T', ' ')} update"`, `git push origin master`, diff --git a/src/get_hotband.js b/src/get_hotband.js index 8b5d31d..db1d14d 100644 --- a/src/get_hotband.js +++ b/src/get_hotband.js @@ -54,9 +54,9 @@ function saveJSON({ now, fileNameSuffix, object, compress = true, uncompress = t // console.log( "year, month, day, hour, minute: " + year + ", " + month + ", " + day + ", " + hour + ", " + minute); // 创建当前文件夹 - let folder = `${DATA_FOLDER}/${year}/${month}/${day}`; + let folder = `${DATA_FOLDER}/${fileNameSuffix}/${year}/${month}/${day}`; createFolder(folder); - let fileName = `${folder}/${year}${month}${day}_${hour}${minute}_${fileNameSuffix}`; + let fileName = `${folder}/${year}${month}${day}_${hour}${minute}`; // 生成文件名 // '2022-07-23T10:11:38.650Z' => '20220723_1011' @@ -229,9 +229,9 @@ async function main() { now: now, fileNameSuffix: `simplify`, object: data, - // compress: true, + compress: true, // uncompress: true, - compress: false, + // compress: false, uncompress: false, }); From 1ee3fc6f325c0aba35da7a69186b76877241fa1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E5=B0=8F=E5=A2=A8?= <2291200076@qq.com> Date: Sun, 24 Jul 2022 01:57:10 +0800 Subject: [PATCH 09/50] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=20PUSH=5FTO=5FGIT=20?= =?UTF-8?q?=E7=8E=AF=E5=A2=83=E5=8F=98=E9=87=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env | 6 +++++- index.js | 9 +++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/.env b/.env index 494a7a5..59caf61 100644 --- a/.env +++ b/.env @@ -9,4 +9,8 @@ DATA_FOLDER=data # 是否在程序刚一启动时就抓取一次数据 # 1为是 -EXECUTE_AT_STARTUP=1 \ No newline at end of file +EXECUTE_AT_STARTUP=1 + +# 数据是否推送到Git仓库 +# 1为是 +PUSH_TO_GIT=0 \ No newline at end of file diff --git a/index.js b/index.js index 7692bc9..96881a4 100644 --- a/index.js +++ b/index.js @@ -14,6 +14,7 @@ dotenv.config('./.env'); // 导入 .env 文件中的环境变量 const DEBUG_MODE = process.env.DEBUG_MODE == true; const EXECUTE_AT_STARTUP = process.env.EXECUTE_AT_STARTUP == true; +const PUSH_TO_GIT = process.env.PUSH_TO_GIT == true; const ROOT_PATH = path.join(__dirname, process.env.DATA_FOLDER ?? 'data'); @@ -45,6 +46,12 @@ console.log("Start running ..."); async function start() { // 爬取热搜数据 await get_hotband.main(); + + // 调试模式下 + if (DEBUG_MODE) { + // 推送到 Git 仓库 + await pushToGitRepo(); + } } // 调试模式下，程序一启动就首先运行一次 @@ -63,6 +70,8 @@ const scheduleJob = schedule.scheduleJob('05 * * * * *', start); * 定时将热搜数据推送到 Git 仓库 */ async function pushToGitRepo() { + if (!PUSH_TO_GIT) return; + let commands = [ 'git status', 'git pull', From 70e39f735cb80d1c5c90aae70cda65334b78f2f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E5=B0=8F=E5=A2=A8?= <2291200076@qq.com> Date: Sun, 24 Jul 2022 10:55:19 +0800 Subject: [PATCH 10/50] =?UTF-8?q?=E7=8E=AF=E5=A2=83=E5=8F=98=E9=87=8F=20.e?= =?UTF-8?q?nv=20=E6=96=87=E4=BB=B6=E4=B8=8D=E6=B7=BB=E5=8A=A0=E5=88=B0git?= =?UTF-8?q?=E4=BB=93=E5=BA=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env => .env.example | 0 .gitignore | 5 +++-- 2 files changed, 3 insertions(+), 2 deletions(-) rename .env => .env.example (100%) diff --git a/.env b/.env.example similarity index 100% rename from .env rename to .env.example diff --git a/.gitignore b/.gitignore index 5d77e60..9fc2270 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,8 @@ .DS_Store -test.js +data/* +.env node_modules .VSCodeCounter -data/* +test.js From 93bfc6c2cf08fbdedce5d0aa15db4e8970cb550d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E5=B0=8F=E5=A2=A8?= <2291200076@qq.com> Date: Sun, 24 Jul 2022 14:30:08 +0800 Subject: [PATCH 11/50] =?UTF-8?q?=E8=AF=B7=E6=B1=82=E5=A4=B1=E8=B4=A5?= =?UTF-8?q?=E9=87=8D=E8=AF=951=E6=AC=A1=EF=BC=8C=E5=B9=B6=E4=B8=94?= =?UTF-8?q?=E4=BF=9D=E5=AD=98=E8=AF=B7=E6=B1=82=E5=A4=B1=E8=B4=A5=E7=9A=84?= =?UTF-8?q?=E5=86=85=E5=AE=B9=E4=BE=BF=E4=BA=8E=E5=90=8E=E7=BB=AD=E5=88=86?= =?UTF-8?q?=E6=9E=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/get_hotband.js | 334 +++++++++++++++++++++++---------------------- 1 file changed, 173 insertions(+), 161 deletions(-) diff --git a/src/get_hotband.js b/src/get_hotband.js index db1d14d..f3caa97 100644 --- a/src/get_hotband.js +++ b/src/get_hotband.js @@ -74,178 +74,190 @@ async function main() { let now = new Date(requestTimestamp + 8 * 3600 * 1000).toISOString(); let result = await getApiResult("https://weibo.com/ajax/statuses/hot_band"); + if (result.ok != 1) { + console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), "请求成功，但服务器处理失败，正在重试。"); + result = await getApiResult("https://weibo.com/ajax/statuses/hot_band"); + if (result.ok != 1) { + console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), "请求成功，但服务器处理失败，保存失败信息。"); + // ok 不为 1，那么久直接保存便于后续分析，不进行后续处理 + saveJSON({ + now: now, + fileNameSuffix: `origin-error`, + object: result, + compress: true, + uncompress: false + }); + return; + } + } + + console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), "请求成功"); // console.log("result", result); - if (result.ok != 1) { - console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), "请求成功，但服务器处理失败。"); - } else { - console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), "请求成功"); + /** + * 保存原始数据 + */ + saveJSON({ + now: now, + fileNameSuffix: `origin`, + object: result, + compress: true, + uncompress: false + }); - /** - * 保存原始数据 - */ - saveJSON({ - now: now, - fileNameSuffix: `origin`, - object: result, - compress: true, - uncompress: false - }); - - let data = JSON.parse(JSON.stringify(result.data)); + let data = JSON.parse(JSON.stringify(result.data)); - /** - * 过滤掉不需要的数据 - */ - // hotgov - delete data.hotgov["mblog"]; - // 重复字段只保留一个 - delete data.hotgov["note"]; // note word - delete data.hotgov["small_icon_desc"]; // icon_desc small_icon_desc - delete data.hotgov["small_icon_desc_color"]; // icon_desc_color small_icon_desc_color + /** + * 过滤掉不需要的数据 + */ + // hotgov + delete data.hotgov["mblog"]; + // 重复字段只保留一个 + delete data.hotgov["note"]; // note word + delete data.hotgov["small_icon_desc"]; // icon_desc small_icon_desc + delete data.hotgov["small_icon_desc_color"]; // icon_desc_color small_icon_desc_color - // band_list - for (let i = 0; i < data.band_list.length; i++) { - const item = data.band_list[i]; + // band_list + for (let i = 0; i < data.band_list.length; i++) { + const item = data.band_list[i]; - // 过滤广告 - if (item.is_ad) { - data.band_list.splice(i, 1); - i--; - } - - // 过滤空字段 - delete item["ad_info"]; - - // 重复字段只保留一个 - delete item["note"]; // note word - delete item["icon_desc"]; delete item["small_icon_desc"]; // label_name icon_desc small_icon_desc - delete item["small_icon_desc_color"]; // icon_desc_color small_icon_desc_color - delete item["flag_desc"]; // flag_desc subject_label 这两个有值的时候相同，没有值的时候，前一个为 undefined，后一个为 "" + // 过滤广告 + if (item.is_ad) { + data.band_list.splice(i, 1); + i--; } + // 过滤空字段 + delete item["ad_info"]; - /** - * 获取需要的数据，进行转换 - */ - let convert = []; - data.band_list.forEach(item => { - let detail = ""; - let pic_ids = []; - if (item.mblog) { // 有些热搜没有 mblog - var regex = /(<([^>]+)>)/ig - detail = item.mblog.text.replace(regex, ""); - if (item.mblog.pics) { - pic_ids = item.mblog.pics.map(pic => `${pic}`); - } - } - convert.push({ - // 热搜排行顺序 - rank: item.rank, - realpos: item.realpos, - - // 热搜信息 - word: item.word, // 热搜标题 - word_scheme: item.word_scheme, // 热搜话题 "#热搜标题#" - emoticon: item.emoticon, // 热搜小表情，如 "[泪]" - label_name: item.label_name, // 热搜标签，如 "爆" "热" "新" "" - onboard_time: item.onboard_time, // 热搜上线时间，秒级时间戳，如 1658565575 - - /** - * 热搜数据 - * - * 大部分的 num 和 raw_hot 是相同的，页面上显示的是 num，可能是人工调控的热搜 - * - * 两者差值通过观测似乎最大是 1250000 - * 例如 【爆】唐山打架事件8名违法嫌疑人已到案 这条热搜一开始 delta 首先不断增大，最大达到 1250000 - * 然后热搜数量增加到 12600000 左右的时候，delta 逐渐减小到 1040000 左右 - */ - num: item.num, - raw_hot: item.raw_hot, - detla: item.num - item.raw_hot, // 计算值 - - url: `https://s.weibo.com/weibo?q=${encodeURIComponent(item.word_scheme)}`, // 热搜话题链接 - - // 分类 - category: item.category ? item.category.split(',') : "", - subject_label: item.subject_label, - - // 其他 - more: { - is_new: item.is_new, - subject_querys: item.subject_querys, - mid: item.mid, - icon_desc_color: item.icon_desc_color, - detail: detail, - }, - }); - }); - saveJSON({ - now: now, - fileNameSuffix: `final`, - object: convert, - compress: true, - // uncompress: true, - uncompress: false, - }); - - - /** - * 只统计微博调控信息 - */ - let convert2 = []; - let total = 0; - data.band_list.forEach(item => { - total += item.num; - total -= item.raw_hot; - if (item.num - item.raw_hot == 0) return; - convert2.push([ - `[${item.realpos}] ${item.word}【${item.label_name}】`, - `原始:${item.raw_hot} 显示:${item.num} 调控: ${item.num - item.raw_hot}` - ]); - }); - saveJSON({ - now: now, - fileNameSuffix: `regulation`, - object: { - total_delta: total, // 所有调控值之和 - data: convert2 - }, - compress: false, - uncompress: true - }); - - - /** - * 保存预处理后数据 - */ - // 过滤掉不需要的数据 - // band_list - data.band_list.forEach(function (item) { - delete item["mblog"]; - }); - saveJSON({ - now: now, - fileNameSuffix: `simplify`, - object: data, - compress: true, - // uncompress: true, - // compress: false, - uncompress: false, - }); - - - /** - * 更新最新的 - */ - fs.writeFileSync(`${DATA_FOLDER}/latest.json`, JSON.stringify({ - update_time: requestTimestamp, - update_time_friendly: now.substring(0, 19).replace(/T/g, " "), - regulation: convert2, - data: convert - })); + // 重复字段只保留一个 + delete item["note"]; // note word + delete item["icon_desc"]; delete item["small_icon_desc"]; // label_name icon_desc small_icon_desc + delete item["small_icon_desc_color"]; // icon_desc_color small_icon_desc_color + delete item["flag_desc"]; // flag_desc subject_label 这两个有值的时候相同，没有值的时候，前一个为 undefined，后一个为 "" } + + + /** + * 获取需要的数据，进行转换 + */ + let convert = []; + data.band_list.forEach(item => { + let detail = ""; + let pic_ids = []; + if (item.mblog) { // 有些热搜没有 mblog + var regex = /(<([^>]+)>)/ig + detail = item.mblog.text.replace(regex, ""); + if (item.mblog.pics) { + pic_ids = item.mblog.pics.map(pic => `${pic}`); + } + } + convert.push({ + // 热搜排行顺序 + rank: item.rank, + realpos: item.realpos, + + // 热搜信息 + word: item.word, // 热搜标题 + word_scheme: item.word_scheme, // 热搜话题 "#热搜标题#" + emoticon: item.emoticon, // 热搜小表情，如 "[泪]" + label_name: item.label_name, // 热搜标签，如 "爆" "热" "新" "" + onboard_time: item.onboard_time, // 热搜上线时间，秒级时间戳，如 1658565575 + + /** + * 热搜数据 + * + * 大部分的 num 和 raw_hot 是相同的，页面上显示的是 num，可能是人工调控的热搜 + * + * 两者差值通过观测似乎最大是 1250000 + * 例如 【爆】唐山打架事件8名违法嫌疑人已到案 这条热搜一开始 delta 首先不断增大，最大达到 1250000 + * 然后热搜数量增加到 12600000 左右的时候，delta 逐渐减小到 1040000 左右 + */ + num: item.num, + raw_hot: item.raw_hot, + detla: item.num - item.raw_hot, // 计算值 + + url: `https://s.weibo.com/weibo?q=${encodeURIComponent(item.word_scheme)}`, // 热搜话题链接 + + // 分类 + category: item.category ? item.category.split(',') : "", + subject_label: item.subject_label, + + // 其他 + more: { + is_new: item.is_new, + subject_querys: item.subject_querys, + mid: item.mid, + icon_desc_color: item.icon_desc_color, + detail: detail, + }, + }); + }); + saveJSON({ + now: now, + fileNameSuffix: `final`, + object: convert, + compress: true, + // uncompress: true, + uncompress: false, + }); + + + /** + * 只统计微博调控信息 + */ + let convert2 = []; + let total = 0; + data.band_list.forEach(item => { + total += item.num; + total -= item.raw_hot; + if (item.num - item.raw_hot == 0) return; + convert2.push([ + `[${item.realpos}] ${item.word}【${item.label_name}】`, + `原始:${item.raw_hot} 显示:${item.num} 调控: ${item.num - item.raw_hot}` + ]); + }); + saveJSON({ + now: now, + fileNameSuffix: `regulation`, + object: { + total_delta: total, // 所有调控值之和 + data: convert2 + }, + compress: false, + uncompress: true + }); + + + /** + * 保存预处理后数据 + */ + // 过滤掉不需要的数据 + // band_list + data.band_list.forEach(function (item) { + delete item["mblog"]; + }); + saveJSON({ + now: now, + fileNameSuffix: `simplify`, + object: data, + compress: true, + // uncompress: true, + // compress: false, + uncompress: false, + }); + + + /** + * 更新最新的 + */ + fs.writeFileSync(`${DATA_FOLDER}/latest.json`, JSON.stringify({ + update_time: requestTimestamp, + update_time_friendly: now.substring(0, 19).replace(/T/g, " "), + regulation: convert2, + data: convert + })); } exports.main = main; \ No newline at end of file From 7a9c20d0382973ab61ca1c881e60654446badb48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E5=B0=8F=E5=A2=A8?= <2291200076@qq.com> Date: Sun, 24 Jul 2022 14:34:14 +0800 Subject: [PATCH 12/50] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=89=B9=E5=A4=84?= =?UTF-8?q?=E7=90=86=E5=91=BD=E4=BB=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pm2 restart.bat | 1 + pm2 restart.sh | 1 + pm2 start.bat | 1 + pm2 start.sh | 1 + pm2 stop.bat | 2 ++ pm2 stop.sh | 2 ++ 6 files changed, 8 insertions(+) create mode 100644 pm2 restart.bat create mode 100644 pm2 restart.sh create mode 100644 pm2 start.bat create mode 100644 pm2 start.sh create mode 100644 pm2 stop.bat create mode 100644 pm2 stop.sh diff --git a/pm2 restart.bat b/pm2 restart.bat new file mode 100644 index 0000000..9e9eb8b --- /dev/null +++ b/pm2 restart.bat @@ -0,0 +1 @@ +pm2 restart weibo-hotband-bot \ No newline at end of file diff --git a/pm2 restart.sh b/pm2 restart.sh new file mode 100644 index 0000000..9e9eb8b --- /dev/null +++ b/pm2 restart.sh @@ -0,0 +1 @@ +pm2 restart weibo-hotband-bot \ No newline at end of file diff --git a/pm2 start.bat b/pm2 start.bat new file mode 100644 index 0000000..bdbca0d --- /dev/null +++ b/pm2 start.bat @@ -0,0 +1 @@ +pm2 start index.js --name weibo-hotband-bot \ No newline at end of file diff --git a/pm2 start.sh b/pm2 start.sh new file mode 100644 index 0000000..bdbca0d --- /dev/null +++ b/pm2 start.sh @@ -0,0 +1 @@ +pm2 start index.js --name weibo-hotband-bot \ No newline at end of file diff --git a/pm2 stop.bat b/pm2 stop.bat new file mode 100644 index 0000000..b3c0f7f --- /dev/null +++ b/pm2 stop.bat @@ -0,0 +1,2 @@ +pm2 stop weibo-hotband-bot +pm2 delete weibo-hotband-bot \ No newline at end of file diff --git a/pm2 stop.sh b/pm2 stop.sh new file mode 100644 index 0000000..b3c0f7f --- /dev/null +++ b/pm2 stop.sh @@ -0,0 +1,2 @@ +pm2 stop weibo-hotband-bot +pm2 delete weibo-hotband-bot \ No newline at end of file From b1546c355ea1074c4d1a6742042c2f5b9997fadd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E5=B0=8F=E5=A2=A8?= <2291200076@qq.com> Date: Mon, 25 Jul 2022 09:45:39 +0800 Subject: [PATCH 13/50] add README.md --- README.md | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..e6c8d35 --- /dev/null +++ b/README.md @@ -0,0 +1,49 @@ +# 微博热搜数据爬取工具 + +> 本仓库中代码仅供学习研究使用，不得用于违法用途。 + +## 简介 + +您可以将本项目代码部署在服务器上（在本地运行也可），程序会每隔一分钟拉取一次微博热搜数据，并保存为 `json` 格式文件。 + +## 部署 + +1. 克隆仓库（或直接下载压缩包） + +```bash +git clone https://git.only4.work/coder-xiaomo/weibo-hotband +``` + +2. 安装依赖 + +```bash +npm i +``` + +3. 修改配置文件 + +TODO + +4. 启动项目 + +```bash +# 直接运行 +# node index.js + +# 使用 pm2 +# pm2 start index.js --name weibo-hotband-bot +``` + +5. 停止项目 + +```bash +# 使用 node index.js 命令直接运行的项目可以通过 `Ctrl + C` 停止 + +# 使用 pm2 运行的可以使用以下两行命令来停止和从列表中删除项目 +# pm2 stop weibo-hotband-bot +# pm2 delete weibo-hotband-bot +``` + +## 说明 + +TODO From 3d4246cc45c7f562c209d5821fbd9200d0f8aa7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E5=B0=8F=E5=A2=A8?= <2291200076@qq.com> Date: Mon, 25 Jul 2022 10:01:27 +0800 Subject: [PATCH 14/50] =?UTF-8?q?=E6=9B=B4=E6=96=B0=20'README.md'?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e6c8d35..5584af8 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,25 @@ # 微博热搜数据爬取工具 > 本仓库中代码仅供学习研究使用，不得用于违法用途。 +> +> 数据来自微博，本项目不对数据真实性做验证，使用数据时请遵守微博平台的相关限制要求。 ## 简介 您可以将本项目代码部署在服务器上（在本地运行也可），程序会每隔一分钟拉取一次微博热搜数据，并保存为 `json` 格式文件。 +## 数据来源 + +微博热搜接口：https://weibo.com/ajax/statuses/hot_band + +该接口来自微博web端，网址：https://weibo.com/newlogin?tabtype=search + +## 运行环境 + +原理上来说 Windows 下和 Linux 都可运行，目前仅在 Windows 下测试过，暂未在 Linux 系统下测试。 + +项目使用 node 开发，以下部署流程默认您已安装了 `Git`、`Nodejs`。 + ## 部署 1. 克隆仓库（或直接下载压缩包） @@ -22,7 +36,7 @@ npm i 3. 修改配置文件 -TODO +将项目目录下的 `.env.example` 文件复制一份，并改名为 `.env`，使用文本编辑器打开（例如：记事本、VS Code、vim等均可），根据其中的注释说明来进行配置即可。 4. 启动项目 From ec0ed8824449edecaff788de2db4a834beb54198 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E5=B0=8F=E5=A2=A8?= <2291200076@qq.com> Date: Mon, 25 Jul 2022 10:37:49 +0800 Subject: [PATCH 15/50] =?UTF-8?q?=E5=90=AF=E5=8A=A8=E6=97=B6=E6=A3=80?= =?UTF-8?q?=E6=9F=A5.env=E6=96=87=E4=BB=B6=E6=98=AF=E5=90=A6=E5=AD=98?= =?UTF-8?q?=E5=9C=A8=EF=BC=9B=E6=9B=B4=E6=96=B0README?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 21 +++++++++++++++++++-- index.js | 6 ++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5584af8..a5cfa21 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # 微博热搜数据爬取工具 > 本仓库中代码仅供学习研究使用，不得用于违法用途。 -> +> > 数据来自微博，本项目不对数据真实性做验证，使用数据时请遵守微博平台的相关限制要求。 ## 简介 @@ -38,6 +38,11 @@ npm i 将项目目录下的 `.env.example` 文件复制一份，并改名为 `.env`，使用文本编辑器打开（例如：记事本、VS Code、vim等均可），根据其中的注释说明来进行配置即可。 +> 如果不创建 .env 文件，项目启动时会报如下错误并退出。 +> ```bash +> [ERROR] .env file not found! +> ``` + 4. 启动项目 ```bash @@ -60,4 +65,16 @@ npm i ## 说明 -TODO +项目爬取的数据默认保存在项目目录下的 data 文件夹中，您也可以通过修改 `.env` 文件中的 `DATA_FOLDER` 参数值来自定义数据保存路径。 + +在程序运行后，该文件夹下会出现 `latest.json` 文件及其余几个文件夹，这些子文件夹中的文件按照以下格式保存：`年/月/日/年月日_时分.json`。 + +每次爬取后，`latest.json`中的数据都会被覆盖为最新的热搜数据。 + +`origin` 文件夹中的数据是通过Api接口获取到的原始数据，没有经过任何处理。 + +`simplify` 文件夹中的数据是在原始数据的基础上，去除了部分冗余数据。 + +`final` 文件夹中的数据是从原始数据中抽离出的有用数据，并重新整理得到的。 + +`regulation` 文件夹中的数据主要用于观测原始值与显示值不同的热搜，这部分热搜猜测可能是经过微博平台调控的。（这部分数据没有太大意义，可以忽略） diff --git a/index.js b/index.js index 96881a4..24049c0 100644 --- a/index.js +++ b/index.js @@ -4,10 +4,16 @@ const dotenv = require('dotenv'); const schedule = require('node-schedule'); const path = require('path'); const os = require('os'); +const fs = require('fs'); /** * 环境变量 */ + if (!fs.existsSync('.env')) { + // 如果没有 .env 文件，则报错并退出 + console.error('[ERROR] .env file not found!'); + return; +} process.env = {}; // 清除系统自带的环境变量 dotenv.config('./.env'); // 导入 .env 文件中的环境变量 // console.log(process.env); From 08615413c98f735c6e3fc5c2dacaa8be92747a13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E5=B0=8F=E5=A2=A8?= <2291200076@qq.com> Date: Mon, 25 Jul 2022 18:01:26 +0800 Subject: [PATCH 16/50] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E7=83=AD=E6=90=9C?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E9=A2=84=E8=A7=88=E7=BD=91=E9=A1=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 4 ++ html/index.html | 101 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 html/index.html diff --git a/README.md b/README.md index a5cfa21..61635db 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,10 @@ 您可以将本项目代码部署在服务器上（在本地运行也可），程序会每隔一分钟拉取一次微博热搜数据，并保存为 `json` 格式文件。 +## 数据预览 + +在部署并启动项目后，您可以在浏览器中打开 `html/index.html` 文件实时预览当前热搜。 + ## 数据来源 微博热搜接口：https://weibo.com/ajax/statuses/hot_band diff --git a/html/index.html b/html/index.html new file mode 100644 index 0000000..29c67f9 --- /dev/null +++ b/html/index.html @@ -0,0 +1,101 @@ + + + + + + + + 微博热搜 + + + + +

+
+ + + + \ No newline at end of file From 317510a794138b0dc3005e89d0bb870a8c437732 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E5=B0=8F=E5=A2=A8?= <2291200076@qq.com> Date: Mon, 25 Jul 2022 22:26:15 +0800 Subject: [PATCH 17/50] =?UTF-8?q?=E6=9B=B4=E6=96=B0html=E9=A1=B5=E9=9D=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- html/index.html | 223 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 160 insertions(+), 63 deletions(-) diff --git a/html/index.html b/html/index.html index 29c67f9..5a5d3fa 100644 --- a/html/index.html +++ b/html/index.html @@ -1,5 +1,5 @@ - + @@ -11,90 +11,187 @@ width: 100%; text-align: center; border-spacing: 0; + border: 0.4px solid black; + } + + #list tr { + /* height: 50px; */ } #list td { margin: 0; - border: 0.5px solid black; + border: 0.4px solid black; + } + + /* 热搜的 label 样式 */ + .hotband-label { + color: white; + padding: 3px; + border-radius: 6px; + font-size: 10px; + + + display: inline-block; } -

+
+

微博热搜榜

+
+
+ 过滤： + + +
+
+

+ +
From 61bcb1f2988a3654ca71570b0c8529514c322f46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E5=B0=8F=E5=A2=A8?= <2291200076@qq.com> Date: Tue, 26 Jul 2022 13:40:19 +0800 Subject: [PATCH 18/50] =?UTF-8?q?=E6=9B=B4=E6=96=B0html?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- html/index.html | 145 +++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 125 insertions(+), 20 deletions(-) diff --git a/html/index.html b/html/index.html index 5a5d3fa..80508ed 100644 --- a/html/index.html +++ b/html/index.html @@ -15,7 +15,7 @@ } #list tr { - /* height: 50px; */ + height: min(1.85rem, 50px); } #list td { @@ -41,13 +41,41 @@

微博热搜榜

- 过滤： + 显示字段： + + + + + +
+ + + | + + +

@@ -61,22 +89,70 @@ // 拉取下来的数据 let hotBandData; - // 刷新按钮 + // 按钮 + const btnShowAll = document.getElementById('btn_show_all'); + const btnShowNone = document.getElementById('btn_show_none'); + const btnShowConcise = document.getElementById('btn_show_concise'); + const btnShowDefault = document.getElementById('btn_show_default'); + const btnShowDetailed = document.getElementById('btn_show_detailed'); + const btnRefresh = document.getElementById('btn_refresh'); // 复选框 const filterCheckbox = document.getElementsByClassName("filter_checkbox"); + + const showEmoticon = document.getElementById("show_emoticon"); + const showNum = document.getElementById("show_num"); + const showCategory = document.getElementById("show_category"); + const showOnboardTime = document.getElementById("show_onboard_time"); + const showIsNew = document.getElementById("show_is_new"); const showDetail = document.getElementById("show_detail"); const showMid = document.getElementById("show_mid"); // 绑定按钮点击事件 + btnShowAll.addEventListener('click', function () { + for (let i = 0; i < filterCheckbox.length; i++) { + const element = filterCheckbox[i]; + element.checked = true; + } + render(); + }); + btnShowNone.addEventListener('click', function () { + for (let i = 0; i < filterCheckbox.length; i++) { + const element = filterCheckbox[i]; + element.checked = false; + } + render(); + }); + btnShowConcise.addEventListener('click', function () { + for (let i = 0; i < filterCheckbox.length; i++) { + const element = filterCheckbox[i]; + element.checked = element.getAttribute('concise-checked') === 'true'; + } + render(); + }); + btnShowDefault.addEventListener('click', function () { + for (let i = 0; i < filterCheckbox.length; i++) { + const element = filterCheckbox[i]; + element.checked = element.getAttribute('checked') === 'true'; + } + render(); + }); + btnShowDetailed.addEventListener('click', function () { + for (let i = 0; i < filterCheckbox.length; i++) { + const element = filterCheckbox[i]; + element.checked = element.getAttribute('detailed-checked') === 'true'; + } + render(); + }); + btnRefresh.onclick = function () { getData(); document.getElementById("update-finish-info").style.display = ""; btnRefresh.style.display = "none"; setTimeout(function () { document.getElementById("update-finish-info").style.display = "none"; - btnRefresh.style.display = ""; + btnRefresh.style.display = ""; }, 1000); }; @@ -84,10 +160,27 @@ for (let i = 0; i < filterCheckbox.length; i++) { console.log(filterCheckbox[i]); filterCheckbox[i].onchange = function () { - render(hotBandData); + render(); }; } + // 根据屏幕判断要显示哪些字段 + // 此时还未拉取数据，所以进入 render 函数会直接返回，不会多次渲染 + let initWidth = document.body.offsetWidth; + console.log(initWidth); + if (initWidth < 400) { + btnShowNone.click(); + btnShowNone.innerHTML += "(默认)"; + } else if (initWidth < 600) { + btnShowConcise.click(); + btnShowConcise.innerHTML += "(默认)"; + } else if (initWidth < 1900) { + btnShowDefault.click(); + btnShowDefault.innerHTML += "(默认)"; + } else { + btnShowDetailed.click(); + btnShowDetailed.innerHTML += "(默认)"; + } // 网页加载后加载榜单 getData(); @@ -103,18 +196,20 @@ if (xhr.readyState == 4 && xhr.status == 200) { hotBandData = JSON.parse(xhr.responseText); console.log(hotBandData); - render(hotBandData); + + // 更新时间 + document.getElementById("latestUpdateTime").innerHTML = + "数据拉取时间：" + new Date().toLocaleString() + "
" + + "热榜更新时间：" + new Date(hotBandData.update_time).toLocaleString(); + + // 渲染榜单 + render(); } } } - function render(hotBandData) { - /** - * 更新时间 - */ - document.getElementById("latestUpdateTime").innerHTML = - "数据拉取时间：" + new Date().toLocaleString() + "
" + - "热榜更新时间：" + new Date(hotBandData.update_time).toLocaleString(); + function render() { + if (!hotBandData) return; /** * 渲染热搜列表 @@ -127,11 +222,11 @@ 编号 热搜 - 表情 - 热度
(展示/真实) - 分类 - 热搜上线时间 - 是否新热搜 + ${showEmoticon.checked ? "表情" : ""} + ${showNum.checked ? '热度
(展示/真实)' : ""} + ${showCategory.checked ? "分类" : ""} + ${showOnboardTime.checked ? "上线时间" : ""} + ${showIsNew.checked ? "是否新热搜" : ""} ${showDetail.checked ? "热搜详情" : ""} ${showMid.checked ? "mid" : ""} @@ -154,9 +249,12 @@ + ${showEmoticon.checked ? ` ${hotBand.emoticon} + ` : ""} + ${showNum.checked ? ` ${hotDelta == 0 ? hotBand.num : `${hotBand.num} / ${hotBand.raw_hot}`}
@@ -166,15 +264,22 @@ + ` : ""} + ${showCategory.checked ? ` ${hotBand.category.map((c) => `${c}`).join('；')} + ` : ""} + ${showOnboardTime.checked ? ` ${new Date(hotBand.onboard_time * 1000).toLocaleString()} + ` : ""} + ${showIsNew.checked ? ` ${hotBand.more.is_new == 1 ? "是" : ""} + ` : ""} ${showDetail.checked ? ` From 48fa9025cd6d74106537d09873e6eb0684f07299 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E5=B0=8F=E5=A2=A8?= <2291200076@qq.com> Date: Tue, 26 Jul 2022 14:00:15 +0800 Subject: [PATCH 19/50] =?UTF-8?q?=E8=87=AA=E5=8A=A8=E6=8B=89=E5=8F=96?= =?UTF-8?q?=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- html/index.html | 37 +++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/html/index.html b/html/index.html index 80508ed..e471524 100644 --- a/html/index.html +++ b/html/index.html @@ -80,6 +80,10 @@

+
+ + + \ No newline at end of file diff --git a/html/index.html b/html/weibo.html similarity index 96% rename from html/index.html rename to html/weibo.html index 2c34c89..7e45e89 100644 --- a/html/index.html +++ b/html/weibo.html @@ -29,8 +29,6 @@ padding: 3px; border-radius: 6px; font-size: 10px; - - display: inline-block; } @@ -79,12 +77,18 @@

- - - + + + + + + + + +
\ No newline at end of file From a76c39b030dfeba6d90feaca43c01dc61474609b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E5=B0=8F=E5=A2=A8?= <2291200076@qq.com> Date: Thu, 28 Jul 2022 21:25:58 +0800 Subject: [PATCH 24/50] =?UTF-8?q?=E5=B0=8F=E8=B0=83=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- index.js | 4 ++-- src/{get_hotband.js => get_weibo_hotband.js} | 0 2 files changed, 2 insertions(+), 2 deletions(-) rename src/{get_hotband.js => get_weibo_hotband.js} (100%) diff --git a/index.js b/index.js index 24049c0..c10bffe 100644 --- a/index.js +++ b/index.js @@ -36,7 +36,7 @@ if (DEBUG_MODE) { /** * 引入模块 */ -const get_hotband = require('./src/get_hotband'); +const get_weibo_hotband = require('./src/get_weibo_hotband'); const execute_command = require('./src/execute_command'); @@ -51,7 +51,7 @@ console.log("Start running ..."); */ async function start() { // 爬取热搜数据 - await get_hotband.main(); + await get_weibo_hotband.main(); // 调试模式下 if (DEBUG_MODE) { diff --git a/src/get_hotband.js b/src/get_weibo_hotband.js similarity index 100% rename from src/get_hotband.js rename to src/get_weibo_hotband.js From cf28f475e310d4dd7186414dc60591883aca4f5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E5=B0=8F=E5=A2=A8?= <2291200076@qq.com> Date: Thu, 28 Jul 2022 22:35:07 +0800 Subject: [PATCH 25/50] =?UTF-8?q?=E6=8A=BD=E7=A6=BB=E5=87=BA=E5=85=AC?= =?UTF-8?q?=E5=85=B1=E6=96=B9=E6=B3=95=EF=BC=9B=E6=B7=BB=E5=8A=A0B?= =?UTF-8?q?=E7=AB=99=E7=83=AD=E6=90=9C=E7=88=AC=E5=8F=96=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- html/weibo.html | 2 +- index.js | 2 + src/get_bilibili_hotband.js | 114 ++++++++++++++++++++++++++++++++++++ src/get_weibo_hotband.js | 72 ++++++----------------- src/utils/utils.js | 49 ++++++++++++++++ 5 files changed, 184 insertions(+), 55 deletions(-) create mode 100644 src/get_bilibili_hotband.js create mode 100644 src/utils/utils.js diff --git a/html/weibo.html b/html/weibo.html index 7e45e89..be75868 100644 --- a/html/weibo.html +++ b/html/weibo.html @@ -227,7 +227,7 @@ function getData() { var xhr = new XMLHttpRequest(); - xhr.open("GET", "../data/latest.json?t=" + Date.now(), true); + xhr.open("GET", "../data/weibo/latest.json?t=" + Date.now(), true); xhr.send(); xhr.onreadystatechange = function () { if (xhr.readyState !== 4) return; diff --git a/index.js b/index.js index c10bffe..f786500 100644 --- a/index.js +++ b/index.js @@ -37,6 +37,7 @@ if (DEBUG_MODE) { * 引入模块 */ const get_weibo_hotband = require('./src/get_weibo_hotband'); +const get_bilibili_hotband = require('./src/get_bilibili_hotband'); const execute_command = require('./src/execute_command'); @@ -52,6 +53,7 @@ console.log("Start running ..."); async function start() { // 爬取热搜数据 await get_weibo_hotband.main(); + await get_bilibili_hotband.main(); // 调试模式下 if (DEBUG_MODE) { diff --git a/src/get_bilibili_hotband.js b/src/get_bilibili_hotband.js new file mode 100644 index 0000000..9d29daa --- /dev/null +++ b/src/get_bilibili_hotband.js @@ -0,0 +1,114 @@ +'use strict'; + +const request = require('request'); +const fs = require('fs'); +const path = require('path'); + +const utils = require('./utils/utils'); + +const DATA_FOLDER = path.join(path.dirname(__dirname), process.env.DATA_FOLDER ?? 'data', 'bilibili'); +console.log("DATA_FOLDER", DATA_FOLDER); +utils.createFolder(DATA_FOLDER); // 程序运行就保证 data 目录存在 + +// 请求微博热搜 APi 接口 +async function getApiResult(url) { + var return_data = await new Promise((resolve) => { + request({ + method: 'GET', + url: url, + json: true, + }, (error, response, result) => { + if (!error && (response.statusCode == 200)) { + // 请求成功 + resolve(result); + } else { + // 请求失败 + console.log(`error is ${error}`); + resolve("error"); + } + }); + }); + // console.log(`return_data is ${JSON.stringify(return_data)}`); + return return_data; +} + +async function main() { + let requestTimestamp = Date.now(); + let now = new Date(requestTimestamp + 8 * 3600 * 1000).toISOString(); + + let result = await getApiResult("https://app.bilibili.com/x/v2/search/trending/ranking"); + if (result.code != 0) { + console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), "bilibili", "请求成功，但服务器处理失败，正在重试。"); + result = await getApiResult("https://app.bilibili.com/x/v2/search/trending/ranking"); + if (result.ok != 1) { + console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), "bilibili", "请求成功，但服务器处理失败，保存失败信息。"); + // ok 不为 1，那么久直接保存便于后续分析，不进行后续处理 + utils.saveJSON({ + saveFolder: DATA_FOLDER, + now: now, + fileNameSuffix: `origin-error`, + object: result, + compress: true, + uncompress: false + }); + return; + } + } + + console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), "bilibili", "请求成功"); + // console.log("result", result); + + let data = result.data; + + // 去除 trackid + delete data["trackid"]; + // console.log(data); + + /** + * 保存原始数据 + */ + utils.saveJSON({ + saveFolder: DATA_FOLDER, + now: now, + fileNameSuffix: `origin`, + object: result, + compress: true, + uncompress: false + }); + + + /** + * 获取需要的数据，进行转换 + */ + let convert = []; + data.list.forEach(item => { + // { + // "position": 1, + // "keyword": "关键词", + // "show_name": "热搜名称", + // "word_type": 8, + // "icon": "热搜的图标，也可能没有", + // "hot_id": 7399 // 热搜id + // } + convert.push(item); + }); + utils.saveJSON({ + saveFolder: DATA_FOLDER, + now: now, + fileNameSuffix: `final`, + object: convert, + compress: true, + uncompress: false, + }); + + /** + * 更新最新的 + */ + fs.writeFileSync(`${DATA_FOLDER}/latest.json`, JSON.stringify({ + update_time: requestTimestamp, + update_time_friendly: now.substring(0, 19).replace(/T/g, " "), + data: data + })); +} + +exports.main = main; diff --git a/src/get_weibo_hotband.js b/src/get_weibo_hotband.js index 1eec48c..e5a11c2 100644 --- a/src/get_weibo_hotband.js +++ b/src/get_weibo_hotband.js @@ -4,11 +4,11 @@ const request = require('request'); const fs = require('fs'); const path = require('path'); -const DATA_FOLDER = path.join(path.dirname(__dirname), process.env.DATA_FOLDER ?? 'data'); -console.log("DATA_FOLDER", DATA_FOLDER); -createFolder(DATA_FOLDER); // 程序运行就保证 data 目录存在 +const utils = require('./utils/utils'); -const LATEST_DATA_ONLY = process.env.LATEST_DATA_ONLY == true; +const DATA_FOLDER = path.join(path.dirname(__dirname), process.env.DATA_FOLDER ?? 'data', 'weibo'); +console.log("DATA_FOLDER", DATA_FOLDER); +utils.createFolder(DATA_FOLDER); // 程序运行就保证 data 目录存在 // 请求微博热搜 APi 接口 async function getApiResult(url) { @@ -32,59 +32,19 @@ async function getApiResult(url) { return return_data; } -// 创建目录 -async function createFolder(folderToCreate) { - let currentFolder = folderToCreate.replace(/\\/g, '/'); - let parentFolder = currentFolder.substring(0, currentFolder.lastIndexOf('/')); - if (!fs.existsSync(currentFolder)) { - // 文件夹不存在，创建文件夹 - createFolder(parentFolder); // 保证父级文件夹存在 - fs.mkdirSync(currentFolder); // 创建当前级文件夹 - } else { - // 否则就什么也不做 - } -} - -// 保存 JSON -function saveJSON({ now, fileNameSuffix, object, compress = true, uncompress = true }) { - if (LATEST_DATA_ONLY) return; - - let year = now.substring(0, 4); - let month = now.substring(5, 7); - let day = now.substring(8, 10); - let hour = now.substring(11, 13); - let minute = now.substring(14, 16); - // console.log(now); - // console.log( "year, month, day, hour, minute: " + year + ", " + month + ", " + day + ", " + hour + ", " + minute); - - // 创建当前文件夹 - let folder = `${DATA_FOLDER}/${fileNameSuffix}/${year}/${month}/${day}`; - createFolder(folder); - let fileName = `${folder}/${year}${month}${day}_${hour}${minute}`; - - // 生成文件名 - // '2022-07-23T10:11:38.650Z' => '20220723_1011' - // let fileName = now.replace(/T/, '_').replace(/:\d{2}.\d{3}Z/, '').replace(/[-:]/g, ''); - // console.log(`fileName is ${fileName}`); - - if (compress) - fs.writeFileSync(`${fileName}.min.json`, JSON.stringify(object)); - if (uncompress) - fs.writeFileSync(`${fileName}.json`, JSON.stringify(object, "", "\t")); -} - async function main() { let requestTimestamp = Date.now(); let now = new Date(requestTimestamp + 8 * 3600 * 1000).toISOString(); let result = await getApiResult("https://weibo.com/ajax/statuses/hot_band"); if (result.ok != 1) { - console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), "请求成功，但服务器处理失败，正在重试。"); + console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), "weibo", "请求成功，但服务器处理失败，正在重试。"); result = await getApiResult("https://weibo.com/ajax/statuses/hot_band"); if (result.ok != 1) { - console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), "请求成功，但服务器处理失败，保存失败信息。"); + console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), "weibo", "请求成功，但服务器处理失败，保存失败信息。"); // ok 不为 1，那么久直接保存便于后续分析，不进行后续处理 - saveJSON({ + utils.saveJSON({ + saveFolder: DATA_FOLDER, now: now, fileNameSuffix: `origin-error`, object: result, @@ -95,13 +55,14 @@ async function main() { } } - console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), "请求成功"); + console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), "weibo", "请求成功"); // console.log("result", result); /** * 保存原始数据 */ - saveJSON({ + utils.saveJSON({ + saveFolder: DATA_FOLDER, now: now, fileNameSuffix: `origin`, object: result, @@ -198,7 +159,8 @@ async function main() { }, }); }); - saveJSON({ + utils.saveJSON({ + saveFolder: DATA_FOLDER, now: now, fileNameSuffix: `final`, object: convert, @@ -222,7 +184,8 @@ async function main() { `原始:${item.raw_hot} 显示:${item.num} 调控: ${item.num - item.raw_hot}` ]); }); - saveJSON({ + utils.saveJSON({ + saveFolder: DATA_FOLDER, now: now, fileNameSuffix: `regulation`, object: { @@ -242,7 +205,8 @@ async function main() { data.band_list.forEach(function (item) { delete item["mblog"]; }); - saveJSON({ + utils.saveJSON({ + saveFolder: DATA_FOLDER, now: now, fileNameSuffix: `simplify`, object: data, @@ -264,4 +228,4 @@ async function main() { })); } -exports.main = main; \ No newline at end of file +exports.main = main; diff --git a/src/utils/utils.js b/src/utils/utils.js new file mode 100644 index 0000000..82aa0aa --- /dev/null +++ b/src/utils/utils.js @@ -0,0 +1,49 @@ +const fs = require('fs'); + +const LATEST_DATA_ONLY = process.env.LATEST_DATA_ONLY == true; + +// 创建目录 +async function createFolder(folderToCreate) { + let currentFolder = folderToCreate.replace(/\\/g, '/'); + let parentFolder = currentFolder.substring(0, currentFolder.lastIndexOf('/')); + if (!fs.existsSync(currentFolder)) { + // 文件夹不存在，创建文件夹 + createFolder(parentFolder); // 保证父级文件夹存在 + fs.mkdirSync(currentFolder); // 创建当前级文件夹 + } else { + // 否则就什么也不做 + } +} + +// 保存 JSON +function saveJSON({ saveFolder, now, fileNameSuffix, object, compress = true, uncompress = true }) { + if (LATEST_DATA_ONLY) return; + + let year = now.substring(0, 4); + let month = now.substring(5, 7); + let day = now.substring(8, 10); + let hour = now.substring(11, 13); + let minute = now.substring(14, 16); + // console.log(now); + // console.log( "year, month, day, hour, minute: " + year + ", " + month + ", " + day + ", " + hour + ", " + minute); + + // 创建当前文件夹 + let folder = `${saveFolder}/${fileNameSuffix}/${year}/${month}/${day}`; + createFolder(folder); + let fileName = `${folder}/${year}${month}${day}_${hour}${minute}`; + + // 生成文件名 + // '2022-07-23T10:11:38.650Z' => '20220723_1011' + // let fileName = now.replace(/T/, '_').replace(/:\d{2}.\d{3}Z/, '').replace(/[-:]/g, ''); + // console.log(`fileName is ${fileName}`); + + if (compress) + fs.writeFileSync(`${fileName}.min.json`, JSON.stringify(object)); + if (uncompress) + fs.writeFileSync(`${fileName}.json`, JSON.stringify(object, "", "\t")); +} + +module.exports = { + createFolder, + saveJSON, +} \ No newline at end of file From 1c70c884067624f3c87fa5ceb877386ff36aa2fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E5=B0=8F=E5=A2=A8?= <2291200076@qq.com> Date: Thu, 28 Jul 2022 23:13:21 +0800 Subject: [PATCH 26/50] =?UTF-8?q?B=E7=AB=99=20=E5=89=8D=E7=AB=AF=E9=A1=B5?= =?UTF-8?q?=E5=AE=8C=E5=96=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- html/bilibili.html | 239 +++++++++++++++++++++++++++++++++++- html/weibo.html | 3 +- src/get_bilibili_hotband.js | 3 +- 3 files changed, 239 insertions(+), 6 deletions(-) diff --git a/html/bilibili.html b/html/bilibili.html index 604ae10..c09ee7b 100644 --- a/html/bilibili.html +++ b/html/bilibili.html @@ -40,11 +40,22 @@
显示字段： + + +
| -
@@ -55,8 +66,7 @@ @@ -64,6 +74,229 @@
+ diff --git a/html/weibo.html b/html/weibo.html index be75868..3c21e8b 100644 --- a/html/weibo.html +++ b/html/weibo.html @@ -82,8 +82,7 @@ diff --git a/src/get_bilibili_hotband.js b/src/get_bilibili_hotband.js index 9d29daa..3de3a4a 100644 --- a/src/get_bilibili_hotband.js +++ b/src/get_bilibili_hotband.js @@ -107,7 +107,8 @@ async function main() { fs.writeFileSync(`${DATA_FOLDER}/latest.json`, JSON.stringify({ update_time: requestTimestamp, update_time_friendly: now.substring(0, 19).replace(/T/g, " "), - data: data + data: data.list, + exp_str: data.exp_str, })); } From e7692bac65d402857fe85bec03d04a5b926174b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E5=B0=8F=E5=A2=A8?= <2291200076@qq.com> Date: Fri, 29 Jul 2022 14:08:23 +0800 Subject: [PATCH 27/50] =?UTF-8?q?B=E7=AB=99=E7=83=AD=E6=90=9C=20=E6=96=87?= =?UTF-8?q?=E5=AD=97=E6=94=B9=E5=B0=8F=EF=BC=9B=E5=A4=96=E5=B1=82=E5=A5=97?= =?UTF-8?q?=E5=A3=B3=E9=A1=B5=E9=9D=A2=E6=A0=B7=E5=BC=8F=E5=B0=8F=E8=B0=83?= =?UTF-8?q?=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- html/bilibili.html | 3 ++- index.html | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/html/bilibili.html b/html/bilibili.html index c09ee7b..f698d7a 100644 --- a/html/bilibili.html +++ b/html/bilibili.html @@ -21,6 +21,7 @@ #list td { margin: 0; border: 0.4px solid black; + font-size: 14px; } /* 热搜的 label 样式 */ @@ -269,7 +270,7 @@ ${hotBand.position} - +
${getIconText(hotBand.icon)[0]} diff --git a/index.html b/index.html index b25fc40..deedc9d 100644 --- a/index.html +++ b/index.html @@ -45,6 +45,8 @@ #float-container.bt .navbar-item { text-align: center; cursor: pointer; + width: 80px; + user-select: none; } #float-container.bt .navbar-item .navbar-image { From c062151ea32ab16f25e12c58e5f20aaddc915366 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E5=B0=8F=E5=A2=A8?= <2291200076@qq.com> Date: Fri, 29 Jul 2022 14:44:23 +0800 Subject: [PATCH 28/50] =?UTF-8?q?=E6=8A=BD=E7=A6=BBhtml=E5=85=AC=E5=85=B1?= =?UTF-8?q?=E6=A0=B7=E5=BC=8F=EF=BC=9B=E5=A5=97=E5=A3=B3=E9=A1=B5=E9=9D=A2?= =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E4=B8=8D=E5=88=B7=E6=96=B0=E9=A1=B5=E9=9D=A2?= =?UTF-8?q?=E5=88=87=E6=8D=A2=E9=A1=B5=E9=9D=A2=EF=BC=8C=E4=BB=A5=E5=8F=8A?= =?UTF-8?q?url=20target=E5=8F=82=E6=95=B0=E7=AD=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- html/assets/css/main.css | 24 ++++++++++++++++ html/bilibili.html | 23 +-------------- html/weibo.html | 27 +---------------- index.html | 62 +++++++++++++++++++++++++++++++++++++--- 4 files changed, 84 insertions(+), 52 deletions(-) create mode 100644 html/assets/css/main.css diff --git a/html/assets/css/main.css b/html/assets/css/main.css new file mode 100644 index 0000000..a2c74fd --- /dev/null +++ b/html/assets/css/main.css @@ -0,0 +1,24 @@ +#list { + width: 100%; + text-align: center; + border-spacing: 0; + border: 0.4px solid black; +} + +#list tr { + height: min(1.85rem, 50px); +} + +#list td { + margin: 0; + border: 0.4px solid black; +} + +/* 热搜的 label 样式 */ +.hotband-label { + color: white; + padding: 3px; + border-radius: 6px; + font-size: 10px; + display: inline-block; +} diff --git a/html/bilibili.html b/html/bilibili.html index f698d7a..12a4aac 100644 --- a/html/bilibili.html +++ b/html/bilibili.html @@ -6,32 +6,11 @@ B站热搜 + diff --git a/html/weibo.html b/html/weibo.html index 3c21e8b..275ceb2 100644 --- a/html/weibo.html +++ b/html/weibo.html @@ -6,32 +6,7 @@ 微博热搜 - + diff --git a/index.html b/index.html index deedc9d..abfc0b2 100644 --- a/index.html +++ b/index.html @@ -25,6 +25,7 @@ height: 100%; } + .iframe-container, iframe { width: 100%; height: 100%; @@ -69,13 +70,15 @@
- +
+ +
-
+

微博

-
+

B站

@@ -83,16 +86,67 @@
From a068c3bb6e93f752503355334d840a39c889ca95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E5=B0=8F=E5=A2=A8?= <2291200076@qq.com> Date: Fri, 29 Jul 2022 15:27:08 +0800 Subject: [PATCH 29/50] =?UTF-8?q?B=E7=AB=99=E7=83=AD=E6=90=9C=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0=E7=9B=B4=E6=92=AD=E4=B8=AD=E6=A0=87=E7=AD=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- html/bilibili.html | 1 + 1 file changed, 1 insertion(+) diff --git a/html/bilibili.html b/html/bilibili.html index 12a4aac..85786e9 100644 --- a/html/bilibili.html +++ b/html/bilibili.html @@ -255,6 +255,7 @@ ${getIconText(hotBand.icon)[0]}
${hotBand.show_name} + ${hotBand.show_live_icon ? `直播中` : ""} From 68812bc7ca3f45002c3406f7cf8f55ca43dd648e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E5=B0=8F=E5=A2=A8?= <2291200076@qq.com> Date: Fri, 29 Jul 2022 15:28:02 +0800 Subject: [PATCH 30/50] =?UTF-8?q?=E6=9B=B4=E6=96=B0README?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 61635db..0e570aa 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ -# 微博热搜数据爬取工具 +# 热搜数据爬取工具 -> 本仓库中代码仅供学习研究使用，不得用于违法用途。 +> 本仓库中代码仅供学习研究使用，不得用于违法用途，学习使用完毕后请于24小时内删除。 > -> 数据来自微博，本项目不对数据真实性做验证，使用数据时请遵守微博平台的相关限制要求。 +> 数据来自微博、B站热搜榜单，本项目不对数据真实性做验证，使用数据时请遵守相关平台的相关限制要求。 ## 简介 -您可以将本项目代码部署在服务器上（在本地运行也可），程序会每隔一分钟拉取一次微博热搜数据，并保存为 `json` 格式文件。 +您可以将本项目代码部署在服务器上（在本地运行也可），程序会每隔一分钟拉取一次热搜数据，并保存为 `json` 格式文件。 ## 数据预览 @@ -16,7 +16,11 @@ 微博热搜接口：https://weibo.com/ajax/statuses/hot_band -该接口来自微博web端，网址：https://weibo.com/newlogin?tabtype=search +> 该接口来自页面：https://weibo.com/newlogin?tabtype=search + +B站热搜接口：https://app.bilibili.com/x/v2/search/trending/ranking + +> 该接口来自页面：https://www.bilibili.com/blackboard/activity-trending-topic.html ## 运行环境 @@ -43,6 +47,7 @@ npm i 将项目目录下的 `.env.example` 文件复制一份，并改名为 `.env`，使用文本编辑器打开（例如：记事本、VS Code、vim等均可），根据其中的注释说明来进行配置即可。 > 如果不创建 .env 文件，项目启动时会报如下错误并退出。 +> > ```bash > [ERROR] .env file not found! > ``` From 05ea84f47e5b140f95121befdcdaf9e22720f9b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E5=B0=8F=E5=A2=A8?= <2291200076@qq.com> Date: Fri, 29 Jul 2022 15:47:38 +0800 Subject: [PATCH 31/50] =?UTF-8?q?=E8=AF=B7=E6=B1=82API=E6=8A=BD=E7=A6=BB?= =?UTF-8?q?=E5=87=BA=E7=8B=AC=E7=AB=8B=E6=96=B9=E6=B3=95=20requestUtils?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/get_bilibili_hotband.js | 38 ++++++------------------- src/get_weibo_hotband.js | 42 +++++++--------------------- src/utils/{utils.js => fileUtils.js} | 0 src/utils/requestUtils.js | 27 ++++++++++++++++++ 4 files changed, 45 insertions(+), 62 deletions(-) rename src/utils/{utils.js => fileUtils.js} (100%) create mode 100644 src/utils/requestUtils.js diff --git a/src/get_bilibili_hotband.js b/src/get_bilibili_hotband.js index 3de3a4a..d39c5f1 100644 --- a/src/get_bilibili_hotband.js +++ b/src/get_bilibili_hotband.js @@ -1,49 +1,27 @@ 'use strict'; -const request = require('request'); const fs = require('fs'); const path = require('path'); -const utils = require('./utils/utils'); +const fileUtils = require('./utils/fileUtils'); +const requestUtils = require('./utils/requestUtils'); const DATA_FOLDER = path.join(path.dirname(__dirname), process.env.DATA_FOLDER ?? 'data', 'bilibili'); console.log("DATA_FOLDER", DATA_FOLDER); -utils.createFolder(DATA_FOLDER); // 程序运行就保证 data 目录存在 - -// 请求微博热搜 APi 接口 -async function getApiResult(url) { - var return_data = await new Promise((resolve) => { - request({ - method: 'GET', - url: url, - json: true, - }, (error, response, result) => { - if (!error && (response.statusCode == 200)) { - // 请求成功 - resolve(result); - } else { - // 请求失败 - console.log(`error is ${error}`); - resolve("error"); - } - }); - }); - // console.log(`return_data is ${JSON.stringify(return_data)}`); - return return_data; -} +fileUtils.createFolder(DATA_FOLDER); // 程序运行就保证 data 目录存在 async function main() { let requestTimestamp = Date.now(); let now = new Date(requestTimestamp + 8 * 3600 * 1000).toISOString(); - let result = await getApiResult("https://app.bilibili.com/x/v2/search/trending/ranking"); + let result = await requestUtils.getApiResult("https://app.bilibili.com/x/v2/search/trending/ranking"); if (result.code != 0) { console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), "bilibili", "请求成功，但服务器处理失败，正在重试。"); - result = await getApiResult("https://app.bilibili.com/x/v2/search/trending/ranking"); + result = await requestUtils.getApiResult("https://app.bilibili.com/x/v2/search/trending/ranking"); if (result.ok != 1) { console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), "bilibili", "请求成功，但服务器处理失败，保存失败信息。"); // ok 不为 1，那么久直接保存便于后续分析，不进行后续处理 - utils.saveJSON({ + fileUtils.saveJSON({ saveFolder: DATA_FOLDER, now: now, fileNameSuffix: `origin-error`, @@ -67,7 +45,7 @@ async function main() { /** * 保存原始数据 */ - utils.saveJSON({ + fileUtils.saveJSON({ saveFolder: DATA_FOLDER, now: now, fileNameSuffix: `origin`, @@ -92,7 +70,7 @@ async function main() { // } convert.push(item); }); - utils.saveJSON({ + fileUtils.saveJSON({ saveFolder: DATA_FOLDER, now: now, fileNameSuffix: `final`, diff --git a/src/get_weibo_hotband.js b/src/get_weibo_hotband.js index e5a11c2..d482d81 100644 --- a/src/get_weibo_hotband.js +++ b/src/get_weibo_hotband.js @@ -1,49 +1,27 @@ 'use strict'; -const request = require('request'); const fs = require('fs'); const path = require('path'); -const utils = require('./utils/utils'); +const fileUtils = require('./utils/fileUtils'); +const requestUtils = require('./utils/requestUtils'); const DATA_FOLDER = path.join(path.dirname(__dirname), process.env.DATA_FOLDER ?? 'data', 'weibo'); console.log("DATA_FOLDER", DATA_FOLDER); -utils.createFolder(DATA_FOLDER); // 程序运行就保证 data 目录存在 - -// 请求微博热搜 APi 接口 -async function getApiResult(url) { - var return_data = await new Promise((resolve) => { - request({ - method: 'GET', - url: url, - json: true, - }, (error, response, result) => { - if (!error && (response.statusCode == 200)) { - // 请求成功 - resolve(result); - } else { - // 请求失败 - console.log(`error is ${error}`); - resolve("error"); - } - }); - }); - // console.log(`return_data is ${JSON.stringify(return_data)}`); - return return_data; -} +fileUtils.createFolder(DATA_FOLDER); // 程序运行就保证 data 目录存在 async function main() { let requestTimestamp = Date.now(); let now = new Date(requestTimestamp + 8 * 3600 * 1000).toISOString(); - let result = await getApiResult("https://weibo.com/ajax/statuses/hot_band"); + let result = await requestUtils.getApiResult("https://weibo.com/ajax/statuses/hot_band"); if (result.ok != 1) { console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), "weibo", "请求成功，但服务器处理失败，正在重试。"); - result = await getApiResult("https://weibo.com/ajax/statuses/hot_band"); + result = await requestUtils.getApiResult("https://weibo.com/ajax/statuses/hot_band"); if (result.ok != 1) { console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), "weibo", "请求成功，但服务器处理失败，保存失败信息。"); // ok 不为 1，那么久直接保存便于后续分析，不进行后续处理 - utils.saveJSON({ + fileUtils.saveJSON({ saveFolder: DATA_FOLDER, now: now, fileNameSuffix: `origin-error`, @@ -61,7 +39,7 @@ async function main() { /** * 保存原始数据 */ - utils.saveJSON({ + fileUtils.saveJSON({ saveFolder: DATA_FOLDER, now: now, fileNameSuffix: `origin`, @@ -159,7 +137,7 @@ async function main() { }, }); }); - utils.saveJSON({ + fileUtils.saveJSON({ saveFolder: DATA_FOLDER, now: now, fileNameSuffix: `final`, @@ -184,7 +162,7 @@ async function main() { `原始:${item.raw_hot} 显示:${item.num} 调控: ${item.num - item.raw_hot}` ]); }); - utils.saveJSON({ + fileUtils.saveJSON({ saveFolder: DATA_FOLDER, now: now, fileNameSuffix: `regulation`, @@ -205,7 +183,7 @@ async function main() { data.band_list.forEach(function (item) { delete item["mblog"]; }); - utils.saveJSON({ + fileUtils.saveJSON({ saveFolder: DATA_FOLDER, now: now, fileNameSuffix: `simplify`, diff --git a/src/utils/utils.js b/src/utils/fileUtils.js similarity index 100% rename from src/utils/utils.js rename to src/utils/fileUtils.js diff --git a/src/utils/requestUtils.js b/src/utils/requestUtils.js new file mode 100644 index 0000000..ef37af4 --- /dev/null +++ b/src/utils/requestUtils.js @@ -0,0 +1,27 @@ +const request = require('request'); + +// 请求 APi 接口 +async function getApiResult(url) { + var return_data = await new Promise((resolve) => { + request({ + method: 'GET', + url: url, + json: true, + }, (error, response, result) => { + if (!error && (response.statusCode == 200)) { + // 请求成功 + resolve(result); + } else { + // 请求失败 + console.log(`error is ${error}`); + resolve("error"); + } + }); + }); + // console.log(`return_data is ${JSON.stringify(return_data)}`); + return return_data; +} + +module.exports = { + getApiResult, +} \ No newline at end of file From 5bb9ac6235d6ffd48ea813a8bdf57e543e315f98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E5=B0=8F=E5=A2=A8?= <2291200076@qq.com> Date: Fri, 29 Jul 2022 16:15:45 +0800 Subject: [PATCH 32/50] =?UTF-8?q?weibo,bilibili=E5=90=8E=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?hotband=EF=BC=9B=E4=BF=AE=E6=94=B9html=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- html/{bilibili.html => bilibili_hotband.html} | 2 +- html/{weibo.html => weibo_hotband.html} | 2 +- index.html | 6 +++--- src/get_bilibili_hotband.js | 15 +++++++++------ src/get_weibo_hotband.js | 15 +++++++++------ 5 files changed, 23 insertions(+), 17 deletions(-) rename html/{bilibili.html => bilibili_hotband.html} (99%) rename html/{weibo.html => weibo_hotband.html} (99%) diff --git a/html/bilibili.html b/html/bilibili_hotband.html similarity index 99% rename from html/bilibili.html rename to html/bilibili_hotband.html index 85786e9..364158d 100644 --- a/html/bilibili.html +++ b/html/bilibili_hotband.html @@ -192,7 +192,7 @@ function getData() { var xhr = new XMLHttpRequest(); - xhr.open("GET", "../data/bilibili/latest.json?t=" + Date.now(), true); + xhr.open("GET", "../data/bilibili-hotband/latest.json?t=" + Date.now(), true); xhr.send(); xhr.onreadystatechange = function () { if (xhr.readyState !== 4) return; diff --git a/html/weibo.html b/html/weibo_hotband.html similarity index 99% rename from html/weibo.html rename to html/weibo_hotband.html index 275ceb2..dfb9105 100644 --- a/html/weibo.html +++ b/html/weibo_hotband.html @@ -201,7 +201,7 @@ function getData() { var xhr = new XMLHttpRequest(); - xhr.open("GET", "../data/weibo/latest.json?t=" + Date.now(), true); + xhr.open("GET", "../data/weibo-hotband/latest.json?t=" + Date.now(), true); xhr.send(); xhr.onreadystatechange = function () { if (xhr.readyState !== 4) return; diff --git a/index.html b/index.html index abfc0b2..f1af8d0 100644 --- a/index.html +++ b/index.html @@ -74,11 +74,11 @@
-
+

微博

-
+

B站

@@ -123,7 +123,7 @@ // 根据 URL 参数切换页面 function locationToPageByUrlParam() { - let pages = ['weibo', 'bilibili']; + let pages = ['weibo_hotband', 'bilibili_hotband']; // 获取 url 中的 target 参数 let url = new URL(window.location.href); let target = url.searchParams.get('target'); diff --git a/src/get_bilibili_hotband.js b/src/get_bilibili_hotband.js index d39c5f1..2a38c57 100644 --- a/src/get_bilibili_hotband.js +++ b/src/get_bilibili_hotband.js @@ -6,7 +6,10 @@ const path = require('path'); const fileUtils = require('./utils/fileUtils'); const requestUtils = require('./utils/requestUtils'); -const DATA_FOLDER = path.join(path.dirname(__dirname), process.env.DATA_FOLDER ?? 'data', 'bilibili'); +const API_URL = "https://app.bilibili.com/x/v2/search/trending/ranking"; +const SUB_FOLDER = "bilibili-hotband"; + +const DATA_FOLDER = path.join(path.dirname(__dirname), process.env.DATA_FOLDER ?? 'data', SUB_FOLDER); console.log("DATA_FOLDER", DATA_FOLDER); fileUtils.createFolder(DATA_FOLDER); // 程序运行就保证 data 目录存在 @@ -14,12 +17,12 @@ async function main() { let requestTimestamp = Date.now(); let now = new Date(requestTimestamp + 8 * 3600 * 1000).toISOString(); - let result = await requestUtils.getApiResult("https://app.bilibili.com/x/v2/search/trending/ranking"); + let result = await requestUtils.getApiResult(API_URL); if (result.code != 0) { - console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), "bilibili", "请求成功，但服务器处理失败，正在重试。"); - result = await requestUtils.getApiResult("https://app.bilibili.com/x/v2/search/trending/ranking"); + console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), SUB_FOLDER, "请求成功，但服务器处理失败，正在重试。"); + result = await requestUtils.getApiResult(API_URL); if (result.ok != 1) { - console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), "bilibili", "请求成功，但服务器处理失败，保存失败信息。"); + console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), SUB_FOLDER, "请求成功，但服务器处理失败，保存失败信息。"); // ok 不为 1，那么久直接保存便于后续分析，不进行后续处理 fileUtils.saveJSON({ saveFolder: DATA_FOLDER, @@ -33,7 +36,7 @@ async function main() { } } - console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), "bilibili", "请求成功"); + console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), SUB_FOLDER, "请求成功"); // console.log("result", result); let data = result.data; diff --git a/src/get_weibo_hotband.js b/src/get_weibo_hotband.js index d482d81..cb08c35 100644 --- a/src/get_weibo_hotband.js +++ b/src/get_weibo_hotband.js @@ -6,7 +6,10 @@ const path = require('path'); const fileUtils = require('./utils/fileUtils'); const requestUtils = require('./utils/requestUtils'); -const DATA_FOLDER = path.join(path.dirname(__dirname), process.env.DATA_FOLDER ?? 'data', 'weibo'); +const API_URL = "https://weibo.com/ajax/statuses/hot_band"; +const SUB_FOLDER = "weibo-hotband"; + +const DATA_FOLDER = path.join(path.dirname(__dirname), process.env.DATA_FOLDER ?? 'data', SUB_FOLDER); console.log("DATA_FOLDER", DATA_FOLDER); fileUtils.createFolder(DATA_FOLDER); // 程序运行就保证 data 目录存在 @@ -14,12 +17,12 @@ async function main() { let requestTimestamp = Date.now(); let now = new Date(requestTimestamp + 8 * 3600 * 1000).toISOString(); - let result = await requestUtils.getApiResult("https://weibo.com/ajax/statuses/hot_band"); + let result = await requestUtils.getApiResult(API_URL); if (result.ok != 1) { - console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), "weibo", "请求成功，但服务器处理失败，正在重试。"); - result = await requestUtils.getApiResult("https://weibo.com/ajax/statuses/hot_band"); + console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), SUB_FOLDER, "请求成功，但服务器处理失败，正在重试。"); + result = await requestUtils.getApiResult(API_URL); if (result.ok != 1) { - console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), "weibo", "请求成功，但服务器处理失败，保存失败信息。"); + console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), SUB_FOLDER, "请求成功，但服务器处理失败，保存失败信息。"); // ok 不为 1，那么久直接保存便于后续分析，不进行后续处理 fileUtils.saveJSON({ saveFolder: DATA_FOLDER, @@ -33,7 +36,7 @@ async function main() { } } - console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), "weibo", "请求成功"); + console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), SUB_FOLDER, "请求成功"); // console.log("result", result); /** From f2dcfe274e9cb93e359323ad67e72543d35bbabc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E5=B0=8F=E5=A2=A8?= <2291200076@qq.com> Date: Fri, 29 Jul 2022 16:21:55 +0800 Subject: [PATCH 33/50] =?UTF-8?q?=E6=B7=BB=E5=8A=A0B=E7=AB=99=E6=8E=92?= =?UTF-8?q?=E8=A1=8C=E6=A6=9C=E6=8E=A5=E5=8F=A3=E6=8B=89=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 5 +++ index.js | 5 ++- src/get_bilibili_rank.js | 87 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 src/get_bilibili_rank.js diff --git a/README.md b/README.md index 0e570aa..c50ab09 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,11 @@ B站热搜接口：https://app.bilibili.com/x/v2/search/trending/ranking > 该接口来自页面：https://www.bilibili.com/blackboard/activity-trending-topic.html +B站排行榜接口：https://api.bilibili.com/x/web-interface/ranking/v2?type=all + +> 该接口来自页面：https://www.bilibili.com/v/popular/rank/all +> (切换到其他榜单再切换回来会调用此接口) + ## 运行环境 原理上来说 Windows 下和 Linux 都可运行，目前仅在 Windows 下测试过，暂未在 Linux 系统下测试。 diff --git a/index.js b/index.js index f786500..3ba44d1 100644 --- a/index.js +++ b/index.js @@ -9,7 +9,7 @@ const fs = require('fs'); /** * 环境变量 */ - if (!fs.existsSync('.env')) { +if (!fs.existsSync('.env')) { // 如果没有 .env 文件，则报错并退出 console.error('[ERROR] .env file not found!'); return; @@ -38,6 +38,8 @@ if (DEBUG_MODE) { */ const get_weibo_hotband = require('./src/get_weibo_hotband'); const get_bilibili_hotband = require('./src/get_bilibili_hotband'); +const get_bilibili_rank = require('./src/get_bilibili_rank'); + const execute_command = require('./src/execute_command'); @@ -54,6 +56,7 @@ async function start() { // 爬取热搜数据 await get_weibo_hotband.main(); await get_bilibili_hotband.main(); + await get_bilibili_rank.main(); // 调试模式下 if (DEBUG_MODE) { diff --git a/src/get_bilibili_rank.js b/src/get_bilibili_rank.js new file mode 100644 index 0000000..dec154b --- /dev/null +++ b/src/get_bilibili_rank.js @@ -0,0 +1,87 @@ +'use strict'; + +const fs = require('fs'); +const path = require('path'); + +const fileUtils = require('./utils/fileUtils'); +const requestUtils = require('./utils/requestUtils'); + +const API_URL = "https://api.bilibili.com/x/web-interface/ranking/v2?type=all"; +const SUB_FOLDER = "bilibili-rank"; + +const DATA_FOLDER = path.join(path.dirname(__dirname), process.env.DATA_FOLDER ?? 'data', SUB_FOLDER); +console.log("DATA_FOLDER", DATA_FOLDER); +fileUtils.createFolder(DATA_FOLDER); // 程序运行就保证 data 目录存在 + +async function main() { + let requestTimestamp = Date.now(); + let now = new Date(requestTimestamp + 8 * 3600 * 1000).toISOString(); + + let result = await requestUtils.getApiResult(API_URL); + if (result.code != 0) { + console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), SUB_FOLDER, "请求成功，但服务器处理失败，正在重试。"); + result = await requestUtils.getApiResult(API_URL); + if (result.ok != 1) { + console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), SUB_FOLDER, "请求成功，但服务器处理失败，保存失败信息。"); + // ok 不为 1，那么久直接保存便于后续分析，不进行后续处理 + fileUtils.saveJSON({ + saveFolder: DATA_FOLDER, + now: now, + fileNameSuffix: `origin-error`, + object: result, + compress: true, + uncompress: false + }); + return; + } + } + + console.log(new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), SUB_FOLDER, "请求成功"); + // console.log("result", result); + + let data = result.data; + + // // 去除 trackid + // delete data["trackid"]; + // console.log(data); + + /** + * 保存原始数据 + */ + fileUtils.saveJSON({ + saveFolder: DATA_FOLDER, + now: now, + fileNameSuffix: `origin`, + object: result, + compress: true, + uncompress: false + }); + + // /** + // * 获取需要的数据，进行转换 + // */ + // let convert = []; + // data.list.forEach(item => { + // convert.push(item); + // }); + // fileUtils.saveJSON({ + // saveFolder: DATA_FOLDER, + // now: now, + // fileNameSuffix: `final`, + // object: convert, + // compress: true, + // uncompress: false, + // }); + + /** + * 更新最新的 + */ + fs.writeFileSync(`${DATA_FOLDER}/latest.json`, JSON.stringify({ + update_time: requestTimestamp, + update_time_friendly: now.substring(0, 19).replace(/T/g, " "), + note: data.note, + data: data.list, + })); +} + +exports.main = main; From 8f808a0422acb900a171f36dd2bc70c5fa31084d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E5=B0=8F=E5=A2=A8?= <2291200076@qq.com> Date: Fri, 29 Jul 2022 16:31:06 +0800 Subject: [PATCH 34/50] =?UTF-8?q?=E6=B7=BB=E5=8A=A0B=E7=AB=99=E6=8E=92?= =?UTF-8?q?=E8=A1=8C=E6=A6=9C=E9=A1=B5=E9=9D=A2=EF=BC=88=E5=86=85=E5=AE=B9?= =?UTF-8?q?=E6=98=AF=E5=BE=AE=E5=8D=9A=E7=83=AD=E6=90=9C=E7=9A=84=EF=BC=8C?= =?UTF-8?q?=E9=9C=80=E8=A6=81=E6=9B=B4=E6=96=B0=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- html/assets/image/icon_rank.png | Bin 0 -> 1951 bytes html/bilibili_rank.html | 326 ++++++++++++++++++++++++++++++++ index.html | 16 +- 3 files changed, 336 insertions(+), 6 deletions(-) create mode 100644 html/assets/image/icon_rank.png create mode 100644 html/bilibili_rank.html diff --git a/html/assets/image/icon_rank.png b/html/assets/image/icon_rank.png new file mode 100644 index 0000000000000000000000000000000000000000..b50baa0c6e1ac399e9aa9624450a0bc999387f67 GIT binary patch literal 1951 zcmV;Q2VnS#P)Px+SxH1eRCodHoL!6)RTO~FnQ1o|C|gkxS1{}bvqnWw6CVU2y1W?QM2HVc;txWM z@c|ZNeCm@i^7Ej?2>vBT6SD{}CK@AXKus`aF=S12Ln1^7;t#mHMKs&a9M8Gkc78h3 znYPov?c8LyGq>m7Ip^#BzxND669Z*PgY4UP8bm7yVGs!h`LiF1<^w_s0QpY>Wq}|| zh*K;aT{EZB@mv)-CMq? zN0*ubadql3vJ<8}XGVf2mftssZJO-+BPaJj88HZ?z9=r98JylZ4vC$z7YjtVWNj zy5Ib^eH(aC_gSaG5g?+o=q@%i1W|7Nbw9}!VcH4kVfS*1JMplk`LSg0zds4F;) z8SsVFH8Qc80p_ELJzEY0Wf};m(B??B>fDm!1pyJKd9L?hP@7Z{kn~nYE_b36+Z-<_ z?l@_<^B?s-9DyaaDI3{VH@xD#FQCF8utUZp4dD?=#$ZwRFkCp^ICd#edB z=FS1^TL9V}1FhAI*=TYErKMW%!)fXYn1{XvxNr%8yLI~F1i-{UtyW969S5ZZ^+!^I z=JWurz6~%v&&M#X)#tgS9RCTxbQW1{a52EpN-#ekRcZ0|rJ${8{ept9Nyyr0O)C~OiumQF=cf^b=1nbBSmxa>xb zE(fHox)oskDnRa7>D*slNt++XY4VByyVm6u_wY6g6D?0&UBLl_L_Q?3!A|PzI1i`o!6<^s22pcIuF(5k? z+qJOK)bV~>WPE865H_0feJC97x4wz5c!I$$59}ABDaR0mV=t&T2dQZipyh$oRm6iSe|6p7bN9f5e0FI31G1+n61m_$X{;%7jb z(7O2L!z_w!Rtk`kW0xX7`7rS`dMb<jm%_ZO0A)2Xsh6T07hGZ#96QLH76RU|fM=x+N$ZA} zYU`QFbMPbag1_C<3mKnFT3rNOV34N zV7J==NKRIXGn!qXo36!n`{^>V-2q5Nu%{k>j>&bGg)#9`blY48NZc5er=IKzbxDw8 zz{+M17tS(3!bF`HW>38?;UhibUavFKV!tOK`9j%8MKN`+_?Z+ZN!*>RZK%0F*x!(POh|GOKK1n}U4Bgh3!65mKIc zh+Ag^BxM8J62{w5Y_He`iGV#6Tr=^5PW3)iO$C>nuyDlossKrX*ezTrR0vXU#f`C} zh5*u)v(91K9JlGA+8}o#Y55bW^u1hUUstbxa0|4_z33)7DqRD5D?gR1u43+t;+LDa ziFn+{ugCF#;uRIYV9AS$M_l}hE)OVvQR*dc%2|f}LD=P?-c$W*$CEcYx$!*Qu`hY^ lg*7sw!*`P_JLxrn{{xa)`|sA=GYbF!002ovPDHLkV1g#Nk_!L; literal 0 HcmV?d00001 diff --git a/html/bilibili_rank.html b/html/bilibili_rank.html new file mode 100644 index 0000000..dfb9105 --- /dev/null +++ b/html/bilibili_rank.html @@ -0,0 +1,326 @@ + + + + + + + + 微博热搜 + + + + +
+

微博热搜榜

+
+
+ 显示字段： + + + + + + + +
+ + + | + + + +
+
+

+ + + + + + + + + +
+ + + + \ No newline at end of file diff --git a/index.html b/index.html index f1af8d0..9f27256 100644 --- a/index.html +++ b/index.html @@ -39,7 +39,6 @@ background-color: #fdfdfd; box-shadow: 0 -2px 10px 0 rgb(0 0 0 / 10%); display: grid; - grid-template-columns: repeat(2, 1fr); place-items: center; } @@ -73,18 +72,24 @@
-
+
-

微博

+

微博热搜

+
+
+ +

B站排行

- -

B站

+ +

B站热搜

+ + diff --git a/html/bilibili_rank.html b/html/bilibili_rank.html index f9acf90..0f916b8 100644 --- a/html/bilibili_rank.html +++ b/html/bilibili_rank.html @@ -86,7 +86,10 @@
-

— 到底啦 —

+

+ — 到底啦 —
+ 数据来源: https://www.bilibili.com/v/popular/rank/all +