coder-xiaomo/tools
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity GitHub Gitee

update

Browse Source
This commit is contained in:
程序员小墨 2022-10-29 16:09:01 +08:00
parent b93e1598e6
commit 8a5cd1225a
4 changed files with 10 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hifini_music/sql/statistic.sql Normal file
View File

@ -0,0 +1,2 @@
SELECT count(*) FROM hifini_thread WHERE music_title='' and music_pic='' and music_url='';
SELECT count(*) FROM hifini_thread WHERE music_url like 'get_music.php?key=%' and music_real_url='';

2
netease_music/sql/neteasemusic.sql
View File

@ -150,7 +150,7 @@ CREATE TABLE `playlist` (
`playlist_create_time` bigint(20) UNSIGNED NULL DEFAULT NULL COMMENT '创建日期', `playlist_create_time` bigint(20) UNSIGNED NULL DEFAULT NULL COMMENT '创建日期',
`playlist_update_time` bigint(20) UNSIGNED NULL DEFAULT NULL COMMENT '更新日期', `playlist_update_time` bigint(20) UNSIGNED NULL DEFAULT NULL COMMENT '更新日期',
`track_count` int(10) UNSIGNED NOT NULL COMMENT '歌单歌曲数', `track_count` int(10) UNSIGNED NOT NULL COMMENT '歌单歌曲数',
`play_count` int(10) UNSIGNED NOT NULL COMMENT '歌单播放数', `play_count` bigint(20) UNSIGNED NOT NULL COMMENT '歌单播放数',
`subscribed_count` int(10) UNSIGNED NOT NULL COMMENT '歌单收藏数', `subscribed_count` int(10) UNSIGNED NOT NULL COMMENT '歌单收藏数',
`share_count` int(10) UNSIGNED NOT NULL COMMENT '歌单分享数', `share_count` int(10) UNSIGNED NOT NULL COMMENT '歌单分享数',
`comment_count` int(10) UNSIGNED NOT NULL COMMENT '歌单评论数', `comment_count` int(10) UNSIGNED NOT NULL COMMENT '歌单评论数',

2
netease_music/sql/structure.sql
View File

@ -67,7 +67,7 @@ CREATE TABLE `playlist` (
`playlist_update_time` bigint(20) unsigned DEFAULT NULL COMMENT '更新日期', `playlist_update_time` bigint(20) unsigned DEFAULT NULL COMMENT '更新日期',
-- 数据 -- 数据
`track_count` int(10) unsigned NOT NULL COMMENT '歌单歌曲数', `track_count` int(10) unsigned NOT NULL COMMENT '歌单歌曲数',
`play_count` int(10) unsigned NOT NULL COMMENT '歌单播放数', `play_count` bigint(20) unsigned NOT NULL COMMENT '歌单播放数',
`subscribed_count` int(10) unsigned NOT NULL COMMENT '歌单收藏数', `subscribed_count` int(10) unsigned NOT NULL COMMENT '歌单收藏数',
`share_count` int(10) unsigned NOT NULL COMMENT '歌单分享数', `share_count` int(10) unsigned NOT NULL COMMENT '歌单分享数',
`comment_count` int(10) unsigned NOT NULL COMMENT '歌单评论数', `comment_count` int(10) unsigned NOT NULL COMMENT '歌单评论数',

9
netease_music/src/getInfo/playlistUtils.js
View File

@ -14,7 +14,7 @@ const dataManager = require('../dataManager');
async function fetchAll({ args }) { async function fetchAll({ args }) {
// 睡眠时间设置长一些，不然容易触发500错误 // 睡眠时间设置长一些，不然容易触发500错误
global.sleepTime = Math.max(500, global.sleepTime); global.sleepTime = Math.max(1000, global.sleepTime);
console.log("global.sleepTime", global.sleepTime); console.log("global.sleepTime", global.sleepTime);
console.log("playlist 需要一口气爬完，中途不能停止，否则下次又要重头爬（歌单不会重复爬取，但是分页列表会）"); console.log("playlist 需要一口气爬完，中途不能停止，否则下次又要重头爬（歌单不会重复爬取，但是分页列表会）");
@ -28,8 +28,8 @@ async function fetchAll({ args }) {
for (let i = 0; i < cate.length; i++) { for (let i = 0; i < cate.length; i++) {
const categoryName = cate[i]; const categoryName = cate[i];
try { try {
console.log("开始爬取分类：", categoryName); console.log(`开始爬取分类：${categoryName}(i=${i})`);
await fetchCategory({ categoryName: categoryName, progress: `${i + 1}/${cate.length}` }); await fetchCategory({ categoryName: `${categoryName}`, progress: `${i + 1}/${cate.length}` });
} catch (err) { } catch (err) {
console.error(err); console.error(err);
} }
@ -94,7 +94,10 @@ async function fetchCategory({ categoryName, progress }) {
} }
} else { } else {
console.log("失败url", url); console.log("失败url", url);
fs.writeFileSync(path.join(__dirname, "../../temp", `[error]discover-playlist.html`), html);
await dbUtils.query('INSERT INTO log (`id`, `name`, `msg`) VALUES (?, ?, ?)', [0, 'playlist_fetch', `失败 可能是爬太快了\n${url}`]); await dbUtils.query('INSERT INTO log (`id`, `name`, `msg`) VALUES (?, ?, ?)', [0, 'playlist_fetch', `失败 可能是爬太快了\n${url}`]);
await sleepUtils.sleep(40 * 1000); // 等待40s再继续爬取
continue;
} }
// 最有一页判断标识 // 最有一页判断标识