diff --git a/netease_music/sql/statistic.sql b/netease_music/sql/statistic.sql index 6b813f0..ac1d469 100644 --- a/netease_music/sql/statistic.sql +++ b/netease_music/sql/statistic.sql @@ -1,3 +1,20 @@ +INSERT INTO analysis (`key`, `value`) VALUES ('songCount', (SELECT count(*) as count FROM song) ) ON DUPLICATE KEY UPDATE `value` = VALUES(`value`); +INSERT INTO analysis (`key`, `value`) VALUES ('songWaiting', (SELECT count(DISTINCT song_id) AS count + FROM ( SELECT song_id FROM song_artist_relation UNION SELECT song_id FROM song_album_relation ) t_tmp + WHERE song_id NOT IN ( SELECT song_id FROM song )) ) ON DUPLICATE KEY UPDATE `value` = VALUES(`value`); +INSERT INTO analysis (`key`, `value`) VALUES ('albumCount', (SELECT count(*) as count FROM album) ) ON DUPLICATE KEY UPDATE `value` = VALUES(`value`); +INSERT INTO analysis (`key`, `value`) VALUES ('albumWaiting', (SELECT count( DISTINCT album_id ) as count FROM song_album_relation WHERE album_id NOT IN ( SELECT album_id FROM album )) ) ON DUPLICATE KEY UPDATE `value` = VALUES(`value`); +INSERT INTO analysis (`key`, `value`) VALUES ('artistCount', (SELECT count(*) AS count FROM artist) ) ON DUPLICATE KEY UPDATE `value` = VALUES(`value`); +INSERT INTO analysis (`key`, `value`) VALUES ('artistWaiting', (SELECT count( DISTINCT artist_id ) as count FROM song_artist_relation WHERE artist_id NOT IN ( SELECT artist_id FROM artist )) ) ON DUPLICATE KEY UPDATE `value` = VALUES(`value`); +INSERT INTO analysis (`key`, `value`) VALUES ('lyricCount', (SELECT count(*) AS count FROM lyric) ) ON DUPLICATE KEY UPDATE `value` = VALUES(`value`); +INSERT INTO analysis (`key`, `value`) VALUES ('commentCount', (SELECT count( DISTINCT song_id ) AS count FROM comment) ) ON DUPLICATE KEY UPDATE `value` = VALUES(`value`); +INSERT INTO analysis (`key`, `value`) VALUES ('commentTotalCount', (SELECT count(*) AS count FROM comment) ) ON DUPLICATE KEY UPDATE `value` = VALUES(`value`); +INSERT INTO analysis (`key`, `value`) VALUES ('userCount', (SELECT count(*) AS count FROM user) ) ON DUPLICATE KEY UPDATE `value` = VALUES(`value`); +INSERT INTO analysis (`key`, `value`) VALUES ('songAlbumCount', (SELECT count(*) AS count FROM song_album_relation) ) ON DUPLICATE KEY UPDATE `value` = VALUES(`value`); +INSERT INTO analysis (`key`, `value`) VALUES ('songArtistCount', (SELECT count(*) AS count FROM song_artist_relation) ) ON DUPLICATE KEY UPDATE `value` = VALUES(`value`); + + + -- 查看需要爬取的 song 的分布 SELECT cast( format( t_tmp.song_id / 10000000, 0) * 10000000 as UNSIGNED ) as s, count(*) as count FROM ( @@ -38,6 +55,7 @@ GROUP BY s ORDER BY s DESC + -- optimize table optimize table album; optimize table artist; @@ -50,6 +68,8 @@ optimize table song_album_relation; optimize table song_artist_relation; optimize table user; + + -- 查询单个数据库里面各个表所占磁盘空间大小包括其索引的大小 SELECT table_schema AS '数据库', diff --git a/netease_music/sql/structure.sql b/netease_music/sql/structure.sql index 11e4e68..a7f56b6 100644 --- a/netease_music/sql/structure.sql +++ b/netease_music/sql/structure.sql @@ -129,4 +129,11 @@ CREATE TABLE `log` ( `msg` varchar(200) NOT NULL COMMENT '出错信息', `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '爬取时间', KEY `id` (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; \ No newline at end of file +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; + +CREATE TABLE `analysis` ( + `key` varchar(255) NOT NULL COMMENT '参数名', + `value` varchar(255) DEFAULT NULL COMMENT '参数值', + `modify_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最后更新时间', + UNIQUE KEY `key` (`key`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; diff --git a/netease_music/src/index.js b/netease_music/src/index.js index f3ae84c..382886d 100644 --- a/netease_music/src/index.js +++ b/netease_music/src/index.js @@ -143,7 +143,8 @@ async function watch() { promiseList.push(new Promise(async (resolve, reject) => { // console.log(`query ${sql.name} ...`); let sqlStartTime = Date.now(); - let result = await dbUtils.query(sql.sql, []); + // let result = await dbUtils.query(sql.sql, []); + let result = await dbUtils.query(`SELECT \`value\` as count FROM analysis WHERE \`key\`='${sql.name}'`); let sqlTimeSpent = Date.now() - sqlStartTime; sqlsTimeSpent += sqlTimeSpent; newWatchParam[sql.name] = result[0].count; diff --git a/netease_music/watch.js b/netease_music/watch.js index 1b6431d..0a26925 100644 --- a/netease_music/watch.js +++ b/netease_music/watch.js @@ -1,4 +1,4 @@ -let keepWatching = true; +let keepWatching = false; if (keepWatching) { global.useMysqlPool = true; global.connectionLimit = 15;