netease_music 临时脚本暂存

2023-12-30 01:10:07 +08:00 · 2023-12-30 01:10:07 +08:00 · f72dad74c3
commit f72dad74c3
parent e2a092be3c
5 changed files with 290 additions and 5 deletions
--- a/netease_music/manual-script/#
+++ b/netease_music/manual-script/#
@ -0,0 +1,76 @@
+node index --utils lyric --min 2100000000 --limit 10000
+node index --utils lyric --min 2090000000 --max 2100000000 --limit 10000
+node index --utils lyric --min 2080000000 --max 2090000000 --limit 10000
+node index --utils lyric --min 2070000000 --max 2080000000 --limit 10000
+node index --utils lyric --min 2000000000 --max 2070000000 --limit 10000
+node index --utils lyric --min 1970000000 --max 2000000000 --limit 10000
+node index --utils lyric --min 1960000000 --max 1970000000 --limit 10000
+node index --utils lyric --min 1950000000 --max 1960000000 --limit 10000
+node index --utils lyric --min 1940000000 --max 1950000000 --limit 10000
+
+
+
+node index --utils lyric --min 1930000000 --max 1940000000 --limit 10000
+node index --utils lyric --min 1920000000 --max 1930000000 --limit 10000
+node index --utils lyric --min 1910000000 --max 1920000000 --limit 10000
+node index --utils lyric --min 1900000000 --max 1910000000 --limit 10000
+node index --utils lyric --min 1890000000 --max 1900000000 --limit 10000
+node index --utils lyric --min 1880000000 --max 1890000000 --limit 10000
+node index --utils lyric --min 1870000000 --max 1880000000 --limit 10000
+node index --utils lyric --min 1860000000 --max 1870000000 --limit 10000
+node index --utils lyric --min 1850000000 --max 1860000000 --limit 10000
+node index --utils lyric --min 1840000000 --max 1850000000 --limit 10000
+
+
+
+node index --utils lyric --min 1830000000 --max 1840000000 --limit 10000
+node index --utils lyric --min 1820000000 --max 1830000000 --limit 10000
+node index --utils lyric --min 1810000000 --max 1820000000 --limit 10000
+node index --utils lyric --min 1800000000 --max 1810000000 --limit 10000
+
+node index --utils lyric --min 1490000000 --max 1500000000 --limit 10000
+node index --utils lyric --min 1480000000 --max 1490000000 --limit 10000
+node index --utils lyric --min 1470000000 --max 1480000000 --limit 10000
+node index --utils lyric --min 1460000000 --max 1470000000 --limit 10000
+node index --utils lyric --min 1450000000 --max 1460000000 --limit 10000
+
+
+
+node index --utils lyric --min 1440000000 --max 1450000000 --limit 10000
+#part2
+node index --utils lyric --min 1430000000 --max 1440000000 --limit 10000
+node index --utils lyric --min 1420000000 --max 1430000000 --limit 10000
+node index --utils lyric --min 1410000000 --max 1420000000 --limit 10000
+node index --utils lyric --min 1400000000 --max 1410000000 --limit 10000
+node index --utils lyric --min 1390000000 --max 1400000000 --limit 10000
+node index --utils lyric --min 1380000000 --max 1390000000 --limit 10000
+node index --utils lyric --min 1370000000 --max 1380000000 --limit 10000
+node index --utils lyric --min 1360000000 --max 1370000000 --limit 10000
+node index --utils lyric --min 1350000000 --max 1360000000 --limit 10000
+
+
+node index --utils lyric --min 530000000 --max 1350000000 --limit 10000
+node index --utils lyric --max 530000000 --limit 10000
+
+
+
+
+
+
+
+
+
+node index --utils comment --min      200000 --max      400000 --limit 10000
+
+
+
+node index --utils comment --min    15000000 --max    20000000 --limit 10000
+node index --utils comment --min    20000000 --max    25000000 --limit 10000
+node index --utils comment --min    25000000 --max    30000000 --limit 10000
+node index --utils comment --min    30000000 --max    35000000 --limit 10000
+node index --utils comment --min    35000000 --max    40000000 --limit 10000
+
+node index --utils comment --min   400000000 --max   500000000 --limit 10000
+node index --utils comment --min   500000000 --max   600000000 --limit 10000
+
+node index --utils comment --min   800000000 --max   900000000 --limit 10000
--- a/netease_music/manual-script/#
+++ b/netease_music/manual-script/#
@ -0,0 +1,37 @@
+-- 统计等待爬取的数据条数 2023.12.25
+SELECT 'comment' as wait_fetch, count(*) as `count` FROM `comment_progress` where current_status = 0
+UNION ALL
+SELECT 'album',  count(*) FROM `wait_fetch_album`
+UNION ALL
+SELECT 'artist', count(*) FROM `wait_fetch_artist`
+UNION ALL
+SELECT 'lyric',  count(*) FROM `wait_fetch_lyric`
+
+-- 查看需要爬取的 comment 的分布
+SELECT cast( FLOOR( song_id / 10000000 ) * 10000000 as UNSIGNED ) as s, count(*) as count
+FROM comment_progress
+WHERE current_status != 2
+GROUP BY s
+ORDER BY s DESC;
+
+-- 查看需要爬取的 lyric 的分布
+SELECT cast( FLOOR( id / 10000000 ) * 10000000 as UNSIGNED ) as s, count(*) as count
+FROM wait_fetch_lyric
+GROUP BY s
+ORDER BY s DESC;
+
+-- 查看需要爬取的 album 的分布
+SELECT cast( FLOOR( id / 1000000 ) * 1000000 as UNSIGNED ) as s, count(*) as count
+FROM wait_fetch_album
+GROUP BY s
+ORDER BY s DESC;
+
+-- 查看需要爬取的 artist 的分布
+SELECT cast( FLOOR(id / 1000000 ) * 1000000 as UNSIGNED ) as s, count(*) as count
+FROM wait_fetch_artist
+GROUP BY s
+ORDER BY s DESC;
+
+
+
+
--- a/netease_music/manual-script/#
+++ b/netease_music/manual-script/#
@ -0,0 +1,40 @@
+# https://shell.aliyun.com/
+git clone https://git.only4.work/coder-xiaomo/tools
+cd tools
+echo '{"mysql":{"charset":"utf8mb4","host":"124.220.172.110","user":"root","password":"123456","port":5204,"database":"","connectTimeout": 3600000,"acquireTimeout": 3600000,"timeout": 3600000}}' > config.json
+npm config set registry https://registry.npmmirror.com
+
+cat config.json
+npm config get registry
+
+npm i
+
+# cd netease_music
+cd ~/tools/netease_music/
+
+# comment_3
+node index --utils comment --min     3500000 --max     3750000 --limit 10000 &
+node index --utils comment --min     3750000 --max     4000000 --limit 10000 &
+node index --utils comment --min     4000000 --max     4250000 --limit 10000 &
+node index --utils comment --min     4250000 --max     4500000 --limit 10000 &
+node index --utils comment --min     4500000 --max     4750000 --limit 10000 &
+node index --utils comment --min     4750000 --max     5000000 --limit 10000 &
+node index --utils comment --min     5000000 --max     5250000 --limit 10000 &
+node index --utils comment --min     5250000 --max     5500000 --limit 10000 &
+# node index --utils comment --min     5500000 --max     6000000 --limit 10000
+
+node index --utils album --min 170000000 --limit 10000
+node index --utils album --min 169000000 --max 170000000 --limit 10000
+node index --utils album --min 168000000 --max 169000000 --limit 10000
+node index --utils album --min 167000000 --max 168000000 --limit 10000
+node index --utils album --min 166000000 --max 167000000 --limit 10000
+node index --utils album --min 165000000 --max 166000000 --limit 10000
+node index --utils album --min 164000000 --max 165000000 --limit 10000
+node index --utils album --min 163000000 --max 164000000 --limit 10000
+node index --utils album --min 162000000 --max 163000000 --limit 10000
+node index --utils album --min 161000000 --max 162000000 --limit 10000
+node index --utils album --min 160000000 --max 161000000 --limit 10000
+
+node index --utils artist --min 33000000 --max 34000000 --limit 10000 &
+node index --utils artist --max 33000000 --limit 10000 &
+
--- a/netease_music/manual-script/#
+++ b/netease_music/manual-script/#
@ -0,0 +1,37 @@
+# https://shell.aliyun.com/
+git clone https://git.only4.work/coder-xiaomo/tools
+cd tools
+echo '{"mysql":{"charset":"utf8mb4","host":"124.220.172.110","user":"root","password":"123456","port":5204,"database":"","connectTimeout": 3600000,"acquireTimeout": 3600000,"timeout": 3600000}}' > config.json
+npm config set registry https://registry.npmmirror.com
+
+cat config.json
+npm config get registry
+
+npm i
+
+# cd netease_music
+cd ~/tools/netease_music/
+
+# comment_3
+node index --utils comment --min  2000000000 --max  2020000000 --limit 10000 &
+node index --utils comment --min  2020000000 --max  2040000000 --limit 10000 &
+node index --utils comment --min  2040000000 --max  2060000000 --limit 10000 &
+node index --utils comment --min  2060000000 --max  2080000000 --limit 10000 &
+node index --utils comment --min  2080000000 --max  2100000000 --limit 10000 &
+node index --utils comment --min  2100000000 --max  2120000000 --limit 10000 &
+# node index --utils comment --min  2120000000 --max  2500000000 --limit 10000
+# node index --utils comment --min  2500000000                   --limit 10000
+
+node index --utils album --min 159000000 --max 160000000 --limit 10000
+node index --utils album --min 158000000 --max 159000000 --limit 10000
+node index --utils album --min 157000000 --max 158000000 --limit 10000
+node index --utils album --min 156000000 --max 157000000 --limit 10000
+node index --utils album --min 155000000 --max 156000000 --limit 10000
+node index --utils album --min 154000000 --max 155000000 --limit 10000
+node index --utils album --min 153000000 --max 154000000 --limit 10000
+node index --utils album --min 152000000 --max 153000000 --limit 10000
+node index --utils album --min 151000000 --max 152000000 --limit 10000
+node index --utils album --min 150000000 --max 151000000 --limit 10000
+
+node index --utils artist --min 35000000 --max 36000000 --limit 10000 &
+node index --utils artist --min 34000000 --max 35000000 --limit 10000 &
--- a/netease_music/manual-script/auto
+++ b/netease_music/manual-script/auto
@ -1,13 +1,12 @@
-# 本地
 # cd ./netease_music
-cd tools/netease_music/
+# cd tools/netease_music/

 # 【家台式机ING】
 start cmd /k "node index --utils assistant"
 start cmd /k "node index --utils song"
-start cmd /k "node index --utils artist --limit 10000"
-start cmd /k "node index --utils album --limit 10000"
-start cmd /k "node index --utils lyric --limit 10000"
+# start cmd /k "node index --utils artist --limit 10000"
+# start cmd /k "node index --utils album --limit 10000"
+# start cmd /k "node index --utils lyric --limit 10000"
 # start cmd /k "node index --utils comment --limit 10000"

 start cmd /k "node index --utils playlist"
@ -171,3 +170,99 @@ node index --utils comment --min  2080000000 --max  2100000000 --limit 10000 &
 node index --utils comment --min  2100000000 --max  2120000000 --limit 10000 &
 # node index --utils comment --min  2120000000 --max  2500000000 --limit 10000
 # node index --utils comment --min  2500000000                   --limit 10000
+
+
+
+# album
+# 阿里云1
+node index --utils album --min 170000000 --limit 10000
+node index --utils album --min 169000000 --max 170000000 --limit 10000
+node index --utils album --min 168000000 --max 169000000 --limit 10000
+node index --utils album --min 167000000 --max 168000000 --limit 10000
+node index --utils album --min 166000000 --max 167000000 --limit 10000
+node index --utils album --min 165000000 --max 166000000 --limit 10000
+node index --utils album --min 164000000 --max 165000000 --limit 10000
+node index --utils album --min 163000000 --max 164000000 --limit 10000
+node index --utils album --min 162000000 --max 163000000 --limit 10000
+node index --utils album --min 161000000 --max 162000000 --limit 10000
+node index --utils album --min 160000000 --max 161000000 --limit 10000
+
+# 阿里云2
+node index --utils album --min 159000000 --max 160000000 --limit 10000
+node index --utils album --min 158000000 --max 159000000 --limit 10000
+node index --utils album --min 157000000 --max 158000000 --limit 10000
+node index --utils album --min 156000000 --max 157000000 --limit 10000
+node index --utils album --min 155000000 --max 156000000 --limit 10000
+node index --utils album --min 154000000 --max 155000000 --limit 10000
+node index --utils album --min 153000000 --max 154000000 --limit 10000
+node index --utils album --min 152000000 --max 153000000 --limit 10000
+node index --utils album --min 151000000 --max 152000000 --limit 10000
+node index --utils album --min 150000000 --max 151000000 --limit 10000
+
+# 【待分配】
+node index --utils album --min 149000000 --max 150000000 --limit 10000
+node index --utils album --min 148000000 --max 149000000 --limit 10000
+node index --utils album --min 147000000 --max 148000000 --limit 10000
+node index --utils album --min 146000000 --max 147000000 --limit 10000
+node index --utils album --min 145000000 --max 146000000 --limit 10000
+node index --utils album --min 144000000 --max 145000000 --limit 10000
+node index --utils album --min 143000000 --max 144000000 --limit 10000
+node index --utils album --min 142000000 --max 143000000 --limit 10000
+node index --utils album --min 141000000 --max 142000000 --limit 10000
+node index --utils album --min 140000000 --max 141000000 --limit 10000
+
+# 【待分配】
+node index --utils album --min 139000000 --max 140000000 --limit 10000
+node index --utils album --min 138000000 --max 139000000 --limit 10000
+node index --utils album --min 137000000 --max 138000000 --limit 10000
+node index --utils album --min 136000000 --max 137000000 --limit 10000
+node index --utils album --min 135000000 --max 136000000 --limit 10000
+node index --utils album --min 134000000 --max 135000000 --limit 10000
+node index --utils album --min 133000000 --max 134000000 --limit 10000
+node index --utils album --min 132000000 --max 133000000 --limit 10000
+node index --utils album --min 131000000 --max 132000000 --limit 10000
+node index --utils album --min 130000000 --max 131000000 --limit 10000
+
+# 【待分配】
+node index --utils album --min 129000000 --max 130000000 --limit 10000
+node index --utils album --min 128000000 --max 129000000 --limit 10000
+node index --utils album --min 127000000 --max 128000000 --limit 10000
+node index --utils album --min 126000000 --max 127000000 --limit 10000
+node index --utils album --min 125000000 --max 126000000 --limit 10000
+node index --utils album --min 124000000 --max 125000000 --limit 10000
+node index --utils album --min 123000000 --max 124000000 --limit 10000
+node index --utils album --min 122000000 --max 123000000 --limit 10000
+node index --utils album --min 121000000 --max 122000000 --limit 10000
+node index --utils album --min 120000000 --max 121000000 --limit 10000
+
+# 家笔记本
+start cmd /k "node index --utils album --min 99000000 --max 120000000 --limit 10000     "
+start cmd /k "node index --utils album --min 98000000 --max 99000000 --limit 10000      "
+start cmd /k "node index --utils album --min 97000000 --max 98000000 --limit 10000      "
+start cmd /k "node index --utils album --min 96000000 --max 97000000 --limit 10000      "
+start cmd /k "node index --utils album --min 95000000 --max 96000000 --limit 10000      "
+start cmd /k "node index --utils album --min 94000000 --max 95000000 --limit 10000      "
+start cmd /k "node index --utils album --min 93000000 --max 94000000 --limit 10000      "
+start cmd /k "node index --utils album --min 92000000 --max 93000000 --limit 10000      "
+start cmd /k "node index --utils album --min 91000000 --max 92000000 --limit 10000      "
+start cmd /k "node index --utils album --max 91000000 --limit 10000                     "
+
+
+
+# artist
+# 【待分配】
+node index --utils artist --min 49000000 --limit 10000
+node index --utils artist --min 48000000 --max 49000000 --limit 10000
+node index --utils artist --min 47000000 --max 48000000 --limit 10000
+node index --utils artist --min 46000000 --max 47000000 --limit 10000
+node index --utils artist --min 37000000 --max 46000000 --limit 10000
+node index --utils artist --min 36000000 --max 37000000 --limit 10000
+
+# 阿里云2
+node index --utils artist --min 35000000 --max 36000000 --limit 10000 &
+node index --utils artist --min 34000000 --max 35000000 --limit 10000 &
+
+# 阿里云1
+node index --utils artist --min 33000000 --max 34000000 --limit 10000 &
+node index --utils artist --max 33000000 --limit 10000 &
+