diff --git a/netease_music/manual-script/# 本机(临时).sh b/netease_music/manual-script/# 本机(临时).sh new file mode 100644 index 0000000..94121cd --- /dev/null +++ b/netease_music/manual-script/# 本机(临时).sh @@ -0,0 +1,76 @@ +node index --utils lyric --min 2100000000 --limit 10000 +node index --utils lyric --min 2090000000 --max 2100000000 --limit 10000 +node index --utils lyric --min 2080000000 --max 2090000000 --limit 10000 +node index --utils lyric --min 2070000000 --max 2080000000 --limit 10000 +node index --utils lyric --min 2000000000 --max 2070000000 --limit 10000 +node index --utils lyric --min 1970000000 --max 2000000000 --limit 10000 +node index --utils lyric --min 1960000000 --max 1970000000 --limit 10000 +node index --utils lyric --min 1950000000 --max 1960000000 --limit 10000 +node index --utils lyric --min 1940000000 --max 1950000000 --limit 10000 + + + +node index --utils lyric --min 1930000000 --max 1940000000 --limit 10000 +node index --utils lyric --min 1920000000 --max 1930000000 --limit 10000 +node index --utils lyric --min 1910000000 --max 1920000000 --limit 10000 +node index --utils lyric --min 1900000000 --max 1910000000 --limit 10000 +node index --utils lyric --min 1890000000 --max 1900000000 --limit 10000 +node index --utils lyric --min 1880000000 --max 1890000000 --limit 10000 +node index --utils lyric --min 1870000000 --max 1880000000 --limit 10000 +node index --utils lyric --min 1860000000 --max 1870000000 --limit 10000 +node index --utils lyric --min 1850000000 --max 1860000000 --limit 10000 +node index --utils lyric --min 1840000000 --max 1850000000 --limit 10000 + + + +node index --utils lyric --min 1830000000 --max 1840000000 --limit 10000 +node index --utils lyric --min 1820000000 --max 1830000000 --limit 10000 +node index --utils lyric --min 1810000000 --max 1820000000 --limit 10000 +node index --utils lyric --min 1800000000 --max 1810000000 --limit 10000 + +node index --utils lyric --min 1490000000 --max 1500000000 --limit 10000 +node index --utils lyric --min 1480000000 --max 1490000000 --limit 10000 +node index --utils lyric --min 1470000000 --max 1480000000 --limit 10000 +node index --utils lyric --min 1460000000 --max 1470000000 --limit 10000 +node index --utils lyric --min 1450000000 --max 1460000000 --limit 10000 + + + +node index --utils lyric --min 1440000000 --max 1450000000 --limit 10000 +#part2 +node index --utils lyric --min 1430000000 --max 1440000000 --limit 10000 +node index --utils lyric --min 1420000000 --max 1430000000 --limit 10000 +node index --utils lyric --min 1410000000 --max 1420000000 --limit 10000 +node index --utils lyric --min 1400000000 --max 1410000000 --limit 10000 +node index --utils lyric --min 1390000000 --max 1400000000 --limit 10000 +node index --utils lyric --min 1380000000 --max 1390000000 --limit 10000 +node index --utils lyric --min 1370000000 --max 1380000000 --limit 10000 +node index --utils lyric --min 1360000000 --max 1370000000 --limit 10000 +node index --utils lyric --min 1350000000 --max 1360000000 --limit 10000 + + +node index --utils lyric --min 530000000 --max 1350000000 --limit 10000 +node index --utils lyric --max 530000000 --limit 10000 + + + + + + + + + +node index --utils comment --min 200000 --max 400000 --limit 10000 + + + +node index --utils comment --min 15000000 --max 20000000 --limit 10000 +node index --utils comment --min 20000000 --max 25000000 --limit 10000 +node index --utils comment --min 25000000 --max 30000000 --limit 10000 +node index --utils comment --min 30000000 --max 35000000 --limit 10000 +node index --utils comment --min 35000000 --max 40000000 --limit 10000 + +node index --utils comment --min 400000000 --max 500000000 --limit 10000 +node index --utils comment --min 500000000 --max 600000000 --limit 10000 + +node index --utils comment --min 800000000 --max 900000000 --limit 10000 diff --git a/netease_music/manual-script/# 统计SQL(临时).sql b/netease_music/manual-script/# 统计SQL(临时).sql new file mode 100644 index 0000000..0809146 --- /dev/null +++ b/netease_music/manual-script/# 统计SQL(临时).sql @@ -0,0 +1,37 @@ +-- 统计等待爬取的数据条数 2023.12.25 +SELECT 'comment' as wait_fetch, count(*) as `count` FROM `comment_progress` where current_status = 0 +UNION ALL +SELECT 'album', count(*) FROM `wait_fetch_album` +UNION ALL +SELECT 'artist', count(*) FROM `wait_fetch_artist` +UNION ALL +SELECT 'lyric', count(*) FROM `wait_fetch_lyric` + +-- 查看需要爬取的 comment 的分布 +SELECT cast( FLOOR( song_id / 10000000 ) * 10000000 as UNSIGNED ) as s, count(*) as count +FROM comment_progress +WHERE current_status != 2 +GROUP BY s +ORDER BY s DESC; + +-- 查看需要爬取的 lyric 的分布 +SELECT cast( FLOOR( id / 10000000 ) * 10000000 as UNSIGNED ) as s, count(*) as count +FROM wait_fetch_lyric +GROUP BY s +ORDER BY s DESC; + +-- 查看需要爬取的 album 的分布 +SELECT cast( FLOOR( id / 1000000 ) * 1000000 as UNSIGNED ) as s, count(*) as count +FROM wait_fetch_album +GROUP BY s +ORDER BY s DESC; + +-- 查看需要爬取的 artist 的分布 +SELECT cast( FLOOR(id / 1000000 ) * 1000000 as UNSIGNED ) as s, count(*) as count +FROM wait_fetch_artist +GROUP BY s +ORDER BY s DESC; + + + + diff --git a/netease_music/manual-script/# 阿里云1(临时).sh b/netease_music/manual-script/# 阿里云1(临时).sh new file mode 100644 index 0000000..d159e59 --- /dev/null +++ b/netease_music/manual-script/# 阿里云1(临时).sh @@ -0,0 +1,40 @@ +# https://shell.aliyun.com/ +git clone https://git.only4.work/coder-xiaomo/tools +cd tools +echo '{"mysql":{"charset":"utf8mb4","host":"124.220.172.110","user":"root","password":"123456","port":5204,"database":"","connectTimeout": 3600000,"acquireTimeout": 3600000,"timeout": 3600000}}' > config.json +npm config set registry https://registry.npmmirror.com + +cat config.json +npm config get registry + +npm i + +# cd netease_music +cd ~/tools/netease_music/ + +# comment_3 +node index --utils comment --min 3500000 --max 3750000 --limit 10000 & +node index --utils comment --min 3750000 --max 4000000 --limit 10000 & +node index --utils comment --min 4000000 --max 4250000 --limit 10000 & +node index --utils comment --min 4250000 --max 4500000 --limit 10000 & +node index --utils comment --min 4500000 --max 4750000 --limit 10000 & +node index --utils comment --min 4750000 --max 5000000 --limit 10000 & +node index --utils comment --min 5000000 --max 5250000 --limit 10000 & +node index --utils comment --min 5250000 --max 5500000 --limit 10000 & +# node index --utils comment --min 5500000 --max 6000000 --limit 10000 + +node index --utils album --min 170000000 --limit 10000 +node index --utils album --min 169000000 --max 170000000 --limit 10000 +node index --utils album --min 168000000 --max 169000000 --limit 10000 +node index --utils album --min 167000000 --max 168000000 --limit 10000 +node index --utils album --min 166000000 --max 167000000 --limit 10000 +node index --utils album --min 165000000 --max 166000000 --limit 10000 +node index --utils album --min 164000000 --max 165000000 --limit 10000 +node index --utils album --min 163000000 --max 164000000 --limit 10000 +node index --utils album --min 162000000 --max 163000000 --limit 10000 +node index --utils album --min 161000000 --max 162000000 --limit 10000 +node index --utils album --min 160000000 --max 161000000 --limit 10000 + +node index --utils artist --min 33000000 --max 34000000 --limit 10000 & +node index --utils artist --max 33000000 --limit 10000 & + diff --git a/netease_music/manual-script/# 阿里云2(临时).sh b/netease_music/manual-script/# 阿里云2(临时).sh new file mode 100644 index 0000000..08d0cbc --- /dev/null +++ b/netease_music/manual-script/# 阿里云2(临时).sh @@ -0,0 +1,37 @@ +# https://shell.aliyun.com/ +git clone https://git.only4.work/coder-xiaomo/tools +cd tools +echo '{"mysql":{"charset":"utf8mb4","host":"124.220.172.110","user":"root","password":"123456","port":5204,"database":"","connectTimeout": 3600000,"acquireTimeout": 3600000,"timeout": 3600000}}' > config.json +npm config set registry https://registry.npmmirror.com + +cat config.json +npm config get registry + +npm i + +# cd netease_music +cd ~/tools/netease_music/ + +# comment_3 +node index --utils comment --min 2000000000 --max 2020000000 --limit 10000 & +node index --utils comment --min 2020000000 --max 2040000000 --limit 10000 & +node index --utils comment --min 2040000000 --max 2060000000 --limit 10000 & +node index --utils comment --min 2060000000 --max 2080000000 --limit 10000 & +node index --utils comment --min 2080000000 --max 2100000000 --limit 10000 & +node index --utils comment --min 2100000000 --max 2120000000 --limit 10000 & +# node index --utils comment --min 2120000000 --max 2500000000 --limit 10000 +# node index --utils comment --min 2500000000 --limit 10000 + +node index --utils album --min 159000000 --max 160000000 --limit 10000 +node index --utils album --min 158000000 --max 159000000 --limit 10000 +node index --utils album --min 157000000 --max 158000000 --limit 10000 +node index --utils album --min 156000000 --max 157000000 --limit 10000 +node index --utils album --min 155000000 --max 156000000 --limit 10000 +node index --utils album --min 154000000 --max 155000000 --limit 10000 +node index --utils album --min 153000000 --max 154000000 --limit 10000 +node index --utils album --min 152000000 --max 153000000 --limit 10000 +node index --utils album --min 151000000 --max 152000000 --limit 10000 +node index --utils album --min 150000000 --max 151000000 --limit 10000 + +node index --utils artist --min 35000000 --max 36000000 --limit 10000 & +node index --utils artist --min 34000000 --max 35000000 --limit 10000 & diff --git a/netease_music/manual-script/auto - 0 local.sh b/netease_music/manual-script/auto - 0 local.sh index ae39cb9..4e3b38f 100644 --- a/netease_music/manual-script/auto - 0 local.sh +++ b/netease_music/manual-script/auto - 0 local.sh @@ -1,13 +1,12 @@ -# 本地 # cd ./netease_music -cd tools/netease_music/ +# cd tools/netease_music/ # 【家台式机ING】 start cmd /k "node index --utils assistant" start cmd /k "node index --utils song" -start cmd /k "node index --utils artist --limit 10000" -start cmd /k "node index --utils album --limit 10000" -start cmd /k "node index --utils lyric --limit 10000" +# start cmd /k "node index --utils artist --limit 10000" +# start cmd /k "node index --utils album --limit 10000" +# start cmd /k "node index --utils lyric --limit 10000" # start cmd /k "node index --utils comment --limit 10000" start cmd /k "node index --utils playlist" @@ -171,3 +170,99 @@ node index --utils comment --min 2080000000 --max 2100000000 --limit 10000 & node index --utils comment --min 2100000000 --max 2120000000 --limit 10000 & # node index --utils comment --min 2120000000 --max 2500000000 --limit 10000 # node index --utils comment --min 2500000000 --limit 10000 + + + +# album +# 阿里云1 +node index --utils album --min 170000000 --limit 10000 +node index --utils album --min 169000000 --max 170000000 --limit 10000 +node index --utils album --min 168000000 --max 169000000 --limit 10000 +node index --utils album --min 167000000 --max 168000000 --limit 10000 +node index --utils album --min 166000000 --max 167000000 --limit 10000 +node index --utils album --min 165000000 --max 166000000 --limit 10000 +node index --utils album --min 164000000 --max 165000000 --limit 10000 +node index --utils album --min 163000000 --max 164000000 --limit 10000 +node index --utils album --min 162000000 --max 163000000 --limit 10000 +node index --utils album --min 161000000 --max 162000000 --limit 10000 +node index --utils album --min 160000000 --max 161000000 --limit 10000 + +# 阿里云2 +node index --utils album --min 159000000 --max 160000000 --limit 10000 +node index --utils album --min 158000000 --max 159000000 --limit 10000 +node index --utils album --min 157000000 --max 158000000 --limit 10000 +node index --utils album --min 156000000 --max 157000000 --limit 10000 +node index --utils album --min 155000000 --max 156000000 --limit 10000 +node index --utils album --min 154000000 --max 155000000 --limit 10000 +node index --utils album --min 153000000 --max 154000000 --limit 10000 +node index --utils album --min 152000000 --max 153000000 --limit 10000 +node index --utils album --min 151000000 --max 152000000 --limit 10000 +node index --utils album --min 150000000 --max 151000000 --limit 10000 + +# 【待分配】 +node index --utils album --min 149000000 --max 150000000 --limit 10000 +node index --utils album --min 148000000 --max 149000000 --limit 10000 +node index --utils album --min 147000000 --max 148000000 --limit 10000 +node index --utils album --min 146000000 --max 147000000 --limit 10000 +node index --utils album --min 145000000 --max 146000000 --limit 10000 +node index --utils album --min 144000000 --max 145000000 --limit 10000 +node index --utils album --min 143000000 --max 144000000 --limit 10000 +node index --utils album --min 142000000 --max 143000000 --limit 10000 +node index --utils album --min 141000000 --max 142000000 --limit 10000 +node index --utils album --min 140000000 --max 141000000 --limit 10000 + +# 【待分配】 +node index --utils album --min 139000000 --max 140000000 --limit 10000 +node index --utils album --min 138000000 --max 139000000 --limit 10000 +node index --utils album --min 137000000 --max 138000000 --limit 10000 +node index --utils album --min 136000000 --max 137000000 --limit 10000 +node index --utils album --min 135000000 --max 136000000 --limit 10000 +node index --utils album --min 134000000 --max 135000000 --limit 10000 +node index --utils album --min 133000000 --max 134000000 --limit 10000 +node index --utils album --min 132000000 --max 133000000 --limit 10000 +node index --utils album --min 131000000 --max 132000000 --limit 10000 +node index --utils album --min 130000000 --max 131000000 --limit 10000 + +# 【待分配】 +node index --utils album --min 129000000 --max 130000000 --limit 10000 +node index --utils album --min 128000000 --max 129000000 --limit 10000 +node index --utils album --min 127000000 --max 128000000 --limit 10000 +node index --utils album --min 126000000 --max 127000000 --limit 10000 +node index --utils album --min 125000000 --max 126000000 --limit 10000 +node index --utils album --min 124000000 --max 125000000 --limit 10000 +node index --utils album --min 123000000 --max 124000000 --limit 10000 +node index --utils album --min 122000000 --max 123000000 --limit 10000 +node index --utils album --min 121000000 --max 122000000 --limit 10000 +node index --utils album --min 120000000 --max 121000000 --limit 10000 + +# 家笔记本 +start cmd /k "node index --utils album --min 99000000 --max 120000000 --limit 10000 " +start cmd /k "node index --utils album --min 98000000 --max 99000000 --limit 10000 " +start cmd /k "node index --utils album --min 97000000 --max 98000000 --limit 10000 " +start cmd /k "node index --utils album --min 96000000 --max 97000000 --limit 10000 " +start cmd /k "node index --utils album --min 95000000 --max 96000000 --limit 10000 " +start cmd /k "node index --utils album --min 94000000 --max 95000000 --limit 10000 " +start cmd /k "node index --utils album --min 93000000 --max 94000000 --limit 10000 " +start cmd /k "node index --utils album --min 92000000 --max 93000000 --limit 10000 " +start cmd /k "node index --utils album --min 91000000 --max 92000000 --limit 10000 " +start cmd /k "node index --utils album --max 91000000 --limit 10000 " + + + +# artist +# 【待分配】 +node index --utils artist --min 49000000 --limit 10000 +node index --utils artist --min 48000000 --max 49000000 --limit 10000 +node index --utils artist --min 47000000 --max 48000000 --limit 10000 +node index --utils artist --min 46000000 --max 47000000 --limit 10000 +node index --utils artist --min 37000000 --max 46000000 --limit 10000 +node index --utils artist --min 36000000 --max 37000000 --limit 10000 + +# 阿里云2 +node index --utils artist --min 35000000 --max 36000000 --limit 10000 & +node index --utils artist --min 34000000 --max 35000000 --limit 10000 & + +# 阿里云1 +node index --utils artist --min 33000000 --max 34000000 --limit 10000 & +node index --utils artist --max 33000000 --limit 10000 & +