coder-xiaomo/tools
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity GitHub Gitee

update

Browse Source
This commit is contained in:
程序员小墨 2022-10-01 16:16:39 +08:00
parent 37fe49c53c
commit 64834ae7f9
9 changed files with 35 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
index.js
View File

@ -1,5 +1,2 @@
// const dbUtils = require('./utils/dbUtils');
// const dbPoolUtils = require('./utils/dbPoolUtils');
const neteaseMusic = require('./netease_music/index'); const neteaseMusic = require('./netease_music/index');
neteaseMusic.main(); neteaseMusic.main();

6
netease_music/index.js
View File

@ -49,7 +49,7 @@ async function startGetMusic(sleepTime) {
const songId = songIds[i]; const songId = songIds[i];
console.log(`${i}/${songIds.length} | song: ${songId} | ${await statistics()}`); console.log(`${i}/${songIds.length} | song: ${songId} | ${await statistics()}`);
try { try {
await songInfoUtils.get({ songId: songId }); await songInfoUtils.fetch({ songId: songId });
} catch (err) { } catch (err) {
console.error(err); console.error(err);
} }
@ -69,7 +69,7 @@ async function startGetMusic(sleepTime) {
const albumId = albumIds[i]; const albumId = albumIds[i];
console.log(`${i}/${albumIds.length} | album: ${albumId} | ${await statistics()}`); console.log(`${i}/${albumIds.length} | album: ${albumId} | ${await statistics()}`);
try { try {
await albumInfoUtils.get({ albumId: albumId }); await albumInfoUtils.fetch({ albumId: albumId });
} catch (err) { } catch (err) {
console.error(err); console.error(err);
} }
@ -89,7 +89,7 @@ async function startGetMusic(sleepTime) {
const artistId = artistIds[i]; const artistId = artistIds[i];
console.log(`${i}/${artistIds.length} | artist: ${artistId} | ${await statistics()}`); console.log(`${i}/${artistIds.length} | artist: ${artistId} | ${await statistics()}`);
try { try {
await artistInfoUtils.get({ artistId: artistId }); await artistInfoUtils.fetch({ artistId: artistId });
} catch (err) { } catch (err) {
console.error(err); console.error(err);
} }

19
netease_music/sql/structure.sql
View File

@ -8,7 +8,7 @@ CREATE TABLE `song` (
`create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '爬取时间', `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '爬取时间',
`modify_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最后更新时间', `modify_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最后更新时间',
PRIMARY KEY (`song_id`) PRIMARY KEY (`song_id`)
) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; ) ENGINE = InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
CREATE TABLE `artist` ( CREATE TABLE `artist` (
`artist_id` int(10) unsigned NOT NULL COMMENT '歌手id', `artist_id` int(10) unsigned NOT NULL COMMENT '歌手id',
@ -19,7 +19,7 @@ CREATE TABLE `artist` (
`create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '爬取时间', `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '爬取时间',
`modify_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最后更新时间', `modify_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最后更新时间',
PRIMARY KEY (`artist_id`) PRIMARY KEY (`artist_id`)
) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; ) ENGINE = InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
CREATE TABLE `album` ( CREATE TABLE `album` (
`album_id` int(10) unsigned NOT NULL COMMENT '专辑id', `album_id` int(10) unsigned NOT NULL COMMENT '专辑id',
@ -31,7 +31,7 @@ CREATE TABLE `album` (
`create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '爬取时间', `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '爬取时间',
`modify_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最后更新时间', `modify_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最后更新时间',
PRIMARY KEY (`album_id`) PRIMARY KEY (`album_id`)
) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; ) ENGINE = InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
CREATE TABLE `song_album_relation` ( CREATE TABLE `song_album_relation` (
`song_id` int(10) unsigned NOT NULL COMMENT '歌曲id', `song_id` int(10) unsigned NOT NULL COMMENT '歌曲id',
@ -39,7 +39,7 @@ CREATE TABLE `song_album_relation` (
`create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '爬取时间', `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '爬取时间',
`modify_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最后更新时间', `modify_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最后更新时间',
PRIMARY KEY `song_id` (`song_id`,`album_id`) PRIMARY KEY `song_id` (`song_id`,`album_id`)
) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; ) ENGINE = InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
CREATE TABLE `song_artist_relation` ( CREATE TABLE `song_artist_relation` (
`song_id` int(10) unsigned NOT NULL COMMENT '歌曲id', `song_id` int(10) unsigned NOT NULL COMMENT '歌曲id',
@ -47,4 +47,13 @@ CREATE TABLE `song_artist_relation` (
`create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '爬取时间', `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '爬取时间',
`modify_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最后更新时间', `modify_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最后更新时间',
PRIMARY KEY `song_id` (`song_id`,`artist_id`) PRIMARY KEY `song_id` (`song_id`,`artist_id`)
) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; ) ENGINE = InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
CREATE TABLE `lyric` (
`song_id` int(10) unsigned NOT NULL COMMENT '歌曲id',
`lyric` text CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '歌词',
`version` int(10) unsigned NOT NULL COMMENT '版本号',
`create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '爬取时间',
`modify_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最后更新时间',
PRIMARY KEY (`song_id`)
) ENGINE = InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;

8
netease_music/src/getInfo/albumInfoUtils.js
View File

@ -6,7 +6,7 @@ const requestUtils = require('../../../utils/requestUtils');
const sleepUtils = require('../../../utils/sleepUtils'); const sleepUtils = require('../../../utils/sleepUtils');
// 获取专辑详情 // 获取专辑详情
async function get({ albumId }) { async function fetch({ albumId }) {
let result = await dbUtils.query('SELECT count(*) as count FROM album WHERE album_id = ?', [albumId]); let result = await dbUtils.query('SELECT count(*) as count FROM album WHERE album_id = ?', [albumId]);
if (result[0].count > 0) { if (result[0].count > 0) {
console.log(`数据库中已有数据，跳过 albumId: ${albumId}`); console.log(`数据库中已有数据，跳过 albumId: ${albumId}`);
@ -24,9 +24,9 @@ async function get({ albumId }) {
let url = `https://music.163.com/album?id=${albumId}`; let url = `https://music.163.com/album?id=${albumId}`;
try { try {
// var html = fs.readFileSync(path.join(__dirname, "../temp", `album-${albumId}.html`), 'utf8'); // var html = fs.readFileSync(path.join(__dirname, "../../temp", `album-${albumId}.html`), 'utf8');
var html = await requestUtils.getApiResult(url); var html = await requestUtils.getApiResult(url);
// fs.writeFileSync(path.join(__dirname, "../temp", `album-${albumId}.html`), html); // fs.writeFileSync(path.join(__dirname, "../../temp", `album-${albumId}.html`), html);
} catch (errors) { } catch (errors) {
console.error(errors); console.error(errors);
return; return;
@ -85,5 +85,5 @@ async function get({ albumId }) {
} }
module.exports = { module.exports = {
get: get, fetch: fetch,
} }

8
netease_music/src/getInfo/artistInfoUtils.js
View File

@ -6,7 +6,7 @@ const requestUtils = require('../../../utils/requestUtils');
const sleepUtils = require('../../../utils/sleepUtils'); const sleepUtils = require('../../../utils/sleepUtils');
// 获取音乐人详情 // 获取音乐人详情
async function get({ artistId }) { async function fetch({ artistId }) {
let result = await dbUtils.query('SELECT count(*) as count FROM artist WHERE artist_id = ?', [artistId]); let result = await dbUtils.query('SELECT count(*) as count FROM artist WHERE artist_id = ?', [artistId]);
if (result[0].count > 0) { if (result[0].count > 0) {
console.log(`数据库中已有数据，跳过 artistId: ${artistId}`); console.log(`数据库中已有数据，跳过 artistId: ${artistId}`);
@ -25,9 +25,9 @@ async function get({ artistId }) {
let url = `https://music.163.com/artist?id=${artistId}`; let url = `https://music.163.com/artist?id=${artistId}`;
try { try {
// var html = fs.readFileSync(path.join(__dirname, "../temp", `artist-${artistId}.html`), 'utf8'); // var html = fs.readFileSync(path.join(__dirname, "../../temp", `artist-${artistId}.html`), 'utf8');
var html = await requestUtils.getApiResult(url); var html = await requestUtils.getApiResult(url);
// fs.writeFileSync(path.join(__dirname, "../temp", `artist-${artistId}.html`), html); // fs.writeFileSync(path.join(__dirname, "../../temp", `artist-${artistId}.html`), html);
} catch (errors) { } catch (errors) {
console.error(errors); console.error(errors);
return; return;
@ -78,5 +78,5 @@ async function get({ artistId }) {
} }
module.exports = { module.exports = {
get: get, fetch: fetch,
} }

8
netease_music/src/getInfo/songInfoUtils.js
View File

@ -6,7 +6,7 @@ const requestUtils = require('../../../utils/requestUtils');
const sleepUtils = require('../../../utils/sleepUtils'); const sleepUtils = require('../../../utils/sleepUtils');
// 获取音乐详情 // 获取音乐详情
async function get({ songId }) { async function fetch({ songId }) {
let result = await dbUtils.query('SELECT count(*) as count FROM song WHERE song_id = ?', [songId]); let result = await dbUtils.query('SELECT count(*) as count FROM song WHERE song_id = ?', [songId]);
if (result[0].count > 0) { if (result[0].count > 0) {
console.log(`数据库中已有数据，跳过 songId: ${songId}`); console.log(`数据库中已有数据，跳过 songId: ${songId}`);
@ -30,9 +30,9 @@ async function get({ songId }) {
let url = `https://music.163.com/song?id=${songId}`; let url = `https://music.163.com/song?id=${songId}`;
try { try {
// var html = fs.readFileSync(path.join(__dirname, "../temp", `song-${songId}.html`), 'utf8'); // var html = fs.readFileSync(path.join(__dirname, "../../temp", `song-${songId}.html`), 'utf8');
var html = await requestUtils.getApiResult(url); var html = await requestUtils.getApiResult(url);
// fs.writeFileSync(path.join(__dirname, "../temp", `song-${songId}.html`), html); // fs.writeFileSync(path.join(__dirname, "../../temp", `song-${songId}.html`), html);
} catch (errors) { } catch (errors) {
console.error(errors); console.error(errors);
return; return;
@ -102,5 +102,5 @@ async function get({ songId }) {
} }
module.exports = { module.exports = {
get: get, fetch: fetch,
} }

8
netease_music/src/getInfo/userInfoUtils.js
View File

@ -6,14 +6,14 @@ const requestUtils = require('../../../utils/requestUtils');
const sleepUtils = require('../../../utils/sleepUtils'); const sleepUtils = require('../../../utils/sleepUtils');
// 获取用户详情 // 获取用户详情
async function get({ userId }) { async function fetch({ userId }) {
let url = `https://music.163.com/user/home?id=${userId}`; let url = `https://music.163.com/user/home?id=${userId}`;
try { try {
var html = fs.readFileSync(path.join(__dirname, "../temp", ` user-${userId}.html`), 'utf8'); var html = fs.readFileSync(path.join(__dirname, "../../temp", ` user-${userId}.html`), 'utf8');
} catch (errors) { } catch (errors) {
var html = await requestUtils.getApiResult(url); var html = await requestUtils.getApiResult(url);
fs.writeFileSync(path.join(__dirname, "../temp", ` user-${userId}.html`), html); fs.writeFileSync(path.join(__dirname, "../../temp", ` user-${userId}.html`), html);
} }
// console.log(html); // console.log(html);
@ -21,5 +21,5 @@ async function get({ userId }) {
} }
module.exports = { module.exports = {
get: get, fetch: fetch,
} }

0
temp/.gitignore → netease_music/temp/.gitignore vendored
View File

2
test.js Normal file
View File

@ -0,0 +1,2 @@
const neteaseMusic = require('./netease_music/index');
neteaseMusic.test();