coder-xiaomo/tools
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity GitHub Gitee

添加sql导入脚本生成工具

Browse Source
This commit is contained in:
小墨
2022-10-26 20:44:07 +08:00
parent 66d5f89b02
commit 19e69f9bfd
3 changed files with 50 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
netease_music/sql/export/export_sql_generator.js
View File

@@ -1,7 +1,6 @@
const fs = require('fs');
const path = require('path');
const absPath = `D:/sql_export`;
// 数字转成字符串，同时在前面填充
@@ -12,38 +11,31 @@ function fill(num, fillers, length) {
return result;
}
// #############################################
const export_gap = 50000000;
const partition_gap = 50000000;
const table = "comment_1024";
const index = "comment_id";
const exportTablePrefix = "comment_export_";
// const export_gap = 50000000;
// const partition_gap = 50000000;
// const table = "comment_1024";
// const index = "comment_id";
// const exportTablePrefix = "comment_export_";
// 数字转成字符串，同时在前面填充
function fill(num, fillers, length) {
var result = `${num}`;
if (result.length < length)
result = new Array(length - result.length + 1).join(fillers) + result;
return result;
}
// let sqlArr1 = [];
// let sqlArr2 = [];
// let sqlArr3 = [];
// for (let i = 200; i < 300; i++) {
// // for (let i = 100; i < 200; i++) {
// let where = `${index} >= ${fill(i * export_gap, ' ', 12)} and ${index} < ${fill((i + 1) * export_gap, ' ', 12)}`;
let sqlArr1 = [];
let sqlArr2 = [];
let sqlArr3 = [];
for (let i = 200; i < 300; i++) {
// for (let i = 100; i < 200; i++) {
let where = `${index} >= ${fill(i * export_gap, ' ', 12)} and ${index} < ${fill((i + 1) * export_gap, ' ', 12)}`;
// let sql_create_table = `create table ${exportTablePrefix}${fill(i, '0', 4)} select * from ${table} where ${where};`;
// sqlArr1.push(sql_create_table);
let sql_create_table = `create table ${exportTablePrefix}${fill(i, '0', 4)} select * from ${table} where ${where};`;
sqlArr1.push(sql_create_table);
// let sql_delete_rows = `DELETE FROM ${table} WHERE ${where};`;
// sqlArr2.push(sql_delete_rows);
let sql_delete_rows = `DELETE FROM ${table} WHERE ${where};`;
sqlArr2.push(sql_delete_rows);
let sql_partition = ` PARTITION p${fill(i, '0', 4)} VALUES LESS THAN (${fill((i + 1) * partition_gap, ' ', 12)})`;
sqlArr3.push(sql_partition);
}
sqlArr3.push(` PARTITION p_max VALUES LESS THAN MAXVALUE`);
// let sql_partition = ` PARTITION p${fill(i, '0', 4)} VALUES LESS THAN (${fill((i + 1) * partition_gap, ' ', 12)})`;
// sqlArr3.push(sql_partition);
// }
// sqlArr3.push(` PARTITION p_max VALUES LESS THAN MAXVALUE`);
// 建表
// console.log(sqlArr1.join('\n'));
@@ -64,9 +56,9 @@ sqlArr3.push(` PARTITION p_max VALUES LESS THAN MAXVALUE`);
// #############################################
// 使用 mysqldump 分块导出数据表
let rangeTxtName = "album"; // 分布区间 "song" "album" "artist"
const dumpTable = "album"; // "comment_progress";
const fieldName = `album_id`;
let rangeTxtName = "song"; // 分布区间 "song" "album" "artist" "user"
const dumpTable = "lyric"; // "comment_progress";
const fieldName = `song_id`;
var a = fs.readFileSync(path.join(__dirname, `distribution_range/${rangeTxtName}.txt`), "utf-8").trim().split("\n").reverse().map(i => i.trim());
// console.log(a);
let outputArr = [`@echo off`, `D:`, `cd D:/Program/Development/Environment/phpstudy_pro/Extensions/MySQL8.0.12/bin`];

26
netease_music/sql/import/import_sql_generator.js Normal file
View File

@@ -0,0 +1,26 @@
const absPath = `D:/sql_export`;
// 数字转成字符串，同时在前面填充
function fill(num, fillers, length) {
var result = `${num}`;
if (result.length < length)
result = new Array(length - result.length + 1).join(fillers) + result;
return result;
}
// 使用 mysql 导入数据表
let outputArr = [
[
`@echo off`, `D:`, `cd D:/Program/Development/Environment/phpstudy_pro/Extensions/MySQL8.0.12/bin`,
`mysql -hlocalhost -uroot -proot neteasemusic`
].join('\n'),
`use neteasemusic;`
];
let firstIndex = 0;
let lastIndex = 115;
for (let i = firstIndex; i <= lastIndex; i++) {
outputArr.push(`source ${absPath}/comment/comment_${fill(i, '0', 4)}.sql`);
}
outputArr.push("echo done.");
console.log(outputArr.join('\n'));

1
netease_music/todo.txt
View File

@@ -31,6 +31,7 @@ node index --utils playlist
后期：
删除song_playlist_relation表中rcmd_reason字段（全是空字符串）
歌单定时更新（rel表中添加一个del字段，先将歌单下面的全部置为删除状态，再插入的时候把已有歌曲的标记重新修改为正常状态）