小报童-一人公司内容爬取

2024-06-25 02:42:25 +08:00
parent ac429e4cbf
commit 12afb942a2
7 changed files with 5310 additions and 0 deletions
--- a/小报童/index.js
+++ b/小报童/index.js
@@ -0,0 +1,52 @@
+const send_request = require('./utils/send_request');
+const fs = require('fs');
+const path = require('path');
+
+// 2024.06.25
+
+function getList() {
+    // https://api.xiaobot.net/paper/oneCompany/post?limit=20&offset=0&tag_name=&keyword=&order_by=created_at+undefined
+    const url = 'https://api.xiaobot.net/paper/oneCompany/post'
+    const data = {
+        limit: 200,
+        offset: 0,
+        tag_name: '',
+        keyword: '',
+        order_by: "created_at " + undefined,
+    }
+    send_request.call('GET', url, data, (e) => {
+        const response = e.data
+        const data = response.data
+        // console.log('data', data)
+        console.log('data', data.length, data.map(d => JSON.stringify(d).substring(0, 100)))
+
+        const uuidTitleMap = {}
+        for (const item of data) {
+            uuidTitleMap[item.uuid] = item.title
+        }
+
+        for (const item of data) {
+            let contentHtml = item.content
+            for (const uuid in uuidTitleMap) {
+                // https://xiaobot.net/post/83e3da77-0c09-4ecb-a8dd-3d7e5d191a6d
+                contentHtml = contentHtml.replace('https://xiaobot.net/post/' + uuid, './' + uuidTitleMap[uuid] + '.html')
+            }
+            const html =`<!DOCTYPE html>
+<html lang="zh-cn">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>${item.title}</title>
+    <link rel="stylesheet" href="../assets/reader.css">
+</head>
+<body>
+    <div class="post" style="padding: 20px;">${contentHtml}</div>
+</body>
+</html>`
+            fs.writeFileSync(path.join(__dirname, 'output', item.title + '.json'), JSON.stringify(item, null, 4), 'utf-8')
+            fs.writeFileSync(path.join(__dirname, 'output', item.title + '.html'), html, 'utf-8')
+        }
+    })
+}
+
+getList()