This commit is contained in:
zyronon 2024-03-15 18:17:52 +08:00
parent a603080852
commit 415895a2d3
22 changed files with 211958 additions and 0 deletions

2108
node/data2.json Normal file

File diff suppressed because it is too large Load Diff

17
node/package.json Normal file
View File

@ -0,0 +1,17 @@
{
"name": "node",
"version": "1.0.0",
"description": "",
"main": "index.js",
"type": "module",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"author": "",
"license": "ISC",
"dependencies": {
"axios": "^1.6.7",
"nanoid": "^5.0.6",
"request": "^2.88.2"
}
}

346
node/pnpm-lock.yaml Normal file
View File

@ -0,0 +1,346 @@
lockfileVersion: '6.0'
settings:
autoInstallPeers: true
excludeLinksFromLockfile: false
dependencies:
axios:
specifier: ^1.6.7
version: 1.6.7
nanoid:
specifier: ^5.0.6
version: 5.0.6
request:
specifier: ^2.88.2
version: 2.88.2
packages:
/ajv@6.12.6:
resolution: {integrity: sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==}
dependencies:
fast-deep-equal: 3.1.3
fast-json-stable-stringify: 2.1.0
json-schema-traverse: 0.4.1
uri-js: 4.4.1
dev: false
/asn1@0.2.6:
resolution: {integrity: sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==}
dependencies:
safer-buffer: 2.1.2
dev: false
/assert-plus@1.0.0:
resolution: {integrity: sha512-NfJ4UzBCcQGLDlQq7nHxH+tv3kyZ0hHQqF5BO6J7tNJeP5do1llPr8dZ8zHonfhAu0PHAdMkSo+8o0wxg9lZWw==}
engines: {node: '>=0.8'}
dev: false
/asynckit@0.4.0:
resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==}
dev: false
/aws-sign2@0.7.0:
resolution: {integrity: sha512-08kcGqnYf/YmjoRhfxyu+CLxBjUtHLXLXX/vUfx9l2LYzG3c1m61nrpyFUZI6zeS+Li/wWMMidD9KgrqtGq3mA==}
dev: false
/aws4@1.12.0:
resolution: {integrity: sha512-NmWvPnx0F1SfrQbYwOi7OeaNGokp9XhzNioJ/CSBs8Qa4vxug81mhJEAVZwxXuBmYB5KDRfMq/F3RR0BIU7sWg==}
dev: false
/axios@1.6.7:
resolution: {integrity: sha512-/hDJGff6/c7u0hDkvkGxR/oy6CbCs8ziCsC7SqmhjfozqiJGc8Z11wrv9z9lYfY4K8l+H9TpjcMDX0xOZmx+RA==}
dependencies:
follow-redirects: 1.15.6
form-data: 4.0.0
proxy-from-env: 1.1.0
transitivePeerDependencies:
- debug
dev: false
/bcrypt-pbkdf@1.0.2:
resolution: {integrity: sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==}
dependencies:
tweetnacl: 0.14.5
dev: false
/caseless@0.12.0:
resolution: {integrity: sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==}
dev: false
/combined-stream@1.0.8:
resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==}
engines: {node: '>= 0.8'}
dependencies:
delayed-stream: 1.0.0
dev: false
/core-util-is@1.0.2:
resolution: {integrity: sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ==}
dev: false
/dashdash@1.14.1:
resolution: {integrity: sha512-jRFi8UDGo6j+odZiEpjazZaWqEal3w/basFjQHQEwVtZJGDpxbH1MeYluwCS8Xq5wmLJooDlMgvVarmWfGM44g==}
engines: {node: '>=0.10'}
dependencies:
assert-plus: 1.0.0
dev: false
/delayed-stream@1.0.0:
resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==}
engines: {node: '>=0.4.0'}
dev: false
/ecc-jsbn@0.1.2:
resolution: {integrity: sha512-eh9O+hwRHNbG4BLTjEl3nw044CkGm5X6LoaCf7LPp7UU8Qrt47JYNi6nPX8xjW97TKGKm1ouctg0QSpZe9qrnw==}
dependencies:
jsbn: 0.1.1
safer-buffer: 2.1.2
dev: false
/extend@3.0.2:
resolution: {integrity: sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==}
dev: false
/extsprintf@1.3.0:
resolution: {integrity: sha512-11Ndz7Nv+mvAC1j0ktTa7fAb0vLyGGX+rMHNBYQviQDGU0Hw7lhctJANqbPhu9nV9/izT/IntTgZ7Im/9LJs9g==}
engines: {'0': node >=0.6.0}
dev: false
/fast-deep-equal@3.1.3:
resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==}
dev: false
/fast-json-stable-stringify@2.1.0:
resolution: {integrity: sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==}
dev: false
/follow-redirects@1.15.6:
resolution: {integrity: sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==}
engines: {node: '>=4.0'}
peerDependencies:
debug: '*'
peerDependenciesMeta:
debug:
optional: true
dev: false
/forever-agent@0.6.1:
resolution: {integrity: sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw==}
dev: false
/form-data@2.3.3:
resolution: {integrity: sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==}
engines: {node: '>= 0.12'}
dependencies:
asynckit: 0.4.0
combined-stream: 1.0.8
mime-types: 2.1.35
dev: false
/form-data@4.0.0:
resolution: {integrity: sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==}
engines: {node: '>= 6'}
dependencies:
asynckit: 0.4.0
combined-stream: 1.0.8
mime-types: 2.1.35
dev: false
/getpass@0.1.7:
resolution: {integrity: sha512-0fzj9JxOLfJ+XGLhR8ze3unN0KZCgZwiSSDz168VERjK8Wl8kVSdcu2kspd4s4wtAa1y/qrVRiAA0WclVsu0ng==}
dependencies:
assert-plus: 1.0.0
dev: false
/har-schema@2.0.0:
resolution: {integrity: sha512-Oqluz6zhGX8cyRaTQlFMPw80bSJVG2x/cFb8ZPhUILGgHka9SsokCCOQgpveePerqidZOrT14ipqfJb7ILcW5Q==}
engines: {node: '>=4'}
dev: false
/har-validator@5.1.5:
resolution: {integrity: sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w==}
engines: {node: '>=6'}
deprecated: this library is no longer supported
dependencies:
ajv: 6.12.6
har-schema: 2.0.0
dev: false
/http-signature@1.2.0:
resolution: {integrity: sha512-CAbnr6Rz4CYQkLYUtSNXxQPUH2gK8f3iWexVlsnMeD+GjlsQ0Xsy1cOX+mN3dtxYomRy21CiOzU8Uhw6OwncEQ==}
engines: {node: '>=0.8', npm: '>=1.3.7'}
dependencies:
assert-plus: 1.0.0
jsprim: 1.4.2
sshpk: 1.18.0
dev: false
/is-typedarray@1.0.0:
resolution: {integrity: sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA==}
dev: false
/isstream@0.1.2:
resolution: {integrity: sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g==}
dev: false
/jsbn@0.1.1:
resolution: {integrity: sha512-UVU9dibq2JcFWxQPA6KCqj5O42VOmAY3zQUfEKxU0KpTGXwNoCjkX1e13eHNvw/xPynt6pU0rZ1htjWTNTSXsg==}
dev: false
/json-schema-traverse@0.4.1:
resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==}
dev: false
/json-schema@0.4.0:
resolution: {integrity: sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==}
dev: false
/json-stringify-safe@5.0.1:
resolution: {integrity: sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==}
dev: false
/jsprim@1.4.2:
resolution: {integrity: sha512-P2bSOMAc/ciLz6DzgjVlGJP9+BrJWu5UDGK70C2iweC5QBIeFf0ZXRvGjEj2uYgrY2MkAAhsSWHDWlFtEroZWw==}
engines: {node: '>=0.6.0'}
dependencies:
assert-plus: 1.0.0
extsprintf: 1.3.0
json-schema: 0.4.0
verror: 1.10.0
dev: false
/mime-db@1.52.0:
resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==}
engines: {node: '>= 0.6'}
dev: false
/mime-types@2.1.35:
resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==}
engines: {node: '>= 0.6'}
dependencies:
mime-db: 1.52.0
dev: false
/nanoid@5.0.6:
resolution: {integrity: sha512-rRq0eMHoGZxlvaFOUdK1Ev83Bd1IgzzR+WJ3IbDJ7QOSdAxYjlurSPqFs9s4lJg29RT6nPwizFtJhQS6V5xgiA==}
engines: {node: ^18 || >=20}
hasBin: true
dev: false
/oauth-sign@0.9.0:
resolution: {integrity: sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==}
dev: false
/performance-now@2.1.0:
resolution: {integrity: sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==}
dev: false
/proxy-from-env@1.1.0:
resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==}
dev: false
/psl@1.9.0:
resolution: {integrity: sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==}
dev: false
/punycode@2.3.1:
resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==}
engines: {node: '>=6'}
dev: false
/qs@6.5.3:
resolution: {integrity: sha512-qxXIEh4pCGfHICj1mAJQ2/2XVZkjCDTcEgfoSQxc/fYivUZxTkk7L3bDBJSoNrEzXI17oUO5Dp07ktqE5KzczA==}
engines: {node: '>=0.6'}
dev: false
/request@2.88.2:
resolution: {integrity: sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==}
engines: {node: '>= 6'}
deprecated: request has been deprecated, see https://github.com/request/request/issues/3142
dependencies:
aws-sign2: 0.7.0
aws4: 1.12.0
caseless: 0.12.0
combined-stream: 1.0.8
extend: 3.0.2
forever-agent: 0.6.1
form-data: 2.3.3
har-validator: 5.1.5
http-signature: 1.2.0
is-typedarray: 1.0.0
isstream: 0.1.2
json-stringify-safe: 5.0.1
mime-types: 2.1.35
oauth-sign: 0.9.0
performance-now: 2.1.0
qs: 6.5.3
safe-buffer: 5.2.1
tough-cookie: 2.5.0
tunnel-agent: 0.6.0
uuid: 3.4.0
dev: false
/safe-buffer@5.2.1:
resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==}
dev: false
/safer-buffer@2.1.2:
resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==}
dev: false
/sshpk@1.18.0:
resolution: {integrity: sha512-2p2KJZTSqQ/I3+HX42EpYOa2l3f8Erv8MWKsy2I9uf4wA7yFIkXRffYdsx86y6z4vHtV8u7g+pPlr8/4ouAxsQ==}
engines: {node: '>=0.10.0'}
hasBin: true
dependencies:
asn1: 0.2.6
assert-plus: 1.0.0
bcrypt-pbkdf: 1.0.2
dashdash: 1.14.1
ecc-jsbn: 0.1.2
getpass: 0.1.7
jsbn: 0.1.1
safer-buffer: 2.1.2
tweetnacl: 0.14.5
dev: false
/tough-cookie@2.5.0:
resolution: {integrity: sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==}
engines: {node: '>=0.8'}
dependencies:
psl: 1.9.0
punycode: 2.3.1
dev: false
/tunnel-agent@0.6.0:
resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==}
dependencies:
safe-buffer: 5.2.1
dev: false
/tweetnacl@0.14.5:
resolution: {integrity: sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==}
dev: false
/uri-js@4.4.1:
resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==}
dependencies:
punycode: 2.3.1
dev: false
/uuid@3.4.0:
resolution: {integrity: sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==}
deprecated: Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details.
hasBin: true
dev: false
/verror@1.10.0:
resolution: {integrity: sha512-ZZKSmDAEFOijERBLkmYfJ+vmk3w+7hOLYDNkRCuRuMJGEmqYNCNLyBBFwWKVMhfwaEF3WOd0Zlw86U/WC/+nYw==}
engines: {'0': node >=0.6.0}
dependencies:
assert-plus: 1.0.0
core-util-is: 1.0.2
extsprintf: 1.3.0
dev: false

45
node/process-get-img.js Normal file
View File

@ -0,0 +1,45 @@
import fs from 'fs'
import request from 'request'
import path from 'path'
import axios from 'axios'
import {nanoid} from 'nanoid'
let fileName = './data2.json'
let saveFileStr = fs.readFileSync(fileName, "utf8");
let inputData = JSON.parse(saveFileStr);
const downloadImage = (src, dest, callback) => {
request.head(src, (err, res, body) => {
if (err) {
console.log(err);
return;
}
src &&
request(src)
.pipe(fs.createWriteStream(dest))
.on("close", () => {
callback && callback(null, dest);
});
});
};
let saveFilePath = './imgs/'
async function sleep(val) {
return new Promise(resolve => {
setTimeout(resolve, val)
})
}
inputData.slice(0, 2).map(async v => {
await v.cover_url.map(async a => {
let coverUrl = a.url_list[a.url_list.length - 1]
a.url_list = [coverUrl]
console.log('coverUrl', coverUrl)
await sleep(1500)
downloadImage(coverUrl, saveFilePath + nanoid() + '.png')
})
})
fs.writeFileSync(fileName, JSON.stringify(inputData, null, 2));

2174
src/assets/data/data.js Normal file

File diff suppressed because it is too large Load Diff

17995
src/assets/data/input.json Normal file

File diff suppressed because it is too large Load Diff

2890
src/assets/data/ouput.json Normal file

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,85 @@
import fs from 'fs'
let fileName = './post/user-81069823274.json'
const str = fs.readFileSync("./input.json", "utf8");
let inputData = JSON.parse(str);
const saveFileStr = fs.readFileSync(fileName, "utf8");
let saveFileData = JSON.parse(saveFileStr);
// console.log('list', inputData.has_more)
// console.log('f', fs)
inputData.aweme_list.map(v => {
Object.keys(v).map(k => {
if (![
'aweme_id',
'desc',
'create_time',
'music',
'video',
'share_url',
'statistics',
'status',
'text_extra',
'is_top',
'share_info',
'duration',
'image_infos',
'risk_infos',
'position',
'author_user_id',
'prevent_download',
'long_video',
'aweme_control',
'images',
'suggest_words',
].includes(k)) {
delete v[k]
}
})
Object.keys(v.music).map(k => {
if (![
'id',
'title',
'author',
'cover_medium',
'cover_thumb',
'play_url',
'duration',
'user_count',
'owner_id',
'owner_nickname',
'is_original'
].includes(k)) {
delete v.music[k]
}
})
Object.keys(v.video).map(k => {
if (![
'play_addr',
'cover',
'height',
'width',
'ratio',
'use_static_cover',
'duration',
'horizontal_type',
'animated_cover',
].includes(k)) {
delete v.video[k]
}
})
})
saveFileData = saveFileData.concat(inputData.aweme_list)
fs.writeFileSync(
"./ouput.json",
JSON.stringify(inputData.aweme_list, null, 2)
);
fs.writeFileSync(
fileName,
JSON.stringify(saveFileData, null, 2)
);

View File

@ -0,0 +1,63 @@
import fs from 'fs'
let fileName = './data2.json'
let saveFileStr = fs.readFileSync(fileName, "utf8");
let inputData = JSON.parse(saveFileStr);
inputData.map(v => {
Object.keys(v).map(k => {
if (![
'avatar_168x168',
'avatar_300x300',
'aweme_count',
'birthday_hide_level',
'can_show_group_card',
'commerce_user_level',
'province',
'city',
'country',
'district',
'favoriting_count',
'follow_status',
'follower_count',
'follower_request_status',
'follower_status',
'following_count',
'forward_count',
'public_collects_count',
'share_info',
'mplatform_followers_count',
'nickname',
'max_follower_count',
'gender',
'cover_colour',
'cover_url',
'commerce_info',
'commerce_user_info',
'short_id',
'signature',
'total_favorited',
'uid',
'unique_id',
'user_age',
'white_cover_url',
'card_entries',
'cover_and_head_image_info',
'ip_location',
].includes(k)) {
delete v[k]
}
})
})
// saveFileData = saveFileData.concat(inputData.aweme_list)
// fs.writeFileSync(
// "./ouput.json",
// JSON.stringify(inputData.aweme_list, null, 2)
// );
fs.writeFileSync(
// fileName,
'./data2.json',
JSON.stringify(inputData, null, 2)
);