139 lines
3.9 KiB
TypeScript
139 lines
3.9 KiB
TypeScript
import pMap from "p-map";
|
|
import pRetry from "p-retry";
|
|
import fse from "fs-extra"
|
|
import yauzl from "yauzl";
|
|
import got from "got";
|
|
import env from "dotenv"
|
|
import { MultiBar } from "cli-progress";
|
|
import {URL, fileURLToPath } from "node:url";
|
|
import { LOGGER } from "./logger.js";
|
|
|
|
export async function readzipentry(zipfile: yauzl.ZipFile, entry: yauzl.Entry):Promise<string|Buffer> {
|
|
const data: Buffer<ArrayBufferLike>[] = [];
|
|
return await new Promise((reslove, reject) => {
|
|
zipfile.openReadStream(entry, (err, e) => {
|
|
e.on("data", (chunk: Buffer) => {
|
|
data.push(chunk);
|
|
}).on("end", () => {
|
|
reslove(data.toString());
|
|
});
|
|
});
|
|
});
|
|
}
|
|
|
|
export async function fastdownload(data:[string,string]){
|
|
const multibar = new MultiBar({
|
|
format: ' {bar} | {filename} | {value}/{total}',
|
|
noTTYOutput: true,
|
|
notTTYSchedule: 10*1000,
|
|
})
|
|
const totalBar = multibar.create(data.length, 0, {filename: '总文件数'})
|
|
return await pMap(data,async(e)=>{
|
|
const size:number = await (async()=>{const head = (await got.head(e[0])).headers['content-length'];if(head){return Number(head)}else{return 0}})()
|
|
const bar = multibar.create(size, 0, {filename: e[1]})
|
|
try{
|
|
await pRetry(async()=>{
|
|
if(!fse.existsSync(e[1])){
|
|
await got.get(e[0],{
|
|
responseType:"buffer",
|
|
}).on('downloadProgress',(progress)=>{
|
|
bar.update(progress.transferred)
|
|
}).then((res)=>{
|
|
fse.outputFileSync(e[1],res.rawBody)
|
|
})
|
|
}
|
|
},{retries:3})
|
|
}catch(e){
|
|
LOGGER.error({err:e})
|
|
}finally{
|
|
bar.stop()
|
|
totalBar.increment()
|
|
multibar.remove(bar)
|
|
}
|
|
},{concurrency:16})
|
|
}
|
|
|
|
export async function mr_fastdownload(data:[string,string,string]){
|
|
const multibar = new MultiBar({
|
|
format: ' {bar} | {filename} | {value}/{total}',
|
|
noTTYOutput: true,
|
|
notTTYSchedule: 10*1000,
|
|
});
|
|
const totalBar = multibar.create(data.length, 0, { filename: '总文件数' });
|
|
return await pMap(data, async (e) => {
|
|
const bar = multibar.create(Number(e[2]), 0, { filename: e[1] });
|
|
try {
|
|
await pRetry(async () => {
|
|
if (!fse.existsSync(e[1])) {
|
|
await got.get(e[0], {
|
|
responseType: "buffer",
|
|
}).on('downloadProgress', (progress) => {
|
|
bar.update(progress.transferred);
|
|
}).then((res) => {
|
|
fse.outputFileSync(e[1], res.rawBody);
|
|
});
|
|
}
|
|
}, { retries: 3 });
|
|
}
|
|
catch (e) {
|
|
LOGGER.error({ err: e });
|
|
}
|
|
finally {
|
|
bar.stop();
|
|
totalBar.increment();
|
|
multibar.remove(bar);
|
|
}
|
|
}, { concurrency: 16 });
|
|
}
|
|
|
|
export async function xfastdownload(
|
|
url: string,
|
|
path: string,
|
|
concurrency: number = 1
|
|
) {
|
|
let e = [];
|
|
e.push([url, path]);
|
|
await pMap(
|
|
e,
|
|
async (e) => {
|
|
try {
|
|
return await pRetry(
|
|
async () => {
|
|
if (
|
|
url !== null &&
|
|
url !== "" &&
|
|
path !== null &&
|
|
!fse.existsSync(e[1])
|
|
) {
|
|
const res = (await got.get(e[0])).rawBody; //下载文件
|
|
await fse.outputFile(e[1], res); //保存文件
|
|
}
|
|
},
|
|
{
|
|
retries: 2,
|
|
}
|
|
);
|
|
} catch (error) {
|
|
LOGGER.error({ err: error });
|
|
}
|
|
},
|
|
{
|
|
concurrency: concurrency,
|
|
}
|
|
);
|
|
}
|
|
|
|
export const usemirror = (()=>{
|
|
env.config()
|
|
const mirror = process.env.MIRROR
|
|
if(mirror){
|
|
return false
|
|
}else{
|
|
return true
|
|
}
|
|
})()
|
|
|
|
export const isDevelopment = (()=>{
|
|
env.config()
|
|
return process.env.DEVELOPMENT
|
|
})() |