fix：DeEarth unzip出错时无法处理 refactor：支持从MCIM获取head
This commit is contained in:
Tianpao
@@ -35,6 +35,7 @@ fs.mkdirSync(movepath)
|
||||
}
|
||||
|
||||
export async function DeEarth(modpath: string, movepath: string) {
|
||||
try{
|
||||
const zip = new AdmZip(modpath).getEntries();
|
||||
//for (let i = 0; i < zip.length; i++) {
|
||||
//const e = zip[i]
|
||||
@@ -126,6 +127,9 @@ export async function DeEarth(modpath: string, movepath: string) {
|
||||
}
|
||||
}
|
||||
}
|
||||
}catch(error){
|
||||
LOGGER.error("DeEarth: "+error)
|
||||
}
|
||||
}
|
||||
|
||||
async function FastGot(url: string) {
|
||||
|
||||
@@ -1,22 +1,25 @@
|
||||
import pMap from "p-map";
|
||||
import pRetry from "p-retry";
|
||||
import fs from "node:fs"
|
||||
import fse from "fs-extra"
|
||||
import fs from "node:fs";
|
||||
import fse from "fs-extra";
|
||||
import yauzl from "yauzl";
|
||||
import got from "got";
|
||||
import env from "dotenv"
|
||||
import env from "dotenv";
|
||||
import { MultiBar } from "cli-progress";
|
||||
import {URL, fileURLToPath } from "node:url";
|
||||
import { URL, fileURLToPath } from "node:url";
|
||||
import { LOGGER } from "./logger.js";
|
||||
|
||||
export async function readzipentry(zipfile: yauzl.ZipFile, entry: yauzl.Entry):Promise<string> {
|
||||
export async function readzipentry(
|
||||
zipfile: yauzl.ZipFile,
|
||||
entry: yauzl.Entry
|
||||
): Promise<string> {
|
||||
const data: Buffer<ArrayBufferLike>[] = [];
|
||||
return await new Promise((reslove, reject) => {
|
||||
zipfile.openReadStream(entry, (err, e) => {
|
||||
e.on("data", (chunk: Buffer) => {
|
||||
data.push(chunk);
|
||||
}).on("end", () => {
|
||||
if (data.length !== 0&&typeof data !== "string"){
|
||||
if (data.length !== 0 && typeof data !== "string") {
|
||||
reslove(Buffer.concat(data).toString());
|
||||
}
|
||||
});
|
||||
@@ -24,119 +27,155 @@ export async function readzipentry(zipfile: yauzl.ZipFile, entry: yauzl.Entry):P
|
||||
});
|
||||
}
|
||||
|
||||
export async function fastdownload(data:[string,string]){
|
||||
export async function fastdownload(data: [string, string]) {
|
||||
const multibar = new MultiBar({
|
||||
format: ' {bar} | {filename} | {value}/{total}',
|
||||
noTTYOutput: true,
|
||||
notTTYSchedule: 10*1000,
|
||||
})
|
||||
const totalBar = multibar.create(data.length, 0, {filename: '总文件数'})
|
||||
return await pMap(data,async(e)=>{
|
||||
const size:number = await (async()=>{const head = (await got.head(e[0].replace("https://mod.mcimirror.top","https://edge.forgecdn.net"))).headers['content-length'];if(head){return Number(head)}else{return 0}})()
|
||||
const bar = multibar.create(size, 0, {filename: e[1]})
|
||||
try{
|
||||
await pRetry(async()=>{
|
||||
if(!fs.existsSync(e[1])){
|
||||
await got.get(e[0],{
|
||||
responseType:"buffer",
|
||||
}).on('downloadProgress',(progress)=>{
|
||||
bar.update(progress.transferred)
|
||||
}).then((res)=>{
|
||||
fse.outputFileSync(e[1],res.rawBody)
|
||||
})
|
||||
}
|
||||
},{retries:3})
|
||||
}catch(e){
|
||||
LOGGER.error({err:e})
|
||||
}finally{
|
||||
bar.stop()
|
||||
totalBar.increment()
|
||||
multibar.remove(bar)
|
||||
format: " {bar} | {filename} | {value}/{total}",
|
||||
noTTYOutput: true,
|
||||
notTTYSchedule: 10 * 1000,
|
||||
});
|
||||
const totalBar = multibar.create(data.length, 0, { filename: "总文件数" });
|
||||
return await pMap(
|
||||
data,
|
||||
async (e) => {
|
||||
const size: number = await (async () => {
|
||||
const head = (
|
||||
await got.head(
|
||||
e[0] /*.replace("https://mod.mcimirror.top","https://edge.forgecdn.net")*/,
|
||||
{ headers: { "user-agent": "DeEarthX" } }
|
||||
)
|
||||
).headers["content-length"];
|
||||
if (head) {
|
||||
return Number(head);
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
})();
|
||||
const bar = multibar.create(size, 0, { filename: e[1] });
|
||||
try {
|
||||
await pRetry(
|
||||
async () => {
|
||||
if (!fs.existsSync(e[1])) {
|
||||
await got
|
||||
.get(e[0], {
|
||||
responseType: "buffer",
|
||||
headers: {
|
||||
"user-agent": "DeEarthX",
|
||||
},
|
||||
})
|
||||
.on("downloadProgress", (progress) => {
|
||||
bar.update(progress.transferred);
|
||||
})
|
||||
.then((res) => {
|
||||
fse.outputFileSync(e[1], res.rawBody);
|
||||
});
|
||||
}
|
||||
},
|
||||
{ retries: 3 }
|
||||
);
|
||||
} catch (e) {
|
||||
LOGGER.error({ err: e });
|
||||
} finally {
|
||||
bar.stop();
|
||||
totalBar.increment();
|
||||
multibar.remove(bar);
|
||||
}
|
||||
},{concurrency:16})
|
||||
},
|
||||
{ concurrency: 16 }
|
||||
);
|
||||
}
|
||||
|
||||
export async function mr_fastdownload(data:[string,string,string]){
|
||||
export async function mr_fastdownload(data: [string, string, string]) {
|
||||
const multibar = new MultiBar({
|
||||
format: ' {bar} | {filename} | {value}/{total}',
|
||||
noTTYOutput: true,
|
||||
notTTYSchedule: 10*1000,
|
||||
format: " {bar} | {filename} | {value}/{total}",
|
||||
noTTYOutput: true,
|
||||
notTTYSchedule: 10 * 1000,
|
||||
});
|
||||
const totalBar = multibar.create(data.length, 0, { filename: '总文件数' });
|
||||
return await pMap(data, async (e) => {
|
||||
const totalBar = multibar.create(data.length, 0, { filename: "总文件数" });
|
||||
return await pMap(
|
||||
data,
|
||||
async (e) => {
|
||||
const bar = multibar.create(Number(e[2]), 0, { filename: e[1] });
|
||||
try {
|
||||
await pRetry(async () => {
|
||||
if (!fse.existsSync(e[1])) {
|
||||
await got.get(e[0], {
|
||||
responseType: "buffer",
|
||||
}).on('downloadProgress', (progress) => {
|
||||
bar.update(progress.transferred);
|
||||
}).then((res) => {
|
||||
fse.outputFileSync(e[1], res.rawBody);
|
||||
});
|
||||
}
|
||||
}, { retries: 3 });
|
||||
await pRetry(
|
||||
async () => {
|
||||
if (!fse.existsSync(e[1])) {
|
||||
await got
|
||||
.get(e[0], {
|
||||
responseType: "buffer",
|
||||
headers: {
|
||||
"user-agent": "DeEarthX",
|
||||
},
|
||||
})
|
||||
.on("downloadProgress", (progress) => {
|
||||
bar.update(progress.transferred);
|
||||
})
|
||||
.then((res) => {
|
||||
fse.outputFileSync(e[1], res.rawBody);
|
||||
});
|
||||
}
|
||||
},
|
||||
{ retries: 3 }
|
||||
);
|
||||
} catch (e) {
|
||||
LOGGER.error({ err: e });
|
||||
} finally {
|
||||
bar.stop();
|
||||
totalBar.increment();
|
||||
multibar.remove(bar);
|
||||
}
|
||||
catch (e) {
|
||||
LOGGER.error({ err: e });
|
||||
}
|
||||
finally {
|
||||
bar.stop();
|
||||
totalBar.increment();
|
||||
multibar.remove(bar);
|
||||
}
|
||||
}, { concurrency: 16 });
|
||||
},
|
||||
{ concurrency: 16 }
|
||||
);
|
||||
}
|
||||
|
||||
export async function xfastdownload(
|
||||
url: string,
|
||||
path: string,
|
||||
concurrency: number = 1
|
||||
) {
|
||||
let e = [];
|
||||
e.push([url, path]);
|
||||
await pMap(
|
||||
e,
|
||||
async (e) => {
|
||||
try {
|
||||
return await pRetry(
|
||||
async () => {
|
||||
if (
|
||||
url !== null &&
|
||||
url !== "" &&
|
||||
path !== null &&
|
||||
!fs.existsSync(e[1])
|
||||
) {
|
||||
const res = (await got.get(e[0])).rawBody; //下载文件
|
||||
await fse.outputFile(e[1], res); //保存文件
|
||||
}
|
||||
},
|
||||
{
|
||||
retries: 2,
|
||||
url: string,
|
||||
path: string,
|
||||
concurrency: number = 1
|
||||
) {
|
||||
let e = [];
|
||||
e.push([url, path]);
|
||||
await pMap(
|
||||
e,
|
||||
async (e) => {
|
||||
try {
|
||||
return await pRetry(
|
||||
async () => {
|
||||
if (
|
||||
url !== null &&
|
||||
url !== "" &&
|
||||
path !== null &&
|
||||
!fs.existsSync(e[1])
|
||||
) {
|
||||
const res = (await got.get(e[0])).rawBody; //下载文件
|
||||
await fse.outputFile(e[1], res); //保存文件
|
||||
}
|
||||
);
|
||||
} catch (error) {
|
||||
LOGGER.error({ err: error });
|
||||
}
|
||||
},
|
||||
{
|
||||
concurrency: concurrency,
|
||||
},
|
||||
{
|
||||
retries: 2,
|
||||
}
|
||||
);
|
||||
} catch (error) {
|
||||
LOGGER.error({ err: error });
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
export const usemirror = (()=>{
|
||||
env.config({debug:false})
|
||||
const mirror = process.env.MIRROR
|
||||
if(mirror){
|
||||
return false
|
||||
}else{
|
||||
return true
|
||||
},
|
||||
{
|
||||
concurrency: concurrency,
|
||||
}
|
||||
})()
|
||||
);
|
||||
}
|
||||
|
||||
export const isDevelopment = (()=>{
|
||||
env.config()
|
||||
return process.env.DEVELOPMENT
|
||||
})()
|
||||
export const usemirror = (() => {
|
||||
env.config({ debug: false });
|
||||
const mirror = process.env.MIRROR;
|
||||
if (mirror) {
|
||||
return false;
|
||||
} else {
|
||||
return true;
|
||||
}
|
||||
})();
|
||||
|
||||
export const isDevelopment = (() => {
|
||||
env.config();
|
||||
return process.env.DEVELOPMENT;
|
||||
})();
|
||||
|
||||
Reference in New Issue
Block a user