move more filesystem handling into _shared.js
This commit is contained in:
parent
88b945ad54
commit
e4253463a4
11
_shared.js
11
_shared.js
|
@ -1,4 +1,6 @@
|
|||
import * as crypto from 'node:crypto';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
|
||||
export const make_query = (params) => {
|
||||
const filtered = {};
|
||||
|
@ -21,19 +23,22 @@ export const make_request = async (path, params) => {
|
|||
return resp;
|
||||
};
|
||||
|
||||
export const download_to = async (url, path, date) => {
|
||||
export const download_to = async (url, dest, date) => {
|
||||
const resp = await fetch(url);
|
||||
if (resp.status !== 200) {
|
||||
console.log(url);
|
||||
throw resp;
|
||||
}
|
||||
const buffer = Buffer.from(await resp.arrayBuffer());
|
||||
await fs.promises.writeFile(path, buffer);
|
||||
const tmp = crypto.randomUUID();
|
||||
await fs.promises.writeFile(tmp, buffer);
|
||||
date ??= resp.headers.get('last-modified');
|
||||
if (date) {
|
||||
const date_obj = new Date(date);
|
||||
await fs.promises.utimes(path, date_obj, date_obj);
|
||||
await fs.promises.utimes(tmp, date_obj, date_obj);
|
||||
}
|
||||
await fs.promises.mkdir(path.dirname(dest), { recursive: true });
|
||||
await fs.promises.rename(tmp, dest);
|
||||
};
|
||||
|
||||
export const exists = (path) => fs.promises.access(path).then(() => true, () => false);
|
||||
|
|
|
@ -1,6 +1,5 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
import * as fs from 'node:fs';
|
||||
import { download_to, exists, make_request } from './_shared.js';
|
||||
|
||||
for (const tags of process.argv.slice(2)) {
|
||||
|
@ -17,17 +16,12 @@ for (const tags of process.argv.slice(2)) {
|
|||
page = `b${page_posts[319].id}`;
|
||||
}
|
||||
|
||||
await fs.promises.mkdir(tags, { recursive: true });
|
||||
|
||||
const tmp = `${tags}/tmp`;
|
||||
|
||||
for (let i = 0; i < posts.length; i++) {
|
||||
const post = posts[i];
|
||||
const path = `${tags}/${post.id}-${post.file.md5}.${post.file.ext}`;
|
||||
if (!await exists(path)) {
|
||||
console.log(i + 1 + '/' + posts.length, path);
|
||||
await download_to(`https://static1.e621.net/data/${post.file.md5.slice(0, 2)}/${post.file.md5.slice(2, 4)}/${post.file.md5}.${post.file.ext}`, tmp, post.created_at);
|
||||
await fs.promises.rename(tmp, path);
|
||||
await download_to(`https://static1.e621.net/data/${post.file.md5.slice(0, 2)}/${post.file.md5.slice(2, 4)}/${post.file.md5}.${post.file.ext}`, path, post.created_at);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in New Issue