mirrors/barkey
1
0
Fork 0
mirror of https://codeberg.org/yeentown/barkey.git synced 2025-07-07 04:26:58 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

reduce log spam from import/export processes

Browse source
This commit is contained in:
Hazelnoot 2025-05-22 09:56:37 -04:00
parent 12acad0654
commit 8e30744bb9
17 changed files with 114 additions and 78 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
packages/backend/src/queue/processors/ExportAccountDataProcessorService.ts
View file

@ -86,21 +86,23 @@ export class ExportAccountDataProcessorService {
@bindThis
public async process(job: Bull.Job): Promise<void> {
this.logger.info('Exporting Account Data...');
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) {
this.logger.debug(`Skip: user ${job.data.user.id} does not exist`);
return;
}
const profile = await this.userProfilesRepository.findOneBy({ userId: job.data.user.id });
if (profile == null) {
this.logger.debug(`Skip: user ${job.data.user.id} has no profile`);
return;
}
this.logger.info(`Exporting account data for ${job.data.user.id} ...`);
const [path, cleanup] = await createTempDir();
this.logger.info(`Temp dir is ${path}`);
this.logger.debug(`Temp dir is ${path}`);
// User Export
@ -745,12 +747,12 @@ export class ExportAccountDataProcessorService {
zlib: { level: 0 },
});
archiveStream.on('close', async () => {
this.logger.succ(`Exported to: ${archivePath}`);
this.logger.debug(`Exported to path: ${archivePath}`);
const fileName = 'data-request-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.zip';
const driveFile = await this.driveService.addFile({ user, path: archivePath, name: fileName, force: true });
this.logger.succ(`Exported to: ${driveFile.id}`);
this.logger.debug(`Exported to drive: ${driveFile.id}`);
cleanup();
archiveCleanup();
if (profile.email) {

6
packages/backend/src/queue/processors/ExportAntennasProcessorService.ts
View file

@ -45,8 +45,12 @@ export class ExportAntennasProcessorService {
public async process(job: Bull.Job<DBExportAntennasData>): Promise<void> {
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) {
this.logger.debug(`Skip: user ${job.data.user.id} does not exist`);
return;
}
this.logger.info(`Exporting antennas of ${job.data.user.id} ...`);
const [path, cleanup] = await createTemp();
const stream = fs.createWriteStream(path, { flags: 'a' });
const write = (input: string): Promise<void> => {
@ -96,7 +100,7 @@ export class ExportAntennasProcessorService {
const fileName = 'antennas-' + DateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.json';
const driveFile = await this.driveService.addFile({ user, path, name: fileName, force: true, ext: 'json' });
this.logger.succ('Exported to: ' + driveFile.id);
this.logger.debug('Exported to: ' + driveFile.id);
this.notificationService.createNotification(user.id, 'exportCompleted', {
exportedEntity: 'antenna',

11
packages/backend/src/queue/processors/ExportBlockingProcessorService.ts
View file

@ -40,17 +40,18 @@ export class ExportBlockingProcessorService {
@bindThis
public async process(job: Bull.Job<DbJobDataWithUser>): Promise<void> {
this.logger.info(`Exporting blocking of ${job.data.user.id} ...`);
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) {
this.logger.debug(`Skip: user ${job.data.user.id} does not exist`);
return;
}
this.logger.info(`Exporting blocking of ${job.data.user.id} ...`);
// Create temp file
const [path, cleanup] = await createTemp();
this.logger.info(`Temp file is ${path}`);
this.logger.debug(`Temp file is ${path}`);
try {
const stream = fs.createWriteStream(path, { flags: 'a' });
@ -105,12 +106,12 @@ export class ExportBlockingProcessorService {
}
stream.end();
this.logger.succ(`Exported to: ${path}`);
this.logger.debug(`Exported to: ${path}`);
const fileName = 'blocking-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.csv';
const driveFile = await this.driveService.addFile({ user, path, name: fileName, force: true, ext: 'csv' });
this.logger.succ(`Exported to: ${driveFile.id}`);
this.logger.debug(`Exported to: ${driveFile.id}`);
this.notificationService.createNotification(user.id, 'exportCompleted', {
exportedEntity: 'blocking',

11
packages/backend/src/queue/processors/ExportClipsProcessorService.ts
View file

@ -51,17 +51,18 @@ export class ExportClipsProcessorService {
@bindThis
public async process(job: Bull.Job<DbJobDataWithUser>): Promise<void> {
this.logger.info(`Exporting clips of ${job.data.user.id} ...`);
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) {
this.logger.debug(`Skip: user ${job.data.user.id} does not exist`);
return;
}
this.logger.info(`Exporting clips of ${job.data.user.id} ...`);
// Create temp file
const [path, cleanup] = await createTemp();
this.logger.info(`Temp file is ${path}`);
this.logger.debug(`Temp file is ${path}`);
try {
const stream = Writable.toWeb(fs.createWriteStream(path, { flags: 'a' }));
@ -75,12 +76,12 @@ export class ExportClipsProcessorService {
await writer.write(']');
await writer.close();
this.logger.succ(`Exported to: ${path}`);
this.logger.debug(`Exported to: ${path}`);
const fileName = 'clips-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.json';
const driveFile = await this.driveService.addFile({ user, path, name: fileName, force: true, ext: 'json' });
this.logger.succ(`Exported to: ${driveFile.id}`);
this.logger.debug(`Exported to: ${driveFile.id}`);
this.notificationService.createNotification(user.id, 'exportCompleted', {
exportedEntity: 'clip',

11
packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts
View file

@ -45,16 +45,17 @@ export class ExportCustomEmojisProcessorService {
@bindThis
public async process(job: Bull.Job): Promise<void> {
this.logger.info('Exporting custom emojis ...');
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) {
this.logger.debug(`Skip: user ${job.data.user.id} does not exist`);
return;
}
this.logger.info(`Exporting custom emojis of ${job.data.user.id} ...`);
const [path, cleanup] = await createTempDir();
this.logger.info(`Temp dir is ${path}`);
this.logger.debug(`Temp dir is ${path}`);
const metaPath = path + '/meta.json';
@ -130,12 +131,12 @@ export class ExportCustomEmojisProcessorService {
zlib: { level: 0 },
});
archiveStream.on('close', async () => {
this.logger.succ(`Exported to: ${archivePath}`);
this.logger.debug(`Exported to: ${archivePath}`);
const fileName = 'custom-emojis-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.zip';
const driveFile = await this.driveService.addFile({ user, path: archivePath, name: fileName, force: true });
this.logger.succ(`Exported to: ${driveFile.id}`);
this.logger.debug(`Exported to: ${driveFile.id}`);
this.notificationService.createNotification(user.id, 'exportCompleted', {
exportedEntity: 'customEmoji',

11
packages/backend/src/queue/processors/ExportFavoritesProcessorService.ts
View file

@ -45,17 +45,18 @@ export class ExportFavoritesProcessorService {
@bindThis
public async process(job: Bull.Job<DbJobDataWithUser>): Promise<void> {
this.logger.info(`Exporting favorites of ${job.data.user.id} ...`);
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) {
this.logger.debug(`Skip: user ${job.data.user.id} does not exist`);
return;
}
this.logger.info(`Exporting favorites of ${job.data.user.id} ...`);
// Create temp file
const [path, cleanup] = await createTemp();
this.logger.info(`Temp file is ${path}`);
this.logger.debug(`Temp file is ${path}`);
try {
const stream = fs.createWriteStream(path, { flags: 'a' });
@ -119,12 +120,12 @@ export class ExportFavoritesProcessorService {
await write(']');
stream.end();
this.logger.succ(`Exported to: ${path}`);
this.logger.debug(`Exported to: ${path}`);
const fileName = 'favorites-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.json';
const driveFile = await this.driveService.addFile({ user, path, name: fileName, force: true, ext: 'json' });
this.logger.succ(`Exported to: ${driveFile.id}`);
this.logger.debug(`Exported to: ${driveFile.id}`);
this.notificationService.createNotification(user.id, 'exportCompleted', {
exportedEntity: 'favorite',

11
packages/backend/src/queue/processors/ExportFollowingProcessorService.ts
View file

@ -44,17 +44,18 @@ export class ExportFollowingProcessorService {
@bindThis
public async process(job: Bull.Job<DbExportFollowingData>): Promise<void> {
this.logger.info(`Exporting following of ${job.data.user.id} ...`);
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) {
this.logger.debug(`Skip: user ${job.data.user.id} does not exist`);
return;
}
this.logger.info(`Exporting following of ${job.data.user.id} ...`);
// Create temp file
const [path, cleanup] = await createTemp();
this.logger.info(`Temp file is ${path}`);
this.logger.debug(`Temp file is ${path}`);
try {
const stream = fs.createWriteStream(path, { flags: 'a' });
@ -109,12 +110,12 @@ export class ExportFollowingProcessorService {
}
stream.end();
this.logger.succ(`Exported to: ${path}`);
this.logger.debug(`Exported to: ${path}`);
const fileName = 'following-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.csv';
const driveFile = await this.driveService.addFile({ user, path, name: fileName, force: true, ext: 'csv' });
this.logger.succ(`Exported to: ${driveFile.id}`);
this.logger.debug(`Exported to: ${driveFile.id}`);
this.notificationService.createNotification(user.id, 'exportCompleted', {
exportedEntity: 'following',

11
packages/backend/src/queue/processors/ExportMutingProcessorService.ts
View file

@ -40,17 +40,18 @@ export class ExportMutingProcessorService {
@bindThis
public async process(job: Bull.Job<DbJobDataWithUser>): Promise<void> {
this.logger.info(`Exporting muting of ${job.data.user.id} ...`);
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) {
this.logger.debug(`Skip: user ${job.data.user.id} does not exist`);
return;
}
this.logger.debug(`Exporting muting of ${job.data.user.id} ...`);
// Create temp file
const [path, cleanup] = await createTemp();
this.logger.info(`Temp file is ${path}`);
this.logger.debug(`Temp file is ${path}`);
try {
const stream = fs.createWriteStream(path, { flags: 'a' });
@ -106,12 +107,12 @@ export class ExportMutingProcessorService {
}
stream.end();
this.logger.succ(`Exported to: ${path}`);
this.logger.debug(`Exported to: ${path}`);
const fileName = 'mute-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.csv';
const driveFile = await this.driveService.addFile({ user, path, name: fileName, force: true, ext: 'csv' });
this.logger.succ(`Exported to: ${driveFile.id}`);
this.logger.debug(`Exported to: ${driveFile.id}`);
this.notificationService.createNotification(user.id, 'exportCompleted', {
exportedEntity: 'muting',

11
packages/backend/src/queue/processors/ExportNotesProcessorService.ts
View file

@ -120,17 +120,18 @@ export class ExportNotesProcessorService {
@bindThis
public async process(job: Bull.Job<DbJobDataWithUser>): Promise<void> {
this.logger.info(`Exporting notes of ${job.data.user.id} ...`);
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) {
this.logger.debug(`Skip: user ${job.data.user.id} does not exist`);
return;
}
this.logger.info(`Exporting notes of ${job.data.user.id} ...`);
// Create temp file
const [path, cleanup] = await createTemp();
this.logger.info(`Temp file is ${path}`);
this.logger.debug(`Temp file is ${path}`);
try {
// メモリが足りなくならないようにストリームで処理する
@ -146,12 +147,12 @@ export class ExportNotesProcessorService {
.pipeThrough(new TextEncoderStream())
.pipeTo(new FileWriterStream(path));
this.logger.succ(`Exported to: ${path}`);
this.logger.debug(`Exported to: ${path}`);
const fileName = 'notes-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.json';
const driveFile = await this.driveService.addFile({ user, path, name: fileName, force: true, ext: 'json' });
this.logger.succ(`Exported to: ${driveFile.id}`);
this.logger.debug(`Exported to: ${driveFile.id}`);
this.notificationService.createNotification(user.id, 'exportCompleted', {
exportedEntity: 'note',

11
packages/backend/src/queue/processors/ExportUserListsProcessorService.ts
View file

@ -43,13 +43,14 @@ export class ExportUserListsProcessorService {
@bindThis
public async process(job: Bull.Job<DbJobDataWithUser>): Promise<void> {
this.logger.info(`Exporting user lists of ${job.data.user.id} ...`);
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) {
this.logger.debug(`Skip: user ${job.data.user.id} does not exist`);
return;
}
this.logger.info(`Exporting user lists of ${job.data.user.id} ...`);
const lists = await this.userListsRepository.findBy({
userId: user.id,
});
@ -57,7 +58,7 @@ export class ExportUserListsProcessorService {
// Create temp file
const [path, cleanup] = await createTemp();
this.logger.info(`Temp file is ${path}`);
this.logger.debug(`Temp file is ${path}`);
try {
const stream = fs.createWriteStream(path, { flags: 'a' });
@ -85,12 +86,12 @@ export class ExportUserListsProcessorService {
}
stream.end();
this.logger.succ(`Exported to: ${path}`);
this.logger.debug(`Exported to: ${path}`);
const fileName = 'user-lists-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.csv';
const driveFile = await this.driveService.addFile({ user, path, name: fileName, force: true, ext: 'csv' });
this.logger.succ(`Exported to: ${driveFile.id}`);
this.logger.debug(`Exported to: ${driveFile.id}`);
this.notificationService.createNotification(user.id, 'exportCompleted', {
exportedEntity: 'userList',

15
packages/backend/src/queue/processors/ImportAntennasProcessorService.ts
View file

@ -8,7 +8,7 @@ import _Ajv from 'ajv';
import { IdService } from '@/core/IdService.js';
import { GlobalEventService } from '@/core/GlobalEventService.js';
import Logger from '@/logger.js';
import type { AntennasRepository } from '@/models/_.js';
import type { AntennasRepository, UsersRepository } from '@/models/_.js';
import { DI } from '@/di-symbols.js';
import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
@ -59,6 +59,9 @@ export class ImportAntennasProcessorService {
@Inject(DI.antennasRepository)
private antennasRepository: AntennasRepository,
@Inject(DI.usersRepository)
private usersRepository: UsersRepository,
private queueLoggerService: QueueLoggerService,
private idService: IdService,
private globalEventService: GlobalEventService,
@ -68,6 +71,14 @@ export class ImportAntennasProcessorService {
@bindThis
public async process(job: Bull.Job<DBAntennaImportJobData>): Promise<void> {
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) {
this.logger.debug(`Skip: user ${job.data.user.id} does not exist`);
return;
}
this.logger.debug(`Importing blocking of ${job.data.user.id} ...`);
const now = new Date();
try {
for (const antenna of job.data.antenna) {
@ -92,7 +103,7 @@ export class ImportAntennasProcessorService {
withReplies: antenna.withReplies,
withFile: antenna.withFile,
});
this.logger.succ('Antenna created: ' + result.id);
this.logger.debug('Antenna created: ' + result.id);
this.globalEventService.publishInternalEvent('antennaCreated', result);
}
} catch (err: any) {

10
packages/backend/src/queue/processors/ImportBlockingProcessorService.ts
View file

@ -40,10 +40,9 @@ export class ImportBlockingProcessorService {
@bindThis
public async process(job: Bull.Job<DbUserImportJobData>): Promise<void> {
this.logger.info(`Importing blocking of ${job.data.user.id} ...`);
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) {
this.logger.debug(`Skip: user ${job.data.user.id} does not exist`);
return;
}
@ -51,14 +50,17 @@ export class ImportBlockingProcessorService {
id: job.data.fileId,
});
if (file == null) {
this.logger.debug(`Skip: file ${job.data.fileId} does not exist`);
return;
}
this.logger.debug(`Importing blocking of ${job.data.user.id} ...`);
const csv = await this.downloadService.downloadTextFile(file.url);
const targets = csv.trim().split('\n');
this.queueService.createImportBlockingToDbJob({ id: user.id }, targets);
this.logger.succ('Import jobs created');
this.logger.debug('Import jobs created');
}
@bindThis
@ -93,7 +95,7 @@ export class ImportBlockingProcessorService {
// skip myself
if (target.id === job.data.user.id) return;
this.logger.info(`Block ${target.id} ...`);
this.logger.debug(`Block ${target.id} ...`);
this.queueService.createBlockJob([{ from: { id: user.id }, to: { id: target.id }, silent: true }]);
} catch (e) {

11
packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts
View file

@ -46,18 +46,19 @@ export class ImportCustomEmojisProcessorService {
@bindThis
public async process(job: Bull.Job<DbUserImportJobData>): Promise<void> {
this.logger.info('Importing custom emojis ...');
const file = await this.driveFilesRepository.findOneBy({
id: job.data.fileId,
});
if (file == null) {
this.logger.debug(`Skip: file ${job.data.fileId} does not exist`);
return;
}
this.logger.info(`Importing custom emojis from ${file.id} (${file.name}) ...`);
const [path, cleanup] = await createTempDir();
this.logger.info(`Temp dir is ${path}`);
this.logger.debug(`Temp dir is ${path}`);
const destPath = path + '/emojis.zip';
@ -73,7 +74,7 @@ export class ImportCustomEmojisProcessorService {
const outputPath = path + '/emojis';
try {
this.logger.succ(`Unzipping to ${outputPath}`);
this.logger.debug(`Unzipping to ${outputPath}`);
ZipReader.withDestinationPath(outputPath).viaBuffer(await fs.promises.readFile(destPath));
const metaRaw = fs.readFileSync(outputPath + '/meta.json', 'utf-8');
const meta = JSON.parse(metaRaw);
@ -126,7 +127,7 @@ export class ImportCustomEmojisProcessorService {
cleanup();
this.logger.succ('Imported');
this.logger.debug('Imported');
} catch (e) {
this.logger.error('Error importing custom emojis:', e as Error);
cleanup();

10
packages/backend/src/queue/processors/ImportFollowingProcessorService.ts
View file

@ -40,10 +40,9 @@ export class ImportFollowingProcessorService {
@bindThis
public async process(job: Bull.Job<DbUserImportJobData>): Promise<void> {
this.logger.info(`Importing following of ${job.data.user.id} ...`);
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) {
this.logger.debug(`Skip: user ${job.data.user.id} does not exist`);
return;
}
@ -51,14 +50,17 @@ export class ImportFollowingProcessorService {
id: job.data.fileId,
});
if (file == null) {
this.logger.debug(`Skip: file ${job.data.fileId} does not exist`);
return;
}
this.logger.info(`Importing following of ${job.data.user.id} ...`);
const csv = await this.downloadService.downloadTextFile(file.url);
const targets = csv.trim().split('\n');
this.queueService.createImportFollowingToDbJob({ id: user.id }, targets, job.data.withReplies);
this.logger.succ('Import jobs created');
this.logger.debug('Import jobs created');
}
@bindThis
@ -93,7 +95,7 @@ export class ImportFollowingProcessorService {
// skip myself
if (target.id === job.data.user.id) return;
this.logger.info(`Follow ${target.id} ${job.data.withReplies ? 'with replies' : 'without replies'} ...`);
this.logger.debug(`Follow ${target.id} ${job.data.withReplies ? 'with replies' : 'without replies'} ...`);
this.queueService.createFollowJob([{ from: user, to: { id: target.id }, silent: true, withReplies: job.data.withReplies }]);
} catch (e) {

10
packages/backend/src/queue/processors/ImportMutingProcessorService.ts
View file

@ -41,10 +41,9 @@ export class ImportMutingProcessorService {
@bindThis
public async process(job: Bull.Job<DbUserImportJobData>): Promise<void> {
this.logger.info(`Importing muting of ${job.data.user.id} ...`);
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) {
this.logger.debug(`Skip: user ${job.data.user.id} does not exist`);
return;
}
@ -52,9 +51,12 @@ export class ImportMutingProcessorService {
id: job.data.fileId,
});
if (file == null) {
this.logger.debug(`Skip: file ${job.data.fileId} does not exist`);
return;
}
this.logger.info(`Importing muting of ${job.data.user.id} ...`);
const csv = await this.downloadService.downloadTextFile(file.url);
let linenum = 0;
@ -89,7 +91,7 @@ export class ImportMutingProcessorService {
// skip myself
if (target.id === job.data.user.id) continue;
this.logger.info(`Mute[${linenum}] ${target.id} ...`);
this.logger.debug(`Mute[${linenum}] ${target.id} ...`);
await this.userMutingService.mute(user, target);
} catch (e) {
@ -97,6 +99,6 @@ export class ImportMutingProcessorService {
}
}
this.logger.succ('Imported');
this.logger.debug('Imported');
}
}

22
packages/backend/src/queue/processors/ImportNotesProcessorService.ts
View file

@ -159,10 +159,9 @@ export class ImportNotesProcessorService {
@bindThis
public async process(job: Bull.Job<DbNoteImportJobData>): Promise<void> {
this.logger.info(`Starting note import of ${job.data.user.id} ...`);
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) {
this.logger.debug(`Skip: user ${job.data.user.id} does not exist`);
return;
}
@ -170,9 +169,12 @@ export class ImportNotesProcessorService {
id: job.data.fileId,
});
if (file == null) {
this.logger.debug(`Skip: file ${job.data.fileId} does not exist`);
return;
}
this.logger.info(`Starting note import of ${job.data.user.id} ...`);
let folder = await this.driveFoldersRepository.findOneBy({ name: 'Imports', userId: job.data.user.id });
if (folder == null) {
await this.driveFoldersRepository.insert({ id: this.idService.gen(), name: 'Imports', userId: job.data.user.id });
@ -184,7 +186,7 @@ export class ImportNotesProcessorService {
if (type === 'Twitter' || file.name.startsWith('twitter') && file.name.endsWith('.zip')) {
const [path, cleanup] = await createTempDir();
this.logger.info(`Temp dir is ${path}`);
this.logger.debug(`Temp dir is ${path}`);
const destPath = path + '/twitter.zip';
@ -198,7 +200,7 @@ export class ImportNotesProcessorService {
const outputPath = path + '/twitter';
try {
this.logger.succ(`Unzipping to ${outputPath}`);
this.logger.debug(`Unzipping to ${outputPath}`);
ZipReader.withDestinationPath(outputPath).viaBuffer(await fsp.readFile(destPath));
const unprocessedTweets = this.parseTwitterFile(await fsp.readFile(outputPath + '/data/tweets.js', 'utf-8'));
@ -212,7 +214,7 @@ export class ImportNotesProcessorService {
} else if (type === 'Facebook' || file.name.startsWith('facebook-') && file.name.endsWith('.zip')) {
const [path, cleanup] = await createTempDir();
this.logger.info(`Temp dir is ${path}`);
this.logger.debug(`Temp dir is ${path}`);
const destPath = path + '/facebook.zip';
@ -226,7 +228,7 @@ export class ImportNotesProcessorService {
const outputPath = path + '/facebook';
try {
this.logger.succ(`Unzipping to ${outputPath}`);
this.logger.debug(`Unzipping to ${outputPath}`);
ZipReader.withDestinationPath(outputPath).viaBuffer(await fsp.readFile(destPath));
const postsJson = await fsp.readFile(outputPath + '/your_activity_across_facebook/posts/your_posts__check_ins__photos_and_videos_1.json', 'utf-8');
const posts = JSON.parse(postsJson);
@ -243,7 +245,7 @@ export class ImportNotesProcessorService {
} else if (file.name.endsWith('.zip')) {
const [path, cleanup] = await createTempDir();
this.logger.info(`Temp dir is ${path}`);
this.logger.debug(`Temp dir is ${path}`);
const destPath = path + '/unknown.zip';
@ -257,7 +259,7 @@ export class ImportNotesProcessorService {
const outputPath = path + '/unknown';
try {
this.logger.succ(`Unzipping to ${outputPath}`);
this.logger.debug(`Unzipping to ${outputPath}`);
ZipReader.withDestinationPath(outputPath).viaBuffer(await fsp.readFile(destPath));
const isInstagram = type === 'Instagram' || fs.existsSync(outputPath + '/instagram_live') || fs.existsSync(outputPath + '/instagram_ads_and_businesses');
const isOutbox = type === 'Mastodon' || fs.existsSync(outputPath + '/outbox.json');
@ -301,7 +303,7 @@ export class ImportNotesProcessorService {
} else if (job.data.type === 'Misskey' || file.name.startsWith('notes-') && file.name.endsWith('.json')) {
const [path, cleanup] = await createTemp();
this.logger.info(`Temp dir is ${path}`);
this.logger.debug(`Temp dir is ${path}`);
try {
await fsp.writeFile(path, '', 'utf-8');
@ -318,7 +320,7 @@ export class ImportNotesProcessorService {
cleanup();
}
this.logger.succ('Import jobs created');
this.logger.debug('Import jobs created');
}
@bindThis

8
packages/backend/src/queue/processors/ImportUserListsProcessorService.ts
View file

@ -49,10 +49,9 @@ export class ImportUserListsProcessorService {
@bindThis
public async process(job: Bull.Job<DbUserImportJobData>): Promise<void> {
this.logger.info(`Importing user lists of ${job.data.user.id} ...`);
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) {
this.logger.debug(`Skip: user ${job.data.user.id} does not exist`);
return;
}
@ -60,9 +59,12 @@ export class ImportUserListsProcessorService {
id: job.data.fileId,
});
if (file == null) {
this.logger.debug(`Skip: file ${job.data.fileId} does not exist`);
return;
}
this.logger.info(`Importing user lists of ${job.data.user.id} ...`);
const csv = await this.downloadService.downloadTextFile(file.url);
let linenum = 0;
@ -107,6 +109,6 @@ export class ImportUserListsProcessorService {
}
}
this.logger.succ('Imported');
this.logger.debug('Imported');
}
}