Function processMangaInBatches

processMangaInBatches<T>(
    entries: KenmeiManga[],
    processFn: (batch: KenmeiManga[]) => Promise<T[]>,
    batchSize?: number,
): Promise<T[]>

Process manga entries in batches to avoid memory issues and enable progress tracking.

Type Parameters

T

Parameters

entries: KenmeiManga[]
Array of Kenmei manga entries.
processFn: (batch: KenmeiManga[]) => Promise<T[]>
Async function to process each batch.
batchSize: number = 50
Batch size (default: 50).

Returns Promise<T[]>

Aggregated results from all batches.

Source

export async function processMangaInBatches<T>(
  entries: KenmeiManga[],
  processFn: (batch: KenmeiManga[]) => Promise<T[]>,
  batchSize = 50,
): Promise<T[]> {
  return withGroupAsync(
    `[KenmeiProcessor] Process Batches (${entries.length} entries)`,
    async () => {
      const results: T[] = [];

      // Process in batches
      for (let i = 0; i < entries.length; i += batchSize) {
        const batch = entries.slice(i, i + batchSize);
        if (batch.length === 0) continue;
        const batchResults = await processFn(batch);
        if (batchResults && batchResults.length > 0)
          results.push(...batchResults);
      }

      return results;
    },
  );
}