Function batchMatchManga

Source

export async function batchMatchManga(
  mangaList: KenmeiManga[],
  token?: string,
  config: Partial<SearchServiceConfig> = {},
  progressCallback?: (
    current: number,
    total: number,
    currentTitle?: string,
  ) => void,
  shouldCancel?: () => boolean,
  abortSignal?: AbortSignal,
): Promise<MangaMatchResult[]> {
  // Ensure we have the latest cache data
  syncWithClientCache();

  const searchConfig = { ...DEFAULT_SEARCH_CONFIG, ...config };
  const results: MangaMatchResult[] = [];

  // Create a set to track which manga have been reported in the progress
  const reportedIndices = new Set<number>();

  // Function to check if the operation should be cancelled
  const checkCancellation = () => {
    // Check the abort signal first
    if (abortSignal && abortSignal.aborted) {
      console.log("Batch matching process aborted by abort signal");
      throw new Error("Operation aborted by abort signal");
    }

    // Then check the cancellation function
    if (shouldCancel && shouldCancel()) {
      console.log("Batch matching process cancelled by user");
      throw new Error("Operation cancelled by user");
    }

    return false;
  };

  // Update progress with deduplication
  const updateProgress = (index: number, title?: string) => {
    if (progressCallback && !reportedIndices.has(index)) {
      reportedIndices.add(index);
      progressCallback(reportedIndices.size, mangaList.length, title);
    }
  };

  try {
    // First, check which manga are already in cache
    const cachedResults: Record<number, AniListManga[]> = {};
    const uncachedManga: { index: number; manga: KenmeiManga }[] = [];

    // Track manga IDs if we have them (for batch fetching)
    const knownMangaIds: { index: number; id: number }[] = [];

    // If we're bypassing cache, treat all manga as uncached
    if (searchConfig.bypassCache) {
      console.log(
        `🚨 FRESH SEARCH: Bypassing cache for all ${mangaList.length} manga titles`,
      );

      // Put all manga in the uncached list
      mangaList.forEach((manga, index) => {
        uncachedManga.push({ index, manga });
      });
    } else {
      console.log(`Checking cache for ${mangaList.length} manga titles...`);

      // Check cache for all manga first
      mangaList.forEach((manga, index) => {
        const cacheKey = generateCacheKey(manga.title);

        // If manga has a known AniList ID, we can batch fetch it
        if (manga.anilistId && Number.isInteger(manga.anilistId)) {
          knownMangaIds.push({ index, id: manga.anilistId });
        }
        // Otherwise check the cache
        else if (isCacheValid(cacheKey)) {
          // This manga is in cache
          cachedResults[index] = mangaCache[cacheKey].manga;
          console.log(`Found cached results for: ${manga.title}`);

          // Immediately update progress for cached manga
          updateProgress(index, manga.title);
        } else {
          // This manga needs to be fetched by search
          uncachedManga.push({ index, manga });
        }
      });

      console.log(
        `Found ${Object.keys(cachedResults).length} cached manga, ${knownMangaIds.length} have known IDs, ${uncachedManga.length} require searching`,
      );
    }

    // Check for cancellation
    checkCancellation();

    // First, fetch all manga with known IDs in batches (only if not bypassing cache)
    if (knownMangaIds.length > 0 && !searchConfig.bypassCache) {
      const ids = knownMangaIds.map((item) => item.id);
      console.log(`Fetching ${ids.length} manga with known IDs...`);

      // Get manga by IDs in batches, passing the abort signal
      const batchedManga = await getBatchedMangaIds(
        ids,
        token,
        shouldCancel,
        abortSignal,
      );

      // Create a map of ID to manga for easier lookup
      const mangaMap = new Map<number, AniListManga>();
      batchedManga.forEach((manga) => mangaMap.set(manga.id, manga));

      // Store the results in cachedResults by their original index
      knownMangaIds.forEach((item) => {
        const manga = mangaMap.get(item.id);
        if (manga) {
          cachedResults[item.index] = [manga]; // Store as array of one manga for consistency

          // Also store in the general cache to help future searches
          const title = mangaList[item.index].title;
          const cacheKey = generateCacheKey(title);
          mangaCache[cacheKey] = {
            manga: [manga],
            timestamp: Date.now(),
          };

          // Update progress for each found manga
          updateProgress(item.index, title);
        } else {
          // Manga ID was not found, add to uncached list for title search
          uncachedManga.push({
            index: item.index,
            manga: mangaList[item.index],
          });
        }
      });

      // Check for cancellation
      checkCancellation();
    }

    // Now process remaining uncached manga with strict concurrency control
    if (uncachedManga.length > 0) {
      // Create a semaphore to strictly limit concurrency - process one manga at a time
      const MAX_CONCURRENT = 1;
      let activeCount = 0;

      // Track processed manga to prevent duplicates
      const processedMangas = new Set<number>();

      // Create a queue that will be processed one by one
      const queue = [...uncachedManga];

      // Create a promise that we can use to wait for all processing to complete
      let resolve: (value: void | PromiseLike<void>) => void;
      let reject: (reason?: unknown) => void;
      const completionPromise = new Promise<void>((res, rej) => {
        resolve = res;
        reject = rej;
      });

      // Track if we've been cancelled
      let isCancelled = false;

      // Function to check if we're done processing all manga
      const checkIfDone = () => {
        if ((queue.length === 0 && activeCount === 0) || isCancelled) {
          resolve();
        }
      };

      // Function to start processing the next manga in the queue
      const processNext = async () => {
        // Check for cancellation
        try {
          if (checkCancellation()) {
            isCancelled = true;
            resolve(); // Resolve to unblock the main thread
            return;
          }
        } catch (error) {
          isCancelled = true;
          reject(error);
          return;
        }

        // If the queue is empty or we're cancelled, we're done
        if (queue.length === 0 || isCancelled) {
          checkIfDone();
          return;
        }

        // If we're at max concurrency, wait
        if (activeCount >= MAX_CONCURRENT) {
          return;
        }

        // Get the next manga from the queue
        const { index, manga } = queue.shift()!;

        // Skip if this manga has already been processed
        if (processedMangas.has(index)) {
          processNext();
          return;
        }

        // Mark this manga as being processed
        processedMangas.add(index);
        activeCount++;

        try {
          // Check cancellation again before searching
          if (checkCancellation()) {
            throw new Error("Operation cancelled by user");
          }

          // Double-check cache one more time before searching
          const cacheKey = generateCacheKey(manga.title);
          if (!searchConfig.bypassCache && isCacheValid(cacheKey)) {
            cachedResults[index] = mangaCache[cacheKey].manga;
            console.log(
              `Using cache for ${manga.title} (found during processing)`,
            );
            // Update progress for this manga
            updateProgress(index, manga.title);
          } else {
            // Search for this manga
            console.log(
              `Searching for manga: ${manga.title} (${reportedIndices.size}/${mangaList.length})`,
            );

            // Update progress for this manga before search
            updateProgress(index, manga.title);

            // Check cancellation again before making the API call
            if (checkCancellation()) {
              throw new Error("Operation cancelled by user");
            }

            const potentialMatches = await searchMangaByTitle(
              manga.title,
              token,
              searchConfig,
              abortSignal, // Pass the abort signal to the search function
            );

            // Store the results
            cachedResults[index] = potentialMatches.map((match) => match.manga);
          }
        } catch (error) {
          // Check if this was a cancellation
          if (
            error instanceof Error &&
            (error.message.includes("cancelled") ||
              error.message.includes("aborted"))
          ) {
            console.error(`Search cancelled for manga: ${manga.title}`);
            isCancelled = true;
            reject(error); // Reject to stop the process
            return;
          }

          console.error(`Error searching for manga: ${manga.title}`, error);
          // Store empty result on error
          cachedResults[index] = [];
        } finally {
          // Decrement the active count and process the next manga
          activeCount--;

          // Don't try to process more if we've been cancelled
          if (!isCancelled) {
            processNext();
          }

          // Check if we're done
          checkIfDone();
        }
      };

      // Start processing up to MAX_CONCURRENT manga
      for (let i = 0; i < Math.min(MAX_CONCURRENT, uncachedManga.length); i++) {
        processNext();
      }

      try {
        // Wait for all processing to complete
        await completionPromise;
      } catch (error) {
        console.log("Processing cancelled:", error);

        // If we got here due to cancellation, return the partial results we've managed to gather
        if (
          error instanceof Error &&
          (error.message.includes("cancelled") ||
            error.message.includes("aborted"))
        ) {
          console.log(
            `Cancellation completed, returning ${results.length} partial results`,
          );

          // Process whatever results we have so far
          for (let i = 0; i < mangaList.length; i++) {
            if (cachedResults[i]) {
              const manga = mangaList[i];
              const potentialMatches = cachedResults[i].map((anilistManga) => ({
                manga: anilistManga,
                confidence: calculateConfidence(manga.title, anilistManga),
              }));

              results.push({
                kenmeiManga: manga,
                anilistMatches: potentialMatches,
                selectedMatch:
                  potentialMatches.length > 0
                    ? potentialMatches[0].manga
                    : undefined,
                status: "pending",
              });
            }
          }

          return results;
        }

        // If it's a different kind of error, rethrow it
        throw error;
      }

      // Check for cancellation after the batch completes
      checkCancellation();
    }

    // First fill in the results array to match the mangaList length
    for (let i = 0; i < mangaList.length; i++) {
      results[i] = {
        kenmeiManga: mangaList[i],
        anilistMatches: [],
        status: "pending",
      } as MangaMatchResult; // Use empty arrays instead of null
    }

    // Fill in the results for manga we have matches for
    for (let i = 0; i < mangaList.length; i++) {
      // Check for cancellation periodically
      if (i % 10 === 0) {
        checkCancellation();
      }

      const manga = mangaList[i];
      const potentialMatches = cachedResults[i] || [];

      // Update progress for any remaining manga
      updateProgress(i, manga.title);

      // Fix mapping to create proper MangaMatch objects
      const potentialMatchesFixed = potentialMatches.map((match) => ({
        manga: match,
        confidence: calculateConfidence(manga.title, match),
      }));

      results[i] = {
        kenmeiManga: manga,
        anilistMatches: potentialMatchesFixed,
        selectedMatch:
          potentialMatchesFixed.length > 0
            ? potentialMatchesFixed[0].manga
            : undefined,
        status: "pending",
      };
    }

    // Filter out any null entries (though there shouldn't be any)
    return results.filter((result) => result !== null);
  } catch (error) {
    console.error("Error in batch matching process:", error);

    // If we got here due to cancellation, return whatever partial results we have
    if (
      error instanceof Error &&
      (error.message.includes("cancelled") || error.message.includes("aborted"))
    ) {
      console.log(
        `Cancellation detected, returning ${results.length} partial results`,
      );
      return results;
    }

    // Otherwise rethrow the error
    throw error;
  }
}
Copy