fix: perform format operation in chunks (#115)

* feat: perform format operation in chunks

* move mapSettledWithConcurrency to utils.ts

* swap concurrency calc method

* add back missing config check

* Update src/index.ts

Co-authored-by: fisker Cheung <lionkay@gmail.com>

* Remove useless calculations

Co-authored-by: fisker Cheung <lionkay@gmail.com>

---------

Co-authored-by: fisker Cheung <lionkay@gmail.com>

Author: cellison-figma

src/index.ts

@@ -1,6 +1,7 @@
 import fs from "node:fs/promises";
 import isBinaryPath from "is-binary-path";
 import stringify from "json-sorted-stringify";
+import os from "node:os";
 import path from "node:path";
 import process from "node:process";
 import Cache from "./cache.js";
@@ -21,6 +22,7 @@ import {
   getProjectPath,
   getStdin,
   getTargetsPaths,
+  mapSettledWithConcurrency,
 } from "./utils.js";
 import {
   fastRelativePath,
@@ -179,34 +181,32 @@ async function runGlobs(options: Options, pluginsDefaultOptions: PluginsOptions,
   const cache = shouldCache ? new Cache(cacheVersion, projectPath, getCacheRootPath(rootPath), options, stdout) : undefined;
   const prettier = await makePrettier(options, cache);
 
-  //TODO: Maybe do work in chunks here, as keeping too many formatted files in memory can be a problem
-  const filesResults = await Promise.allSettled(
-    filesPathsTargets.map(async (filePath) => {
-      const isIgnored = () => (ignoreManual ? ignoreManual(filePath) : getIgnoreResolved(filePath, ignoreNames));
-      const isCacheable = () => cache?.has(filePath, isIgnored);
-      const isExplicitlyIncluded = () => filesExplicitPathsSet.has(filePath);
-      const isForceIncluded = options.dump && isExplicitlyIncluded();
-      const isExcluded = cache ? !(await isCacheable()) : await isIgnored();
-      if (!isForceIncluded && isExcluded) return;
-      const getFormatOptions = async (): Promise<FormatOptions> => {
-        const editorConfig = options.editorConfig ? getEditorConfigFormatOptions(await getEditorConfigResolved(filePath, editorConfigNames)) : {};
-        const prettierConfig = prettierManualConfig || (options.config ? await getPrettierConfigResolved(filePath, prettierConfigNames) : {});
-        const formatOptions = { ...editorConfig, ...prettierConfig, ...options.formatOptions };
-        return formatOptions;
-      };
-      try {
-        if (options.check || options.list) {
-          return await prettier.checkWithPath(filePath, getFormatOptions, cliContextConfig, pluginsDefaultOptions, pluginsCustomOptions);
-        } else if (options.write) {
-          return await prettier.writeWithPath(filePath, getFormatOptions, cliContextConfig, pluginsDefaultOptions, pluginsCustomOptions);
-        } else {
-          return await prettier.formatWithPath(filePath, getFormatOptions, cliContextConfig, pluginsDefaultOptions, pluginsCustomOptions);
-        }
-      } finally {
-        spinner?.update(fastRelativePath(rootPath, filePath));
+  const concurrency = options.parallel ? (options.parallelWorkers ?? Math.max(1, (os.availableParallelism?.() ?? os.cpus().length) - 1)) : 1;
+  const filesResults = await mapSettledWithConcurrency(filesPathsTargets, concurrency, async (filePath) => {
+    const isIgnored = () => (ignoreManual ? ignoreManual(filePath) : getIgnoreResolved(filePath, ignoreNames));
+    const isCacheable = () => cache?.has(filePath, isIgnored);
+    const isExplicitlyIncluded = () => filesExplicitPathsSet.has(filePath);
+    const isForceIncluded = options.dump && isExplicitlyIncluded();
+    const isExcluded = cache ? !(await isCacheable()) : await isIgnored();
+    if (!isForceIncluded && isExcluded) return;
+    const getFormatOptions = async (): Promise<FormatOptions> => {
+      const editorConfig = options.editorConfig ? getEditorConfigFormatOptions(await getEditorConfigResolved(filePath, editorConfigNames)) : {};
+      const prettierConfig = prettierManualConfig || (options.config ? await getPrettierConfigResolved(filePath, prettierConfigNames) : {});
+      const formatOptions = { ...editorConfig, ...prettierConfig, ...options.formatOptions };
+      return formatOptions;
+    };
+    try {
+      if (options.check || options.list) {
+        return await prettier.checkWithPath(filePath, getFormatOptions, cliContextConfig, pluginsDefaultOptions, pluginsCustomOptions);
+      } else if (options.write) {
+        return await prettier.writeWithPath(filePath, getFormatOptions, cliContextConfig, pluginsDefaultOptions, pluginsCustomOptions);
+      } else {
+        return await prettier.formatWithPath(filePath, getFormatOptions, cliContextConfig, pluginsDefaultOptions, pluginsCustomOptions);
       }
-    }),
-  );
+    } finally {
+      spinner?.update(fastRelativePath(rootPath, filePath));
+    }
+  });
 
   spinner?.stop("Checking formatting...");
 

src/utils.ts

@@ -635,6 +635,36 @@ function zipObjectUnless<T extends Key, U>(keys: T[], values: U[], unless: (valu
   return map;
 }
 
+async function mapSettledWithConcurrency<T, R>(
+  values: readonly T[],
+  concurrency: number,
+  mapper: (value: T, index: number) => Promise<R>,
+): Promise<PromiseSettledResult<R>[]> {
+  const results = new Array<PromiseSettledResult<R>>(values.length);
+  let nextIndex = 0;
+
+  async function runNext(): Promise<void> {
+    while (true) {
+      const index = nextIndex++;
+      if (index >= values.length) return;
+
+      try {
+        results[index] = {
+          status: "fulfilled",
+          value: await mapper(values[index], index),
+        };
+      } catch (reason) {
+        results[index] = { status: "rejected", reason };
+      }
+    }
+  }
+
+  const runnersCount = Math.min(values.length, concurrency);
+  await Promise.all(Array.from({ length: runnersCount }, runNext));
+
+  return results;
+}
+
 export {
   castArray,
   fastJoinedPath,
@@ -671,6 +701,7 @@ export {
   isString,
   isTruthy,
   isUndefined,
+  mapSettledWithConcurrency,
   memoize,
   negate,
   noop,

test/__tests__/map-settled-with-concurrency.js

@@ -0,0 +1,22 @@
+import { expect, test } from "@jest/globals";
+import { mapSettledWithConcurrency } from "../../dist/utils.js";
+
+test("bounds concurrency and preserves settled-result order", async () => {
+  let active = 0;
+  let maxActive = 0;
+
+  const results = await mapSettledWithConcurrency([0, 1, 2, 3, 4, 5], 3, async (value) => {
+    active += 1;
+    maxActive = Math.max(maxActive, active);
+    await new Promise((resolve) => setImmediate(resolve));
+    active -= 1;
+    if (value === 2) throw new Error("expected failure");
+    return value * 2;
+  });
+
+  expect(maxActive).toBe(3);
+  expect(results.map((result) => result.status)).toEqual(["fulfilled", "fulfilled", "rejected", "fulfilled", "fulfilled", "fulfilled"]);
+  expect(results[0]).toEqual({ status: "fulfilled", value: 0 });
+  expect(results[2]).toMatchObject({ status: "rejected", reason: new Error("expected failure") });
+  expect(results[5]).toEqual({ status: "fulfilled", value: 10 });
+});