feat: initial implementation (#1)

feat: initial code
2026-09-30 13:32:57 +08:00 · 2026-02-13 19:14:26 -05:00
parent 720d6f6a8f
commit f19b3a7e84
11 changed files with 2253 additions and 0 deletions
@@ -0,0 +1,40 @@
+name: Lint & Format
+
+on:
+  workflow_dispatch:
+  pull_request:
+  push:
+    branches: [main]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions: {}
+
+jobs:
+  format-lint:
+    name: Lint & Format
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+        with:
+          persist-credentials: false
+      - name: Setup pnpm
+        uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061 # v4.2.0
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@efa25f7f19611383d5b0ccf2d1c8914531636bf9
+        with:
+          toolchain: stable
+          components: clippy, rustfmt
+      - name: Install dependencies
+        run: pnpm install
+      - name: Check linting
+        run: pnpm rust:lint
+        if: ${{ !cancelled() }}
+      - name: Check formatting
+        run: pnpm rust:format
+        if: ${{ !cancelled() }}
@@ -0,0 +1,17 @@
+node_modules/
+dist/
+*.node
+*.node.js
+target/
+Cargo.lock
+.DS_Store
+*.swp
+*.swo
+*~
+.idea/
+.vscode/
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+*.tgz
@@ -0,0 +1,8 @@
+{
+  "singleQuote": true,
+  "trailingComma": "all",
+  "printWidth": 120,
+  "semi": true,
+  "organizeImportsSkipDestructiveCodeActions": true,
+  "plugins": ["prettier-plugin-organize-imports"]
+}
@@ -0,0 +1,28 @@
+[package]
+name = "walkrs"
+version = "0.0.1"
+edition = "2024"
+publish = false
+
+[lib]
+crate-type = ["cdylib", "rlib"]
+
+[dependencies]
+napi = { version = "2", features = ["async"] }
+napi-derive = "2"
+tokio = { version = "1", features = ["full"] }
+ignore = "0.4"
+globset = "0.4"
+num_cpus = "1"
+anyhow = "1"
+serde = { version = "1", features = ["derive"] }
+chrono = "0.4"
+
+[build-dependencies]
+built = { version = "0.7", features = ["chrono"] }
+
+[profile.release]
+lto = true
+codegen-units = 1
+opt-level = 3
+strip = true
@@ -0,0 +1,38 @@
+# @immich/walkrs
+
+High-performance file tree walker for Node.js, built with Rust and the battle-tested [ignore](https://github.com/BurntSushi/ripgrep/tree/master/crates/ignore) crate from ripgrep.
+
+## Background
+
+This project grew out of the need for fast and reliable external library scanning in [Immich](https://github.com/immich-app/immich). Immich needed to scan very large photo libraries efficiently, often containing hundreds of thousands of files across complex directory structures.
+
+By leveraging Rust's performance and the same ignore logic used in ripgrep (one of the fastest file search tools available), walkrs delivers exceptional speed and reliability for file tree traversal.
+
+## Installation
+
+```bash
+pnpm add @immich/walkrs
+```
+
+## Usage
+
+```typescript
+import { walk } from '@immich/walkrs';
+
+// Simple usage - walk a directory
+const files = await walk({
+  root_paths: ['/path/to/scan'],
+});
+
+// Advanced usage with filtering
+const photos = await walk({
+  root_paths: ['/photos', '/backup/photos'],
+  supported_extensions: ['.jpg', '.png', '.heic', '.webp'],
+  exclusion_patterns: ['**/.stfolder/**'],
+  include_hidden: false,
+});
+```
+
+## Performance
+
+walkrs is designed to handle massive directory trees efficiently. It is greatly affected by multithreading: In benchmarks we have scanned 11M files in under 30 seconds over NFS on a machine with 32 CPU threads available. When restricting walkrs to a single thread, the time for the same task goes up to 208 seconds. Compare this with the single-threaded fast-glob which uses 360 seconds for the same task.
@@ -0,0 +1,80 @@
+import js from '@eslint/js';
+import eslintPluginPrettierRecommended from 'eslint-plugin-prettier/recommended';
+import eslintPluginUnicorn from 'eslint-plugin-unicorn';
+import globals from 'globals';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import typescriptEslint from 'typescript-eslint';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+export default typescriptEslint.config([
+  eslintPluginUnicorn.configs.recommended,
+  eslintPluginPrettierRecommended,
+  js.configs.recommended,
+  typescriptEslint.configs.recommended,
+  {
+    ignores: ['eslint.config.mjs'],
+  },
+  {
+    languageOptions: {
+      globals: {
+        ...globals.node,
+      },
+
+      parser: typescriptEslint.parser,
+      ecmaVersion: 5,
+      sourceType: 'module',
+
+      parserOptions: {
+        project: 'tsconfig.json',
+        tsconfigRootDir: __dirname,
+      },
+    },
+
+    rules: {
+      '@typescript-eslint/interface-name-prefix': 'off',
+      '@typescript-eslint/explicit-function-return-type': 'off',
+      '@typescript-eslint/explicit-module-boundary-types': 'off',
+      '@typescript-eslint/no-explicit-any': 'off',
+      '@typescript-eslint/no-floating-promises': 'error',
+      'unicorn/prevent-abbreviations': 'off',
+      'unicorn/filename-case': 'off',
+      'unicorn/no-null': 'off',
+      'unicorn/prefer-top-level-await': 'off',
+      'unicorn/prefer-event-target': 'off',
+      'unicorn/no-thenable': 'off',
+      'unicorn/import-style': 'off',
+      'unicorn/prefer-structured-clone': 'off',
+      'unicorn/no-for-loop': 'off',
+      '@typescript-eslint/await-thenable': 'error',
+      '@typescript-eslint/no-misused-promises': 'error',
+      'require-await': 'off',
+      '@typescript-eslint/require-await': 'error',
+      curly: 2,
+      'prettier/prettier': 0,
+      'object-shorthand': ['error', 'always'],
+
+      'no-restricted-imports': [
+        'error',
+        {
+          patterns: [
+            {
+              group: ['.*'],
+              message: 'Relative imports are not allowed.',
+            },
+          ],
+        },
+      ],
+
+      '@typescript-eslint/no-unused-vars': [
+        'warn',
+        {
+          argsIgnorePattern: '^_',
+          varsIgnorePattern: '^_',
+        },
+      ],
+    },
+  },
+]);
@@ -0,0 +1,80 @@
+{
+  "name": "@immich/walkrs",
+  "version": "0.0.1",
+  "description": "Fast file tree walker for Node.js, built with ripgrep's ignore crate",
+  "private": true,
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "packageManager": "pnpm@10.28.0+sha512.05df71d1421f21399e053fde567cea34d446fa02c76571441bfc1c7956e98e363088982d940465fd34480d4d90a0668bc12362f8aa88000a64e83d0b0e47be48",
+  "engines": {
+    "pnpm": ">=10.0.0"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "require": "./dist/index.js",
+      "import": "./dist/index.js"
+    }
+  },
+  "napi": {
+    "name": "walkrs",
+    "triples": {
+      "additional": [
+        "aarch64-apple-darwin",
+        "x86_64-apple-darwin",
+        "aarch64-unknown-linux-gnu",
+        "x86_64-unknown-linux-gnu",
+        "aarch64-unknown-linux-musl",
+        "x86_64-unknown-linux-musl",
+        "x86_64-pc-windows-msvc"
+      ]
+    },
+    "package": {
+      "name": "@immich/walkrs"
+    }
+  },
+  "license": "GNU Affero General Public License version 3",
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "napi build --platform --release dist",
+    "build:debug": "napi build --platform --debug dist",
+    "build:release": "napi build --platform --release dist",
+    "prepublishOnly": "pnpm run build:release",
+    "test": "node --loader tsx ./test.ts",
+    "typecheck": "tsc --noEmit",
+    "ts:check": "tsc --noEmit",
+    "ts:format": "prettier --check .",
+    "ts:format:fix": "prettier --write .",
+    "ts:lint": "eslint \"src/**/*.ts\" --max-warnings 0",
+    "rust:format": "cargo fmt -- --check",
+    "rust:format:fix": "cargo fmt",
+    "rust:lint": "cargo clippy --all-targets --all-features -- -D warnings",
+    "format": "pnpm run ts:format && pnpm run rust:format",
+    "format:fix": "pnpm run ts:format:fix && pnpm run rust:format:fix"
+  },
+  "devDependencies": {
+    "@eslint/js": "^9.8.0",
+    "eslint": "^9.14.0",
+    "eslint-config-prettier": "^10.1.8",
+    "eslint-plugin-prettier": "^5.1.3",
+    "eslint-plugin-unicorn": "^62.0.0",
+    "@napi-rs/cli": "^2.18.0",
+    "@types/node": "^24.10.11",
+    "globals": "^16.0.0",
+    "prettier": "^3.7.4",
+    "prettier-plugin-organize-imports": "^4.0.0",
+    "prettier-plugin-rust": "^0.1.9",
+    "typescript-eslint": "^8.28.0",
+    "tsx": "^4.7.0",
+    "typescript": "^5.9.2"
+  },
+  "volta": {
+    "node": "24.13.0"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/immich-app/immich"
+  }
+}
@@ -0,0 +1,36 @@
+# Rust formatting configuration to match TypeScript style
+edition = "2021"
+
+# Use 2 spaces for indentation (matching prettier)
+hard_tabs = false
+tab_spaces = 2
+
+# Match prettier's printWidth of 120
+max_width = 120
+
+# Formatting options
+newline_style = "Unix"
+use_small_heuristics = "Default"
+reorder_imports = true
+reorder_modules = true
+remove_nested_parens = true
+
+# Function and type formatting
+fn_single_line = false
+where_single_line = false
+
+# Import formatting
+imports_granularity = "Crate"
+
+# Trailing comma (matching prettier's trailingComma: all)
+trailing_comma = "Vertical"
+
+# Comment formatting
+wrap_comments = false
+format_code_in_doc_comments = false
+normalize_comments = false
+
+# Misc
+format_strings = false
+format_macro_matchers = true
+format_macro_bodies = true
@@ -0,0 +1,174 @@
+use napi::bindgen_prelude::*;
+use napi_derive::napi;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, Ordering};
+
+#[napi(object)]
+pub struct WalkOptions {
+  #[napi(ts_type = "string[]")]
+  pub paths: Vec<String>,
+
+  #[napi(ts_type = "boolean | undefined")]
+  pub include_hidden: Option<bool>,
+
+  #[napi(ts_type = "string[] | undefined")]
+  pub exclusion_patterns: Option<Vec<String>>,
+
+  #[napi(ts_type = "string[] | undefined")]
+  pub extensions: Option<Vec<String>>,
+}
+
+#[napi(ts_return_type = "Promise<string[]>")]
+pub async fn walk(options: WalkOptions) -> Result<Vec<String>> {
+  if options.paths.is_empty() {
+    return Ok(Vec::new());
+  }
+
+  let root_paths = options.paths.clone();
+  let include_hidden = options.include_hidden.unwrap_or(false);
+  let exclusion_patterns = options.exclusion_patterns.unwrap_or_default();
+  let extensions = options.extensions.unwrap_or_default();
+
+  tokio::task::spawn_blocking(move || find_files(&root_paths, include_hidden, &exclusion_patterns, &extensions))
+    .await
+    .map_err(|e| Error::new(Status::GenericFailure, format!("Task join error: {}", e)))?
+}
+
+struct BatchSender {
+  batch: Vec<String>,
+  tx: std::sync::mpsc::Sender<Vec<String>>,
+  limit: usize,
+}
+
+impl BatchSender {
+  fn new(tx: std::sync::mpsc::Sender<Vec<String>>, limit: usize) -> Self {
+    Self {
+      batch: Vec::with_capacity(limit),
+      tx,
+      limit,
+    }
+  }
+
+  fn send(&mut self, item: String) -> std::result::Result<(), ()> {
+    self.batch.push(item);
+    if self.batch.len() >= self.limit {
+      self.flush()?;
+    }
+    Ok(())
+  }
+
+  fn flush(&mut self) -> std::result::Result<(), ()> {
+    if !self.batch.is_empty() {
+      let batch = std::mem::replace(&mut self.batch, Vec::with_capacity(self.limit));
+      self.tx.send(batch).map_err(|_| ())?;
+    }
+    Ok(())
+  }
+}
+
+impl Drop for BatchSender {
+  fn drop(&mut self) {
+    let _ = self.flush();
+  }
+}
+
+fn find_files(
+  root_paths: &[String],
+  include_hidden: bool,
+  exclusion_patterns: &[String],
+  extensions: &[String],
+) -> Result<Vec<String>> {
+  let mut exclusion_set = globset::GlobSetBuilder::new();
+  for pattern in exclusion_patterns {
+    if let Ok(glob) = globset::Glob::new(pattern) {
+      exclusion_set.add(glob);
+    }
+  }
+  let exclusion_set = match exclusion_set.build() {
+    Ok(set) => set,
+    Err(e) => {
+      eprintln!("Error building exclusion patterns: {}", e);
+      globset::GlobSetBuilder::new().build().unwrap()
+    }
+  };
+  let exclusion_set = Arc::new(exclusion_set);
+
+  let ext_set: std::collections::HashSet<String> = extensions
+    .iter()
+    .map(|ext| ext.strip_prefix('.').unwrap_or(ext).to_lowercase())
+    .collect();
+  let ext_set = Arc::new(ext_set);
+
+  let (tx, rx) = std::sync::mpsc::channel::<Vec<String>>();
+  let quit_flag = Arc::new(AtomicBool::new(false));
+
+  let mut walker_builder = ignore::WalkBuilder::new(&root_paths[0]);
+  for path in &root_paths[1..] {
+    walker_builder.add(path);
+  }
+
+  let walker = walker_builder
+    .git_ignore(false)
+    .hidden(!include_hidden)
+    .parents(false)
+    .ignore(false)
+    .git_global(false)
+    .git_exclude(false)
+    .build_parallel();
+
+  walker.run(|| {
+    let tx = tx.clone();
+    let quit_flag = quit_flag.clone();
+    let exclusion_set = Arc::clone(&exclusion_set);
+    let ext_set = Arc::clone(&ext_set);
+    let mut batch_sender = BatchSender::new(tx, 256);
+
+    Box::new(move |entry_result| {
+      if quit_flag.load(Ordering::Relaxed) {
+        return ignore::WalkState::Quit;
+      }
+
+      let Ok(entry) = entry_result else {
+        return ignore::WalkState::Continue;
+      };
+
+      let path = entry.path();
+
+      if !exclusion_set.is_empty() && !exclusion_set.matches(path).is_empty() {
+        return ignore::WalkState::Skip;
+      }
+
+      let Some(file_type) = entry.file_type() else {
+        return ignore::WalkState::Continue;
+      };
+
+      if !file_type.is_file() {
+        return ignore::WalkState::Continue;
+      }
+
+      if !ext_set.is_empty()
+        && !path
+          .extension()
+          .and_then(|e| e.to_str())
+          .is_some_and(|ext| ext_set.contains(&ext.to_lowercase()))
+      {
+        return ignore::WalkState::Continue;
+      }
+
+      if batch_sender.send(path.to_string_lossy().into_owned()).is_err() {
+        quit_flag.store(true, Ordering::Relaxed);
+        return ignore::WalkState::Quit;
+      }
+      ignore::WalkState::Continue
+    })
+  });
+
+  drop(tx);
+
+  let mut all_files = Vec::new();
+  while let Ok(batch) = rx.recv() {
+    all_files.extend(batch);
+  }
+
+  Ok(all_files)
+}
@@ -0,0 +1,20 @@
+{
+  "compilerOptions": {
+    "target": "ES2020",
+    "module": "ESNext",
+    "lib": ["ES2020"],
+    "declaration": true,
+    "declarationDir": "./dist",
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "resolveJsonModule": true,
+    "moduleResolution": "bundler",
+    "allowSyntheticDefaultImports": true
+  },
+  "include": ["."],
+  "exclude": ["node_modules", "dist"]
+}