javascript node.js promise node-promisify

Is this valid syntax for node/promisify async function?

I accidentally typed await(await stat(content... and it worked. Not sure if this is valid syntax, or there is a better way to do it? I'm trying to read all the files that are directories and do not match my regex.

const fs = require('fs')
const path = require('path')
const content = path.resolve('.') + '/docs' + '/'
const util = require('util');
const stat = util.promisify(fs.stat)
const readDir = util.promisify(fs.readdir)
const directories = 'docs/';
const exclude = new RegExp(/^(adir|\.somedir)/,'i');
let newFiles = {}

async function main(){
    const ls = await readDir(directories)
    console.log('starting....');
    let newArray = []
     for (let index = 0; index < ls.length; index++) {
               let x =  await (await stat(content + ls[index])).isDirectory()
               let file = ls[index]
               if (x && !(exclude.test(file))){newArray.push(file)}
               console.log('x is ',x);
        }   
    console.log('new filtered array: ', newArray);
}

Solution

My advice would be not to put all of your eggs in one basket. We can write an ultra fast ls function using Node's fs.Dirent objects and bypass the need for a slow fs.stat call on each file -

// fsext.js

import { readdir } from "fs/promises"
import { join } from "path"

async function* ls (path = ".")
{ yield { dir: path }
  for (const dirent of await readdir(path, { withFileTypes: true }))
    if (dirent.isDirectory())
      yield *ls(join(path, dirent.name))
    else
      yield { file: join(path, dirent.name) }
}

async function toArray (iter)
{ const r = []
  for await (const v of iter)
    r.push(v)
  return r
}

export { ls, toArray }

// main.js

import { ls, toArray } from "./fsext.js"

toArray(ls("./node_modules")).then(console.log, console.error)

To test it out, let's add some popular npm packages so we have a large hierarchy to test our our program. We'll install the lot and count the number of directories and files -

$ npm install async chalk commander debug express immutable lodash moment prop-types react react-dom request webpack

$ find ./node_modules | wc -l

Now let's run our program and time it -

$ time node main.js

[
  { dir: './node_modules' },
  { dir: 'node_modules/.bin' },
  { file: 'node_modules/.bin/acorn' },
  { file: 'node_modules/.bin/browserslist' },
  { file: 'node_modules/.bin/loose-envify' },
  { file: 'node_modules/.bin/mime' },
  { file: 'node_modules/.bin/sshpk-conv' },
  { file: 'node_modules/.bin/sshpk-sign' },
  { file: 'node_modules/.bin/sshpk-verify' },
  { file: 'node_modules/.bin/terser' },
  { file: 'node_modules/.bin/uuid' },
  { file: 'node_modules/.bin/webpack' },
  { file: 'node_modules/.package-lock.json' },
  { dir: 'node_modules/@types' },
  { dir: 'node_modules/@types/eslint' },
  { file: 'node_modules/@types/eslint/LICENSE' },
  { file: 'node_modules/@types/eslint/README.md' },
  { file: 'node_modules/@types/eslint/helpers.d.ts' },
  { file: 'node_modules/@types/eslint/index.d.ts' },
  { dir: 'node_modules/@types/eslint/lib' },
   ... 5433 more items
]

node main.js  0.09s user 0.02s system 116% cpu 0.099 total

dirs

If we only want directories, we can write dirs as a simple specialization of our generic ls -

// fsext.js (continued)

async function* dirs (path)
{ for await (const f of ls(path))
    if (f.dir)
      yield f.dir
}

$ find ./node_modules -type d | wc -l

Now compare it against our program

// main.js

import { dirs, toArray } from "./fsext.js"

toArray(dirs("./node_modules")).then(console.log, console.error)

$ time node.main.js

[
  './node_modules',
  'node_modules/.bin',
  'node_modules/@types',
  'node_modules/@types/eslint',
  'node_modules/@types/eslint/lib',
  'node_modules/@types/eslint/lib/rules',
  'node_modules/@types/eslint/rules',
  'node_modules/@types/eslint-scope',
  'node_modules/@types/estree',
  'node_modules/@types/json-schema',
  'node_modules/@types/node',
  'node_modules/@types/node/assert',
  'node_modules/@types/node/dns',
  'node_modules/@types/node/fs',
  'node_modules/@types/node/stream',
  'node_modules/@types/node/timers',
  'node_modules/@types/node/ts3.6',
  'node_modules/@webassemblyjs',
  'node_modules/@webassemblyjs/ast',
  'node_modules/@webassemblyjs/ast/esm',
  ... 437 more items
]

node main2.js  0.09s user 0.02s system 108% cpu 0.099 total

exclude

If we want to exclude certain directories or files, we can write it generically as well -

// fsext.js (continued)

async function* exclude (iter, test)
{ for await (const v of iter)
    if (Boolean(test(v)))
      continue
    else
      yield v
}

// main.js

import { dirs, exclude, toArray } from "./fsext.js"

toArray(exclude(dirs("./node_modules"), v => /@/.test(v)))
  .then(console.log, console.error)

$ time node main.js

[
  './node_modules',
  'node_modules/.bin',
  'node_modules/accepts',
  'node_modules/acorn',
  'node_modules/acorn/bin',
  'node_modules/acorn/dist',
  'node_modules/ajv',
  'node_modules/ajv/dist',
  'node_modules/ajv/lib',
  'node_modules/ajv/lib/compile',
  'node_modules/ajv/lib/dot',
  'node_modules/ajv/lib/dotjs',
  'node_modules/ajv/lib/refs',
  'node_modules/ajv/scripts',
  'node_modules/ajv-keywords',
  'node_modules/ajv-keywords/keywords',
  'node_modules/ajv-keywords/keywords/dot',
  'node_modules/ajv-keywords/keywords/dotjs',
  'node_modules/ansi-styles',
  'node_modules/array-flatten',
  ... 351 more items
]

node main.js  0.09s user 0.02s system 105% cpu 0.104 total

reorganize

In our file system extensions module, fsext, we wrote two functions that work on any iterables, not just the ls or dirs. I would suggest breaking these out into their own iter module. This type of reorganization helps decouple concerns and maximize code reuse throughout your entire program -

// iter.js

async function* empty () {}

async function* exclude (iter = empty(), test = Boolean)
{ for await (const v of iter)
    if (Boolean(test(v)))
      continue
    else
      yield v
}

async function toArray (iter = empty())
{ const r = []
  for await (const v of iter)
    r.push(v)
  return r
}

export { empty, exclude, toArray }

// fsext.js

import { readdir } from "fs/promises"
import { join } from "path"

async function* ls (path = ".")
{ yield { dir: path }
  for (const dirent of await readdir(path, { withFileTypes: true }))
    if (dirent.isDirectory())
      yield *ls(join(path, dirent.name))
    else
      yield { file: join(path, dirent.name) }
}

async function* dirs (path)
{ for await (const f of ls(path))
    if (f.dir)
      yield f.dir
}

async function* files (path)
{ for await (const f of ls(path))
    if (f.file)
      yield f.file
}

export { ls, dirs, files }

// main.js

import { dirs } from "./fsext.js"
import { exclude, toArray } from "./iter.js"

const somePath = "..."
const someTest = v => ...

toArray(exclude(dirs(somePath), someTest))
  .then(console.log, console.error)

search

Looking for a specific file or folder? Read on in this Q&A to implement search.