node.jsnpmyarn-workspacespulumimise

Recursive install of node_modules in Pulumi resources

I have the following project I am using Pulumi to manage my various cloud run jobs. I am trying to set up all my package management to use a tool like mise for each of development. The problem is each of the cloud run jobs has their own package.json file and there is no easy way to install all of the node_modules recursively. I have looked at tools like yarn workspaces, or pnpm (with recursive install), but neither works. When I use these tools pulumi up fails with a syntax error. I can only run pulumi up on each job after running npm i in each child directory which is cumbersome.

├── README.md
├── infra
│   ├── job1
│   │   ├── Pulumi.dev.yaml
│   │   ├── Pulumi.prod.yaml
│   │   ├── Pulumi.qa.yaml
│   │   ├── Pulumi.yaml
│   │   ├── index.ts
│   │   ├── package-lock.json
│   │   ├── package.json
│   │   └── tsconfig.json
│   ├── job2
│   │   ├── Pulumi.dev.yaml
│   │   ├── Pulumi.prod.yaml
│   │   ├── Pulumi.qa.yaml
│   │   ├── Pulumi.yaml
│   │   ├── index.ts
│   │   ├── package-lock.json
│   │   ├── package.json
│   │   └── tsconfig.json
│   └── job3
│       ├── Pulumi.dev.yaml
│       ├── Pulumi.prod.yaml
│       ├── Pulumi.qa.yaml
│       ├── Pulumi.yaml
│       ├── index.ts
│       ├── package-lock.json
│       ├── package.json
│       └── tsconfig.json
├── justfile
├── lib
│   └── job5
│       ├── cloudrun.ts
│       ├── package-lock.json
│       ├── package.json
│       └── tsconfig.json
├── mise.toml

Is there a way in which I can call npm i recursively from the main project directory? Is it possible to share the package.json files in some way they all essentially are the same:

{
    "name": "job1",
    "main": "index.ts",
    "version": "1.0.0",
    "devDependencies": {
        "@eslint/eslintrc": "^3.2.0",
        "@eslint/js": "^9.17.0",
        "@eslint/migrate-config": "1.3.5",
        "@types/node": "^18.19.69",
        "@typescript-eslint/eslint-plugin": "^8.19.0",
        "@typescript-eslint/parser": "^8.19.0",
        "eslint": "^9.17.0",
        "typescript": "^5.7.2"
    },
    "dependencies": {
        "@lib/cloudrun": "file:../../lib/job5",
        "@pulumi/docker": "^4.5.8",
        "@pulumi/gcp": "^7.38.0",
        "@pulumi/pulumi": "^3.144.1",
        "@pulumi/random": "^4.16.8",
        "natural-cron": "^1.0.2"
    }
}

What is the best practice here? I am open to any and all solutions.
Solution

  • Try implementing the solution below and see if it helps:

    Monorepo Package Configuration

    {
  "name": "pulumi-infrastructure",
  "private": true,
  "workspaces": [
    "infra/*",
    "lib/*"
  ],
  "scripts": {
    "install-all": "npm install && npm run install:jobs",
    "install:jobs": "node scripts/install-jobs.js",
    "pulumi:up-all": "node scripts/pulumi-up-all.js"
  },
  "devDependencies": {
    "@types/node": "^18.19.69",
    "@typescript-eslint/eslint-plugin": "^8.19.0",
    "@typescript-eslint/parser": "^8.19.0",
    "eslint": "^9.17.0",
    "typescript": "^5.7.2"
  }
}

    NPM Install Script

    // scripts/install-jobs.js
const { execSync } = require('child_process');
const path = require('path');
const fs = require('fs');

// Directories to process
const directories = ['infra', 'lib'];

function installDependencies(dir) {
  const items = fs.readdirSync(dir);
  
  items.forEach(item => {
    const fullPath = path.join(dir, item);
    const stats = fs.statSync(fullPath);
    
    if (stats.isDirectory()) {
      const packageJsonPath = path.join(fullPath, 'package.json');
      
      if (fs.existsSync(packageJsonPath)) {
        console.log(`Installing dependencies in ${fullPath}`);
        try {
          execSync('npm install', {
            cwd: fullPath,
            stdio: 'inherit'
          });
        } catch (error) {
          console.error(`Error installing dependencies in ${fullPath}:`, error);
          process.exit(1);
        }
      }
    }
  });
}

// Process each directory
directories.forEach(dir => {
  console.log(`Processing directory: ${dir}`);
  installDependencies(dir);
});

    Pulumi Up Script

    // scripts/pulumi-up-all.js
const { execSync } = require('child_process');
const path = require('path');
const fs = require('fs');

// Get the Pulumi stack from command line or default to 'dev'
const stack = process.argv[2] || 'dev';

function runPulumiUp(dir) {
  const items = fs.readdirSync(dir);
  
  items.forEach(item => {
    const fullPath = path.join(dir, item);
    const stats = fs.statSync(fullPath);
    
    if (stats.isDirectory()) {
      const pulumiYamlPath = path.join(fullPath, 'Pulumi.yaml');
      
      if (fs.existsSync(pulumiYamlPath)) {
        console.log(`Running pulumi up in ${fullPath}`);
        try {
          execSync(`pulumi up --stack ${stack} --yes`, {
            cwd: fullPath,
            stdio: 'inherit'
          });
        } catch (error) {
          console.error(`Error running pulumi up in ${fullPath}:`, error);
          process.exit(1);
        }
      }
    }
  });
}

// Process infra directory
console.log('Processing infrastructure jobs...');
runPulumiUp('infra');

    Here’s how to implement it:

    Create a root package.json at the project root level that uses workspaces and shared dependencies.

    Move shared devDependencies to the root package.json and keep only project-specific dependencies in the individual job's package.json files.

    Create installation and deployment scripts to handle the recursive operations (follow the steps below).

    First, simplify your job-specific package.json files to only include necessary dependencies:

    {
    "name": "job1",
    "main": "index.ts",
    "version": "1.0.0",
    "dependencies": {
        "@lib/cloudrun": "file:../../lib/job5",
        "@pulumi/docker": "^4.5.8",
        "@pulumi/gcp": "^7.38.0",
        "@pulumi/pulumi": "^3.144.1",
        "@pulumi/random": "^4.16.8",
        "natural-cron": "^1.0.2"
    }
}

    Create the scripts directory and add the installation and Pulumi scripts I provided above.

    You can now use the following commands:

    # Install all dependencies recursively
npm run install-all

# Run Pulumi up for all jobs
npm run pulumi:up-all dev  # or qa/prod for other environments

    Here’s some additional recommendations:

    Add a .npmrc file at the root to ensure consistent package management:

    save-exact=true
legacy-peer-deps=true
workspaces=true

    Consider adding these scripts to your justfile for easier execution:

    install:
    npm run install-all

deploy ENV:
    npm run pulumi:up-all {{ENV}}

    With this approach, shared devDependencies are managed at root level. Each job can still maintain its own specific dependencies, and the scripts provide a consistent way to install dependencies and run Pulumi commands. You can still use mise for version management at the root level.