Building Scalable Node.js APIs

Building scalable Node.js APIs requires careful consideration of architecture, performance, security, and maintainability. This comprehensive guide covers essential patterns and best practices for creating robust backend services that can handle growth and complexity. 🚀

Project Structure and Architecture

Layered Architecture

src/
├── controllers/     # Route handlers
├── services/       # Business logic
├── repositories/   # Data access layer
├── models/        # Data models
├── middleware/    # Custom middleware
├── utils/         # Utility functions
├── config/        # Configuration
├── routes/        # Route definitions
└── app.js         # Application entry point

Example Implementation

// src/controllers/userController.js
const userService = require('../services/userService')
const { validationResult } = require('express-validator')

class UserController {
  async createUser(req, res, next) {
    try {
      const errors = validationResult(req)
      if (!errors.isEmpty()) {
        return res.status(400).json({ errors: errors.array() })
      }

      const userData = req.body
      const user = await userService.createUser(userData)

      res.status(201).json({
        success: true,
        data: user,
        message: 'User created successfully',
      })
    } catch (error) {
      next(error)
    }
  }

  async getUsers(req, res, next) {
    try {
      const { page = 1, limit = 10, search } = req.query
      const options = {
        page: parseInt(page),
        limit: parseInt(limit),
        search,
      }

      const result = await userService.getUsers(options)

      res.json({
        success: true,
        data: result.users,
        pagination: {
          page: result.page,
          limit: result.limit,
          total: result.total,
          pages: result.pages,
        },
      })
    } catch (error) {
      next(error)
    }
  }
}

module.exports = new UserController()

// src/services/userService.js
const userRepository = require('../repositories/userRepository')
const bcrypt = require('bcrypt')
const jwt = require('jsonwebtoken')
const { AppError } = require('../utils/errors')

class UserService {
  async createUser(userData) {
    const existingUser = await userRepository.findByEmail(userData.email)
    if (existingUser) {
      throw new AppError('User already exists', 409)
    }

    const hashedPassword = await bcrypt.hash(userData.password, 12)
    const user = await userRepository.create({
      ...userData,
      password: hashedPassword,
    })

    // Remove password from response
    const { password, ...userWithoutPassword } = user
    return userWithoutPassword
  }

  async getUsers(options) {
    const users = await userRepository.findAll(options)
    const total = await userRepository.count(options.search)

    return {
      users,
      page: options.page,
      limit: options.limit,
      total,
      pages: Math.ceil(total / options.limit),
    }
  }

  async authenticateUser(email, password) {
    const user = await userRepository.findByEmail(email)
    if (!user) {
      throw new AppError('Invalid credentials', 401)
    }

    const isValidPassword = await bcrypt.compare(password, user.password)
    if (!isValidPassword) {
      throw new AppError('Invalid credentials', 401)
    }

    const token = jwt.sign({ userId: user.id, email: user.email }, process.env.JWT_SECRET, {
      expiresIn: '24h',
    })

    return { token, user: { id: user.id, email: user.email, name: user.name } }
  }
}

module.exports = new UserService()

Database Integration and ORM

Prisma Setup

// prisma/schema.prisma
generator client {
  provider = "prisma-client-js"
}

datasource db {
  provider = "postgresql"
  url      = env("DATABASE_URL")
}

model User {
  id        String   @id @default(cuid())
  email     String   @unique
  name      String
  password  String
  role      Role     @default(USER)
  posts     Post[]
  createdAt DateTime @default(now())
  updatedAt DateTime @updatedAt

  @@map("users")
}

model Post {
  id        String   @id @default(cuid())
  title     String
  content   String
  published Boolean  @default(false)
  author    User     @relation(fields: [authorId], references: [id])
  authorId  String
  createdAt DateTime @default(now())
  updatedAt DateTime @updatedAt

  @@map("posts")
}

enum Role {
  USER
  ADMIN
}

// src/repositories/userRepository.js
const { PrismaClient } = require('@prisma/client')
const prisma = new PrismaClient()

class UserRepository {
  async create(userData) {
    return await prisma.user.create({
      data: userData,
      select: {
        id: true,
        email: true,
        name: true,
        role: true,
        createdAt: true,
      },
    })
  }

  async findByEmail(email) {
    return await prisma.user.findUnique({
      where: { email },
      include: {
        posts: {
          select: {
            id: true,
            title: true,
            published: true,
          },
        },
      },
    })
  }

  async findAll(options = {}) {
    const { page = 1, limit = 10, search } = options
    const skip = (page - 1) * limit

    const where = search
      ? {
          OR: [
            { name: { contains: search, mode: 'insensitive' } },
            { email: { contains: search, mode: 'insensitive' } },
          ],
        }
      : {}

    return await prisma.user.findMany({
      where,
      select: {
        id: true,
        email: true,
        name: true,
        role: true,
        createdAt: true,
      },
      skip,
      take: limit,
      orderBy: { createdAt: 'desc' },
    })
  }

  async count(search) {
    const where = search
      ? {
          OR: [
            { name: { contains: search, mode: 'insensitive' } },
            { email: { contains: search, mode: 'insensitive' } },
          ],
        }
      : {}

    return await prisma.user.count({ where })
  }
}

module.exports = new UserRepository()

Authentication and Authorization

JWT Authentication Middleware

// src/middleware/auth.js
const jwt = require('jsonwebtoken')
const { AppError } = require('../utils/errors')
const userRepository = require('../repositories/userRepository')

const authenticateToken = async (req, res, next) => {
  try {
    const authHeader = req.headers.authorization
    const token = authHeader && authHeader.split(' ')[1]

    if (!token) {
      throw new AppError('Access token required', 401)
    }

    const decoded = jwt.verify(token, process.env.JWT_SECRET)
    const user = await userRepository.findById(decoded.userId)

    if (!user) {
      throw new AppError('User not found', 401)
    }

    req.user = user
    next()
  } catch (error) {
    if (error.name === 'JsonWebTokenError') {
      next(new AppError('Invalid token', 401))
    } else if (error.name === 'TokenExpiredError') {
      next(new AppError('Token expired', 401))
    } else {
      next(error)
    }
  }
}

const authorize = (...roles) => {
  return (req, res, next) => {
    if (!roles.includes(req.user.role)) {
      return next(new AppError('Insufficient permissions', 403))
    }
    next()
  }
}

module.exports = { authenticateToken, authorize }

Rate Limiting

// src/middleware/rateLimiter.js
const rateLimit = require('express-rate-limit')
const RedisStore = require('rate-limit-redis')
const Redis = require('ioredis')

const redis = new Redis(process.env.REDIS_URL)

const createRateLimiter = (options = {}) => {
  return rateLimit({
    store: new RedisStore({
      client: redis,
      prefix: 'rl:',
    }),
    windowMs: options.windowMs || 15 * 60 * 1000, // 15 minutes
    max: options.max || 100, // limit each IP to 100 requests per windowMs
    message: {
      error: 'Too many requests from this IP, please try again later',
    },
    standardHeaders: true,
    legacyHeaders: false,
    ...options,
  })
}

// Different rate limits for different endpoints
const generalLimiter = createRateLimiter()
const authLimiter = createRateLimiter({
  windowMs: 15 * 60 * 1000, // 15 minutes
  max: 5, // limit each IP to 5 requests per windowMs
  skipSuccessfulRequests: true,
})

module.exports = { generalLimiter, authLimiter }

Error Handling

Custom Error Classes

// src/utils/errors.js
class AppError extends Error {
  constructor(message, statusCode) {
    super(message)
    this.statusCode = statusCode
    this.status = `${statusCode}`.startsWith('4') ? 'fail' : 'error'
    this.isOperational = true

    Error.captureStackTrace(this, this.constructor)
  }
}

class ValidationError extends AppError {
  constructor(message, field) {
    super(message, 400)
    this.field = field
  }
}

class DatabaseError extends AppError {
  constructor(message) {
    super(message, 500)
  }
}

module.exports = { AppError, ValidationError, DatabaseError }

Global Error Handler

// src/middleware/errorHandler.js
const { AppError } = require('../utils/errors')

const handleCastErrorDB = (err) => {
  const message = `Invalid ${err.path}: ${err.value}`
  return new AppError(message, 400)
}

const handleDuplicateFieldsDB = (err) => {
  const value = err.errmsg.match(/(["'])(\\?.)*?\1/)[0]
  const message = `Duplicate field value: ${value}. Please use another value!`
  return new AppError(message, 400)
}

const handleValidationErrorDB = (err) => {
  const errors = Object.values(err.errors).map((el) => el.message)
  const message = `Invalid input data. ${errors.join('. ')}`
  return new AppError(message, 400)
}

const handleJWTError = () => new AppError('Invalid token. Please log in again!', 401)

const handleJWTExpiredError = () =>
  new AppError('Your token has expired! Please log in again.', 401)

const sendErrorDev = (err, res) => {
  res.status(err.statusCode).json({
    status: err.status,
    error: err,
    message: err.message,
    stack: err.stack,
  })
}

const sendErrorProd = (err, res) => {
  // Operational, trusted error: send message to client
  if (err.isOperational) {
    res.status(err.statusCode).json({
      status: err.status,
      message: err.message,
    })
  } else {
    // Programming or other unknown error: don't leak error details
    console.error('ERROR 💥', err)
    res.status(500).json({
      status: 'error',
      message: 'Something went wrong!',
    })
  }
}

module.exports = (err, req, res, next) => {
  err.statusCode = err.statusCode || 500
  err.status = err.status || 'error'

  if (process.env.NODE_ENV === 'development') {
    sendErrorDev(err, res)
  } else {
    let error = { ...err }
    error.message = err.message

    if (error.name === 'CastError') error = handleCastErrorDB(error)
    if (error.code === 11000) error = handleDuplicateFieldsDB(error)
    if (error.name === 'ValidationError') error = handleValidationErrorDB(error)
    if (error.name === 'JsonWebTokenError') error = handleJWTError()
    if (error.name === 'TokenExpiredError') error = handleJWTExpiredError()

    sendErrorProd(error, res)
  }
}

Performance Optimization

Caching Strategy

// src/utils/cache.js
const Redis = require('ioredis')
const redis = new Redis(process.env.REDIS_URL)

class CacheService {
  constructor() {
    this.defaultTTL = 3600 // 1 hour
  }

  async get(key) {
    try {
      const cached = await redis.get(key)
      return cached ? JSON.parse(cached) : null
    } catch (error) {
      console.error('Cache get error:', error)
      return null
    }
  }

  async set(key, value, ttl = this.defaultTTL) {
    try {
      await redis.setex(key, ttl, JSON.stringify(value))
    } catch (error) {
      console.error('Cache set error:', error)
    }
  }

  async del(key) {
    try {
      await redis.del(key)
    } catch (error) {
      console.error('Cache delete error:', error)
    }
  }

  async invalidatePattern(pattern) {
    try {
      const keys = await redis.keys(pattern)
      if (keys.length > 0) {
        await redis.del(...keys)
      }
    } catch (error) {
      console.error('Cache invalidate error:', error)
    }
  }
}

module.exports = new CacheService()

Database Query Optimization

// src/services/postService.js
const postRepository = require('../repositories/postRepository')
const cacheService = require('../utils/cache')

class PostService {
  async getPosts(options = {}) {
    const cacheKey = `posts:${JSON.stringify(options)}`

    // Try to get from cache first
    let cachedPosts = await cacheService.get(cacheKey)
    if (cachedPosts) {
      return cachedPosts
    }

    // If not in cache, fetch from database
    const posts = await postRepository.findAll(options)

    // Cache the result
    await cacheService.set(cacheKey, posts, 600) // 10 minutes

    return posts
  }

  async createPost(postData) {
    const post = await postRepository.create(postData)

    // Invalidate related caches
    await cacheService.invalidatePattern('posts:*')

    return post
  }
}

module.exports = new PostService()

API Documentation with Swagger

// src/utils/swagger.js
const swaggerJSDoc = require('swagger-jsdoc')
const swaggerUi = require('swagger-ui-express')

const options = {
  definition: {
    openapi: '3.0.0',
    info: {
      title: 'Node.js API',
      version: '1.0.0',
      description: 'A scalable Node.js API',
    },
    servers: [
      {
        url: process.env.API_URL || 'http://localhost:3000',
        description: 'Development server',
      },
    ],
    components: {
      securitySchemes: {
        bearerAuth: {
          type: 'http',
          scheme: 'bearer',
          bearerFormat: 'JWT',
        },
      },
    },
  },
  apis: ['./src/routes/*.js'], // Path to the API files
}

const specs = swaggerJSDoc(options)

module.exports = { specs, swaggerUi }

// src/routes/userRoutes.js
const express = require('express')
const { body } = require('express-validator')
const userController = require('../controllers/userController')
const { authenticateToken, authorize } = require('../middleware/auth')

const router = express.Router()

/**
 * @swagger
 * /api/users:
 *   post:
 *     summary: Create a new user
 *     tags: [Users]
 *     requestBody:
 *       required: true
 *       content:
 *         application/json:
 *           schema:
 *             type: object
 *             required:
 *               - name
 *               - email
 *               - password
 *             properties:
 *               name:
 *                 type: string
 *               email:
 *                 type: string
 *                 format: email
 *               password:
 *                 type: string
 *                 minLength: 6
 *     responses:
 *       201:
 *         description: User created successfully
 *       400:
 *         description: Validation error
 *       409:
 *         description: User already exists
 */
router.post(
  '/',
  [
    body('name').notEmpty().withMessage('Name is required'),
    body('email').isEmail().withMessage('Valid email is required'),
    body('password').isLength({ min: 6 }).withMessage('Password must be at least 6 characters'),
  ],
  userController.createUser
)

/**
 * @swagger
 * /api/users:
 *   get:
 *     summary: Get all users
 *     tags: [Users]
 *     security:
 *       - bearerAuth: []
 *     parameters:
 *       - in: query
 *         name: page
 *         schema:
 *           type: integer
 *         description: Page number
 *       - in: query
 *         name: limit
 *         schema:
 *           type: integer
 *         description: Number of items per page
 *     responses:
 *       200:
 *         description: List of users
 *       401:
 *         description: Unauthorized
 */
router.get('/', authenticateToken, authorize('ADMIN'), userController.getUsers)

module.exports = router

Testing Strategy

Unit Tests

// tests/services/userService.test.js
const userService = require('../../src/services/userService')
const userRepository = require('../../src/repositories/userRepository')
const { AppError } = require('../../src/utils/errors')

jest.mock('../../src/repositories/userRepository')

describe('UserService', () => {
  beforeEach(() => {
    jest.clearAllMocks()
  })

  describe('createUser', () => {
    it('should create a new user successfully', async () => {
      const userData = {
        name: 'John Doe',
        email: 'john@example.com',
        password: 'password123',
      }

      userRepository.findByEmail.mockResolvedValue(null)
      userRepository.create.mockResolvedValue({
        id: '1',
        ...userData,
        password: 'hashedPassword',
      })

      const result = await userService.createUser(userData)

      expect(result).toEqual({
        id: '1',
        name: 'John Doe',
        email: 'john@example.com',
      })
      expect(userRepository.findByEmail).toHaveBeenCalledWith(userData.email)
      expect(userRepository.create).toHaveBeenCalled()
    })

    it('should throw error if user already exists', async () => {
      const userData = {
        name: 'John Doe',
        email: 'john@example.com',
        password: 'password123',
      }

      userRepository.findByEmail.mockResolvedValue({ id: '1' })

      await expect(userService.createUser(userData)).rejects.toThrow(AppError)
      expect(userRepository.create).not.toHaveBeenCalled()
    })
  })
})

Integration Tests

// tests/integration/users.test.js
const request = require('supertest')
const app = require('../../src/app')
const { PrismaClient } = require('@prisma/client')

const prisma = new PrismaClient()

describe('Users API', () => {
  beforeEach(async () => {
    await prisma.user.deleteMany()
  })

  afterAll(async () => {
    await prisma.$disconnect()
  })

  describe('POST /api/users', () => {
    it('should create a new user', async () => {
      const userData = {
        name: 'John Doe',
        email: 'john@example.com',
        password: 'password123',
      }

      const response = await request(app).post('/api/users').send(userData).expect(201)

      expect(response.body).toMatchObject({
        success: true,
        data: {
          name: userData.name,
          email: userData.email,
        },
      })
    })

    it('should return validation error for invalid data', async () => {
      const userData = {
        name: '',
        email: 'invalid-email',
        password: '123',
      }

      const response = await request(app).post('/api/users').send(userData).expect(400)

      expect(response.body.errors).toHaveLength(3)
    })
  })
})

Deployment and DevOps

Docker Configuration

# Dockerfile
FROM node:18-alpine

WORKDIR /app

# Copy package files
COPY package*.json ./

# Install dependencies
RUN npm ci --only=production

# Copy source code
COPY . .

# Generate Prisma client
RUN npx prisma generate

# Create non-root user
RUN addgroup -g 1001 -S nodejs
RUN adduser -S nodejs -u 1001

# Change ownership of the app directory
RUN chown -R nodejs:nodejs /app
USER nodejs

EXPOSE 3000

CMD ["npm", "start"]

# docker-compose.yml
version: '3.8'

services:
  app:
    build: .
    ports:
      - '3000:3000'
    environment:
      - NODE_ENV=production
      - DATABASE_URL=postgresql://user:password@db:5432/myapp
      - REDIS_URL=redis://redis:6379
    depends_on:
      - db
      - redis
    restart: unless-stopped

  db:
    image: postgres:15-alpine
    environment:
      - POSTGRES_DB=myapp
      - POSTGRES_USER=user
      - POSTGRES_PASSWORD=password
    volumes:
      - postgres_data:/var/lib/postgresql/data
    restart: unless-stopped

  redis:
    image: redis:7-alpine
    restart: unless-stopped

volumes:
  postgres_data:

Conclusion

Building scalable Node.js APIs requires attention to:

Clean Architecture: Separate concerns with layered architecture
Proper Error Handling: Comprehensive error management
Security: Authentication, authorization, and rate limiting
Performance: Caching, query optimization, and monitoring
Testing: Unit and integration tests for reliability
Documentation: Clear API documentation
Deployment: Containerization and CI/CD pipelines

These patterns and practices will help you build robust, maintainable APIs that can scale with your application's growth. 🚀