Jan 5, 2024

Building Scalable APIs with Node.js and Express

Building scalable APIs is crucial for modern web applications. Node.js and Express provide an excellent foundation for creating robust, high-performance APIs. At Dev Intelligence, we’ve built numerous scalable APIs using these technologies, and we’re sharing our best practices.

Setting Up Your Express Application

Project Structure

Start with a well-organized project structure:

src/
├── controllers/
│   ├── userController.js
│   └── productController.js
├── middleware/
│   ├── auth.js
│   ├── validation.js
│   └── errorHandler.js
├── models/
│   ├── User.js
│   └── Product.js
├── routes/
│   ├── users.js
│   └── products.js
├── services/
│   ├── userService.js
│   └── emailService.js
├── utils/
│   ├── database.js
│   └── logger.js
└── app.js

Basic Express Setup

const express = require('express');
const cors = require('cors');
const helmet = require('helmet');
const rateLimit = require('express-rate-limit');
const compression = require('compression');

const app = express();

// Security middleware
app.use(helmet());
app.use(cors());

// Compression middleware
app.use(compression());

// Rate limiting
const limiter = rateLimit({
  windowMs: 15 * 60 * 1000, // 15 minutes
  max: 100 // limit each IP to 100 requests per windowMs
});
app.use(limiter);

// Body parsing middleware
app.use(express.json({ limit: '10mb' }));
app.use(express.urlencoded({ extended: true }));

module.exports = app;

API Design Best Practices

RESTful API Design

Follow REST conventions for predictable, intuitive APIs:

// Good RESTful routes
GET    /api/users          // Get all users
GET    /api/users/:id      // Get specific user
POST   /api/users          // Create new user
PUT    /api/users/:id      // Update user
DELETE /api/users/:id      // Delete user

// Avoid non-RESTful patterns
GET    /api/getUsers       // Bad
POST   /api/deleteUser     // Bad

Consistent Response Format

Use a consistent response format across your API:

// Success response
{
  "success": true,
  "data": {
    "id": 1,
    "name": "John Doe",
    "email": "john@example.com"
  },
  "message": "User retrieved successfully"
}

// Error response
{
  "success": false,
  "error": {
    "code": "USER_NOT_FOUND",
    "message": "User with ID 1 not found"
  },
  "timestamp": "2024-01-15T10:30:00Z"
}

HTTP Status Codes

Use appropriate HTTP status codes:

// Success codes
200 OK           // Successful GET, PUT
201 Created      // Successful POST
204 No Content   // Successful DELETE

// Client error codes
400 Bad Request  // Invalid request data
401 Unauthorized // Authentication required
403 Forbidden    // Access denied
404 Not Found    // Resource not found
422 Unprocessable Entity // Validation errors

// Server error codes
500 Internal Server Error // Server error
503 Service Unavailable   // Service down

Middleware Implementation

Authentication Middleware

const jwt = require('jsonwebtoken');

const authenticateToken = (req, res, next) => {
  const authHeader = req.headers['authorization'];
  const token = authHeader && authHeader.split(' ')[1];

  if (!token) {
    return res.status(401).json({
      success: false,
      error: { code: 'TOKEN_REQUIRED', message: 'Access token required' }
    });
  }

  jwt.verify(token, process.env.JWT_SECRET, (err, user) => {
    if (err) {
      return res.status(403).json({
        success: false,
        error: { code: 'INVALID_TOKEN', message: 'Invalid or expired token' }
      });
    }
    req.user = user;
    next();
  });
};

Validation Middleware

const { body, validationResult } = require('express-validator');

const validateUser = [
  body('name').trim().isLength({ min: 2 }).withMessage('Name must be at least 2 characters'),
  body('email').isEmail().normalizeEmail().withMessage('Valid email required'),
  body('password').isLength({ min: 8 }).withMessage('Password must be at least 8 characters'),
  
  (req, res, next) => {
    const errors = validationResult(req);
    if (!errors.isEmpty()) {
      return res.status(422).json({
        success: false,
        error: {
          code: 'VALIDATION_ERROR',
          message: 'Validation failed',
          details: errors.array()
        }
      });
    }
    next();
  }
];

Error Handling Middleware

const errorHandler = (err, req, res, next) => {
  console.error(err.stack);

  // Mongoose validation error
  if (err.name === 'ValidationError') {
    const errors = Object.values(err.errors).map(e => e.message);
    return res.status(422).json({
      success: false,
      error: {
        code: 'VALIDATION_ERROR',
        message: 'Validation failed',
        details: errors
      }
    });
  }

  // JWT errors
  if (err.name === 'JsonWebTokenError') {
    return res.status(401).json({
      success: false,
      error: { code: 'INVALID_TOKEN', message: 'Invalid token' }
    });
  }

  // Default error
  res.status(500).json({
    success: false,
    error: {
      code: 'INTERNAL_ERROR',
      message: 'Something went wrong'
    }
  });
};

Database Integration

Connection Pooling

const { Pool } = require('pg');

const pool = new Pool({
  user: process.env.DB_USER,
  host: process.env.DB_HOST,
  database: process.env.DB_NAME,
  password: process.env.DB_PASSWORD,
  port: process.env.DB_PORT,
  max: 20,
  idleTimeoutMillis: 30000,
  connectionTimeoutMillis: 2000,
});

// Test connection
pool.on('connect', () => {
  console.log('Connected to database');
});

pool.on('error', (err) => {
  console.error('Database connection error:', err);
});

Model Layer

class UserModel {
  static async create(userData) {
    const { name, email, password } = userData;
    const query = `
      INSERT INTO users (name, email, password_hash, created_at)
      VALUES ($1, $2, $3, NOW())
      RETURNING id, name, email, created_at
    `;
    const values = [name, email, password];
    const result = await pool.query(query, values);
    return result.rows[0];
  }

  static async findById(id) {
    const query = 'SELECT id, name, email, created_at FROM users WHERE id = $1';
    const result = await pool.query(query, [id]);
    return result.rows[0];
  }

  static async findByEmail(email) {
    const query = 'SELECT * FROM users WHERE email = $1';
    const result = await pool.query(query, [email]);
    return result.rows[0];
  }
}

Performance Optimization

Caching Strategy

const redis = require('redis');
const client = redis.createClient({
  host: process.env.REDIS_HOST,
  port: process.env.REDIS_PORT
});

const cacheMiddleware = (duration = 300) => {
  return async (req, res, next) => {
    const key = `cache:${req.originalUrl}`;
    
    try {
      const cached = await client.get(key);
      if (cached) {
        return res.json(JSON.parse(cached));
      }
      
      // Store original res.json
      const originalJson = res.json;
      res.json = function(data) {
        // Cache the response
        client.setex(key, duration, JSON.stringify(data));
        originalJson.call(this, data);
      };
      
      next();
    } catch (error) {
      console.error('Cache error:', error);
      next();
    }
  };
};

Async/Await Best Practices

// Good: Parallel execution
const getUserData = async (userId) => {
  const [user, orders, preferences] = await Promise.all([
    UserModel.findById(userId),
    OrderModel.findByUserId(userId),
    PreferenceModel.findByUserId(userId)
  ]);
  
  return { user, orders, preferences };
};

// Good: Sequential when dependent
const processOrder = async (orderData) => {
  const user = await UserModel.findById(orderData.userId);
  if (!user) throw new Error('User not found');
  
  const order = await OrderModel.create(orderData);
  await EmailService.sendOrderConfirmation(user.email, order);
  
  return order;
};

Testing Your API

Unit Tests

const request = require('supertest');
const app = require('../app');

describe('User API', () => {
  test('GET /api/users should return users', async () => {
    const response = await request(app)
      .get('/api/users')
      .expect(200);
    
    expect(response.body.success).toBe(true);
    expect(Array.isArray(response.body.data)).toBe(true);
  });

  test('POST /api/users should create user', async () => {
    const userData = {
      name: 'John Doe',
      email: 'john@example.com',
      password: 'password123'
    };
    
    const response = await request(app)
      .post('/api/users')
      .send(userData)
      .expect(201);
    
    expect(response.body.success).toBe(true);
    expect(response.body.data.name).toBe(userData.name);
  });
});

Deployment Considerations

Environment Configuration

// config/database.js
const config = {
  development: {
    host: 'localhost',
    port: 5432,
    database: 'dev_db'
  },
  production: {
    host: process.env.DB_HOST,
    port: process.env.DB_PORT,
    database: process.env.DB_NAME,
    ssl: { rejectUnauthorized: false }
  }
};

module.exports = config[process.env.NODE_ENV || 'development'];

Health Check Endpoint

app.get('/health', async (req, res) => {
  try {
    // Check database connection
    await pool.query('SELECT 1');
    
    res.json({
      status: 'healthy',
      timestamp: new Date().toISOString(),
      uptime: process.uptime(),
      memory: process.memoryUsage()
    });
  } catch (error) {
    res.status(503).json({
      status: 'unhealthy',
      error: error.message
    });
  }
});

Conclusion

Building scalable APIs with Node.js and Express requires careful attention to architecture, performance, and best practices. By following these guidelines, you can create robust APIs that can handle growth and provide excellent user experiences.

At Dev Intelligence, we specialize in building scalable backend solutions using Node.js, Express, and other modern technologies. Our team has extensive experience in API design, database optimization, and performance tuning.

Ready to build your next API? Contact us to discuss your project requirements and discover how we can help you create a scalable, high-performance API solution.