File Uploads

Two Upload Patterns

Pattern 1: Server-proxied upload
  Browser → POST /upload → Server → S3/MinIO
  Pro: full control, can scan/transform in-flight
  Con: server bandwidth and memory for every upload

Pattern 2: Direct-to-storage (presigned URL)
  Browser → GET /upload-url → Server (returns signed URL)
  Browser → PUT signed-url → S3/MinIO directly
  Pro: zero server bandwidth cost
  Con: validation must happen after the fact (or via object metadata)

Choose based on file size and whether you need in-flight processing. Images → direct upload. Documents that need scanning → proxied.

Proxied Upload (Server Side)

import Busboy from 'busboy';
import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3';
import { randomUUID } from 'crypto';
import type { IncomingMessage, ServerResponse } from 'http';

const s3 = new S3Client({
	endpoint: process.env.MINIO_ENDPOINT,
	region: 'us-east-1',
	credentials: {
		accessKeyId: process.env.MINIO_ACCESS_KEY!,
		secretAccessKey: process.env.MINIO_SECRET_KEY!
	},
	forcePathStyle: true
});

const ALLOWED_TYPES = new Set(['image/jpeg', 'image/png', 'image/webp', 'application/pdf']);
const MAX_BYTES = 10 * 1024 * 1024; // 10MB

export async function handleUpload(req: IncomingMessage, res: ServerResponse, userId: string) {
	return new Promise<{ key: string; url: string }>((resolve, reject) => {
		const bb = Busboy({
			headers: req.headers,
			limits: { fileSize: MAX_BYTES, files: 1 }
		});

		bb.on('file', (fieldname, stream, info) => {
			const { filename, mimeType } = info;

			if (!ALLOWED_TYPES.has(mimeType)) {
				stream.resume(); // drain the stream
				return reject(new Error(`File type not allowed: ${mimeType}`));
			}

			const ext = filename.split('.').pop()?.toLowerCase() ?? 'bin';
			const key = `uploads/${userId}/${randomUUID()}.${ext}`;

			// Stream directly to S3 — no temp file on disk
			const chunks: Buffer[] = [];
			let totalBytes = 0;

			stream.on('data', (chunk: Buffer) => {
				totalBytes += chunk.length;
				chunks.push(chunk);
			});

			stream.on('limit', () => {
				reject(new Error('File too large'));
			});

			stream.on('end', async () => {
				const body = Buffer.concat(chunks);

				await s3.send(
					new PutObjectCommand({
						Bucket: 'user-uploads',
						Key: key,
						Body: body,
						ContentType: mimeType,
						Metadata: {
							'uploaded-by': userId,
							'original-name': encodeURIComponent(filename)
						}
					})
				);

				resolve({
					key,
					url: `${process.env.MINIO_PUBLIC_URL}/user-uploads/${key}`
				});
			});
		});

		bb.on('error', reject);
		req.pipe(bb);
	});
}

Busboy streams the multipart body — no buffering the entire file in memory before hitting S3.

Express / Fastify Integration

// Express
import express from 'express';

const app = express();

app.post('/upload', async (req, res) => {
	try {
		const result = await handleUpload(req, res, req.user.id);
		res.json({ success: true, ...result });
	} catch (err) {
		res.status(400).json({ error: (err as Error).message });
	}
});

// Fastify
import Fastify from 'fastify';
import multipart from '@fastify/multipart';

const fastify = Fastify();
fastify.register(multipart, {
	limits: { fileSize: 10 * 1024 * 1024, files: 1 }
});

fastify.post('/upload', async (req, reply) => {
	const data = await req.file();
	if (!data) return reply.code(400).send({ error: 'No file' });

	const key = `uploads/${req.user.id}/${randomUUID()}`;

	await s3.send(
		new PutObjectCommand({
			Bucket: 'user-uploads',
			Key: key,
			Body: await data.toBuffer(),
			ContentType: data.mimetype
		})
	);

	return { key };
});

Direct Upload (Presigned URL)

import { PutObjectCommand, GetObjectCommand } from '@aws-sdk/client-s3';
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';

// Step 1: client requests an upload URL
app.post('/upload-url', async (req, res) => {
	const { filename, contentType, size } = req.body;

	if (!ALLOWED_TYPES.has(contentType)) {
		return res.status(400).json({ error: 'File type not allowed' });
	}

	if (size > MAX_BYTES) {
		return res.status(400).json({ error: 'File too large' });
	}

	const ext = filename.split('.').pop()?.toLowerCase() ?? 'bin';
	const key = `uploads/${req.user.id}/${randomUUID()}.${ext}`;

	const uploadUrl = await getSignedUrl(
		s3,
		new PutObjectCommand({
			Bucket: 'user-uploads',
			Key: key,
			ContentType: contentType,
			ContentLength: size // enforce exact size — client can't upload more
		}),
		{ expiresIn: 300 }
	); // 5 minutes

	res.json({ uploadUrl, key });
});

// Step 2: after upload, client notifies server to record the file
app.post('/upload-confirm', async (req, res) => {
	const { key } = req.body;

	// Verify the object actually exists in storage
	try {
		await s3.send(new HeadObjectCommand({ Bucket: 'user-uploads', Key: key }));
	} catch {
		return res.status(400).json({ error: 'File not found in storage' });
	}

	// Validate key belongs to this user (check prefix)
	if (!key.startsWith(`uploads/${req.user.id}/`)) {
		return res.status(403).json({ error: 'Forbidden' });
	}

	await db.query(
		'INSERT INTO user_files (user_id, storage_key, created_at) VALUES ($1, $2, NOW())',
		[req.user.id, key]
	);

	res.json({ success: true });
});

// Client side (browser)
async function uploadFile(file: File) {
	// 1. Request upload URL
	const { uploadUrl, key } = await fetch('/upload-url', {
		method: 'POST',
		headers: { 'Content-Type': 'application/json' },
		body: JSON.stringify({
			filename: file.name,
			contentType: file.type,
			size: file.size
		})
	}).then((r) => r.json());

	// 2. Upload directly to storage
	const uploadRes = await fetch(uploadUrl, {
		method: 'PUT',
		body: file,
		headers: { 'Content-Type': file.type }
	});

	if (!uploadRes.ok) throw new Error('Upload failed');

	// 3. Confirm with server
	await fetch('/upload-confirm', {
		method: 'POST',
		headers: { 'Content-Type': 'application/json' },
		body: JSON.stringify({ key })
	});

	return key;
}

File Validation

Never trust the Content-Type header — it comes from the client. Check the actual bytes:

import { fileTypeFromBuffer } from 'file-type';

async function validateFileType(buffer: Buffer, claimedType: string): Promise<string> {
	const detected = await fileTypeFromBuffer(buffer);

	if (!detected) throw new Error('Cannot determine file type');

	// Check magic bytes match the claimed type
	if (detected.mime !== claimedType) {
		throw new Error(`File type mismatch: claimed ${claimedType}, detected ${detected.mime}`);
	}

	if (!ALLOWED_TYPES.has(detected.mime)) {
		throw new Error(`File type not allowed: ${detected.mime}`);
	}

	return detected.mime;
}

# Install
npm install file-type

For images, also validate dimensions:

import sharp from 'sharp';

async function validateImage(buffer: Buffer): Promise<{ width: number; height: number }> {
	const metadata = await sharp(buffer).metadata();

	const MAX_DIMENSION = 8000;
	if ((metadata.width ?? 0) > MAX_DIMENSION || (metadata.height ?? 0) > MAX_DIMENSION) {
		throw new Error('Image dimensions too large');
	}

	return { width: metadata.width!, height: metadata.height! };
}

Image Processing Pipeline

Transform images before storing — resize, convert format, strip metadata:

import sharp from 'sharp';

interface ImageVariant {
	suffix: string;
	width: number;
	height?: number;
	format: 'webp' | 'jpeg';
	quality: number;
}

const VARIANTS: ImageVariant[] = [
	{ suffix: 'thumb', width: 150, height: 150, format: 'webp', quality: 80 },
	{ suffix: 'medium', width: 800, format: 'webp', quality: 85 },
	{ suffix: 'large', width: 1920, format: 'webp', quality: 90 }
];

async function processAndStoreImage(buffer: Buffer, userId: string) {
	const id = randomUUID();
	const uploads: Promise<void>[] = [];

	for (const variant of VARIANTS) {
		const processed = await sharp(buffer)
			.rotate() // auto-rotate from EXIF
			.resize(variant.width, variant.height, { fit: 'cover' })
			[variant.format]({ quality: variant.quality })
			.withMetadata({ orientation: undefined }) // strip EXIF GPS, keep color profile
			.toBuffer();

		const key = `images/${userId}/${id}/${variant.suffix}.${variant.format}`;

		uploads.push(
			s3
				.send(
					new PutObjectCommand({
						Bucket: 'user-uploads',
						Key: key,
						Body: processed,
						ContentType: `image/${variant.format}`,
						CacheControl: 'public, max-age=31536000, immutable' // 1 year — content-addressed
					})
				)
				.then(() => undefined)
		);
	}

	await Promise.all(uploads);

	return {
		id,
		urls: VARIANTS.reduce<Record<string, string>>((acc, v) => {
			acc[v.suffix] = `${process.env.CDN_URL}/images/${userId}/${id}/${v.suffix}.${v.format}`;
			return acc;
		}, {})
	};
}

Virus Scanning

For user-uploaded documents and executables — scan before making accessible:

import NodeClam from 'clamscan';

const clamscan = await new NodeClam().init({
	clamdscan: {
		socket: '/var/run/clamav/clamd.ctl',
		timeout: 60000
	}
});

async function scanBuffer(buffer: Buffer): Promise<void> {
	const { isInfected, viruses } = await clamscan.scanBuffer(buffer);
	if (isInfected) {
		throw new Error(`Malware detected: ${viruses.join(', ')}`);
	}
}

# docker-compose.yml — add ClamAV sidecar
services:
  clamav:
    image: clamav/clamav:latest
    volumes:
      - clamav_data:/var/lib/clamav
      - /var/run/clamav:/var/run/clamav

For high-throughput, scan asynchronously: store to a quarantine bucket, scan via queue, move to the public bucket on pass or delete on fail.

Upload Progress Tracking

// Client: track progress via XHR (fetch doesn't expose upload progress)
function uploadWithProgress(
	file: File,
	uploadUrl: string,
	onProgress: (percent: number) => void
): Promise<void> {
	return new Promise((resolve, reject) => {
		const xhr = new XMLHttpRequest();

		xhr.upload.addEventListener('progress', (e) => {
			if (e.lengthComputable) {
				onProgress(Math.round((e.loaded / e.total) * 100));
			}
		});

		xhr.addEventListener('load', () => {
			xhr.status < 400 ? resolve() : reject(new Error(`Upload failed: ${xhr.status}`));
		});

		xhr.addEventListener('error', () => reject(new Error('Network error')));

		xhr.open('PUT', uploadUrl);
		xhr.setRequestHeader('Content-Type', file.type);
		xhr.send(file);
	});
}

Multipart Upload for Large Files

For files > 100MB, use S3 multipart upload — splits into chunks, uploads in parallel, more resilient to network failures:

import {
	CreateMultipartUploadCommand,
	UploadPartCommand,
	CompleteMultipartUploadCommand,
	AbortMultipartUploadCommand
} from '@aws-sdk/client-s3';

const PART_SIZE = 10 * 1024 * 1024; // 10MB per part

async function multipartUpload(key: string, buffer: Buffer, contentType: string) {
	const { UploadId } = await s3.send(
		new CreateMultipartUploadCommand({
			Bucket: 'user-uploads',
			Key: key,
			ContentType: contentType
		})
	);

	const parts: { ETag: string; PartNumber: number }[] = [];

	try {
		for (let i = 0; i < Math.ceil(buffer.length / PART_SIZE); i++) {
			const start = i * PART_SIZE;
			const end = Math.min(start + PART_SIZE, buffer.length);
			const partNumber = i + 1;

			const { ETag } = await s3.send(
				new UploadPartCommand({
					Bucket: 'user-uploads',
					Key: key,
					UploadId,
					PartNumber: partNumber,
					Body: buffer.subarray(start, end)
				})
			);

			parts.push({ ETag: ETag!, PartNumber: partNumber });
		}

		await s3.send(
			new CompleteMultipartUploadCommand({
				Bucket: 'user-uploads',
				Key: key,
				UploadId,
				MultipartUpload: { Parts: parts }
			})
		);
	} catch (err) {
		// Clean up incomplete upload (avoid storage costs)
		await s3.send(
			new AbortMultipartUploadCommand({
				Bucket: 'user-uploads',
				Key: key,
				UploadId
			})
		);
		throw err;
	}
}

Set a lifecycle rule to abort incomplete multipart uploads automatically:

await s3.send(
	new PutBucketLifecycleConfigurationCommand({
		Bucket: 'user-uploads',
		LifecycleConfiguration: {
			Rules: [
				{
					ID: 'abort-incomplete-multipart',
					Status: 'Enabled',
					Filter: {},
					AbortIncompleteMultipartUpload: { DaysAfterInitiation: 1 }
				}
			]
		}
	})
);