AWS S3

    npm i aws-sdk

S3 - File Upload

    import { ConfigService } from '@infrastructure/config/config.service';
    import { BadRequestException, HttpStatus, Injectable, InternalServerErrorException } from '@nestjs/common';
    import * as AWS from 'aws-sdk';
    import * as exceljs from 'exceljs';
    import { CopyObjectRequest, PutObjectRequest } from 'aws-sdk/clients/s3';

    @Injectable()
    export class S3Service {
        constructor( private readonly configService: ConfigService ) { }
        
        private s3 = new AWS.S3({
            region: this.configService.get('AWS_REGION'),
            credentials: {
                accessKeyId: process.env.AWS_ACCESS_KEY,
                secretAccessKey: process.env.AWS_SECRET_KEY
            }
        })

        private bucketName = this.configService.getS3BucketName();

        private cloudfront = new AWS.CloudFront({
            region: this.configService.get('AWS_REGION'),
                credentials: {
                    accessKeyId: process.env.AWS_ACCESS_KEY,
                    secretAccessKey: process.env.AWS_SECRET_KEY
                }
        })
 
    }

Utitlity Method

    /**
     * Extract the bucket and key from the S3 URI location parameters. If the location is not correctly formatted, throw an error.
     * @param location 
     * @returns 
     */
    private parseS3Location(location: string): { Bucket: string, Key: string } {
        // Assuming the location is a valid S3 URL like "s3://bucket-name/path/to/file"
        const parts = location.replace('s3://', '').split('/');
        const Bucket = parts.shift();
        const Key = parts.join('/');
        if (!Bucket || !Key) {
            throw new BadRequestException({ status: HttpStatus.BAD_REQUEST, message: 'Invalid S3 location format' });
        }
        return { Bucket, Key };
    }

Upload CSV File

    /**
     * Upload a CSV file to the specified S3 bucket using the provided parameters: the file as a buffer, the bucket name, and the key. 
     * If all parameters are correct, the file is uploaded, and the S3 location is returned. Otherwise, an error is thrown.
     * @param file 
     * @param bucketName 
     * @param key 
     * @returns 
     */
    async uploadCsv(file: string, bucketName: string, key: string): Promise<string> {
        try {
            const buffer = Buffer.from(file, 'utf-8');
            const params: PutObjectRequest = {
                    Bucket: bucketName,
                    Key: key,
                    Body: buffer,
                };

            const uploadResult = await this.s3.upload(params).promise();
            return uploadResult.Location;
        } catch (error) {
            console.log('Error uploading CSV file to S3:', error);
            throw new InternalServerErrorException({ status: HttpStatus.INTERNAL_SERVER_ERROR, message: 'Failed to upload CSV file to S3' });
        }
    }

CSV File

    /**
     * Retrieve the data of the CSV file stored in the S3 bucket based on the provided s3FileKeyPath. 
     * If the s3FileKeyPath is valid, return the file's buffer data; otherwise, throw an error.
     * @param location 
     * @returns 
     */
    async getFileFromS3(s3FileKeyPath: string): Promise<Buffer> {
        try {
            // const { Bucket, Key } = this.parseS3Location(location);
            const response = await this.s3.getObject({ Bucket:this.bucketName, Key:s3FileKeyPath }).promise();
            return response.Body as Buffer;
        } catch (error) {
            throw new BadRequestException({ status: HttpStatus.BAD_REQUEST, message: `Failed to get CSV from S3: ${error.message}` });
        }
    }

Upload Buffer

    /**
     * given buffer file and upload it to the specified s3FileKeyPath in the S3 bucket. Return the S3 URL for the uploaded file.
     * @param bufferFile 
     * @param location 
     * @returns 
     */
    async uploadBuffer(bufferFile: exceljs.Buffer, s3FileKeyPath: string): Promise<string> {
        try {
            // const { Bucket, Key } = this.parseS3Location(location);
            const params: PutObjectRequest = {
            Bucket: this.bucketName,
            Key: s3FileKeyPath,
            Body: bufferFile,
            };

            const uploadResult = await this.s3.upload(params).promise();
            return uploadResult.Location;
        } catch (error) {
            console.log('Error uploading CSV file to S3:', error);
            throw new InternalServerErrorException({ status: HttpStatus.INTERNAL_SERVER_ERROR, message: 'Failed to upload CSV file to S3' });
        }
    }

Presigned URL

    /**
     * Generate a pre-signed S3 URL with an expiration of one hour.
     * @param bucket 
     * @param key 
     * @returns 
     */
    async generatePresignedUrl(bucket: string, key: string): Promise<string> {
        // Define parameters for generating pre-signed URL
        const params = {
            Bucket: bucket,
            Key: key,
            Expires: 60*1, // URL expiration time in seconds (adjust as needed)
        };

        try {
            // Generate pre-signed URL
            const presignedUrl = await this.s3.getSignedUrlPromise('putObject', params);
            return presignedUrl;
        } catch (error) {
            // Handle errors
            throw new BadRequestException({ status: HttpStatus.BAD_REQUEST, message: `Failed to generate pre-signed URL: ${error.message}` });
        }
    }

Copy File

    /**
     * This function copies the data from  one bucket and stores it in other bucket.
     * @param sourcePath S3 URI path from which data need to be copied
     * @param destinationBucket bucket where data needs to be stored
     * @returns key where the ibject is stored.
     */
    async copyS3File(sourcePath: string, destinationBucket: string): Promise<string> {
        try {
            const { Bucket, Key } = this.parseS3Location(sourcePath);

            const copyParams : CopyObjectRequest = {
            Bucket: destinationBucket,
            CopySource: `/${Bucket}/${Key}`,
            Key: Key
            };
            
            await this.s3.copyObject(copyParams).promise();

            return Key;
        } catch (error) {
            throw new BadRequestException({ status: HttpStatus.BAD_REQUEST, message: `Failed to Copy from source S3  bucket: ${error.message}` });
        }
    }

Delete S3 File

    private async deleteFileFromS3(bucketName: string, filePath: string): Promise<void> {
        try {
            await this.s3.deleteObject({ Bucket: bucketName, Key: filePath }).promise();
        } catch (error) {
            console.error('Error deleting file from S3:', error);
            throw new BadRequestException({ status: HttpStatus.BAD_REQUEST, message: 'Failed to delete file from S3.' });
        }
    }

Invalidate File in CloudFront

    private async invalidateFileFromCloudFront(distributionId: string, filePath: string): Promise<void> {
        try {
            await this.cloudfront.createInvalidation({
            DistributionId: distributionId,
            InvalidationBatch: {
                CallerReference: Date.now().toString(),
                Paths: {
                Quantity: 1,
                Items: [filePath]
                }
            }
            }).promise();
        } catch (error) {
            console.error('Error invalidating file from CloudFront:', error);
            throw new BadRequestException({status: HttpStatus.BAD_REQUEST,message:'Failed to invalidate file from CloudFront.'});
        }
    }

Remove S3 File

    async removeS3File(bucketName: string, filePath: string, distributionId?: string,): Promise<void> {
        try {
            await this.deleteFileFromS3(bucketName, filePath);
            // await this.invalidateFileFromCloudFront(distributionId, filePath);
            console.log(`File '${filePath}' removed from S3 and invalidated from CloudFront successfully.`);
        } catch (error) {
            console.error('Error removing file:', error);
            throw new BadRequestException({status: HttpStatus.BAD_REQUEST, message:`Failed to delete s3 file: ${error.message}`}); // Rethrow the error to be handled by the caller
        }
    }

Start Multipart Upload

    // Multipart upload

    /**
     * Given a fileName, initiate processing on S3 through createMultipartUpload and return the upload ID.
     * @param fileName 
     * @returns 
     */
    async startMultipartUpload(fileName: string): Promise<{ uploadId: string }> {
        const params: AWS.S3.CreateMultipartUploadRequest = {
            Bucket: this.bucketName,
            Key: fileName
        };

        try {
            const data = await this.s3.createMultipartUpload(params).promise();
            return { uploadId: data.UploadId };
        } catch (error) {
            console.log('Error starting multipart upload:', error);
            throw new BadRequestException({status: HttpStatus.BAD_REQUEST, message:`Failed to starting multipart upload: ${error.message}`});
        }
    }

Get Presigned URL

    /**
     * Generate a pre-signed URL for S3 based on the request parameters.
     * @param fileName 
     * @param partNumber 
     * @param uploadId 
     * @returns 
     */
    async getPresignedUrl(fileName: string, partNumber: number, uploadId: string): Promise<string> {
        
        const params = {
            Bucket: this.bucketName,
            Key: fileName,
            PartNumber: partNumber,
            UploadId: uploadId,
            Expires: 3600 // Presigned URL expiration time in seconds
        };

        try {
            const url = await this.s3.getSignedUrlPromise('uploadPart', params);
            return url;
        } catch (error) {
            console.log('Error generating presigned URL:', error);
            throw new BadRequestException({status: HttpStatus.BAD_REQUEST, message:`Failed to generating presigned URL: ${error.message}`});
        }
    }

Complete Multipart Upload

    /**
     * Finally, complete the multipart upload to S3 using the provided request parameters.
     * @param fileName 
     * @param uploadId 
     * @param partETags 
     */
    async completeMultipartUpload(fileName: string, uploadId: string, partETags: AWS.S3.CompletedPart[]): Promise<void> {
        const params: AWS.S3.CompleteMultipartUploadRequest = {
            Bucket: this.bucketName,
            Key: fileName,
            MultipartUpload: {
                Parts: partETags
            },
            UploadId: uploadId
        };

        try {
            await this.s3.completeMultipartUpload(params).promise();
        } catch (error) {
            console.log('Error completing multipart upload:', error);
            throw new BadRequestException({status: HttpStatus.BAD_REQUEST, message:`Failed to completing multipart upload: ${error.message}`});
        }
    }