AWS: API gateway 502 error randomly with Runtime segmentation faults

I am using AWS and have an api which is called via API gateway which calls a node.js lambda function.

Very often but randomly I get 502 responses but when I immediately try again with the exact same request I get a normal response. So I decided to search the logs to see if I could find any issues.

The following is what I found for 1 of the requests:

RequestId: xxxxx Error: Runtime exited with error: signal: segmentation fault Runtime.ExitError

as well as:

xxxx    ERROR   Uncaught Exception  
{
    "errorType": "Error",
    "errorMessage": "Quit inactivity timeout",
    "code": "PROTOCOL_SEQUENCE_TIMEOUT",
    "fatal": true,
    "timeout": 30000,
    "stack": [
        "Error: Quit inactivity timeout",
        "    at Quit.<anonymous> (/opt/nodejs/node_modules/mysql/lib/protocol/Protocol.js:160:17)",
        "    at Quit.emit (node:events:527:28)",
        "    at Quit.emit (node:domain:475:12)",
        "    at Quit._onTimeout (/opt/nodejs/node_modules/mysql/lib/protocol/sequences/Sequence.js:124:8)",
        "    at Timer._onTimeout (/opt/nodejs/node_modules/mysql/lib/protocol/Timer.js:32:23)",
        "    at listOnTimeout (node:internal/timers:559:17)",
        "    at processTimers (node:internal/timers:502:7)"
    ]
}

the following is my reusable sql connector:

const CustomSecret = require('../secrets/CustomSecret');
const mysql = require("mysql");

module.exports = class MySqlConnect {

    databaseCredObject;

    constructor() {
    }

    async queryDb(sql, args) {

        if (!this.databaseCredObject) {
            await this.fetchSecret();
        }

        let connection = null;
        const connection_settings = {
            host: this.databaseCredObject.host,
            user: this.databaseCredObject.username,
            password: this.databaseCredObject.password,
            database: 'logbook'
        };

        connection = mysql.createConnection(connection_settings);

        return new Promise((resolve, reject) => {
            connection.connect(function (err) {
                if (err) {
                    console.log('error when connecting to db:', err);
                } else {
                    console.log('Connected');
                    connection.query(sql, args, function (err, result) {
                        connection.end();
                        if (err) {
                            return reject(err);
                        }
                        return resolve(result);
                    });
                }
            });
        });
    }

    async fetchSecret() {
        const databaseCredString = await CustomSecret.getSecret('secretname', 'eu-west-2');
        this.databaseCredObject = JSON.parse(databaseCredString);
    }
}

Finally this is an example of my lambda function (shortened version):

const {compress, decompress} = require("compress-json");

const MySqlConnect = require("customPackagePath/MySqlConnect");
const CustomJwt = require("customPackagePath/CustomJwt");
const AWS = require("aws-sdk");
const warmer = require("lambda-warmer");

exports.handler = async (event) => {


    if (await warmer(event)) {
        console.log("Warming");
        return 'warmed';
    }

    let responseCode = 200;
    let response = {};

    response.headers = {
        'Content-Type': 'application/json',
        'Access-Control-Allow-Origin': '*'
    };

    const bodyContent = JSON.parse(event.body);
    const dataType = bodyContent.dataType;
    const webAuth = new CustomJwt();
    const decodedToken = webAuth.decodeToken(event.headers.Authorization);
    const userUUID = decodedToken['uuid'];
    
    
    const connection = new MySqlConnect();
    
    let sql;

    switch (dataType) {
        case 'userPreferences':
            sql = await connection.queryDb('SELECT * FROM user WHERE uuid = ?', [userUUID]);
            break;
    }


    let data = [];

    for (let index in sql) {
        data.push(JSON.parse(JSON.stringify(sql[index])));

    }


    const returnData = {
        data
    };


    let compressed = compress(returnData);

    response.statusCode = responseCode;
    response.body = JSON.stringify(compressed);


    return response;
};

Now I am new to infrastructure stuff. But it seems to me that once a lambda function has been called, its not closing or ending correctly. Also I am using the lambda warmer to keep the functions warm as seen in the lambda code and not sure if that is causing any issues.

Appreciate any help with this as I can't seem to figure it out.

Thanks

After doing more research I decided to add this to my Lambda function:

exports.handler = async (event, context, callback) => {

and the return like this

callback(null, response);

and ever since this issue seems to have been resolved. I am not entirely sure why but for now its looking good :)

Share Edit Delete