Detecting Anomalies in Server Logs with AI

Introduction

Anomaly detection identifies unusual patterns. This guide uses autoencoders in TensorFlow.js to detect anomalies in server logs.

Prerequisites

• Node.js >=16
• TensorFlow.js

Step 1: Install Dependencies

npm install @tensorflow/tfjs @tensorflow/tfjs-node csv-parser

Step 2: Preprocess Logs

Convert logs to numeric features (e.g., request time, status code). Example CSV with response_time,status_code.

Step 3: Build Autoencoder

Create anomaly.js:

const tf = require('@tensorflow/tfjs-node');
const fs = require('fs');
const csv = require('csv-parser');

async function loadData(filePath) {
  const data = [];
  return new Promise((resolve) => {
    fs.createReadStream(filePath)
      .pipe(csv())
      .on('data', (row) => {
        data.push([parseFloat(row.response_time), parseInt(row.status_code)]);
      })
      .on('end', () => resolve(tf.tensor2d(data)));
  });
}

async function train() {
  const dataTensor = await loadData('logs.csv');
  const [trainData,] = tf.split(dataTensor, 2);
  
  const inputDim = trainData.shape[1];
  const encodingDim = 1;

  const input = tf.input({ shape: [inputDim] });
  const encoded = tf.layers.dense({ units: encodingDim, activation: 'relu' }).apply(input);
  const decoded = tf.layers.dense({ units: inputDim }).apply(encoded);

  const autoencoder = tf.model({ inputs: input, outputs: decoded });
  autoencoder.compile({ optimizer: 'adam', loss: 'meanSquaredError' });

  await autoencoder.fit(trainData, trainData, {
    epochs: 50,
    batchSize: 32,
    validationSplit: 0.2,
  });

  return autoencoder;
}

async function detect(autoencoder) {
  const dataTensor = await loadData('logs.csv');
  const predictions = autoencoder.predict(dataTensor);
  const errors = tf.sub(dataTensor, predictions).square().mean(1);

  const errorArray = await errors.array();
  const threshold = tf.tensor(errorArray).mean().add(tf.tensor(errorArray).std()).arraySync();

  errorArray.forEach((error, idx) => {
    if (error > threshold) {
      console.log(`Anomaly at row ${idx}, error ${error}`);
    }
  });
}

async function run() {
  const model = await train();
  await detect(model);
}

run();

Summary

AI-based anomaly detection with autoencoders in TensorFlow.js enables proactive identification of unusual log patterns, enhancing system reliability.