Ntopng webhook + Globe.gl + data filter

.gitignore

@@ -1,3 +1,4 @@
+.idea/
 # ---> Node
 # Logs
 logs

app.js

@@ -0,0 +1,42 @@
+var createError = require('http-errors');
+var express = require('express');
+var path = require('path');
+var cookieParser = require('cookie-parser');
+var logger = require('morgan');
+var cors = require('cors')
+
+var indexRouter = require('./routes/index');
+var usersRouter = require('./routes/users');
+
+var app = express();
+
+// view engine setup
+app.set('views', path.join(__dirname, 'views'));
+app.set('view engine', 'jade');
+app.use(cors())
+app.use(logger('dev'));
+app.use(express.json());
+app.use(express.urlencoded({ extended: false }));
+app.use(cookieParser());
+app.use(express.static(path.join(__dirname, 'public')));
+
+app.use('/', indexRouter);
+app.use('/users', usersRouter);
+
+// catch 404 and forward to error handler
+app.use(function(req, res, next) {
+  next(createError(404));
+});
+
+// error handler
+app.use(function(err, req, res, next) {
+  // set locals, only providing error in development
+  res.locals.message = err.message;
+  res.locals.error = req.app.get('env') === 'development' ? err : {};
+
+  // render the error page
+  res.status(err.status || 500);
+  res.render('error');
+});
+
+module.exports = app;

bin/www

@@ -0,0 +1,90 @@
+#!/usr/bin/env node
+
+/**
+ * Module dependencies.
+ */
+
+var app = require('../app');
+var debug = require('debug')('ntopngwebhook:server');
+var http = require('http');
+
+/**
+ * Get port from environment and store in Express.
+ */
+
+var port = normalizePort(process.env.PORT || '3100');
+app.set('port', port);
+
+/**
+ * Create HTTP server.
+ */
+
+var server = http.createServer(app);
+
+/**
+ * Listen on provided port, on all network interfaces.
+ */
+
+server.listen(port);
+server.on('error', onError);
+server.on('listening', onListening);
+
+/**
+ * Normalize a port into a number, string, or false.
+ */
+
+function normalizePort(val) {
+  var port = parseInt(val, 10);
+
+  if (isNaN(port)) {
+    // named pipe
+    return val;
+  }
+
+  if (port >= 0) {
+    // port number
+    return port;
+  }
+
+  return false;
+}
+
+/**
+ * Event listener for HTTP server "error" event.
+ */
+
+function onError(error) {
+  if (error.syscall !== 'listen') {
+    throw error;
+  }
+
+  var bind = typeof port === 'string'
+    ? 'Pipe ' + port
+    : 'Port ' + port;
+
+  // handle specific listen errors with friendly messages
+  switch (error.code) {
+    case 'EACCES':
+      console.error(bind + ' requires elevated privileges');
+      process.exit(1);
+      break;
+    case 'EADDRINUSE':
+      console.error(bind + ' is already in use');
+      process.exit(1);
+      break;
+    default:
+      throw error;
+  }
+}
+
+/**
+ * Event listener for HTTP server "listening" event.
+ */
+
+function onListening() {
+  var addr = server.address();
+  var bind = typeof addr === 'string'
+    ? 'pipe ' + addr
+    : 'port ' + addr.port;
+  debug('Listening on ' + bind);
+}

model/alerts.js

@@ -0,0 +1,121 @@
+var alerts = [{
+    first_seen: 1643634860,
+    srv_city_name: '',
+    ip_version: 4,
+    action: 'store',
+    pool_id: 0,
+    srv_continent_name: '',
+    score: 10,
+    entity_val: 'flow',
+    vlan_id: 0,
+    cli2srv_bytes: 236,
+    cli_country_name: '',
+    entity_id: 4,
+    srv_asn: 0,
+    l7_proto: 5,
+    is_cli_attacker: false,
+    srv_name: 'pdns6.ultradns.co.uk',
+    srv_ip: '204.74.115.1',
+    proto: 17,
+    json: '{"ntopng.key":184352024,"hash_entry_id":21200,"info":"65.127.154.156.in-addr.arpa","alert_generation": {"script_key":"udp_unidirectional","subdir":"flow"}}',
+    srv_country_name: '',
+    community_id: '1:iKl3Ri2iIm0GunF8jR1DEatNOlw=',
+    alert_id: 26,
+    is_srv_attacker: false,
+    srv_blacklisted: false,
+    alerts_map: '04000000',
+    srv_os: '',
+    cli_localhost: true,
+    cli_asn: 0,
+    srv2cli_packets: 0,
+    cli2srv_packets: 4,
+    tstamp: 1643634921,
+    cli_name: 'opnsense.localdomain',
+    cli_continent_name: '',
+    srv2cli_bytes: 0,
+    l7_cat: 14,
+    'proto.ndpi': 'DNS',
+    ifid: 2,
+    observation_point_id: 0,
+    srv_localhost: false,
+    cli_port: 42530,
+    cli_blacklisted: false,
+    dns_last_query: '65.127.154.156.in-addr.arpa',
+    is_flow_alert: true,
+    srv_port: 53,
+    l7_master_proto: 0,
+    is_cli_victim: false,
+    cli_ip: '62.178.50.96',
+    cli_city_name: '',
+    cli_os: 'Android 11.0',
+    is_srv_victim: false
+  }]; 
+
+
+function addAlert(first_seen, srv_city_name, ip_version, action, pool_id, srv_continent_name, score, entity_val, vlan_id, cli2srv_bytes, cli_country_name, entity_id, srv_asn, l7_proto, is_cli_attacker, srv_name, srv_ip, proto, json, srv_country_name, community_id, alert_id, is_srv_attacker, srv_blacklisted, alerts_map, srv_os, cli_localhost, cli_asn, srv2cli_packets, cli2srv_packets, tstamp, cli_name, cli_continent_name, srv2cli_bytes, l7_cat, ifid, observation_point_id, srv_localhost, cli_port, cli_blacklisted, dns_last_query, is_flow_alert, srv_port, l7_master_proto, is_cli_victim, cli_ip, cli_city_name, cli_os, is_srv_victim){
+// function addAlert(srv_name, srv_ip, srv_port, json, cli_ip, cli_port){
+    var data = {
+        first_seen:first_seen,
+        srv_city_name:srv_city_name,
+        ip_version:ip_version,
+        action:action,
+        pool_id:pool_id,
+        srv_continent_name:srv_continent_name,
+        score:score,
+        entity_val:entity_val,
+        vlan_id:vlan_id,
+        cli2srv_bytes:cli2srv_bytes,
+        cli_country_name:cli_country_name,
+        entity_id:entity_id,
+        srv_asn:srv_asn,
+        l7_proto:l7_proto,
+        is_cli_attacker:is_cli_attacker,
+        srv_name:srv_name,
+        srv_ip:srv_ip,
+        proto:proto,
+        json:json,
+        srv_country_name:srv_country_name,
+        community_id:community_id,
+        alert_id:alert_id,
+        is_srv_attacker:is_srv_attacker,
+        srv_blacklisted:srv_blacklisted,
+        alerts_map:alerts_map,
+        srv_os:srv_os,
+        cli_localhost:cli_localhost,
+        cli_asn:cli_asn,
+        srv2cli_packets:srv2cli_packets,
+        cli2srv_packets:cli2srv_packets,
+        tstamp:tstamp,
+        cli_name:cli_name,
+        cli_continent_name:cli_continent_name,
+        srv2cli_bytes:srv2cli_bytes,
+        l7_cat:l7_cat,
+        'proto.ndpi': 'DNS',
+        ifid:ifid,
+        observation_point_id:observation_point_id,
+        srv_localhost:srv_localhost,
+        cli_port:cli_port,
+        cli_blacklisted:cli_blacklisted,
+        dns_last_query:dns_last_query,
+        is_flow_alert:is_flow_alert,
+        srv_port:srv_port,
+        l7_master_proto:l7_master_proto,
+        is_cli_victim:is_cli_victim,
+        cli_ip:cli_ip,
+        cli_city_name:cli_city_name,
+        cli_os:cli_os,
+        is_srv_victim:is_srv_victim
+      }
+      if(!alerts.some(x => x.cli_ip === data.cli_ip && x.srv_ip === data.srv_ip)) {
+          alerts.push(data);
+      }
+}
+
+function getAlerts(){
+    return alerts;
+}
+
+module.exports = {
+    getAlerts, 
+    addAlert
+};

model/globedata.js

@@ -0,0 +1,102 @@
+var data = {
+    loc: [],
+    arc: []
+}
+
+var colors = {
+    loc: "green",
+    arc: ["green", "red"],
+}
+
+var stroke = {
+    loc: 0.1,
+    arc: 1.1
+}
+
+var geoip = require('geoip-lite');
+
+function addArc(src, dest){
+    const arcName = src + " -> " + dest;
+    if(!data.arc.some(x => x.name === arcName)) {
+        if (!src.includes("192.168.1.")) {
+            var geoSrc = geoip.lookup(src);
+            var startLat = geoSrc.ll[0]
+            var startLng = geoSrc.ll[1]
+        } else {
+            console.log("w")
+        }
+        if (!dest.includes("192.168.1.")) {
+            var geoDest = geoip.lookup(dest);
+            var endLat = geoDest.ll[0]
+            var endLng = geoDest.ll[1]
+        } else {
+            console.log("w")
+        }
+
+        // console.log(geoSrc, geoDest)
+        if (geoSrc == undefined) {
+            startLat = 48.1712
+            startLng = 16.321
+        }
+
+        if (geoDest == undefined) {
+            endLat = 48.1712
+            endLng = 16.321
+        }
+        console.log(endLat, endLng)
+
+        var dat = {
+            name: arcName,
+            startLat: startLat,
+            startLng: startLng,
+            endLat: endLat,
+            endLng: endLng,
+            color: colors.arc,
+            stroke: stroke.arc
+        }
+        console.log(arcName)
+        data.arc.push(dat);
+        addLoc(arcName, startLat, startLng)
+    }
+}
+function addLoc(name, lat, lng){
+    var dat = {
+        name: name,
+        lat: lat,
+        lng: lng,
+        size: stroke.loc,
+        color: colors.loc
+    };
+
+    if(!data.loc.some(x => x.lat === lat && x.lng === lng)) {
+        data.loc.push(dat);
+    }else{
+        const dataIndex = data.loc.findIndex(obj => obj.lat === lat && obj.lng === lng);
+        data.loc[dataIndex].name = data.loc[dataIndex].name + "<br>" + name;
+    }
+}
+
+function getLocColor(){
+    return colors.loc;
+}
+
+function getArcColor(){
+    return colors.arc;
+}
+
+function getData(){
+    return data;
+}
+
+function getLocData(){
+    return data.loc;
+}
+
+function getArcData(){
+    return data.arc;
+}
+
+
+module.exports = {
+    addArc, addLoc, getLocData, getArcColor, getData, getArcData, getLocColor
+}

package.json

@@ -0,0 +1,18 @@
+{
+  "name": "ntopngwebhook",
+  "version": "0.0.0",
+  "private": true,
+  "scripts": {
+    "start": "node ./bin/www"
+  },
+  "dependencies": {
+    "cookie-parser": "~1.4.4",
+    "cors": "^2.8.5",
+    "debug": "~2.6.9",
+    "express": "~4.16.1",
+    "geoip-lite": "^1.4.3",
+    "http-errors": "~1.6.3",
+    "jade": "~1.11.0",
+    "morgan": "~1.9.1"
+  }
+}

public/globe.html

@@ -0,0 +1,60 @@
+<head>
+  <style> body { margin: 0; } </style>
+
+  <script src="//unpkg.com/react/umd/react.production.min.js"></script>
+  <script src="//unpkg.com/react-dom/umd/react-dom.production.min.js"></script>
+  <script src="//unpkg.com/babel-standalone"></script>
+
+  <script src="//unpkg.com/react-globe.gl"></script>
+  <script src="https://unpkg.com/axios/dist/axios.min.js"></script>
+</head>
+
+<body>
+<div id="globeViz"></div>
+
+<script type="text/jsx">
+  // Gen random data
+  var data = {};
+  axios.get("http://localhost:3100/alerts").then(res => {
+    data = res.data;
+    console.log(data)
+    ReactDOM.render(
+    <Globe
+      globeImageUrl="//unpkg.com/three-globe/example/img/earth-night.jpg"
+      backgroundImageUrl="//unpkg.com/three-globe/example/img/night-sky.png"
+      // edges
+      arcsData={data.arc}
+      arcColor={(d) => [d.color[0], d.color[1]]}
+      arcDashLength={(d) => d.stroke - 0.1 + 0.3}
+      // arcDashLength={(d) => d.stroke - 0.1}
+      arcDashGap={(d) => 0.1 + (1 - (d.stroke - 0.1))}
+      arcDashAnimateTime={(d) => (1.1 - d.stroke) * 5000 + 2000}
+      arcStroke={"stroke"}
+      //arcCircularResolution={64}
+      // arcLabel={() => "test"}
+      // labels
+      labelsData={data.loc}
+      labelLat={(d) => d.lat}
+      labelLng={(d) => d.lng}
+      labelText={(d) => d.name}
+      labelSize={(d) => 0.5 + d.size}
+      labelDotRadius={(d) => 0.5 + d.size}
+      labelColor={(d) => d.color}
+      labelResolution={2}
+      // bars
+      hexBinPointsData={data.loc}
+      hexBinPointWeight="size"
+      //hexAltitude={(d) => d.sumWeight / 4}
+      hexBinResolution={4}
+      hexTopColor={(d) => d.color}
+      hexSideColor={(d) => d.color}
+      hexBinMerge={true}
+      enablePointerInteraction={false}
+      />,
+    document.getElementById('globeViz')
+  );
+  }).catch(err => alert(err)) 
+
+  
+</script>
+</body>

public/stylesheets/style.css

@@ -0,0 +1,8 @@
+body {
+  padding: 50px;
+  font: 14px "Lucida Grande", Helvetica, Arial, sans-serif;
+}
+
+a {
+  color: #00B7FF;
+}

routes/index.js

@@ -0,0 +1,28 @@
+var express = require('express');
+var router = express.Router();
+var alerts = require('../model/alerts.js');
+var globe = require('../model/globedata.js');
+
+/* GET home page. */
+router.get('/alerts', function(req, res, next) {
+  res.setHeader("Content-Type", "application/json");
+
+  res.json(globe.getData());
+});
+
+router.post('/ws', function(req, res, next) {
+  var data = req.body.alerts;
+
+  data.forEach(item => {
+    var log =  {
+      src: item.srv_ip,
+      dest: item.cli_ip,
+    }
+    console.log(log)
+    alerts.addAlert(item.first_seen, item.srv_city_name, item.ip_version, item.action, item.pool_id, item.srv_continent_name, item.score, item.entity_val, item.vlan_id, item.cli2srv_bytes, item.cli_country_name, item.entity_id, item.srv_asn, item.l7_proto, item.is_cli_attacker, item.srv_name, item.srv_ip, item.proto, item.json, item.srv_country_name, item.community_id, item.alert_id, item.is_srv_attacker, item.srv_blacklisted, item.alerts_map, item.srv_os, item.cli_localhost, item.cli_asn, item.srv2cli_packets, item.cli2srv_packets, item.tstamp, item.cli_name, item.cli_continent_name, item.srv2cli_bytes, item.l7_cat, item.ifid, item.observation_point_id, item.srv_localhost, item.cli_port, item.cli_blacklisted, item.dns_last_query, item.is_flow_alert, item.srv_port, item.l7_master_proto, item.is_cli_victim, item.cli_ip, item.cli_city_name, item.cli_os, item.is_srv_victim);
+    globe.addArc(item.srv_ip, item.cli_ip)
+  })
+  res.sendStatus(200);
+})
+
+module.exports = router;

routes/users.js

@@ -0,0 +1,9 @@
+var express = require('express');
+var router = express.Router();
+
+/* GET users listing. */
+router.get('/', function(req, res, next) {
+  res.send('respond with a resource');
+});
+
+module.exports = router;

views/error.jade

@@ -0,0 +1,6 @@
+extends layout
+
+block content
+  h1= message
+  h2= error.status
+  pre #{error.stack}

views/index.jade

@@ -0,0 +1,5 @@
+extends layout
+
+block content
+  h1= title
+  p Welcome to #{title}

views/layout.jade

@@ -0,0 +1,7 @@
+doctype html
+html
+  head
+    title= title
+    link(rel='stylesheet', href='/stylesheets/style.css')
+  body
+    block content