diff --git a/install.sh b/install.sh index 6983733885..a81274b412 100755 --- a/install.sh +++ b/install.sh @@ -13,6 +13,11 @@ curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_20.0 sudo apt update sudo apt install zeek +# elk[start] # +echo "deb https://artifacts.elastic.co/packages/7.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-7.x.list search +wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add - +# elk[end] # + # create a symlink to zeek so that slips can find it echo "[+] Executing 'ln -s /opt/zeek/bin/zeek /usr/local/bin/bro'\n" @@ -24,6 +29,7 @@ echo "export PATH=$PATH:/usr/local/zeek/bin" >> ~/.bashrc echo "[+] Installing Slips dependencies ...\n" +sudo apt update sudo apt install -y --no-install-recommends \ python3 \ redis-server \ @@ -45,7 +51,10 @@ sudo apt install -y --no-install-recommends \ golang \ notify-osd \ yara \ - libnotify-bin + libnotify-bin \ + elasticsearch \ + logstash \ + kibana echo "[+] Executing 'python3 -m pip install --upgrade pip'\n" diff --git a/modules/elk/__init__.py b/modules/elk/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/modules/elk/alert.conf b/modules/elk/alert.conf new file mode 100644 index 0000000000..0e86be8e0e --- /dev/null +++ b/modules/elk/alert.conf @@ -0,0 +1,23 @@ +input { + file { + path => "dir/StratosphereLinuxIPS/output/**/alerts.log" + start_position => "beginning" + } +} + +filter { + grok { + match => { "message" => "%{TIMESTAMP_ISO8601:timestamp}\.\d{6}\+\d{2}:\d{2}: Src IP %{IPV4:src_ip}\s+\.\s+Detected\s+%{GREEDYDATA:alert_description}" } + } + date { + match => [ "timestamp", "ISO8601" ] + } +} + +output { + elasticsearch { + hosts => ["localhost:9200"] + index => "myindex-%{+YYYY.MM.dd}" + } + stdout { codec => rubydebug } +} diff --git a/modules/elk/configure.sh b/modules/elk/configure.sh new file mode 100644 index 0000000000..bcdc36574c --- /dev/null +++ b/modules/elk/configure.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Check if Elasticsearch is already configured +if ! grep -q "network.host: 127.0.0.1" /etc/elasticsearch/elasticsearch.yml; then + # Configure Elasticsearch to listen only on the localhost + sudo sed -i 's/#network.host: 192.168.0.1/network.host: 127.0.0.1/' /etc/elasticsearch/elasticsearch.yml +fi + +# Check if Logstash is already configured +if [ ! -f /etc/logstash/conf.d/myconfig.conf ]; then + # Create a Logstash configuration file to receive messages + sudo bash -c 'cat > /etc/logstash/conf.d/myconfig.conf < 5000 + } +} + +output { + elasticsearch { + hosts => ["localhost:9200"] + index => "myindex-%{+YYYY.MM.dd}" + } +} +EOF' +fi + +# Check if Kibana is already configured +if ! grep -q "server.host: 127.0.0.1" /etc/kibana/kibana.yml || \ + ! grep -q "elasticsearch.hosts:" /etc/kibana/kibana.yml || \ + ! grep -q "\s* - http://localhost:9200" /etc/kibana/kibana.yml; then + # Configure Kibana to connect to Elasticsearch on localhost only + sudo sed -i 's/#server.host: "localhost"/server.host: "127.0.0.1"/' /etc/kibana/kibana.yml + sudo sed -i 's/#elasticsearch.hosts:/elasticsearch.hosts:/' /etc/kibana/kibana.yml + sudo sed -i 's/#\s* - http:\/\/localhost:9200/ - http:\/\/localhost:9200/' /etc/kibana/kibana.yml +fi diff --git a/modules/elk/elk.py b/modules/elk/elk.py new file mode 100644 index 0000000000..595acfcd68 --- /dev/null +++ b/modules/elk/elk.py @@ -0,0 +1,287 @@ +# Ths is a template module for you to copy and create your own slips module +# Instructions +# 1. Create a new folder on ./modules with the name of your template. Example: +# mkdir modules/anomaly_detector +# 2. Copy this template file in that folder. +# cp modules/template/template.py modules/anomaly_detector/anomaly_detector.py +# 3. Make it a module +# touch modules/template/__init__.py +# 4. Change the name of the module, description and author in the variables +# 5. The file name of the python module (template.py) MUST be the same as the name of the folder (template) +# 6. The variable 'name' MUST have the public name of this module. This is used to ignore the module +# 7. The name of the class MUST be 'Module', do not change it. + + +# Must imports +from slips_files.common.abstracts import Module +import multiprocessing +from slips_files.core.database.database import __database__ +from slips_files.common.config_parser import ConfigParser +from slips_files.common.slips_utils import utils +import sys +import traceback + +# Your imports +import signal +import subprocess +import time +import os +import json +import psutil + +class Module(Module, multiprocessing.Process): + # Name: short name of the module. Do not use spaces + name = 'elk' + description = 'Setup Elk for visualisation(kibana), as well as to recieve messages from different machines' + authors = ['Prakhar Gupta'] + + def __init__(self, outputqueue, redis_port): + multiprocessing.Process.__init__(self) + # All the printing output should be sent to the outputqueue. + # The outputqueue is connected to another process called OutputProcess + self.outputqueue = outputqueue + __database__.start(redis_port) + # To which channels do you want to subscribe? When a message + # arrives on the channel the module will wakeup + self.c1 = __database__.subscribe('new_service') + # Your variables + self.elasticsearch_host = 'localhost' + self.elasticsearch_port = '9200' + self.kibana_port = '5601' + + def print(self, text, verbose=1, debug=0): + """ + Function to use to print text using the outputqueue of slips. + Slips then decides how, when and where to print this text by taking all the processes into account + :param verbose: + 0 - don't print + 1 - basic operation/proof of work + 2 - log I/O operations and filenames + 3 - log database/profile/timewindow changes + :param debug: + 0 - don't print + 1 - print exceptions + 2 - unsupported and unhandled types (cases that may cause errors) + 3 - red warnings that needs examination - developer warnings + :param text: text to print. Can include format like 'Test {}'.format('here') + """ + + levels = f'{verbose}{debug}' + self.outputqueue.put(f'{levels}|{self.name}|{text}') + + def shutdown_gracefully(self): + # Confirm that the module is done processing + __database__.publish('finished_modules', self.name) + + class ELKStart: + def elk_restart(services): + """Restarts the ELK stack services""" + print("Restarting services...") + for service in services: + subprocess.run(['sudo', 'systemctl', 'restart', service]) + time.sleep(2) + + def temp_start_elk(): + """Starts the ELK stack services temporarily""" + + def stop_services(signum, frame): + """Stops the ELK stack services""" + print("\nStopping services...") + subprocess.run(['sudo', 'systemctl', 'stop', 'kibana']) + time.sleep(2) + subprocess.run(['sudo', 'systemctl', 'stop', 'elasticsearch']) + time.sleep(2) + subprocess.run(['sudo', 'systemctl', 'stop', 'logstash']) + exit() + + # Register the signal handler + signal.signal(signal.SIGINT, stop_services) + + # Start the services + print("Starting services...") + subprocess.run(['sudo', 'systemctl', 'start', 'logstash']) + time.sleep(5) + subprocess.run(['sudo', 'systemctl', 'start', 'elasticsearch']) + time.sleep(5) + subprocess.run(['sudo', 'systemctl', 'start', 'kibana']) + + # Wait for a SIGINT signal (Ctrl+C) + print("Press Ctrl+C to stop the services...") + signal.pause() + + def permanent_start_elk(): + """Starts the ELK stack services permanently""" + + # Start the services + print("Starting services...") + subprocess.run(['sudo', 'systemctl', 'start', 'logstash']) + subprocess.run(['sudo', 'systemctl', 'start', 'elasticsearch']) + subprocess.run(['sudo', 'systemctl', 'start', 'kibana']) + + def auto_startup_elk(): + """Starts the ELK stack services automatically on boot""" + + # Enable the services + print("Enabling services for auto_start...") + subprocess.run(['sudo', 'systemctl', 'enable', 'logstash']) + subprocess.run(['sudo', 'systemctl', 'enable', 'elasticsearch']) + subprocess.run(['sudo', 'systemctl', 'enable', 'kibana']) + + def elk_ram_check(): + """Checks if system has atleast 8GB of free RAM for ELK stack""" + mem = psutil.virtual_memory().available / (1024 ** 3) + if mem < 6: + print("\033[1m\033[37mIt seems that Ram is not sufficient please freeup the ram...\033[0m") + return False + else: + return True + def elk_config(): + curr=os.getcwd() + CONFIG=curr + '/config/elk_config.json' + if os.path.exists(CONFIG): + with open(CONFIG) as f: + data = json.load(f) + else: + data = {} + #ADD comment to the config file + data['comment'] = 'Choose number from 1,2,3,4 to start elk services, where 1 stands for temporary, 2 for permanent, 3 for permanent and auto start on boot, 4 for not to start elk services' + data['other_configuration']='You can check other configurations inside /modules/elk' + data['choice'] = 4 + with open(CONFIG, 'w') as f: + json.dump(data, f, indent=4) + + return data['choice'] + + + def ask_elk(): + """Asks the user how to start the ELK stack services""" + + print("\033[1m\033[37mELK Stack uses atleast 8GB of RAM, so please ensure that you have atleast 8GB of free RAM before starting the services.\033[0m") + + if not(Module.ELKStart().elk_ram_check()): + return + + choice = Module.ELKStart().elk_config() + + if not(choice =='4'): + print("#############################################") + print("Please follow below instructions to be able to show results on Kibana:") + print("") + print("1. Open your web browser and type 'localhost:5601' in the address bar to go to Kibana home page.") + print("2. Click on 3 bars on top left side of the page") + print("3. Inside Management Click on 'Stack Management' in the left-hand navigation menu.") + print("4. In the left hand navigation menu, Click on 'Index Patterns' in 'Kibana' section and then click the 'Create index pattern' button.") + print("5. Click on create index pattern hyperlink. (not neccessary if this option is available)") + print("6. Enter 'myindex-*' as the index pattern name in 'Name' field.") + print("7. Choose '@timestamp' as the Time Filter field name and click 'Create index pattern'.") + print() + print("Now you can see the visualizations on Kibana under 'Discover' section in 'Analytics' subsection when clicked on 3 bars on top left side of the page.") + print("Don't forget to change the timeperiod on top right , to get the results you are searching for.") + print() + print() + print("Logstach is also listening on port 5000, for recieving logs from other machines. You can send logs to it by using the following command as examples:") + print("cat ./alerts.log | nc localhost 5000") + print() + print("Please ensure to configure configure.sh in modules/elk/ to use appropiate network adapter ip.") + print("#################################################") + + + + + # Start the services + if choice == '1': + Module.ELKStart().temp_start_elk() + elif choice == '2': + Module.ELKStart().permanent_start_elk() + elif choice == '3': + Module.ELKStart().permanent_start_elk() + Module.ELKStart().auto_startup_elk() + elif choice == '4': + print("Skipping elk services...") + else: + print("\033[1m\033[37mInvalid Choice, Skipping elk services\033[0m") + + class ELK_config: + def elk_alert_config(): + """Copies the alerts.log file""" + + # Update the alert.conf file + Module.ELK_config().update_alert_conf() + + print("Configuring elk services...") + curr_dir = os.getcwd() + alert_conf = curr_dir + '/modules/elk/alert.conf' + subprocess.run(['sudo', 'cp', alert_conf, '/etc/logstash/conf.d/alert.conf']) + + # Ask the user how to start the services + Module.ELKStart().ask_elk() + + def elk_service_config(): + curr_dir = os.getcwd() + elk_conf_script = curr_dir + '/modules/elk/configure.sh' + print("Configuring alerts.log file...") + subprocess.run(['sudo','bash', elk_conf_script]) + + Module.ELK_config().elk_alert_config() + + def elk_service_check(): + """Configures the ELK stack services""" + #Check if elk services are installed + services = ['logstash', 'elasticsearch', 'kibana'] + for service in services: + result = subprocess.run(['systemctl', 'status', service], capture_output=True, text=True) + if "could not be found" in result.stderr: + print(f"{service} is not installed.") + return + else: + print(f"->{service} is installed.") + + + print() + Module.ELK_config().elk_service_config() + + + def run(self): + # Subscribe to the new_evidence channel and listen for incoming messages + Module.ELK_config().elk_service_check() + self.c1 = __database__.subscribe('new_evidence') + self.print('Subscribed to new_evidence channel. Listening for incoming messages...', 1, 0) + + while True: + try: + # Get message from Redis + message = __database__.get_message(self.c1) + + if message and message['data'] == 'stop_process': + # If the message is to stop the process, then stop gracefully + self.shutdown_gracefully() + return True + + if message and message['channel'] == 'new_evidence': + # Get the evidence from Redis + evidence = message['data'] + self.print('Received new evidence: {}'.format(evidence), 1, 0) + + ###The module is intended to send new evidences when the output is completely generated + # # Send the evidence to Elasticsearch + # is_sent, response = Module.ELKStart().elk_restart(['logstash']) + # if is_sent: + # self.print('Successfully sent evidence to Elasticsearch', 1, 0) + # else: + # self.print('Error sending evidence to Elasticsearch. Response: {}'.format(response), 0, 1) + + except KeyboardInterrupt: + # If keyboard interrupt, then stop gracefully + self.shutdown_gracefully() + return True + except Exception as e: + exception_line = sys.exc_info()[2].tb_lineno + self.print('Problem on the run() line {}. Details: {}'.format(exception_line, e), 0, 1) + return True + + + + + + + diff --git a/requirements.txt b/requirements.txt index 11e74a0108..fec2592b3f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -37,4 +37,5 @@ wheel flask tld tqdm -termcolor \ No newline at end of file +termcolor +pypi-json==0.3.0 \ No newline at end of file