From 522dafba1d21b37618c493db3160e1117bd9e74d Mon Sep 17 00:00:00 2001 From: PrakharAIP GSOC Date: Fri, 31 Mar 2023 17:00:24 +0530 Subject: [PATCH 1/6] Added elk stack --- install.sh | 10 ++- modules/elk/__init__.py | 0 modules/elk/alert.conf | 0 modules/elk/configure.sh | 36 +++++++++ modules/elk/elk_start.py | 159 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 204 insertions(+), 1 deletion(-) create mode 100644 modules/elk/__init__.py create mode 100644 modules/elk/alert.conf create mode 100644 modules/elk/configure.sh create mode 100644 modules/elk/elk_start.py diff --git a/install.sh b/install.sh index 7251a3ab1a..e0e91fa2bc 100755 --- a/install.sh +++ b/install.sh @@ -13,6 +13,11 @@ curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_20.0 sudo apt update sudo apt install zeek +# elk[start] # +echo "deb https://artifacts.elastic.co/packages/7.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-7.x.list search +wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add - +# elk[end] # + # create a symlink to zeek so that slips can find it echo "[+] Executing 'ln -s /opt/zeek/bin/zeek /usr/local/bin/bro'\n" @@ -45,7 +50,10 @@ sudo apt install -y --no-install-recommends \ golang \ notify-osd \ yara \ - libnotify-bin + libnotify-bin \ + elasticsearch \ + logstash \ + kibana echo "[+] Executing 'python3 -m pip install --upgrade pip'\n" diff --git a/modules/elk/__init__.py b/modules/elk/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/modules/elk/alert.conf b/modules/elk/alert.conf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/modules/elk/configure.sh b/modules/elk/configure.sh new file mode 100644 index 0000000000..bcdc36574c --- /dev/null +++ b/modules/elk/configure.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Check if Elasticsearch is already configured +if ! grep -q "network.host: 127.0.0.1" /etc/elasticsearch/elasticsearch.yml; then + # Configure Elasticsearch to listen only on the localhost + sudo sed -i 's/#network.host: 192.168.0.1/network.host: 127.0.0.1/' /etc/elasticsearch/elasticsearch.yml +fi + +# Check if Logstash is already configured +if [ ! -f /etc/logstash/conf.d/myconfig.conf ]; then + # Create a Logstash configuration file to receive messages + sudo bash -c 'cat > /etc/logstash/conf.d/myconfig.conf < 5000 + } +} + +output { + elasticsearch { + hosts => ["localhost:9200"] + index => "myindex-%{+YYYY.MM.dd}" + } +} +EOF' +fi + +# Check if Kibana is already configured +if ! grep -q "server.host: 127.0.0.1" /etc/kibana/kibana.yml || \ + ! grep -q "elasticsearch.hosts:" /etc/kibana/kibana.yml || \ + ! grep -q "\s* - http://localhost:9200" /etc/kibana/kibana.yml; then + # Configure Kibana to connect to Elasticsearch on localhost only + sudo sed -i 's/#server.host: "localhost"/server.host: "127.0.0.1"/' /etc/kibana/kibana.yml + sudo sed -i 's/#elasticsearch.hosts:/elasticsearch.hosts:/' /etc/kibana/kibana.yml + sudo sed -i 's/#\s* - http:\/\/localhost:9200/ - http:\/\/localhost:9200/' /etc/kibana/kibana.yml +fi diff --git a/modules/elk/elk_start.py b/modules/elk/elk_start.py new file mode 100644 index 0000000000..5102588ffd --- /dev/null +++ b/modules/elk/elk_start.py @@ -0,0 +1,159 @@ +import signal +import subprocess +import time +import os +import psutil + +class ELKStart: + def temp_start_elk(): + """Starts the ELK stack services temporarily""" + + def stop_services(signum, frame): + """Stops the ELK stack services""" + print("\nStopping services...") + subprocess.run(['sudo', 'systemctl', 'stop', 'kibana']) + time.sleep(2) + subprocess.run(['sudo', 'systemctl', 'stop', 'elasticsearch']) + time.sleep(2) + subprocess.run(['sudo', 'systemctl', 'stop', 'logstash']) + exit() + + # Register the signal handler + signal.signal(signal.SIGINT, stop_services) + + # Start the services + print("Starting services...") + subprocess.run(['sudo', 'systemctl', 'start', 'logstash']) + time.sleep(5) + subprocess.run(['sudo', 'systemctl', 'start', 'elasticsearch']) + time.sleep(5) + subprocess.run(['sudo', 'systemctl', 'start', 'kibana']) + + # Wait for a SIGINT signal (Ctrl+C) + print("Press Ctrl+C to stop the services...") + signal.pause() + + def permanent_start_elk(): + """Starts the ELK stack services permanently""" + + # Start the services + print("Starting services...") + subprocess.run(['sudo', 'systemctl', 'start', 'logstash']) + subprocess.run(['sudo', 'systemctl', 'start', 'elasticsearch']) + subprocess.run(['sudo', 'systemctl', 'start', 'kibana']) + + def auto_startup_elk(): + """Starts the ELK stack services automatically on boot""" + + # Enable the services + print("Enabling services for auto_start...") + subprocess.run(['sudo', 'systemctl', 'enable', 'logstash']) + subprocess.run(['sudo', 'systemctl', 'enable', 'elasticsearch']) + subprocess.run(['sudo', 'systemctl', 'enable', 'kibana']) + + def elk_ram_check(): + """Checks if system has atleast 8GB of free RAM for ELK stack""" + mem = psutil.virtual_memory().available / (1024 ** 3) + if mem < 6: + print("\033[1m\033[37mIt seems that Ram is not sufficient please freeup the ram...\033[0m") + return False + else: + return True + + def ask_elk(): + """Asks the user how to start the ELK stack services""" + + print("\033[1m\033[37mELK Stack uses atleast 8GB of RAM, so please ensure that you have atleast 8GB of free RAM before starting the services.\033[0m") + + if not(ELKStart.elk_ram_check()): + return + # Ask the user how to start the services + print("How do you want to start elk services?") + print("1. Temporarily") + print("2. Permanently") + print("3. Permanently and Automatically on boot") + print("4. Do not start elk services") + choice = input("Enter your choice: ") + + if not(choice =='4'): + print("#############################################") + print("Please follow below instructions to be able to show results on Kibana:") + print("") + print("1. Open your web browser and type 'localhost:5601' in the address bar to go to Kibana home page.") + print("2. Click on 3 bars on top left side of the page") + print("3. Inside Management Click on 'Stack Management' in the left-hand navigation menu.") + print("4. In the left hand navigation menu, Click on 'Index Patterns' in 'Kibana' section and then click the 'Create index pattern' button.") + print("5. Click on create index pattern hyperlink. (not neccessary if this option is available)") + print("6. Enter 'myindex-*' as the index pattern name in 'Name' field.") + print("7. Choose '@timestamp' as the Time Filter field name and click 'Create index pattern'.") + print() + print("Now you can see the visualizations on Kibana under 'Discover' section in 'Analytics' subsection when clicked on 3 bars on top left side of the page.") + print("Don't forget to change the timeperiod on top right , to get the results you are searching for.") + print("#################################################") + + + # Start the services + if choice == '1': + ELKStart.temp_start_elk() + elif choice == '2': + ELKStart.permanent_start_elk() + elif choice == '3': + ELKStart.permanent_start_elk() + ELKStart.auto_startup_elk() + elif choice == '4': + print("Skipping elk services...") + else: + print("\033[1m\033[37mInvalid Choice, Skipping elk services\033[0m") + +class ELK_config: + def update_alert_conf(): + """Updates the alert.conf file with the curr path""" + curr_dir = os.getcwd() + alert_conf = curr_dir + '/modules/elk/alert.conf' + with open(alert_conf, 'r') as f: + lines = f.readlines() + content = f.read() + new_content = content.replace("dir/StratosphereLinuxIPS", f"{curr_dir}/output/**/alerts.log") + with open(alert_conf, 'w') as f: + f.write(new_content) + + def elk_alert_config(): + """Copies the alerts.log file""" + + # Update the alert.conf file + ELK_config.update_alert_conf() + + print("Configuring elk services...") + curr_dir = os.getcwd() + alert_conf = curr_dir + '/modules/elk/alert.conf' + subprocess.run(['sudo', 'cp', alert_conf, '/etc/logstash/conf.d/alert.conf']) + + # Ask the user how to start the services + ELKStart.ask_elk() + + def elk_service_config(): + curr_dir = os.getcwd() + elk_conf_script = curr_dir + '/modules/elk/configure.sh' + print("Configuring alerts.log file...") + subprocess.run(['sudo','bash', elk_conf_script]) + + ELK_config.elk_alert_config() + + def elk_service_check(): + """Configures the ELK stack services""" + #Check if elk services are installed + services = ['logstash', 'elasticsearch', 'kibana'] + for service in services: + result = subprocess.run(['systemctl', 'status', service], capture_output=True, text=True) + if "could not be found" in result.stderr: + print(f"{service} is not installed.") + return + else: + print(f"->{service} is installed.") + + + print() + ELK_config.elk_service_config() + +if __name__ == '__main__': + ELK_config.elk_service_check() From b614136570affeff5314f1effa5cbdd045062570 Mon Sep 17 00:00:00 2001 From: PrakharAIP GSOC Date: Fri, 31 Mar 2023 17:04:39 +0530 Subject: [PATCH 2/6] added apt update to install.sh --- install.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/install.sh b/install.sh index e0e91fa2bc..f384eca89d 100755 --- a/install.sh +++ b/install.sh @@ -29,6 +29,7 @@ echo "export PATH=$PATH:/usr/local/zeek/bin" >> ~/.bashrc echo "[+] Installing Slips dependencies ...\n" +sudo apt update sudo apt install -y --no-install-recommends \ python3 \ redis-server \ From 12c95382e983cb0b742eed5bea01d83723b28ed4 Mon Sep 17 00:00:00 2001 From: PrakharAIP GSOC Date: Fri, 31 Mar 2023 17:35:38 +0530 Subject: [PATCH 3/6] added examples of recieving messages in logstash from different machines --- modules/elk/elk_start.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/modules/elk/elk_start.py b/modules/elk/elk_start.py index 5102588ffd..424f5affdc 100644 --- a/modules/elk/elk_start.py +++ b/modules/elk/elk_start.py @@ -89,9 +89,17 @@ def ask_elk(): print() print("Now you can see the visualizations on Kibana under 'Discover' section in 'Analytics' subsection when clicked on 3 bars on top left side of the page.") print("Don't forget to change the timeperiod on top right , to get the results you are searching for.") + print() + print() + print("Logstach is also listening on port 5000, for recieving logs from other machines. You can send logs to it by using the following command as examples:") + print("cat ./alerts.log | nc localhost 5000") + print() + print("Please ensure to configure configure.sh in modules/elk/ to use appropiate network adapter ip.") print("#################################################") + + # Start the services if choice == '1': ELKStart.temp_start_elk() From 5bfffd577efcbfc91d5dc484e3f3855da78bcd9f Mon Sep 17 00:00:00 2001 From: PrakharAIP GSOC Date: Fri, 31 Mar 2023 23:47:06 +0530 Subject: [PATCH 4/6] added alert.conf --- modules/elk/alert.conf | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/modules/elk/alert.conf b/modules/elk/alert.conf index e69de29bb2..0e86be8e0e 100644 --- a/modules/elk/alert.conf +++ b/modules/elk/alert.conf @@ -0,0 +1,23 @@ +input { + file { + path => "dir/StratosphereLinuxIPS/output/**/alerts.log" + start_position => "beginning" + } +} + +filter { + grok { + match => { "message" => "%{TIMESTAMP_ISO8601:timestamp}\.\d{6}\+\d{2}:\d{2}: Src IP %{IPV4:src_ip}\s+\.\s+Detected\s+%{GREEDYDATA:alert_description}" } + } + date { + match => [ "timestamp", "ISO8601" ] + } +} + +output { + elasticsearch { + hosts => ["localhost:9200"] + index => "myindex-%{+YYYY.MM.dd}" + } + stdout { codec => rubydebug } +} From f5113abc51910c5c040a39d6c324d141eb95d4aa Mon Sep 17 00:00:00 2001 From: PrakharAIP GSOC Date: Sat, 1 Apr 2023 17:03:12 +0530 Subject: [PATCH 5/6] Modified Code according to template --- modules/elk/elk.py | 274 +++++++++++++++++++++++++++++++++++++++ modules/elk/elk_start.py | 167 ------------------------ 2 files changed, 274 insertions(+), 167 deletions(-) create mode 100644 modules/elk/elk.py delete mode 100644 modules/elk/elk_start.py diff --git a/modules/elk/elk.py b/modules/elk/elk.py new file mode 100644 index 0000000000..b8be100c62 --- /dev/null +++ b/modules/elk/elk.py @@ -0,0 +1,274 @@ +# Ths is a template module for you to copy and create your own slips module +# Instructions +# 1. Create a new folder on ./modules with the name of your template. Example: +# mkdir modules/anomaly_detector +# 2. Copy this template file in that folder. +# cp modules/template/template.py modules/anomaly_detector/anomaly_detector.py +# 3. Make it a module +# touch modules/template/__init__.py +# 4. Change the name of the module, description and author in the variables +# 5. The file name of the python module (template.py) MUST be the same as the name of the folder (template) +# 6. The variable 'name' MUST have the public name of this module. This is used to ignore the module +# 7. The name of the class MUST be 'Module', do not change it. + + +# Must imports +from slips_files.common.abstracts import Module +import multiprocessing +from slips_files.core.database.database import __database__ +from slips_files.common.config_parser import ConfigParser +from slips_files.common.slips_utils import utils +import sys +import traceback + +# Your imports +import signal +import subprocess +import time +import os +import psutil + +class Module(Module, multiprocessing.Process): + # Name: short name of the module. Do not use spaces + name = 'elk' + description = 'Setup Elk for visualisation(kibana), as well as to recieve messages from different machines' + authors = ['Prakhar Gupta'] + + def __init__(self, outputqueue, redis_port): + multiprocessing.Process.__init__(self) + # All the printing output should be sent to the outputqueue. + # The outputqueue is connected to another process called OutputProcess + self.outputqueue = outputqueue + __database__.start(redis_port) + # To which channels do you want to subscribe? When a message + # arrives on the channel the module will wakeup + self.c1 = __database__.subscribe('new_service') + # Your variables + self.elasticsearch_host = 'localhost' + self.elasticsearch_port = '9200' + self.kibana_port = '5601' + + def print(self, text, verbose=1, debug=0): + """ + Function to use to print text using the outputqueue of slips. + Slips then decides how, when and where to print this text by taking all the processes into account + :param verbose: + 0 - don't print + 1 - basic operation/proof of work + 2 - log I/O operations and filenames + 3 - log database/profile/timewindow changes + :param debug: + 0 - don't print + 1 - print exceptions + 2 - unsupported and unhandled types (cases that may cause errors) + 3 - red warnings that needs examination - developer warnings + :param text: text to print. Can include format like 'Test {}'.format('here') + """ + + levels = f'{verbose}{debug}' + self.outputqueue.put(f'{levels}|{self.name}|{text}') + + def shutdown_gracefully(self): + # Confirm that the module is done processing + __database__.publish('finished_modules', self.name) + + class ELKStart: + def elk_restart(services): + """Restarts the ELK stack services""" + print("Restarting services...") + for service in services: + subprocess.run(['sudo', 'systemctl', 'restart', service]) + time.sleep(2) + + def temp_start_elk(): + """Starts the ELK stack services temporarily""" + + def stop_services(signum, frame): + """Stops the ELK stack services""" + print("\nStopping services...") + subprocess.run(['sudo', 'systemctl', 'stop', 'kibana']) + time.sleep(2) + subprocess.run(['sudo', 'systemctl', 'stop', 'elasticsearch']) + time.sleep(2) + subprocess.run(['sudo', 'systemctl', 'stop', 'logstash']) + exit() + + # Register the signal handler + signal.signal(signal.SIGINT, stop_services) + + # Start the services + print("Starting services...") + subprocess.run(['sudo', 'systemctl', 'start', 'logstash']) + time.sleep(5) + subprocess.run(['sudo', 'systemctl', 'start', 'elasticsearch']) + time.sleep(5) + subprocess.run(['sudo', 'systemctl', 'start', 'kibana']) + + # Wait for a SIGINT signal (Ctrl+C) + print("Press Ctrl+C to stop the services...") + signal.pause() + + def permanent_start_elk(): + """Starts the ELK stack services permanently""" + + # Start the services + print("Starting services...") + subprocess.run(['sudo', 'systemctl', 'start', 'logstash']) + subprocess.run(['sudo', 'systemctl', 'start', 'elasticsearch']) + subprocess.run(['sudo', 'systemctl', 'start', 'kibana']) + + def auto_startup_elk(): + """Starts the ELK stack services automatically on boot""" + + # Enable the services + print("Enabling services for auto_start...") + subprocess.run(['sudo', 'systemctl', 'enable', 'logstash']) + subprocess.run(['sudo', 'systemctl', 'enable', 'elasticsearch']) + subprocess.run(['sudo', 'systemctl', 'enable', 'kibana']) + + def elk_ram_check(): + """Checks if system has atleast 8GB of free RAM for ELK stack""" + mem = psutil.virtual_memory().available / (1024 ** 3) + if mem < 6: + print("\033[1m\033[37mIt seems that Ram is not sufficient please freeup the ram...\033[0m") + return False + else: + return True + + def ask_elk(): + """Asks the user how to start the ELK stack services""" + + print("\033[1m\033[37mELK Stack uses atleast 8GB of RAM, so please ensure that you have atleast 8GB of free RAM before starting the services.\033[0m") + + if not(Module.ELKStart().elk_ram_check()): + return + # Ask the user how to start the services + print("How do you want to start elk services?") + print("1. Temporarily") + print("2. Permanently") + print("3. Permanently and Automatically on boot") + print("4. Do not start elk services") + choice = input("Enter your choice: ") + + if not(choice =='4'): + print("#############################################") + print("Please follow below instructions to be able to show results on Kibana:") + print("") + print("1. Open your web browser and type 'localhost:5601' in the address bar to go to Kibana home page.") + print("2. Click on 3 bars on top left side of the page") + print("3. Inside Management Click on 'Stack Management' in the left-hand navigation menu.") + print("4. In the left hand navigation menu, Click on 'Index Patterns' in 'Kibana' section and then click the 'Create index pattern' button.") + print("5. Click on create index pattern hyperlink. (not neccessary if this option is available)") + print("6. Enter 'myindex-*' as the index pattern name in 'Name' field.") + print("7. Choose '@timestamp' as the Time Filter field name and click 'Create index pattern'.") + print() + print("Now you can see the visualizations on Kibana under 'Discover' section in 'Analytics' subsection when clicked on 3 bars on top left side of the page.") + print("Don't forget to change the timeperiod on top right , to get the results you are searching for.") + print() + print() + print("Logstach is also listening on port 5000, for recieving logs from other machines. You can send logs to it by using the following command as examples:") + print("cat ./alerts.log | nc localhost 5000") + print() + print("Please ensure to configure configure.sh in modules/elk/ to use appropiate network adapter ip.") + print("#################################################") + + + + + # Start the services + if choice == '1': + Module.ELKStart().temp_start_elk() + elif choice == '2': + Module.ELKStart().permanent_start_elk() + elif choice == '3': + Module.ELKStart().permanent_start_elk() + Module.ELKStart().auto_startup_elk() + elif choice == '4': + print("Skipping elk services...") + else: + print("\033[1m\033[37mInvalid Choice, Skipping elk services\033[0m") + + class ELK_config: + def elk_alert_config(): + """Copies the alerts.log file""" + + # Update the alert.conf file + Module.ELK_config().update_alert_conf() + + print("Configuring elk services...") + curr_dir = os.getcwd() + alert_conf = curr_dir + '/modules/elk/alert.conf' + subprocess.run(['sudo', 'cp', alert_conf, '/etc/logstash/conf.d/alert.conf']) + + # Ask the user how to start the services + Module.ELKStart().ask_elk() + + def elk_service_config(): + curr_dir = os.getcwd() + elk_conf_script = curr_dir + '/modules/elk/configure.sh' + print("Configuring alerts.log file...") + subprocess.run(['sudo','bash', elk_conf_script]) + + Module.ELK_config().elk_alert_config() + + def elk_service_check(): + """Configures the ELK stack services""" + #Check if elk services are installed + services = ['logstash', 'elasticsearch', 'kibana'] + for service in services: + result = subprocess.run(['systemctl', 'status', service], capture_output=True, text=True) + if "could not be found" in result.stderr: + print(f"{service} is not installed.") + return + else: + print(f"->{service} is installed.") + + + print() + Module.ELK_config().elk_service_config() + + + def run(self): + # Subscribe to the new_evidence channel and listen for incoming messages + Module.ELK_config().elk_service_check() + self.c1 = __database__.subscribe('new_evidence') + self.print('Subscribed to new_evidence channel. Listening for incoming messages...', 1, 0) + + while True: + try: + # Get message from Redis + message = __database__.get_message(self.c1) + + if message and message['data'] == 'stop_process': + # If the message is to stop the process, then stop gracefully + self.shutdown_gracefully() + return True + + if message and message['channel'] == 'new_evidence': + # Get the evidence from Redis + evidence = message['data'] + self.print('Received new evidence: {}'.format(evidence), 1, 0) + + ###The module is intended to send new evidences when the output is completely generated + # # Send the evidence to Elasticsearch + # is_sent, response = Module.ELKStart().elk_restart(['logstash']) + # if is_sent: + # self.print('Successfully sent evidence to Elasticsearch', 1, 0) + # else: + # self.print('Error sending evidence to Elasticsearch. Response: {}'.format(response), 0, 1) + + except KeyboardInterrupt: + # If keyboard interrupt, then stop gracefully + self.shutdown_gracefully() + return True + except Exception as e: + exception_line = sys.exc_info()[2].tb_lineno + self.print('Problem on the run() line {}. Details: {}'.format(exception_line, e), 0, 1) + return True + + + + + + + diff --git a/modules/elk/elk_start.py b/modules/elk/elk_start.py deleted file mode 100644 index 424f5affdc..0000000000 --- a/modules/elk/elk_start.py +++ /dev/null @@ -1,167 +0,0 @@ -import signal -import subprocess -import time -import os -import psutil - -class ELKStart: - def temp_start_elk(): - """Starts the ELK stack services temporarily""" - - def stop_services(signum, frame): - """Stops the ELK stack services""" - print("\nStopping services...") - subprocess.run(['sudo', 'systemctl', 'stop', 'kibana']) - time.sleep(2) - subprocess.run(['sudo', 'systemctl', 'stop', 'elasticsearch']) - time.sleep(2) - subprocess.run(['sudo', 'systemctl', 'stop', 'logstash']) - exit() - - # Register the signal handler - signal.signal(signal.SIGINT, stop_services) - - # Start the services - print("Starting services...") - subprocess.run(['sudo', 'systemctl', 'start', 'logstash']) - time.sleep(5) - subprocess.run(['sudo', 'systemctl', 'start', 'elasticsearch']) - time.sleep(5) - subprocess.run(['sudo', 'systemctl', 'start', 'kibana']) - - # Wait for a SIGINT signal (Ctrl+C) - print("Press Ctrl+C to stop the services...") - signal.pause() - - def permanent_start_elk(): - """Starts the ELK stack services permanently""" - - # Start the services - print("Starting services...") - subprocess.run(['sudo', 'systemctl', 'start', 'logstash']) - subprocess.run(['sudo', 'systemctl', 'start', 'elasticsearch']) - subprocess.run(['sudo', 'systemctl', 'start', 'kibana']) - - def auto_startup_elk(): - """Starts the ELK stack services automatically on boot""" - - # Enable the services - print("Enabling services for auto_start...") - subprocess.run(['sudo', 'systemctl', 'enable', 'logstash']) - subprocess.run(['sudo', 'systemctl', 'enable', 'elasticsearch']) - subprocess.run(['sudo', 'systemctl', 'enable', 'kibana']) - - def elk_ram_check(): - """Checks if system has atleast 8GB of free RAM for ELK stack""" - mem = psutil.virtual_memory().available / (1024 ** 3) - if mem < 6: - print("\033[1m\033[37mIt seems that Ram is not sufficient please freeup the ram...\033[0m") - return False - else: - return True - - def ask_elk(): - """Asks the user how to start the ELK stack services""" - - print("\033[1m\033[37mELK Stack uses atleast 8GB of RAM, so please ensure that you have atleast 8GB of free RAM before starting the services.\033[0m") - - if not(ELKStart.elk_ram_check()): - return - # Ask the user how to start the services - print("How do you want to start elk services?") - print("1. Temporarily") - print("2. Permanently") - print("3. Permanently and Automatically on boot") - print("4. Do not start elk services") - choice = input("Enter your choice: ") - - if not(choice =='4'): - print("#############################################") - print("Please follow below instructions to be able to show results on Kibana:") - print("") - print("1. Open your web browser and type 'localhost:5601' in the address bar to go to Kibana home page.") - print("2. Click on 3 bars on top left side of the page") - print("3. Inside Management Click on 'Stack Management' in the left-hand navigation menu.") - print("4. In the left hand navigation menu, Click on 'Index Patterns' in 'Kibana' section and then click the 'Create index pattern' button.") - print("5. Click on create index pattern hyperlink. (not neccessary if this option is available)") - print("6. Enter 'myindex-*' as the index pattern name in 'Name' field.") - print("7. Choose '@timestamp' as the Time Filter field name and click 'Create index pattern'.") - print() - print("Now you can see the visualizations on Kibana under 'Discover' section in 'Analytics' subsection when clicked on 3 bars on top left side of the page.") - print("Don't forget to change the timeperiod on top right , to get the results you are searching for.") - print() - print() - print("Logstach is also listening on port 5000, for recieving logs from other machines. You can send logs to it by using the following command as examples:") - print("cat ./alerts.log | nc localhost 5000") - print() - print("Please ensure to configure configure.sh in modules/elk/ to use appropiate network adapter ip.") - print("#################################################") - - - - - # Start the services - if choice == '1': - ELKStart.temp_start_elk() - elif choice == '2': - ELKStart.permanent_start_elk() - elif choice == '3': - ELKStart.permanent_start_elk() - ELKStart.auto_startup_elk() - elif choice == '4': - print("Skipping elk services...") - else: - print("\033[1m\033[37mInvalid Choice, Skipping elk services\033[0m") - -class ELK_config: - def update_alert_conf(): - """Updates the alert.conf file with the curr path""" - curr_dir = os.getcwd() - alert_conf = curr_dir + '/modules/elk/alert.conf' - with open(alert_conf, 'r') as f: - lines = f.readlines() - content = f.read() - new_content = content.replace("dir/StratosphereLinuxIPS", f"{curr_dir}/output/**/alerts.log") - with open(alert_conf, 'w') as f: - f.write(new_content) - - def elk_alert_config(): - """Copies the alerts.log file""" - - # Update the alert.conf file - ELK_config.update_alert_conf() - - print("Configuring elk services...") - curr_dir = os.getcwd() - alert_conf = curr_dir + '/modules/elk/alert.conf' - subprocess.run(['sudo', 'cp', alert_conf, '/etc/logstash/conf.d/alert.conf']) - - # Ask the user how to start the services - ELKStart.ask_elk() - - def elk_service_config(): - curr_dir = os.getcwd() - elk_conf_script = curr_dir + '/modules/elk/configure.sh' - print("Configuring alerts.log file...") - subprocess.run(['sudo','bash', elk_conf_script]) - - ELK_config.elk_alert_config() - - def elk_service_check(): - """Configures the ELK stack services""" - #Check if elk services are installed - services = ['logstash', 'elasticsearch', 'kibana'] - for service in services: - result = subprocess.run(['systemctl', 'status', service], capture_output=True, text=True) - if "could not be found" in result.stderr: - print(f"{service} is not installed.") - return - else: - print(f"->{service} is installed.") - - - print() - ELK_config.elk_service_config() - -if __name__ == '__main__': - ELK_config.elk_service_check() From b9a85f01e280c3d928586fdffde0379ee25a5dab Mon Sep 17 00:00:00 2001 From: PrakharAIP GSOC Date: Mon, 3 Apr 2023 13:24:42 +0530 Subject: [PATCH 6/6] Added a config file for elk in config directory --- modules/elk/elk.py | 27 ++++++++++++++++++++------- requirements.txt | 3 ++- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/modules/elk/elk.py b/modules/elk/elk.py index b8be100c62..595acfcd68 100644 --- a/modules/elk/elk.py +++ b/modules/elk/elk.py @@ -26,6 +26,7 @@ import subprocess import time import os +import json import psutil class Module(Module, multiprocessing.Process): @@ -134,6 +135,23 @@ def elk_ram_check(): return False else: return True + def elk_config(): + curr=os.getcwd() + CONFIG=curr + '/config/elk_config.json' + if os.path.exists(CONFIG): + with open(CONFIG) as f: + data = json.load(f) + else: + data = {} + #ADD comment to the config file + data['comment'] = 'Choose number from 1,2,3,4 to start elk services, where 1 stands for temporary, 2 for permanent, 3 for permanent and auto start on boot, 4 for not to start elk services' + data['other_configuration']='You can check other configurations inside /modules/elk' + data['choice'] = 4 + with open(CONFIG, 'w') as f: + json.dump(data, f, indent=4) + + return data['choice'] + def ask_elk(): """Asks the user how to start the ELK stack services""" @@ -142,13 +160,8 @@ def ask_elk(): if not(Module.ELKStart().elk_ram_check()): return - # Ask the user how to start the services - print("How do you want to start elk services?") - print("1. Temporarily") - print("2. Permanently") - print("3. Permanently and Automatically on boot") - print("4. Do not start elk services") - choice = input("Enter your choice: ") + + choice = Module.ELKStart().elk_config() if not(choice =='4'): print("#############################################") diff --git a/requirements.txt b/requirements.txt index 11e74a0108..fec2592b3f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -37,4 +37,5 @@ wheel flask tld tqdm -termcolor \ No newline at end of file +termcolor +pypi-json==0.3.0 \ No newline at end of file