From 9269f6fb800c1b225fc655da53e898517385f393 Mon Sep 17 00:00:00 2001 From: Guy Barnhart-Magen Date: Sun, 10 Feb 2019 21:11:59 +0200 Subject: [PATCH 1/6] Working docker file, need to integrate end to end --- Dockerfile | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..d76544f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,23 @@ +FROM neo4j:3.0 + +#HTTP +EXPOSE 7474 +#HTTPS +EXPOSE 7473 +#Bolt +EXPOSE 7687 + +#deprecated by neo4j +#VOLUME ["/data"] +#VOLUME ["/logs"] + +RUN apt update && apt update -y + +RUN apt install -y git python-pip + +COPY [".", "POLAR/"] + +WORKDIR /var/lib/neo4j/POLAR +RUN ["pip", "install", "-r", "requirements.txt"] + +WORKDIR /var/lib/neo4j \ No newline at end of file From 5d2255258c3f4cb9bb396355040aef45fdd65f9f Mon Sep 17 00:00:00 2001 From: Guy Barnhart-Magen Date: Tue, 12 Feb 2019 21:11:22 +0200 Subject: [PATCH 2/6] fixed flush() to close() as there is no write being done, better to close the fd properly --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 87131b8..c19653e 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ with open(path.join(__folder__, 'README.md')) as ld_file: long_description = ld_file.read() - ld_file.flush() + ld_file.close() setup( name='POLAR', From 04c8480cd2a35434a9b89f7b6bbdc5302072264a Mon Sep 17 00:00:00 2001 From: Guy Barnhart-Magen Date: Tue, 12 Feb 2019 23:52:45 +0200 Subject: [PATCH 3/6] added ignore files for the docker build and updated the .gitignore --- .dockerignore | 6 ++++++ .gitignore | 6 +++++- 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..feb1e98 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +*fs/ +*.zip +.* +*.pdf +screenshots/ +*.sh \ No newline at end of file diff --git a/.gitignore b/.gitignore index bfd906c..3950ea5 100644 --- a/.gitignore +++ b/.gitignore @@ -89,4 +89,8 @@ ENV/ .ropeproject # Pycharm project settings -.idea/ \ No newline at end of file +.idea/ + +#filesystem artifacts +data/ +fs/ From 29b402cb2f0ac0bf1e741ccb4b37d5379b62d54a Mon Sep 17 00:00:00 2001 From: Guy Barnhart-Magen Date: Tue, 12 Feb 2019 23:53:25 +0200 Subject: [PATCH 4/6] basic version seems to work (finally), next step to add multiprocessing --- Dockerfile | 35 ++++++++++++++++++----------------- build.sh | 1 + polar/__init__.py | 6 ++---- run.sh | 20 ++++++++++++++++++++ scan_data_dir.py | 17 +++++++++++++++++ 5 files changed, 58 insertions(+), 21 deletions(-) create mode 100755 build.sh create mode 100755 run.sh create mode 100644 scan_data_dir.py diff --git a/Dockerfile b/Dockerfile index d76544f..2418e76 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,23 +1,24 @@ -FROM neo4j:3.0 +#FROM neo4j:3.0 +FROM phusion/baseimage:0.11 -#HTTP -EXPOSE 7474 -#HTTPS -EXPOSE 7473 -#Bolt -EXPOSE 7687 +#HTTP, HTTPS, Bolt +#EXPOSE 7474 7473 7687 -#deprecated by neo4j -#VOLUME ["/data"] -#VOLUME ["/logs"] +RUN apt update \ + && apt upgrade -y \ + && apt install -y python-pip radare2 -RUN apt update && apt update -y +#WORKDIR /var/lib/neo4j +#ENV PATH /var/lib/neo4j/bin:$PATH +#RUN neo4j start \ +# && sleep 5 +#RUN curl -v -H "Content-Type: application/json" -X POST -d '{"password":"test"}' -u neo4j:neo4j http://localhost:7474/user/neo4j/password -RUN apt install -y git python-pip +WORKDIR /home/POLAR +COPY [".", "."] +RUN python setup.py install \ + && pip install -r requirements.txt -COPY [".", "POLAR/"] +#RUN ["scan_data_dir.py"] -WORKDIR /var/lib/neo4j/POLAR -RUN ["pip", "install", "-r", "requirements.txt"] - -WORKDIR /var/lib/neo4j \ No newline at end of file +CMD ["/bin/bash"] \ No newline at end of file diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..44467ec --- /dev/null +++ b/build.sh @@ -0,0 +1 @@ +docker build -t polar_img:1 . diff --git a/polar/__init__.py b/polar/__init__.py index 0f461da..423c44f 100644 --- a/polar/__init__.py +++ b/polar/__init__.py @@ -16,7 +16,6 @@ # Define an Object that stores the Symbol Properties. # A symbol can either be Used, Provided or Imported. - class Symbol(StructuredNode): name = StringProperty(required=True) user = RelationshipFrom('File', 'uses') @@ -44,7 +43,6 @@ class Function(StructuredNode): # First parameter is the filename (the string that is stored, second argument is the path) # Were we will perform the activities - def get_import_export_radare(filename, path): # We define the node or get it if already exists, for further operations. filenode = File.get_or_create({'name': filename})[0] @@ -117,7 +115,7 @@ def parse_main(args=None): parser.add_argument("-db", "--neo4j-database", help="neo4j database url", dest="db_url", - default="bolt://neo4j:neo4j@localhost:7687") + default=config.DATABASE_URL) parser.add_argument("-d", "--directories", help="Directory to parse", @@ -145,7 +143,7 @@ def disassemble_main(args=None): parser.add_argument("-db", "--neo4j-database", help="neo4j database url", dest="db_url", - default="bolt://neo4j:neo4j@localhost:7687") + default=config.DATABASE_URL) parser.add_argument("-f", "--function-tuples", help="file:function tuples", diff --git a/run.sh b/run.sh new file mode 100755 index 0000000..cce0028 --- /dev/null +++ b/run.sh @@ -0,0 +1,20 @@ +#!/bin/bash +echo "Removing old containers" +docker stop neo4j polar 2>/dev/null +docker rm neo4j polar 2>/dev/null + +echo "setup the neo4j instance" +docker run -d --rm --name neo4j --publish 7474:7474 --publish 7473:7473 --publish 7687:7687 neo4j:3.0 +secs=$((5)) +while [ $secs -gt 0 ]; do + echo -ne "." + sleep 1 + : $((secs--)) +done +curl -H "Content-Type: application/json" -d '{"password":"test"}' -u neo4j:neo4j http://localhost:7474/user/neo4j/password + +echo "push data to the neo4j instance" +docker run -it --name polar --link neo4j polar_img:1 python scan_data_dir.py +echo "complete" +echo +echo "Use neo4j:test to log into http://localhost:7474" \ No newline at end of file diff --git a/scan_data_dir.py b/scan_data_dir.py new file mode 100644 index 0000000..81e8b04 --- /dev/null +++ b/scan_data_dir.py @@ -0,0 +1,17 @@ +from os import listdir +from os.path import isfile, join, islink +from polar import get_import_export_radare, config + +config.DATABASE_URL = "bolt://neo4j:test@neo4j:7687" + +directories = ['data/lib/', + 'data/sbin/', + 'data/usr/bin/', + 'data/usr/lib/', + 'data/usr/sbin/'] + +for directory in directories: + onlyfiles = [f for f in listdir(directory) if isfile(join(directory, f)) and not islink(join(directory, f))] + for file in onlyfiles: + print(file) + get_import_export_radare(file,directory+file) From 18a41f50745c694959ee596ab80529a65db1648d Mon Sep 17 00:00:00 2001 From: Guy Barnhart-Magen Date: Wed, 13 Feb 2019 12:34:42 +0200 Subject: [PATCH 5/6] Docker version fully working, analysis time takes ~13m --- Dockerfile | 7 ++++--- polar/__init__.py | 5 +++-- run.sh | 2 +- scan_data_dir.py | 17 ++++++++++++----- 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2418e76..b64c036 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,10 +15,11 @@ RUN apt update \ #RUN curl -v -H "Content-Type: application/json" -X POST -d '{"password":"test"}' -u neo4j:neo4j http://localhost:7474/user/neo4j/password WORKDIR /home/POLAR -COPY [".", "."] -RUN python setup.py install \ - && pip install -r requirements.txt +COPY ["requirements.txt", "."] +RUN pip install -r requirements.txt +COPY [".", "."] +RUN python setup.py install #RUN ["scan_data_dir.py"] CMD ["/bin/bash"] \ No newline at end of file diff --git a/polar/__init__.py b/polar/__init__.py index 423c44f..0828ae1 100644 --- a/polar/__init__.py +++ b/polar/__init__.py @@ -43,11 +43,12 @@ class Function(StructuredNode): # First parameter is the filename (the string that is stored, second argument is the path) # Were we will perform the activities -def get_import_export_radare(filename, path): +def get_import_export_radare(filepath): + _, filename = path.split(filepath) # We define the node or get it if already exists, for further operations. filenode = File.get_or_create({'name': filename})[0] # By using r2, we open the file - r2 = r2pipe.open(path) + r2 = r2pipe.open(filepath) # And *a*nalyze the *f*unctions r2.cmd('af') # and get *i*nformation, on the *i*mports in a *j*son format diff --git a/run.sh b/run.sh index cce0028..b5530b1 100755 --- a/run.sh +++ b/run.sh @@ -14,7 +14,7 @@ done curl -H "Content-Type: application/json" -d '{"password":"test"}' -u neo4j:neo4j http://localhost:7474/user/neo4j/password echo "push data to the neo4j instance" -docker run -it --name polar --link neo4j polar_img:1 python scan_data_dir.py +time docker run -it --name polar --link neo4j polar_img:1 python scan_data_dir.py echo "complete" echo echo "Use neo4j:test to log into http://localhost:7474" \ No newline at end of file diff --git a/scan_data_dir.py b/scan_data_dir.py index 81e8b04..1dd7230 100644 --- a/scan_data_dir.py +++ b/scan_data_dir.py @@ -1,7 +1,9 @@ -from os import listdir +from os import listdir, getcwd, path from os.path import isfile, join, islink from polar import get_import_export_radare, config +import multiprocessing as mp +cwd = getcwd() config.DATABASE_URL = "bolt://neo4j:test@neo4j:7687" directories = ['data/lib/', @@ -10,8 +12,13 @@ 'data/usr/lib/', 'data/usr/sbin/'] +files = [] for directory in directories: - onlyfiles = [f for f in listdir(directory) if isfile(join(directory, f)) and not islink(join(directory, f))] - for file in onlyfiles: - print(file) - get_import_export_radare(file,directory+file) + onlyfiles = [f for f in listdir(directory) if isfile(join(directory, f)) and not islink(join(directory, f))] + for file in onlyfiles: + files.append(path.join(cwd, directory, file)) + +cpus = mp.cpu_count() +print("Analysing %d files using %d CPUs" % (len(files), cpus)) +pl = mp.Pool(processes=cpus) +results = pl.map(get_import_export_radare, files) From a0b41d7b43e3077795f0c8b22de219656fe135ac Mon Sep 17 00:00:00 2001 From: Guy Barnhart-Magen Date: Wed, 13 Feb 2019 12:55:44 +0200 Subject: [PATCH 6/6] added a readme for what the docker does --- EXAMPLE_Docker.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 EXAMPLE_Docker.md diff --git a/EXAMPLE_Docker.md b/EXAMPLE_Docker.md new file mode 100644 index 0000000..62e25db --- /dev/null +++ b/EXAMPLE_Docker.md @@ -0,0 +1,19 @@ +* Example - Docker +** Structure +The docker scripts uses two containers: neo4j and polar. The neo4j docker holds the frontend GUI with a website at http://localhost:7474/ +username: neo4j +password: test + +once the neo4j container is running, the scripts updates its password to the non default "test" (don't use this for real!) and the launches the polar container. + +the polar container is scanning all files copied to it from the data/ folder - and pushes metadata over bolt:// to the neo4j instance. as the analysis is complete - you can query the information in the frontend. + +* Creating the filesystem +In order to run the Example in Docker you need to follow these steps (or use the scripts): + +** "build.sh" script +The build script build the "polar" conatainer with all needed dependencies, and copies the filesystem from the data/ directory + +** "run.sh" script +starts up the neo4j container, updating the default password. +the it runs the polar container, goes over each file, analyse it and send the information to neo4j. once the analysis is complete you can use the frontend to query the data \ No newline at end of file