diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..feb1e98 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +*fs/ +*.zip +.* +*.pdf +screenshots/ +*.sh \ No newline at end of file diff --git a/.gitignore b/.gitignore index bfd906c..3950ea5 100644 --- a/.gitignore +++ b/.gitignore @@ -89,4 +89,8 @@ ENV/ .ropeproject # Pycharm project settings -.idea/ \ No newline at end of file +.idea/ + +#filesystem artifacts +data/ +fs/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..b64c036 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,25 @@ +#FROM neo4j:3.0 +FROM phusion/baseimage:0.11 + +#HTTP, HTTPS, Bolt +#EXPOSE 7474 7473 7687 + +RUN apt update \ + && apt upgrade -y \ + && apt install -y python-pip radare2 + +#WORKDIR /var/lib/neo4j +#ENV PATH /var/lib/neo4j/bin:$PATH +#RUN neo4j start \ +# && sleep 5 +#RUN curl -v -H "Content-Type: application/json" -X POST -d '{"password":"test"}' -u neo4j:neo4j http://localhost:7474/user/neo4j/password + +WORKDIR /home/POLAR +COPY ["requirements.txt", "."] +RUN pip install -r requirements.txt + +COPY [".", "."] +RUN python setup.py install +#RUN ["scan_data_dir.py"] + +CMD ["/bin/bash"] \ No newline at end of file diff --git a/EXAMPLE_Docker.md b/EXAMPLE_Docker.md new file mode 100644 index 0000000..62e25db --- /dev/null +++ b/EXAMPLE_Docker.md @@ -0,0 +1,19 @@ +* Example - Docker +** Structure +The docker scripts uses two containers: neo4j and polar. The neo4j docker holds the frontend GUI with a website at http://localhost:7474/ +username: neo4j +password: test + +once the neo4j container is running, the scripts updates its password to the non default "test" (don't use this for real!) and the launches the polar container. + +the polar container is scanning all files copied to it from the data/ folder - and pushes metadata over bolt:// to the neo4j instance. as the analysis is complete - you can query the information in the frontend. + +* Creating the filesystem +In order to run the Example in Docker you need to follow these steps (or use the scripts): + +** "build.sh" script +The build script build the "polar" conatainer with all needed dependencies, and copies the filesystem from the data/ directory + +** "run.sh" script +starts up the neo4j container, updating the default password. +the it runs the polar container, goes over each file, analyse it and send the information to neo4j. once the analysis is complete you can use the frontend to query the data \ No newline at end of file diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..44467ec --- /dev/null +++ b/build.sh @@ -0,0 +1 @@ +docker build -t polar_img:1 . diff --git a/polar/__init__.py b/polar/__init__.py index 0f461da..0828ae1 100644 --- a/polar/__init__.py +++ b/polar/__init__.py @@ -16,7 +16,6 @@ # Define an Object that stores the Symbol Properties. # A symbol can either be Used, Provided or Imported. - class Symbol(StructuredNode): name = StringProperty(required=True) user = RelationshipFrom('File', 'uses') @@ -44,12 +43,12 @@ class Function(StructuredNode): # First parameter is the filename (the string that is stored, second argument is the path) # Were we will perform the activities - -def get_import_export_radare(filename, path): +def get_import_export_radare(filepath): + _, filename = path.split(filepath) # We define the node or get it if already exists, for further operations. filenode = File.get_or_create({'name': filename})[0] # By using r2, we open the file - r2 = r2pipe.open(path) + r2 = r2pipe.open(filepath) # And *a*nalyze the *f*unctions r2.cmd('af') # and get *i*nformation, on the *i*mports in a *j*son format @@ -117,7 +116,7 @@ def parse_main(args=None): parser.add_argument("-db", "--neo4j-database", help="neo4j database url", dest="db_url", - default="bolt://neo4j:neo4j@localhost:7687") + default=config.DATABASE_URL) parser.add_argument("-d", "--directories", help="Directory to parse", @@ -145,7 +144,7 @@ def disassemble_main(args=None): parser.add_argument("-db", "--neo4j-database", help="neo4j database url", dest="db_url", - default="bolt://neo4j:neo4j@localhost:7687") + default=config.DATABASE_URL) parser.add_argument("-f", "--function-tuples", help="file:function tuples", diff --git a/run.sh b/run.sh new file mode 100755 index 0000000..b5530b1 --- /dev/null +++ b/run.sh @@ -0,0 +1,20 @@ +#!/bin/bash +echo "Removing old containers" +docker stop neo4j polar 2>/dev/null +docker rm neo4j polar 2>/dev/null + +echo "setup the neo4j instance" +docker run -d --rm --name neo4j --publish 7474:7474 --publish 7473:7473 --publish 7687:7687 neo4j:3.0 +secs=$((5)) +while [ $secs -gt 0 ]; do + echo -ne "." + sleep 1 + : $((secs--)) +done +curl -H "Content-Type: application/json" -d '{"password":"test"}' -u neo4j:neo4j http://localhost:7474/user/neo4j/password + +echo "push data to the neo4j instance" +time docker run -it --name polar --link neo4j polar_img:1 python scan_data_dir.py +echo "complete" +echo +echo "Use neo4j:test to log into http://localhost:7474" \ No newline at end of file diff --git a/scan_data_dir.py b/scan_data_dir.py new file mode 100644 index 0000000..1dd7230 --- /dev/null +++ b/scan_data_dir.py @@ -0,0 +1,24 @@ +from os import listdir, getcwd, path +from os.path import isfile, join, islink +from polar import get_import_export_radare, config +import multiprocessing as mp + +cwd = getcwd() +config.DATABASE_URL = "bolt://neo4j:test@neo4j:7687" + +directories = ['data/lib/', + 'data/sbin/', + 'data/usr/bin/', + 'data/usr/lib/', + 'data/usr/sbin/'] + +files = [] +for directory in directories: + onlyfiles = [f for f in listdir(directory) if isfile(join(directory, f)) and not islink(join(directory, f))] + for file in onlyfiles: + files.append(path.join(cwd, directory, file)) + +cpus = mp.cpu_count() +print("Analysing %d files using %d CPUs" % (len(files), cpus)) +pl = mp.Pool(processes=cpus) +results = pl.map(get_import_export_radare, files) diff --git a/setup.py b/setup.py index 87131b8..c19653e 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ with open(path.join(__folder__, 'README.md')) as ld_file: long_description = ld_file.read() - ld_file.flush() + ld_file.close() setup( name='POLAR',