Skip to content

Add --method and --any-status flags for tricky targets #67

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
arthaud merged 2 commits into from
Aug 28, 2026
Merged

Add --method and --any-status flags for tricky targets #67

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f371f87
Add --method and --any-status flags for tricky targets
Aug 25, 2026
90860da
docs: move --method/--any-status into a Custom arguments section
Aug 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 29 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,11 @@ positional arguments:
optional arguments:
-h, --help show this help message and exit
--proxy PROXY use the specified proxy
-X METHOD, --method METHOD
HTTP method to use for all requests (e.g. GET, POST)
--any-status accept any HTTP status code as long as the body is
non-empty and not HTML (e.g. targets that return 500
but still serve content)
-j JOBS, --jobs JOBS number of simultaneous requests
-r RETRY, --retry RETRY
number of request attempts before giving up
Expand All @@ -57,6 +62,30 @@ optional arguments:
git-dumper http://website.com/.git ~/website
```

### Custom arguments

Some targets don't serve the raw `.git` files in the usual way. The following
arguments help with those cases.

#### HTTP Method

Some targets only serve the raw `.git` files over a non-`GET` method. Use
`-X`/`--method` to change the HTTP method used for every request:

```
git-dumper -X POST http://website.com/.git ~/website
```

#### Ignore HTTP status

Some targets respond with a non-200 status (e.g. `500`) while still returning
the real content. Use `--any-status` to accept any status code as long as the
body is non-empty and not HTML:

```
git-dumper --any-status http://website.com/.git ~/website
```


### Disclaimer

Expand Down
113 changes: 77 additions & 36 deletions git_dumper.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,8 +69,8 @@ def get_indexed_files(response):
return files


def verify_response(response):
if response.status_code != 200:
def verify_response(response, any_status=False):
if not any_status and response.status_code != 200:
return (
False,
"[-] %s/%s responded with status code {code}\n".format(
Expand Down Expand Up @@ -221,7 +221,7 @@ def process_tasks(initial_tasks, worker, jobs, args=(), tasks_done=None):
class DownloadWorker(Worker):
""" Download a list of files """

def init(self, url, directory, retry, timeout, http_headers, client_cert_p12=None, client_cert_p12_password=None):
def init(self, url, directory, retry, timeout, http_headers, client_cert_p12=None, client_cert_p12_password=None, method="GET", any_status=False):
self.session = requests.Session()
self.session.verify = False
self.session.headers = http_headers
Expand All @@ -230,13 +230,14 @@ def init(self, url, directory, retry, timeout, http_headers, client_cert_p12=Non
else:
self.session.mount(url, requests.adapters.HTTPAdapter(max_retries=retry))

def do_task(self, filepath, url, directory, retry, timeout, http_headers, client_cert_p12=None, client_cert_p12_password=None):
def do_task(self, filepath, url, directory, retry, timeout, http_headers, client_cert_p12=None, client_cert_p12_password=None, method="GET", any_status=False):
if os.path.isfile(os.path.join(directory, filepath)):
printf("[-] Already downloaded %s/%s\n", url, filepath)
return []

with closing(
self.session.get(
self.session.request(
method,
"%s/%s" % (url, filepath),
allow_redirects=False,
stream=True,
Expand All @@ -250,7 +251,7 @@ def do_task(self, filepath, url, directory, retry, timeout, http_headers, client
response.status_code,
)

valid, error_message = verify_response(response)
valid, error_message = verify_response(response, any_status)
if not valid:
printf(error_message, url, filepath, file=sys.stderr)
return []
Expand All @@ -269,13 +270,14 @@ def do_task(self, filepath, url, directory, retry, timeout, http_headers, client
class RecursiveDownloadWorker(DownloadWorker):
""" Download a directory recursively """

def do_task(self, filepath, url, directory, retry, timeout, http_headers):
def do_task(self, filepath, url, directory, retry, timeout, http_headers, client_cert_p12=None, client_cert_p12_password=None, method="GET", any_status=False):
if os.path.isfile(os.path.join(directory, filepath)):
printf("[-] Already downloaded %s/%s\n", url, filepath)
return []

with closing(
self.session.get(
self.session.request(
method,
"%s/%s" % (url, filepath),
allow_redirects=False,
stream=True,
Expand Down Expand Up @@ -304,7 +306,7 @@ def do_task(self, filepath, url, directory, retry, timeout, http_headers):
for filename in get_indexed_files(response)
]
else: # file
valid, error_message = verify_response(response)
valid, error_message = verify_response(response, any_status)
if not valid:
printf(error_message, url, filepath, file=sys.stderr)
return []
Expand All @@ -323,15 +325,15 @@ def do_task(self, filepath, url, directory, retry, timeout, http_headers):
class FindRefsWorker(DownloadWorker):
""" Find refs/ """

def do_task(self, filepath, url, directory, retry, timeout, http_headers, client_cert_p12=None, client_cert_p12_password=None):
response = self.session.get(
"%s/%s" % (url, filepath), allow_redirects=False, timeout=timeout
def do_task(self, filepath, url, directory, retry, timeout, http_headers, client_cert_p12=None, client_cert_p12_password=None, method="GET", any_status=False):
response = self.session.request(
method, "%s/%s" % (url, filepath), allow_redirects=False, timeout=timeout
)
printf(
"[-] Fetching %s/%s [%d]\n", url, filepath, response.status_code
)

valid, error_message = verify_response(response)
valid, error_message = verify_response(response, any_status)
if not valid:
printf(error_message, url, filepath, file=sys.stderr)
return []
Expand Down Expand Up @@ -360,13 +362,14 @@ def do_task(self, filepath, url, directory, retry, timeout, http_headers, client
class FindObjectsWorker(DownloadWorker):
""" Find objects """

def do_task(self, obj, url, directory, retry, timeout, http_headers, client_cert_p12=None, client_cert_p12_password=None):
def do_task(self, obj, url, directory, retry, timeout, http_headers, client_cert_p12=None, client_cert_p12_password=None, method="GET", any_status=False):
filepath = ".git/objects/%s/%s" % (obj[:2], obj[2:])

if os.path.isfile(os.path.join(directory, filepath)):
printf("[-] Already downloaded %s/%s\n", url, filepath)
else:
response = self.session.get(
response = self.session.request(
method,
"%s/%s" % (url, filepath),
allow_redirects=False,
timeout=timeout,
Expand All @@ -378,7 +381,7 @@ def do_task(self, obj, url, directory, retry, timeout, http_headers, client_cert
response.status_code,
)

valid, error_message = verify_response(response)
valid, error_message = verify_response(response, any_status)
if not valid:
printf(error_message, url, filepath, file=sys.stderr)
return []
Expand Down Expand Up @@ -411,7 +414,7 @@ def sanitize_file(filepath):
f.write(modified_content)


def fetch_git(url, directory, jobs, retry, timeout, http_headers, branches=None, client_cert_p12=None, client_cert_p12_password=None):
def fetch_git(url, directory, jobs, retry, timeout, http_headers, branches=None, client_cert_p12=None, client_cert_p12_password=None, method="GET", any_status=False):
""" Dump a git repository into the output directory """

assert os.path.isdir(directory), "%s is not a directory" % directory
Expand Down Expand Up @@ -441,7 +444,8 @@ def fetch_git(url, directory, jobs, retry, timeout, http_headers, branches=None,
# check for /.git/HEAD
try:
printf("[-] Testing %s/.git/HEAD ", url)
response = session.get(
response = session.request(
method,
"%s/.git/HEAD" % url,
timeout=timeout,
allow_redirects=False
Expand All @@ -452,7 +456,7 @@ def fetch_git(url, directory, jobs, retry, timeout, http_headers, branches=None,

printf("[%d]\n", response.status_code)

valid, error_message = verify_response(response)
valid, error_message = verify_response(response, any_status)
if not valid:
printf(error_message, url, "/.git/HEAD", file=sys.stderr)
return 1
Expand All @@ -473,12 +477,12 @@ def fetch_git(url, directory, jobs, retry, timeout, http_headers, branches=None,

# check for directory listing
printf("[-] Testing %s/.git/ ", url)
response = session.get("%s/.git/" % url, allow_redirects=False)
response = session.request(method, "%s/.git/" % url, allow_redirects=False)
printf("[%d]\n", response.status_code)


if (
response.status_code == 200
(response.status_code == 200 or any_status)
and is_html(response)
and "HEAD" in get_indexed_files(response)
):
Expand All @@ -487,7 +491,7 @@ def fetch_git(url, directory, jobs, retry, timeout, http_headers, branches=None,
[".git/", ".gitignore"],
RecursiveDownloadWorker,
jobs,
args=(url, directory, retry, timeout, http_headers),
args=(url, directory, retry, timeout, http_headers, client_cert_p12, client_cert_p12_password, method, any_status),
)

os.chdir(directory)
Expand Down Expand Up @@ -525,7 +529,7 @@ def fetch_git(url, directory, jobs, retry, timeout, http_headers, branches=None,
tasks,
DownloadWorker,
jobs,
args=(url, directory, retry, timeout, http_headers, client_cert_p12, client_cert_p12_password),
args=(url, directory, retry, timeout, http_headers, client_cert_p12, client_cert_p12_password, method, any_status),
)

# find refs
Expand Down Expand Up @@ -595,7 +599,7 @@ def fetch_git(url, directory, jobs, retry, timeout, http_headers, branches=None,
tasks,
FindRefsWorker,
jobs,
args=(url, directory, retry, timeout, http_headers, client_cert_p12, client_cert_p12_password),
args=(url, directory, retry, timeout, http_headers, client_cert_p12, client_cert_p12_password, method, any_status),
)

# find packs
Expand All @@ -618,7 +622,7 @@ def fetch_git(url, directory, jobs, retry, timeout, http_headers, branches=None,
tasks,
DownloadWorker,
jobs,
args=(url, directory, retry, timeout, http_headers, client_cert_p12, client_cert_p12_password),
args=(url, directory, retry, timeout, http_headers, client_cert_p12, client_cert_p12_password, method, any_status),
)

# find objects
Expand Down Expand Up @@ -658,10 +662,19 @@ def fetch_git(url, directory, jobs, retry, timeout, http_headers, branches=None,
# use .git/index to find objects
index_path = os.path.join(directory, ".git", "index")
if os.path.exists(index_path):
index = dulwich.index.Index(index_path)
# A corrupt/garbage index (e.g. an error page saved via --any-status)
# must not abort the whole dump; skip it and rely on other sources.
try:
index = dulwich.index.Index(index_path)

for entry in index.iterobjects():
objs.add(entry[1].decode())
for entry in index.iterobjects():
objs.add(entry[1].decode())
except Exception as e:
printf(
"[-] Skipping unparseable .git/index: %s\n",
e,
file=sys.stderr,
)

# use packs to find more objects to fetch, and objects that are packed
pack_file_dir = os.path.join(directory, ".git", "objects", "pack")
Expand All @@ -672,21 +685,31 @@ def fetch_git(url, directory, jobs, retry, timeout, http_headers, branches=None,
pack_idx_path = os.path.join(
pack_file_dir, filename[:-5] + ".idx"
)
pack_data = dulwich.pack.PackData(pack_data_path, object_format=dulwich.object_format.DEFAULT_OBJECT_FORMAT)
pack_idx = dulwich.pack.load_pack_index(pack_idx_path, object_format=dulwich.object_format.DEFAULT_OBJECT_FORMAT)
pack = dulwich.pack.Pack.from_objects(pack_data, pack_idx)

for obj_file in pack.iterobjects():
packed_objs.add(obj_file.sha().hexdigest())
objs |= set(get_referenced_sha1(obj_file))
# A corrupt pack/idx (e.g. an error page saved via --any-status)
# must not abort the dump; skip it and continue.
try:
pack_data = dulwich.pack.PackData(pack_data_path, object_format=dulwich.object_format.DEFAULT_OBJECT_FORMAT)
pack_idx = dulwich.pack.load_pack_index(pack_idx_path, object_format=dulwich.object_format.DEFAULT_OBJECT_FORMAT)
pack = dulwich.pack.Pack.from_objects(pack_data, pack_idx)

for obj_file in pack.iterobjects():
packed_objs.add(obj_file.sha().hexdigest())
objs |= set(get_referenced_sha1(obj_file))
except Exception as e:
printf(
"[-] Skipping unparseable pack %s: %s\n",
filename,
e,
file=sys.stderr,
)

# fetch all objects
printf("[-] Fetching objects\n")
process_tasks(
objs,
FindObjectsWorker,
jobs,
args=(url, directory, retry, timeout, http_headers, client_cert_p12, client_cert_p12_password),
args=(url, directory, retry, timeout, http_headers, client_cert_p12, client_cert_p12_password, method, any_status),
tasks_done=packed_objs,
)

Expand All @@ -713,6 +736,19 @@ def main():
parser.add_argument("url", metavar="URL", help="url")
parser.add_argument("directory", metavar="DIR", help="output directory")
parser.add_argument("--proxy", help="use the specified proxy")
parser.add_argument(
"-X",
"--method",
type=str,
default="GET",
help="HTTP method to use for all requests (e.g. GET, POST)",
)
parser.add_argument(
"--any-status",
action="store_true",
help="accept any HTTP status code as long as the body is non-empty "
"and not HTML (e.g. targets that return 500 but still serve content)",
)
parser.add_argument("--client-cert-p12", help="client certificate in PKCS#12")
parser.add_argument("--client-cert-p12-password", help="password for the client certificate")
parser.add_argument(
Expand Down Expand Up @@ -759,6 +795,9 @@ def main():
)
args = parser.parse_args()

# method
args.method = args.method.upper()

# jobs
if args.jobs < 1:
parser.error("invalid number of jobs, got `%d`" % args.jobs)
Expand Down Expand Up @@ -840,6 +879,8 @@ def main():
args.branches,
args.client_cert_p12,
args.client_cert_p12_password,
args.method,
args.any_status,
)
)

Expand Down