From dd99953e87ae1b24da6624810f7265912d4efc94 Mon Sep 17 00:00:00 2001
From: HafeezOJ <32125238+HafeezOJ@users.noreply.github.com>
Date: Tue, 16 Jun 2026 15:36:32 -0700
Subject: [PATCH 01/13] Get article version files with version files endpoint
---
figshare/Article.py | 41 ++++++++++++++++++++++++++++++++++++++++-
1 file changed, 40 insertions(+), 1 deletion(-)
diff --git a/figshare/Article.py b/figshare/Article.py
index 162e258..54e18e8 100644
--- a/figshare/Article.py
+++ b/figshare/Article.py
@@ -439,6 +439,43 @@ def set_version_metadata(self, version_data, files, private_version_no, version_
}
}
+ """
+ Fetch article version files list, even if version files are private.
+ :param article_id int value.
+ :param version int value.
+ """
+ def get_article_version_files(self, article_id: int, version: int):
+ success = False
+ retries = 1
+ while not success and retries <= self.retries:
+ page = 1
+ try:
+ page_size = 100
+ page_empty = False
+ header = {'Authorization': 'token ' + self.api_token}
+ article_version_files_api = self.api_endpoint + "/articles/{0}/versions/{1}/files".format(article_id, version)
+ article_version_files = []
+ while not page_empty:
+ params = {'page': page, 'page_size': page_size}
+ get_response = requests.get(article_version_files_api, headers=header, params=params, timeout=self.retry_wait)
+ if get_response.status_code == 200:
+ if len(get_response.json()) > 0:
+ article_version_files += get_response.json()
+ page += 1
+ else:
+ page_empty = True
+ success = True
+ else:
+ retries = self.retries_if_error(f"{article_id} Public API not reachable. Retry {retries}",
+ get_response.status_code, retries)
+ if retries > self.retries:
+ break
+ return article_version_files
+ except requests.exceptions.RequestException as e:
+ retries = self.retries_if_error(f"{e}. Retry {retries}", get_response.status_code, retries)
+ if retries > self.retries:
+ break
+
def private_article_for_files(self, version_data):
get_response = requests.get(version_data['url_private_api'],
headers={'Authorization': 'token ' + self.api_token},
@@ -963,11 +1000,13 @@ def __final_process(self, check_files, copy_files, check_dir, version_data, fold
# save json in metadata folder for each version
self.logs.write_log_in_file("info", "Saving json in metadata folder for each version.", True)
success = success & self.__save_json_in_metadata(version_data, folder_name)
+ # article_version_files = self.get_article_version_files(int(version_data['id']), int(version_data['version']))
if check_files and copy_files:
+ article_version_files = self.get_article_version_files(int(version_data['id']), int(version_data['version']))
try:
# download all files and verify hash with downloaded file.
- delete_now = self.__download_files(version_data['files'], version_data, folder_name)
+ delete_now = self.__download_files(article_version_files, version_data, folder_name)
except Exception as e:
self.logs.write_log_in_file("error", f"{str(e)} for {'_'.join(os.path.basename(folder_name).split('_')[0:-1])}" , True)
if self.system_config['continue-on-error'] == "False":
From e7807d9aa03abe7bc3ed488fe1b9d4659f24b1d2 Mon Sep 17 00:00:00 2001
From: HafeezOJ <32125238+HafeezOJ@users.noreply.github.com>
Date: Tue, 16 Jun 2026 17:04:49 -0700
Subject: [PATCH 02/13] Remove dead code
---
figshare/Article.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/figshare/Article.py b/figshare/Article.py
index 54e18e8..e5d5c84 100644
--- a/figshare/Article.py
+++ b/figshare/Article.py
@@ -1000,7 +1000,6 @@ def __final_process(self, check_files, copy_files, check_dir, version_data, fold
# save json in metadata folder for each version
self.logs.write_log_in_file("info", "Saving json in metadata folder for each version.", True)
success = success & self.__save_json_in_metadata(version_data, folder_name)
- # article_version_files = self.get_article_version_files(int(version_data['id']), int(version_data['version']))
if check_files and copy_files:
article_version_files = self.get_article_version_files(int(version_data['id']), int(version_data['version']))
From 5edd73662b6de1f28c856859267a68b86c7bf34b Mon Sep 17 00:00:00 2001
From: HafeezOJ <32125238+HafeezOJ@users.noreply.github.com>
Date: Tue, 16 Jun 2026 17:24:15 -0700
Subject: [PATCH 03/13] Handle potential error in the exception
---
figshare/Article.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/figshare/Article.py b/figshare/Article.py
index e5d5c84..4757693 100644
--- a/figshare/Article.py
+++ b/figshare/Article.py
@@ -472,7 +472,8 @@ def get_article_version_files(self, article_id: int, version: int):
break
return article_version_files
except requests.exceptions.RequestException as e:
- retries = self.retries_if_error(f"{e}. Retry {retries}", get_response.status_code, retries)
+ error_code = get_response.status_code if 'get_response' in locals() else 500
+ retries = self.retries_if_error(f"{e}. Retry {retries}", error_code, retries)
if retries > self.retries:
break
From 70a8e9591c7623a478d85c525a03e3c855f0bf00 Mon Sep 17 00:00:00 2001
From: HafeezOJ <32125238+HafeezOJ@users.noreply.github.com>
Date: Wed, 17 Jun 2026 12:05:47 -0700
Subject: [PATCH 04/13] Move page initialization out of loop
---
figshare/Article.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/figshare/Article.py b/figshare/Article.py
index 4757693..25595ed 100644
--- a/figshare/Article.py
+++ b/figshare/Article.py
@@ -447,8 +447,8 @@ def set_version_metadata(self, version_data, files, private_version_no, version_
def get_article_version_files(self, article_id: int, version: int):
success = False
retries = 1
+ page = 1
while not success and retries <= self.retries:
- page = 1
try:
page_size = 100
page_empty = False
From cb80a634f0949a3a0fcb622bd4b6c3374c9678a0 Mon Sep 17 00:00:00 2001
From: HafeezOJ <32125238+HafeezOJ@users.noreply.github.com>
Date: Wed, 17 Jun 2026 12:07:59 -0700
Subject: [PATCH 05/13] Move page size out of loop
---
figshare/Article.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/figshare/Article.py b/figshare/Article.py
index 25595ed..f154ea3 100644
--- a/figshare/Article.py
+++ b/figshare/Article.py
@@ -447,10 +447,10 @@ def set_version_metadata(self, version_data, files, private_version_no, version_
def get_article_version_files(self, article_id: int, version: int):
success = False
retries = 1
+ page_size = 100
page = 1
while not success and retries <= self.retries:
try:
- page_size = 100
page_empty = False
header = {'Authorization': 'token ' + self.api_token}
article_version_files_api = self.api_endpoint + "/articles/{0}/versions/{1}/files".format(article_id, version)
From 48a02650bb5780fa8a9d8d948fd7b64a8d38e1ba Mon Sep 17 00:00:00 2001
From: HafeezOJ <32125238+HafeezOJ@users.noreply.github.com>
Date: Thu, 18 Jun 2026 16:23:05 -0700
Subject: [PATCH 06/13] Retrieve full file list or nothing
---
figshare/Article.py | 75 +++++++++++++++++++++++++++------------------
1 file changed, 45 insertions(+), 30 deletions(-)
diff --git a/figshare/Article.py b/figshare/Article.py
index f154ea3..c6c1b89 100644
--- a/figshare/Article.py
+++ b/figshare/Article.py
@@ -54,10 +54,15 @@ def __init__(self, config, log, ids):
self.matched_curation_folder_list = []
self.no_matched = 0
self.no_unmatched = 0
- self.already_preserved_counts_dict = {'already_preserved_article_ids': set(), 'already_preserved_versions': 0,
+
+ # This dict is used to store counts of skipped items. Set articles_with_error keeps article_ids at failed during preprocessing.
+ # while set articles_with_processing_error keeps articled_ids that failed during processing and their intersection is o
+ # to avoid double counting.
+ self.skipped_items_counts_dict = {'already_preserved_article_ids': set(), 'already_preserved_versions': 0,
'wasabi_preserved_versions': 0, 'ap_trust_preserved_versions': 0,
'articles_with_error': set(), 'article_versions_with_error': 0, 'articles_staged': 0,
- 'articles_locally_preserved': 0
+ 'articles_with_processing_error': set(),
+ 'articles_locally_preserved': 0, 'articles_versions_with_processing_error': 0
}
self.skipped_article_versions = {}
self.processor = Integration(self.config_obj, self.logs)
@@ -123,7 +128,7 @@ def get_articles(self):
if (retries > self.retries):
break
- return article_data, self.already_preserved_counts_dict
+ return article_data, self.skipped_items_counts_dict
def article_loop(self, articles, page_size, page, article_data):
no_of_article = 0
@@ -277,8 +282,8 @@ def __get_article_metadata_by_version(self, version, article_id):
f"Curation folder for article {article_id} version {version['version']} not found.",
True)
self.logs.write_log_in_file("info", "Aborting execution.", True, True)
- self.already_preserved_counts_dict['articles_with_error'].add(article_id)
- self.already_preserved_counts_dict['article_versions_with_error'] += 1
+ self.skipped_items_counts_dict['articles_with_error'].add(article_id)
+ self.skipped_items_counts_dict['article_versions_with_error'] += 1
self.logs.write_log_in_file("error",
f"Curation folder for article {article_id} version {version['version']} not found."
+ " Article version will be skipped.",
@@ -325,7 +330,7 @@ def __get_article_metadata_by_version(self, version, article_id):
# Comparison in archival staging storage (final local)
if compare_hash(version_md5, version_local_final_preserved_list):
already_preserved = True
- self.already_preserved_counts_dict['articles_locally_preserved'] += 1
+ self.skipped_items_counts_dict['articles_locally_preserved'] += 1
self.logs.write_log_in_file("info",
f"Article {article_id} version {version['version']} "
+ "already preserved in archival staging storage",
@@ -334,7 +339,7 @@ def __get_article_metadata_by_version(self, version, article_id):
# Comparison in archival storage (final remote)
if compare_hash(version_md5, version_final_storage_preserved_list):
already_preserved = in_ap_trust = True
- self.already_preserved_counts_dict['ap_trust_preserved_versions'] += 1
+ self.skipped_items_counts_dict['ap_trust_preserved_versions'] += 1
self.logs.write_log_in_file("info",
f"Article {article_id} version {version['version']} "
+ "already preserved in archival storage.",
@@ -351,7 +356,7 @@ def __get_article_metadata_by_version(self, version, article_id):
# Comparison in alternative archival staging storage (remote)
if compare_hash(version_md5, version_staging_storage_preserved_list):
- self.already_preserved_counts_dict['wasabi_preserved_versions'] += 1
+ self.skipped_items_counts_dict['wasabi_preserved_versions'] += 1
self.logs.write_log_in_file("info", f"Article {article_id} version {version['version']} "
+ "already preserved in alternative archival staging storage.",
True)
@@ -359,8 +364,8 @@ def __get_article_metadata_by_version(self, version, article_id):
in_alternative_remote_storage = True
if already_preserved and upload_item and in_alternative_remote_storage:
- self.already_preserved_counts_dict['already_preserved_article_ids'].add(article_id)
- self.already_preserved_counts_dict['already_preserved_versions'] += 1
+ self.skipped_items_counts_dict['already_preserved_article_ids'].add(article_id)
+ self.skipped_items_counts_dict['already_preserved_versions'] += 1
if in_ap_trust:
for version_hash in version_final_storage_preserved_list:
if version_hash[0] == version_md5 and version_hash[1] != payload_size:
@@ -370,12 +375,12 @@ def __get_article_metadata_by_version(self, version, article_id):
True)
return None
elif not already_preserved and upload_item and in_alternative_remote_storage:
- self.already_preserved_counts_dict['already_preserved_article_ids'].add(article_id)
- self.already_preserved_counts_dict['already_preserved_versions'] += 1
+ self.skipped_items_counts_dict['already_preserved_article_ids'].add(article_id)
+ self.skipped_items_counts_dict['already_preserved_versions'] += 1
return None
elif already_preserved and not upload_item and in_alternative_remote_storage:
- self.already_preserved_counts_dict['already_preserved_article_ids'].add(article_id)
- self.already_preserved_counts_dict['already_preserved_versions'] += 1
+ self.skipped_items_counts_dict['already_preserved_article_ids'].add(article_id)
+ self.skipped_items_counts_dict['already_preserved_versions'] += 1
if in_ap_trust:
for version_hash in version_final_storage_preserved_list:
if version_hash[0] == version_md5 and version_hash[1] != payload_size:
@@ -385,8 +390,8 @@ def __get_article_metadata_by_version(self, version, article_id):
True)
return None
elif already_preserved and not upload_item and not in_alternative_remote_storage:
- self.already_preserved_counts_dict['already_preserved_article_ids'].add(article_id)
- self.already_preserved_counts_dict['already_preserved_versions'] += 1
+ self.skipped_items_counts_dict['already_preserved_article_ids'].add(article_id)
+ self.skipped_items_counts_dict['already_preserved_versions'] += 1
if in_ap_trust:
for version_hash in version_final_storage_preserved_list:
if version_hash[0] == version_md5 and version_hash[1] != payload_size:
@@ -441,6 +446,7 @@ def set_version_metadata(self, version_data, files, private_version_no, version_
"""
Fetch article version files list, even if version files are private.
+ Could return None if request fails, otherwise returns a list
:param article_id int value.
:param version int value.
"""
@@ -453,7 +459,7 @@ def get_article_version_files(self, article_id: int, version: int):
try:
page_empty = False
header = {'Authorization': 'token ' + self.api_token}
- article_version_files_api = self.api_endpoint + "/articles/{0}/versions/{1}/files".format(article_id, version)
+ article_version_files_api = self.api_endpoint + "/articles/{0}/versions/{1}/fies".format(article_id, version)
article_version_files = []
while not page_empty:
params = {'page': page, 'page_size': page_size}
@@ -469,13 +475,13 @@ def get_article_version_files(self, article_id: int, version: int):
retries = self.retries_if_error(f"{article_id} Public API not reachable. Retry {retries}",
get_response.status_code, retries)
if retries > self.retries:
- break
+ return None
return article_version_files
except requests.exceptions.RequestException as e:
error_code = get_response.status_code if 'get_response' in locals() else 500
retries = self.retries_if_error(f"{e}. Retry {retries}", error_code, retries)
if retries > self.retries:
- break
+ return None
def private_article_for_files(self, version_data):
get_response = requests.get(version_data['url_private_api'],
@@ -960,9 +966,9 @@ def find_matched_articles(self, articles):
self.logs.write_log_in_file("info", f"Total matched article versions: {self.no_matched}.", True)
self.logs.write_log_in_file("info", f"Total unmatched article versions: {self.no_unmatched}.", True)
self.logs.write_log_in_file("info", "Total skipped unique articles: "
- + f"{len(self.already_preserved_counts_dict['already_preserved_article_ids'])}.", True)
+ + f"{len(self.skipped_items_counts_dict['already_preserved_article_ids'])}.", True)
self.logs.write_log_in_file("info", "Total skipped article versions: "
- + f"{self.already_preserved_counts_dict['already_preserved_versions']}.", True)
+ + f"{self.skipped_items_counts_dict['already_preserved_versions']}.", True)
if len(set(unmatched_articles)) > 0 or len(self.article_non_match_info) > 0:
self.logs.write_log_in_file("warning", "There were unmatched articles or article versions."
@@ -1004,14 +1010,22 @@ def __final_process(self, check_files, copy_files, check_dir, version_data, fold
if check_files and copy_files:
article_version_files = self.get_article_version_files(int(version_data['id']), int(version_data['version']))
- try:
- # download all files and verify hash with downloaded file.
- delete_now = self.__download_files(article_version_files, version_data, folder_name)
- except Exception as e:
- self.logs.write_log_in_file("error", f"{str(e)} for {'_'.join(os.path.basename(folder_name).split('_')[0:-1])}" , True)
- if self.system_config['continue-on-error'] == "False":
- self.logs.write_log_in_file("info", "Aborting execution.", True, True)
+ if article_version_files is None and len(version_data['files']) > 1:
+ # self.skipped_items_counts_dict['articles_with_processing_error'].add(version_data['id'])
+ if version_data['id'] not in self.skipped_items_counts_dict['articles_with_error']:
+ self.skipped_items_counts_dict['articles_with_processing_error'].add(version_data['id'])
+ self.skipped_items_counts_dict['articles_versions_with_processing_error'] += 1
+ self.logs.write_log_in_file("error", f"Error retrieving file list for article {version_data['id']} {version_no}. Skipping", True)
delete_now = True
+ else:
+ try:
+ # download all files and verify hash with downloaded file.
+ delete_now = self.__download_files(article_version_files, version_data, folder_name)
+ except Exception as e:
+ self.logs.write_log_in_file("error", f"{str(e)} for {'_'.join(os.path.basename(folder_name).split('_')[0:-1])}" , True)
+ if self.system_config['continue-on-error'] == "False":
+ self.logs.write_log_in_file("info", "Aborting execution.", True, True)
+ delete_now = True
# check if download process has error or not.
if (delete_now is False):
@@ -1212,8 +1226,9 @@ def process_articles(self, articles):
if (value_post_process != 0):
self.logs.write_log_in_file("error", f"{version_data['id']} version {version_data['version']} - "
+ "Post-processing script failed.", True)
- return processed_count, self.already_preserved_counts_dict['ap_trust_preserved_versions'], \
- self.already_preserved_counts_dict['wasabi_preserved_versions']
+ return processed_count, self.skipped_items_counts_dict['ap_trust_preserved_versions'], \
+ self.skipped_items_counts_dict['wasabi_preserved_versions'], len(self.skipped_items_counts_dict['articles_with_processing_error']), \
+ self.skipped_items_counts_dict['articles_versions_with_processing_error']
"""
Preservation and Curation directory access check while processing.
From d9b2f75e5cb29602bfcd7832e1f36b0c5e0e9b2f Mon Sep 17 00:00:00 2001
From: HafeezOJ <32125238+HafeezOJ@users.noreply.github.com>
Date: Thu, 18 Jun 2026 16:24:14 -0700
Subject: [PATCH 07/13] Update counts
---
app.py | 23 ++++++++++++-----------
1 file changed, 12 insertions(+), 11 deletions(-)
diff --git a/app.py b/app.py
index d9aca10..ae38e3f 100644
--- a/app.py
+++ b/app.py
@@ -154,14 +154,14 @@ def main():
log.write_log_in_file('info', " ", True)
log.write_log_in_file('info', "------- Fetching articles -------", True)
article_obj = Article(config, log, args.ids)
- article_data, already_preserved_counts_dict = article_obj.get_articles()
+ article_data, skipped_items_counts_dict = article_obj.get_articles()
- already_preserved_articles_count = len(already_preserved_counts_dict['already_preserved_article_ids'])
- already_preserved_versions_count = already_preserved_counts_dict['already_preserved_versions']
- locally_preserved_article_version_count = already_preserved_counts_dict['articles_locally_preserved']
+ already_preserved_articles_count = len(skipped_items_counts_dict['already_preserved_article_ids'])
+ already_preserved_versions_count = skipped_items_counts_dict['already_preserved_versions']
+ locally_preserved_article_version_count = skipped_items_counts_dict['articles_locally_preserved']
- articles_with_error_count = len(already_preserved_counts_dict['articles_with_error'])
- article_versions_with_error_count = already_preserved_counts_dict['article_versions_with_error']
+ articles_with_error_count = len(skipped_items_counts_dict['articles_with_error'])
+ article_versions_with_error_count = skipped_items_counts_dict['article_versions_with_error']
published_articles_count = 0
published_articles_versions_count = 0
published_unpublished_count = 0
@@ -195,8 +195,8 @@ def main():
print(" ")
# Start articles processing after completing fetching data from API
- processed_articles_versions_count, ap_trust_preserved_article_version_count, wasabi_preserved_versions \
- = article_obj.process_articles(article_data)
+ processed_articles_versions_count, ap_trust_preserved_article_version_count, wasabi_preserved_versions, articles_with_processing_error, \
+ articles_versions_with_processing_error = article_obj.process_articles(article_data)
# Start collections processing after completing fetching data from API and articles processing.
processed_collections_versions_count, already_preserved_collections_counts = collection_obj.process_collections(collection_data)
@@ -220,18 +220,19 @@ def main():
log.write_log_in_file('info',
"Total count of already preserved (skipped) articles / article versions: \t\t"
- + f'{already_preserved_articles_count} / {already_preserved_versions_count}',
+ + f'{already_preserved_articles_count} / '
+ + f'{published_articles_versions_count + already_preserved_versions_count + article_versions_with_error_count}',
True)
log.write_log_in_file('info',
"Total count of articles with fetch error / articles: \t\t\t\t"
- + f'{articles_with_error_count} / '
+ + f'{articles_with_error_count + articles_with_processing_error} / '
+ f'{published_articles_count + already_preserved_articles_count + articles_with_error_count}',
True)
log.write_log_in_file('info',
"Total count of article versions with fetch error / article versions: \t\t"
- + f'{article_versions_with_error_count} / '
+ + f'{article_versions_with_error_count + articles_versions_with_processing_error} / '
+ f'{published_articles_versions_count + already_preserved_versions_count + article_versions_with_error_count}',
True)
From 3671368541467bcb8ff5a053b4416d0313571afc Mon Sep 17 00:00:00 2001
From: HafeezOJ <32125238+HafeezOJ@users.noreply.github.com>
Date: Thu, 18 Jun 2026 16:48:10 -0700
Subject: [PATCH 08/13] Remove dead code
---
figshare/Article.py | 2 --
1 file changed, 2 deletions(-)
diff --git a/figshare/Article.py b/figshare/Article.py
index c6c1b89..d3b2f39 100644
--- a/figshare/Article.py
+++ b/figshare/Article.py
@@ -29,7 +29,6 @@ class Article:
def __init__(self, config, log, ids):
self.config_obj = config
figshare_config = self.config_obj.figshare_config()
- # self.aptrust_config = self.config_obj.aptrust_config()
self.system_config = self.config_obj.system_config()
self.api_endpoint = figshare_config["url"]
self.api_token = figshare_config["token"]
@@ -1011,7 +1010,6 @@ def __final_process(self, check_files, copy_files, check_dir, version_data, fold
if check_files and copy_files:
article_version_files = self.get_article_version_files(int(version_data['id']), int(version_data['version']))
if article_version_files is None and len(version_data['files']) > 1:
- # self.skipped_items_counts_dict['articles_with_processing_error'].add(version_data['id'])
if version_data['id'] not in self.skipped_items_counts_dict['articles_with_error']:
self.skipped_items_counts_dict['articles_with_processing_error'].add(version_data['id'])
self.skipped_items_counts_dict['articles_versions_with_processing_error'] += 1
From 9fc55df19ae539c7083d4603966ab81e5940a366 Mon Sep 17 00:00:00 2001
From: HafeezOJ <32125238+HafeezOJ@users.noreply.github.com>
Date: Thu, 18 Jun 2026 17:02:10 -0700
Subject: [PATCH 09/13] Update comment and lint
---
figshare/Article.py | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/figshare/Article.py b/figshare/Article.py
index d3b2f39..ffc17c5 100644
--- a/figshare/Article.py
+++ b/figshare/Article.py
@@ -56,13 +56,13 @@ def __init__(self, config, log, ids):
# This dict is used to store counts of skipped items. Set articles_with_error keeps article_ids at failed during preprocessing.
# while set articles_with_processing_error keeps articled_ids that failed during processing and their intersection is o
- # to avoid double counting.
+ # to avoid double counting. Items in articles_(version)_with_processing_error included published_articles_(version) count.
self.skipped_items_counts_dict = {'already_preserved_article_ids': set(), 'already_preserved_versions': 0,
- 'wasabi_preserved_versions': 0, 'ap_trust_preserved_versions': 0,
- 'articles_with_error': set(), 'article_versions_with_error': 0, 'articles_staged': 0,
- 'articles_with_processing_error': set(),
- 'articles_locally_preserved': 0, 'articles_versions_with_processing_error': 0
- }
+ 'wasabi_preserved_versions': 0, 'ap_trust_preserved_versions': 0,
+ 'articles_with_error': set(), 'article_versions_with_error': 0, 'articles_staged': 0,
+ 'articles_with_processing_error': set(),
+ 'articles_locally_preserved': 0, 'articles_versions_with_processing_error': 0
+ }
self.skipped_article_versions = {}
self.processor = Integration(self.config_obj, self.logs)
@@ -458,7 +458,7 @@ def get_article_version_files(self, article_id: int, version: int):
try:
page_empty = False
header = {'Authorization': 'token ' + self.api_token}
- article_version_files_api = self.api_endpoint + "/articles/{0}/versions/{1}/fies".format(article_id, version)
+ article_version_files_api = self.api_endpoint + "/articles/{0}/versions/{1}/files".format(article_id, version)
article_version_files = []
while not page_empty:
params = {'page': page, 'page_size': page_size}
From d3e6fcf46e2d44fade810df5efe3b2f59938ab1f Mon Sep 17 00:00:00 2001
From: HafeezOJ <32125238+HafeezOJ@users.noreply.github.com>
Date: Mon, 22 Jun 2026 12:03:54 -0700
Subject: [PATCH 10/13] Update skipped versions dict
---
figshare/Article.py | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/figshare/Article.py b/figshare/Article.py
index ffc17c5..5058ac4 100644
--- a/figshare/Article.py
+++ b/figshare/Article.py
@@ -171,8 +171,9 @@ def __get_article_versions(self, article):
article_version = 'v' + str(version['version']).zfill(2) if version['version'] <= 9 \
else 'v' + str(version['version'])
article_id = str(article['id'])
- self.skipped_article_versions[article_id] = []
- self.skipped_article_versions[article_id].append(article_version)
+ if article_id not in self.skipped_article_versions.keys():
+ self.skipped_article_versions[article_id] = set()
+ self.skipped_article_versions[article_id].add(article_version)
continue
metadata.append(version_data)
else:
@@ -1013,6 +1014,9 @@ def __final_process(self, check_files, copy_files, check_dir, version_data, fold
if version_data['id'] not in self.skipped_items_counts_dict['articles_with_error']:
self.skipped_items_counts_dict['articles_with_processing_error'].add(version_data['id'])
self.skipped_items_counts_dict['articles_versions_with_processing_error'] += 1
+ if str(version_data['id']) not in self.skipped_article_versions.keys():
+ self.skipped_article_versions[str(version_data['id'])] = set()
+ self.skipped_article_versions[str(version_data['id'])].add(version_no)
self.logs.write_log_in_file("error", f"Error retrieving file list for article {version_data['id']} {version_no}. Skipping", True)
delete_now = True
else:
From 724956033b1a156cbb23ef30bd4a26080a2de268 Mon Sep 17 00:00:00 2001
From: HafeezOJ <32125238+HafeezOJ@users.noreply.github.com>
Date: Mon, 22 Jun 2026 16:16:13 -0700
Subject: [PATCH 11/13] Move counting failed items to end of processing
---
figshare/Article.py | 28 ++++++++++++++--------------
1 file changed, 14 insertions(+), 14 deletions(-)
diff --git a/figshare/Article.py b/figshare/Article.py
index 5058ac4..378e1a7 100644
--- a/figshare/Article.py
+++ b/figshare/Article.py
@@ -54,12 +54,12 @@ def __init__(self, config, log, ids):
self.no_matched = 0
self.no_unmatched = 0
- # This dict is used to store counts of skipped items. Set articles_with_error keeps article_ids at failed during preprocessing.
- # while set articles_with_processing_error keeps articled_ids that failed during processing and their intersection is o
- # to avoid double counting. Items in articles_(version)_with_processing_error included published_articles_(version) count.
+ # This dict is used to store counts of skipped items. Set articles_with_fetch_error keeps article_ids that failed during preprocessing.
+ # while set articles_with_processing_error keeps articled_ids that failed during processing and their intersection is o to avoid
+ # double counting. Items in articles_(version)_with_processing_error are included in published_articles_(version) count.
self.skipped_items_counts_dict = {'already_preserved_article_ids': set(), 'already_preserved_versions': 0,
'wasabi_preserved_versions': 0, 'ap_trust_preserved_versions': 0,
- 'articles_with_error': set(), 'article_versions_with_error': 0, 'articles_staged': 0,
+ 'articles_with_fetch_error': set(), 'article_versions_with_fetch_error': 0, 'articles_staged': 0,
'articles_with_processing_error': set(),
'articles_locally_preserved': 0, 'articles_versions_with_processing_error': 0
}
@@ -282,8 +282,8 @@ def __get_article_metadata_by_version(self, version, article_id):
f"Curation folder for article {article_id} version {version['version']} not found.",
True)
self.logs.write_log_in_file("info", "Aborting execution.", True, True)
- self.skipped_items_counts_dict['articles_with_error'].add(article_id)
- self.skipped_items_counts_dict['article_versions_with_error'] += 1
+ self.skipped_items_counts_dict['articles_with_fetch_error'].add(article_id)
+ self.skipped_items_counts_dict['article_versions_with_fetch_error'] += 1
self.logs.write_log_in_file("error",
f"Curation folder for article {article_id} version {version['version']} not found."
+ " Article version will be skipped.",
@@ -1011,20 +1011,14 @@ def __final_process(self, check_files, copy_files, check_dir, version_data, fold
if check_files and copy_files:
article_version_files = self.get_article_version_files(int(version_data['id']), int(version_data['version']))
if article_version_files is None and len(version_data['files']) > 1:
- if version_data['id'] not in self.skipped_items_counts_dict['articles_with_error']:
- self.skipped_items_counts_dict['articles_with_processing_error'].add(version_data['id'])
- self.skipped_items_counts_dict['articles_versions_with_processing_error'] += 1
- if str(version_data['id']) not in self.skipped_article_versions.keys():
- self.skipped_article_versions[str(version_data['id'])] = set()
- self.skipped_article_versions[str(version_data['id'])].add(version_no)
- self.logs.write_log_in_file("error", f"Error retrieving file list for article {version_data['id']} {version_no}. Skipping", True)
+ self.logs.write_log_in_file("error", f"Error retrieving file list for article {version_data['id']} {version_no}.", True)
delete_now = True
else:
try:
# download all files and verify hash with downloaded file.
delete_now = self.__download_files(article_version_files, version_data, folder_name)
except Exception as e:
- self.logs.write_log_in_file("error", f"{str(e)} for {'_'.join(os.path.basename(folder_name).split('_')[0:-1])}" , True)
+ self.logs.write_log_in_file("error", f"{str(e)} for {'_'.join(os.path.basename(folder_name).split('_')[0:-1])}." , True)
if self.system_config['continue-on-error'] == "False":
self.logs.write_log_in_file("info", "Aborting execution.", True, True)
delete_now = True
@@ -1060,6 +1054,12 @@ def __final_process(self, check_files, copy_files, check_dir, version_data, fold
# if download process has any errors then delete complete folder
self.logs.write_log_in_file("info", "Download process had an error so complete folder is being deleted.", True)
self.delete_folder(check_dir)
+ if version_data['id'] not in self.skipped_items_counts_dict['articles_with_fetch_error']:
+ self.skipped_items_counts_dict['articles_with_processing_error'].add(version_data['id'])
+ self.skipped_items_counts_dict['articles_versions_with_processing_error'] += 1
+ if str(version_data['id']) not in self.skipped_article_versions.keys():
+ self.skipped_article_versions[str(version_data['id'])] = set()
+ self.skipped_article_versions[str(version_data['id'])].add(format_version(version_data['version']))
success = False
else:
if check_files or copy_files:
From fd97753da4f0169eda4e64267df20b8de6ace6fa Mon Sep 17 00:00:00 2001
From: HafeezOJ <32125238+HafeezOJ@users.noreply.github.com>
Date: Mon, 22 Jun 2026 16:17:31 -0700
Subject: [PATCH 12/13] Match counting variable names with usage
---
app.py | 22 +++++++++++-----------
1 file changed, 11 insertions(+), 11 deletions(-)
diff --git a/app.py b/app.py
index ae38e3f..9202787 100644
--- a/app.py
+++ b/app.py
@@ -160,8 +160,8 @@ def main():
already_preserved_versions_count = skipped_items_counts_dict['already_preserved_versions']
locally_preserved_article_version_count = skipped_items_counts_dict['articles_locally_preserved']
- articles_with_error_count = len(skipped_items_counts_dict['articles_with_error'])
- article_versions_with_error_count = skipped_items_counts_dict['article_versions_with_error']
+ articles_with_fetch_error_count = len(skipped_items_counts_dict['articles_with_fetch_error'])
+ article_versions_with_fetch_error_count = skipped_items_counts_dict['article_versions_with_fetch_error']
published_articles_count = 0
published_articles_versions_count = 0
published_unpublished_count = 0
@@ -174,9 +174,9 @@ def main():
log.write_log_in_file('info', "Fetched: "
+ f"Total articles: {published_unpublished_count}, "
- + f"Published articles: {published_articles_count + already_preserved_articles_count + articles_with_error_count}, "
+ + f"Published articles: {published_articles_count + already_preserved_articles_count + articles_with_fetch_error_count}, "
+ "Published article versions: "
- + f"{published_articles_versions_count + already_preserved_versions_count + article_versions_with_error_count}",
+ + f"{published_articles_versions_count + already_preserved_versions_count + article_versions_with_fetch_error_count}",
True)
print(" ")
@@ -214,26 +214,26 @@ def main():
log.write_log_in_file('info',
"Total published articles/article versions: \t\t\t\t\t"
- + f'{published_articles_count + already_preserved_articles_count + articles_with_error_count} / '
- + f'{published_articles_versions_count + already_preserved_versions_count + article_versions_with_error_count}',
+ + f'{published_articles_count + already_preserved_articles_count + articles_with_fetch_error_count} / '
+ + f'{published_articles_versions_count + already_preserved_versions_count + article_versions_with_fetch_error_count}',
True)
log.write_log_in_file('info',
"Total count of already preserved (skipped) articles / article versions: \t\t"
+ f'{already_preserved_articles_count} / '
- + f'{published_articles_versions_count + already_preserved_versions_count + article_versions_with_error_count}',
+ + f'{published_articles_versions_count + already_preserved_versions_count + article_versions_with_fetch_error_count}',
True)
log.write_log_in_file('info',
"Total count of articles with fetch error / articles: \t\t\t\t"
- + f'{articles_with_error_count + articles_with_processing_error} / '
- + f'{published_articles_count + already_preserved_articles_count + articles_with_error_count}',
+ + f'{articles_with_fetch_error_count + articles_with_processing_error} / '
+ + f'{published_articles_count + already_preserved_articles_count + articles_with_fetch_error_count}',
True)
log.write_log_in_file('info',
"Total count of article versions with fetch error / article versions: \t\t"
- + f'{article_versions_with_error_count + articles_versions_with_processing_error} / '
- + f'{published_articles_versions_count + already_preserved_versions_count + article_versions_with_error_count}',
+ + f'{article_versions_with_fetch_error_count + articles_versions_with_processing_error} / '
+ + f'{published_articles_versions_count + already_preserved_versions_count + article_versions_with_fetch_error_count}',
True)
if article_obj.processor.duplicate_bag_in_preservation_storage_count > 0:
From 19793af89266074bab460447f295899f3b64ba1a Mon Sep 17 00:00:00 2001
From: HafeezOJ <32125238+HafeezOJ@users.noreply.github.com>
Date: Mon, 22 Jun 2026 16:19:43 -0700
Subject: [PATCH 13/13] Update logs summary
---
ReBACH_Logs_Summary_Description.md | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/ReBACH_Logs_Summary_Description.md b/ReBACH_Logs_Summary_Description.md
index d67e615..1a6330c 100644
--- a/ReBACH_Logs_Summary_Description.md
+++ b/ReBACH_Logs_Summary_Description.md
@@ -4,17 +4,17 @@
| Log Message | Description |
|-------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| Total matched unique articles | Total number of articles whose article_ids are found both in ReDATA and curation storage. This excludes already preserved articles.
**Note**: A preserved article can be in either of the archival storages. |
+| Total matched unique articles | Total number of articles whose article_ids are found both in ReDATA and curation storage during preprocessing. This excludes already preserved articles.
**Note**: A preserved article can be in either of the archival storages. |
| Total unmatched unique articles | Total number of articles whose article_ids are found in ReDATA but **not** in curation storage. This excludes already preserved articles. |
| Total matched article versions | Total number of article versions whose article_ids are found in ReDATA and curation storage contains a folder of the version. This excludes already preserved versions. |
| Total unmatched article versions | Total number of article versions whose article_ids are found in ReDATA but curation storage **does not** contain a folder of the version. This excludes already preserved versions. This number should always be zero. If non-zero, check the logs. |
-| Total skipped unique articles | Total number of articles that are already preserved and are not processed. These articles have at least one version in archival staging storage or archival storage . |
-| Total skipped article versions | Total number of article versions that are already preserved and are not processed. |
+| Total skipped unique articles | Total number of articles that are found already preserved during preprocessing and are not processed. These articles have at least one version in archival staging storage or archival storage . |
+| Total skipped article versions | Total number of article versions that are found already preserved during preprocessing and are not processed. |
| Total articles | Total number of articles in ReDATA, both published and unpublished. |
| Total published articles/article versions | Total number of published articles against total number of published versions in ReDATA. |
| Total count of already preserved (skipped) articles / article versions | Total count of already preserved articles against total count of already preserved article versions.
**Note:** These figures include all articles and article versions already preserved in either of the remote archival storages and are skipped during processing. |
-| Total count of articles with fetch error / articles | Total number of articles with error while fetching items either from ReDATA or curation storage against total number of published articles.
**Note**: Articles with fetch error are skipped during processing. |
-| Total count of article versions with fetch error / article versions | Total number of article versions with error while fetching items either from ReDATA or curation storage against total number of published article versions.
**Note**: Article versions with fetch error are skipped during processing. |
+| Total count of articles with fetch error / articles | Total number of articles with error while fetching items either from ReDATA (either API errors or file download errors) or curation storage against total number of published articles.
**Note**: Articles with fetch error are skipped during processing. |
+| Total count of article versions with fetch error / article versions | Total number of article versions with error while fetching items either from ReDATA (either API errors or file download errors) or curation storage against total number of published article versions.
**Note**: Article versions with fetch error are skipped during processing. |
| Total articles versions matched/published (unskipped) | Total number of article versions with folders in curation storage against total number of published article versions in ReDATA.
**Note**: The two numbers should be equal if nothing went wrong. If the first number is less than the second, it means there was an issue matching at least one of the published article versions that are published in ReDATA. The logs should be examined |
| Total articles versions processed/matched | Total number of article versions successfully processed against total number of article versions in ReDATA with a folder in curation storage.
**Note**: The first number will always be less than or equal to the second. If the numbers are not equal, the log should be checked. |
| Total count of already preserved article versions in archival storage | Total number of already preserved article versions in archival storage.
**Note:** These article versions are skipped during processing. |