diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 01a8fe5..479fbe4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2,9 +2,9 @@ name: Python tests on: push: - branches: [ main ] + branches: [ main, add-checkbox ] pull_request: - branches: [ main ] + branches: [ main, add-checkbox ] permissions: contents: read diff --git a/docs/input_file_overview.md b/docs/input_file_overview.md index b3f90ad..750fd98 100644 --- a/docs/input_file_overview.md +++ b/docs/input_file_overview.md @@ -87,6 +87,17 @@ One important column to note is the Hold Date, which is the date until which the release the data automatically after this date. If it is missing, the default value is three days after the date of submission. +### Statements + +This sheet contains statements that the submitter must agree to before submitting data. Please read each +statement carefully and confirm your agreement before submitting. + +Additional information about the Access and Benefit-Sharing statement is available in our +[submission FAQs](https://www.ebi.ac.uk/eva/?Help#submissionPanel&link=confirm-data-shared-openly). + +NOTE: To use your Webin account, you must also log in to your account and do a one time confirmation of the Access and +Benefit-Sharing acknowledgement there. [Webin login](https://www.ebi.ac.uk/ena/submit/webin/login) => Manage Account => Tick the box => Save. + ### Analysis For EVA, an analysis is a grouping of samples and data files. This sheet allows us to link VCF files to a project and diff --git a/eva_sub_cli/etc/EVA_Submission_Example.xlsx b/eva_sub_cli/etc/EVA_Submission_Example.xlsx index bd27487..9433d0c 100644 Binary files a/eva_sub_cli/etc/EVA_Submission_Example.xlsx and b/eva_sub_cli/etc/EVA_Submission_Example.xlsx differ diff --git a/eva_sub_cli/etc/EVA_Submission_template.xlsx b/eva_sub_cli/etc/EVA_Submission_template.xlsx index 3589a12..35294de 100644 Binary files a/eva_sub_cli/etc/EVA_Submission_template.xlsx and b/eva_sub_cli/etc/EVA_Submission_template.xlsx differ diff --git a/eva_sub_cli/etc/eva_schema.json b/eva_sub_cli/etc/eva_schema.json index f1e9ebf..537a8e8 100644 --- a/eva_sub_cli/etc/eva_schema.json +++ b/eva_sub_cli/etc/eva_schema.json @@ -63,6 +63,25 @@ } } }, + "statements": { + "type": "array", + "description": "Submitter acknowledgements of various EVA data statements", + "items": { + "type": "object", + "required": ["acknowledgement", "statement"], + "additionalProperties": false, + "properties": { + "acknowledgement": { + "type": "boolean", + "description": "Whether the submitter confirmed this statement" + }, + "statement": { + "type": "string", + "description": "Full text of the statement as presented to the submitter" + } + } + } + }, "project": { "oneOf": [ { diff --git a/eva_sub_cli/etc/spreadsheet2json_conf.yaml b/eva_sub_cli/etc/spreadsheet2json_conf.yaml index 5d7570f..80594ce 100644 --- a/eva_sub_cli/etc/spreadsheet2json_conf.yaml +++ b/eva_sub_cli/etc/spreadsheet2json_conf.yaml @@ -11,6 +11,7 @@ # List of worksheets to be parsed. worksheets: Submitter Details: submitterDetails + Statements: statements Project: project Analysis: analysis Sample: sample @@ -26,6 +27,14 @@ Submitter Details: optional: Address: address +Statements: + header_row: 2 + required: + Acknowledgement: acknowledgement + Statement: statement + cast: + Acknowledgement: boolean + # List of field for Projects Project: header_row: 3 diff --git a/eva_sub_cli/executables/xlsx2json.py b/eva_sub_cli/executables/xlsx2json.py index ae7edac..144efc2 100644 --- a/eva_sub_cli/executables/xlsx2json.py +++ b/eva_sub_cli/executables/xlsx2json.py @@ -22,6 +22,8 @@ SAMPLE_NAME_KEY = 'BioSample Name' SCIENTIFIC_NAME_KEY = 'Scientific Name' SPECIES = 'species' +STATEMENTS = 'Statements' +STATEMENTS_ACK_HEADER = 'Acknowledgement' logger = logging_config.get_logger(__name__) @@ -72,7 +74,9 @@ def active_worksheet(self, worksheet): if self.worksheets is None: self.valid_worksheets() if worksheet not in self.worksheets: - self.add_error(f'Tried to access an invalid worksheet {worksheet}', sheet=worksheet) + self.add_error(f'Could not find expected worksheet {worksheet}. Download the latest unmodified ' + f'template from https://raw.githubusercontent.com/EBIvariation/eva-sub-cli/main/eva_sub_cli/' + f'etc/EVA_Submission_template.xlsx', sheet=worksheet) return self._active_worksheet = worksheet @@ -178,6 +182,21 @@ def get_rows(self): return rows + def get_data_acknowledgement_json_data(self): + json_key = self.xlsx_conf[WORKSHEETS_KEY_NAME][STATEMENTS] + ack_json_key = self.xlsx_conf[STATEMENTS][REQUIRED_HEADERS_KEY_NAME][STATEMENTS_ACK_HEADER] + + acknowledgements = [] + for row in self.get_rows(): + row.pop('row_num') + json_value = {self.translate_header(STATEMENTS, k): v + for k, v in row.items() if v is not None} + # An unchecked/blank checkbox should record as an explicit False + json_value.setdefault(ack_json_key, False) + acknowledgements.append(json_value) + + return {json_key: acknowledgements} + def get_project_json_data(self): json_key = self.xlsx_conf[WORKSHEETS_KEY_NAME][PROJECT] @@ -265,6 +284,8 @@ def json(self, output_json_file): elif title == SAMPLE: sample_data = self.get_sample_json_data() json_data.update(sample_data) + elif title == STATEMENTS: + json_data.update(self.get_data_acknowledgement_json_data()) else: json_data[self.xlsx_conf[WORKSHEETS_KEY_NAME][title]] = [] for row in self.get_rows(): diff --git a/eva_sub_cli/semantic_metadata.py b/eva_sub_cli/semantic_metadata.py index e994752..3a717a1 100644 --- a/eva_sub_cli/semantic_metadata.py +++ b/eva_sub_cli/semantic_metadata.py @@ -1,4 +1,5 @@ import json +import re from collections import Counter from copy import deepcopy from datetime import datetime, timedelta @@ -13,6 +14,7 @@ from eva_sub_cli.date_utils import check_date PROJECT_KEY = 'project' +STATEMENTS_KEY = 'statements' ANALYSIS_KEY = 'analysis' SAMPLE_KEY = 'sample' FILES_KEY = 'files' @@ -33,6 +35,11 @@ # Samples created before this date are not required to have collection date or geographic location threshold_2023 = datetime(2023, 1, 1) +DATA_ACCESS_STATEMENT = """To the best of my knowledge, the data being submitted are not subject to any + restrictions that prohibit their open sharing and are submitted in compliance with applicable national and + international access and benefit-sharing obligations, in accordance with the EMBL-EBI Terms of Use + (https://www.ebi.ac.uk/about/terms-of-use/).""" + def cast_list(l, type_to_cast=str): for e in l: @@ -64,6 +71,20 @@ def check_all(self): self.check_all_analysis_contain_samples() self.check_all_samples_have_sample_in_vcf() self.check_hold_date() + self.check_statements() + + @staticmethod + def _normalize(text): + return re.sub(r'\s+', ' ', text).strip().lower() + + def check_statements(self): + if STATEMENTS_KEY in self.metadata: + for entry in self.metadata[STATEMENTS_KEY]: + if (self._normalize(entry.get('statement') or '') == self._normalize(DATA_ACCESS_STATEMENT) + and entry.get('acknowledgement')): + return + + self.add_error(f'/{STATEMENTS_KEY}', 'Data Access & Sharing statement not acknowledged') def check_hold_date(self): """Check that holdDate is no later than 2 years from today.""" @@ -329,8 +350,8 @@ def check_all_samples_have_sample_in_vcf(self): json_path = f'/{SAMPLE_KEY}/{idx}/{SAMPLE_IN_VCF_KEY}' analysis_aliases = sample.get(ANALYSIS_ALIAS_KEY, []) if any([self.evidence_type_results.get(analysis_alias, {}).get('evidence_type') != 'allele_frequency' for - analysis_alias in analysis_aliases]): + analysis_alias in analysis_aliases]): # SampleInVCF is required if sample.get(SAMPLE_IN_VCF_KEY) is None or sample.get(SAMPLE_IN_VCF_KEY) == '': - self.add_error(json_path, f'{SAMPLE_IN_VCF_KEY} must be provided when Genotypes are present in the VCF file') - + self.add_error(json_path, + f'{SAMPLE_IN_VCF_KEY} must be provided when Genotypes are present in the VCF file') diff --git a/tests/resources/EVA_Submission_Docker_Test.xlsx b/tests/resources/EVA_Submission_Docker_Test.xlsx index 3c8d97d..cc13fd2 100644 Binary files a/tests/resources/EVA_Submission_Docker_Test.xlsx and b/tests/resources/EVA_Submission_Docker_Test.xlsx differ diff --git a/tests/resources/EVA_Submission_test.xlsx b/tests/resources/EVA_Submission_test.xlsx index 11cf881..51dffbd 100644 Binary files a/tests/resources/EVA_Submission_test.xlsx and b/tests/resources/EVA_Submission_test.xlsx differ diff --git a/tests/resources/EVA_Submission_test_fails.xlsx b/tests/resources/EVA_Submission_test_fails.xlsx index 21a81a0..6d98914 100644 Binary files a/tests/resources/EVA_Submission_test_fails.xlsx and b/tests/resources/EVA_Submission_test_fails.xlsx differ diff --git a/tests/resources/EVA_Submission_test_version_missing.xlsx b/tests/resources/EVA_Submission_test_version_missing.xlsx index 3589a12..227dfe4 100644 Binary files a/tests/resources/EVA_Submission_test_version_missing.xlsx and b/tests/resources/EVA_Submission_test_version_missing.xlsx differ diff --git a/tests/resources/EVA_Submission_test_with_formula.xlsx b/tests/resources/EVA_Submission_test_with_formula.xlsx index cb66435..fb38f5a 100644 Binary files a/tests/resources/EVA_Submission_test_with_formula.xlsx and b/tests/resources/EVA_Submission_test_with_formula.xlsx differ diff --git a/tests/resources/EVA_Submission_test_with_links.xlsx b/tests/resources/EVA_Submission_test_with_links.xlsx index f1b2eb4..f4c0149 100644 Binary files a/tests/resources/EVA_Submission_test_with_links.xlsx and b/tests/resources/EVA_Submission_test_with_links.xlsx differ diff --git a/tests/resources/EVA_Submission_test_with_non_vcf_files.xlsx b/tests/resources/EVA_Submission_test_with_non_vcf_files.xlsx index dbc7408..d4fcded 100644 Binary files a/tests/resources/EVA_Submission_test_with_non_vcf_files.xlsx and b/tests/resources/EVA_Submission_test_with_non_vcf_files.xlsx differ diff --git a/tests/resources/EVA_Submission_test_with_project_accession.xlsx b/tests/resources/EVA_Submission_test_with_project_accession.xlsx index 78c4730..963a888 100644 Binary files a/tests/resources/EVA_Submission_test_with_project_accession.xlsx and b/tests/resources/EVA_Submission_test_with_project_accession.xlsx differ diff --git a/tests/resources/EVA_Submission_test_without_data_acknowledgement_sheet.xlsx b/tests/resources/EVA_Submission_test_without_data_acknowledgement_sheet.xlsx new file mode 100644 index 0000000..a6345d4 Binary files /dev/null and b/tests/resources/EVA_Submission_test_without_data_acknowledgement_sheet.xlsx differ diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py index 72902dd..a4db828 100644 --- a/tests/test_orchestrator.py +++ b/tests/test_orchestrator.py @@ -33,7 +33,7 @@ class TestOrchestrator(unittest.TestCase): mapping_file = os.path.join(test_sub_dir, 'vcf_mapping_file.csv') metadata_json = os.path.join(test_sub_dir, 'sub_metadata.json') metadata_xlsx = os.path.join(test_sub_dir, 'sub_metadata.xlsx') - metadata_xlsx_version = '3.0.1' + metadata_xlsx_version = '3.1.0' metadata_xlsx_with_project_accession = os.path.join(test_sub_dir, 'EVA_Submission_test_with_project_accession.xlsx') metadata_xlsx_version_v2 = '2.0.1' metadata_xlsx_v2 = os.path.join(test_sub_dir, 'EVA_Submission_test_V2.xlsx') diff --git a/tests/test_semantic_metadata.py b/tests/test_semantic_metadata.py index 1105a36..b1bf65d 100644 --- a/tests/test_semantic_metadata.py +++ b/tests/test_semantic_metadata.py @@ -1,12 +1,12 @@ from datetime import datetime, timedelta from unittest import TestCase from unittest.mock import patch -from requests import HTTPError, Response import pytest from ebi_eva_common_pyutils.biosamples_communicators import NoAuthHALCommunicator +from requests import HTTPError, Response -from eva_sub_cli.semantic_metadata import SemanticMetadataChecker +from eva_sub_cli.semantic_metadata import SemanticMetadataChecker, DATA_ACCESS_STATEMENT metadata = { "sample": [ @@ -105,7 +105,8 @@ def test_check_all_project_accessions(self): m_ena_download.side_effect = [True, True, HTTPError('problem downloading', response=Response())] checker.check_all_project_accessions() self.assertEqual(checker.errors, [ - {'property': '/project/childProjects/1', 'description': 'Project PRJEBNA does not exist in ENA or is private'} + {'property': '/project/childProjects/1', + 'description': 'Project PRJEBNA does not exist in ENA or is private'} ]) def test_check_all_taxonomy_codes(self): @@ -174,7 +175,7 @@ def test_check_uniqueness_analysis_alias(self): 'description': 'Analysis alias alias1 is present 2 times in the Analysis Sheet' } ] -) + ) def test_check_all_scientific_names(self): metadata = { @@ -220,7 +221,8 @@ def test_check_all_scientific_names(self): def test_check_existing_biosamples_with_checklist(self): checker = SemanticMetadataChecker(metadata, {}) with patch.object(SemanticMetadataChecker, '_get_biosample', - side_effect=[valid_sample, ValueError, invalid_sample1, invalid_sample2, old_invalid_sample, old_invalid_sample2]) as m_get_sample: + side_effect=[valid_sample, ValueError, invalid_sample1, invalid_sample2, old_invalid_sample, + old_invalid_sample2]) as m_get_sample: checker.check_existing_biosamples() self.assertEqual( checker.errors[0], @@ -236,7 +238,7 @@ def test_check_existing_biosamples_with_checklist(self): 'Error validating existing sample SAME00003: geographic location (country and~1or sea) must be equal to one of the allowed values:')) self.assertEqual(checker.errors[3]['description'], - "Error validating existing sample SAME00004: must have required property 'geographic location (country and/or sea)'") + "Error validating existing sample SAME00004: must have required property 'geographic location (country and/or sea)'") # error message lists long regex for collection date self.assertTrue(checker.errors[4]['description'].startswith( @@ -246,13 +248,17 @@ def test_check_existing_biosamples_with_checklist(self): def test_check_existing_biosamples(self): checker = SemanticMetadataChecker(metadata, {}, sample_checklist=None) with patch.object(NoAuthHALCommunicator, 'follows_link', - side_effect=[valid_sample, ValueError, invalid_sample1, invalid_sample2, old_invalid_sample, old_invalid_sample2]) as m_follows_link: + side_effect=[valid_sample, ValueError, invalid_sample1, invalid_sample2, old_invalid_sample, + old_invalid_sample2]) as m_follows_link: checker.check_existing_biosamples() self.assertEqual(checker.errors, [ - {'description': 'SAME00002 does not exist or is private','property': '/sample/1/bioSampleAccession'}, - {'description': 'Existing sample SAME00003 does not have a valid collection date', 'property': '/sample/2/bioSampleAccession'}, - {'description': 'Existing sample SAME00004 does not have a valid collection date', 'property': '/sample/3/bioSampleAccession'}, - {'description': 'Existing sample SAME00004 does not have a valid geographic location', 'property': '/sample/3/bioSampleAccession'}]) + {'description': 'SAME00002 does not exist or is private', 'property': '/sample/1/bioSampleAccession'}, + {'description': 'Existing sample SAME00003 does not have a valid collection date', + 'property': '/sample/2/bioSampleAccession'}, + {'description': 'Existing sample SAME00004 does not have a valid collection date', + 'property': '/sample/3/bioSampleAccession'}, + {'description': 'Existing sample SAME00004 does not have a valid geographic location', + 'property': '/sample/3/bioSampleAccession'}]) @pytest.mark.skip(reason='Contact BioSample API') def test_check_existing_real_biosamples(self): @@ -313,7 +319,8 @@ def test_check_all_analysis_run_accessions(self): checker.check_all_analysis_run_accessions() assert checker.errors == [ - {'property': '/analysis/1/runAccessions', 'description': 'Run SRR00000000001 does not exist in ENA or is private'}] + {'property': '/analysis/1/runAccessions', + 'description': 'Run SRR00000000001 does not exist in ENA or is private'}] def test_check_all_analysis_contain_samples(self): # all analysis contain samples @@ -414,4 +421,31 @@ def test_check_hold_date(self): metadata = {"project": {}, "sample": [], "analysis": [], "files": []} checker = SemanticMetadataChecker(metadata, {}) checker.check_hold_date() - self.assertEqual(checker.errors, []) \ No newline at end of file + self.assertEqual(checker.errors, []) + + def test_check_statementss(self): + # statement present and acknowledged + metadata = {"statements": [{ + "acknowledgement": True, + "statement": DATA_ACCESS_STATEMENT + }]} + checker = SemanticMetadataChecker(metadata, {}) + checker.check_statements() + self.assertEqual(checker.errors, []) + + # statement present but not acknowledged + metadata = {"statements": [{ + "acknowledgement": False, + "statement": DATA_ACCESS_STATEMENT + }]} + checker = SemanticMetadataChecker(metadata, {}) + checker.check_statements() + self.assertEqual(checker.errors, [{'description': 'Data Access & Sharing statement not acknowledged', + 'property': '/statements'}]) + + # no data present for acknowledgement + metadata = {"statements": []} + checker = SemanticMetadataChecker(metadata, {}) + checker.check_statements() + self.assertEqual(checker.errors, [{'description': 'Data Access & Sharing statement not acknowledged', + 'property': '/statements'}]) diff --git a/tests/test_xlsx2json.py b/tests/test_xlsx2json.py index e6f5d79..cbd485a 100644 --- a/tests/test_xlsx2json.py +++ b/tests/test_xlsx2json.py @@ -7,7 +7,8 @@ import yaml from eva_sub_cli import ETC_DIR -from eva_sub_cli.executables.xlsx2json import XlsxParser, create_xls_template_from_yaml +from eva_sub_cli.executables.xlsx2json import XlsxParser, create_xls_template_from_yaml, STATEMENTS +from eva_sub_cli.semantic_metadata import STATEMENTS_KEY, SemanticMetadataChecker, DATA_ACCESS_STATEMENT class TestXlsReader(TestCase): @@ -24,7 +25,9 @@ def tearDown(self): os.path.join(self.resource_dir, 'EVA_Submission_test_errors.yml'), os.path.join(self.resource_dir, 'EVA_Submission_test_errors_v2.yml'), os.path.join(self.resource_dir, 'EVA_Submission_test_with_project_accession.yml'), - os.path.join(self.resource_dir, 'EVA_Submission_test_with_project_accession_output.json') + os.path.join(self.resource_dir, 'EVA_Submission_test_with_project_accession_output.json'), + os.path.join(self.resource_dir, 'EVA_Submission_test_output_without_data_acknowledgement_sheet.json'), + os.path.join(self.resource_dir, 'EVA_Submission_test_errors_without_data_acknowledgement_sheet.yml') ] for f in files_from_tests: if os.path.exists(f): @@ -59,7 +62,7 @@ def test_conversion_2_json(self) -> None: with open(output_json) as open_file: json_data = json.load(open_file) # assert json file is created with expected data - assert sorted(json_data.keys()) == ['analysis', 'files', 'project', 'sample', 'submitterDetails'] + assert sorted(json_data.keys()) == ['analysis', 'files', 'project', 'sample', 'statements', 'submitterDetails'] self.assertEqual(self.get_expected_json(), json_data) # assert json schema @@ -87,7 +90,7 @@ def test_conversion_2_json_with_formula(self) -> None: with open(output_json) as open_file: json_data = json.load(open_file) # assert json file is created with expected data - assert sorted(json_data.keys()) == ['analysis', 'files', 'project', 'sample', 'submitterDetails'] + assert sorted(json_data.keys()) == ['analysis', 'files', 'project', 'sample', 'statements', 'submitterDetails'] self.assertEqual(self.get_expected_json(), json_data) # assert json schema @@ -115,7 +118,7 @@ def test_conversion_2_json_V2(self) -> None: json_data = json.load(open_file) # assert json file is created with expected data assert sorted(json_data.keys()) == ['analysis', 'files', 'project', 'sample', 'submitterDetails'] - self.assertEqual(self.get_expected_json(), json_data) + self.assertEqual(self.get_expected_json_v2(), json_data) # assert json schema with open(self.eva_schema) as eva_schema_file: @@ -123,6 +126,41 @@ def test_conversion_2_json_V2(self) -> None: jsonschema.validate(json_data, eva_json_schema) + def test_conversion_2_json_with_old_worksheet(self) -> None: + xls_filename = os.path.join(self.resource_dir, 'EVA_Submission_test_without_data_acknowledgement_sheet.xlsx') + self.parser = XlsxParser(xls_filename, self.conf_filename) + output_json = os.path.join(self.resource_dir, 'EVA_Submission_test_output_without_data_acknowledgement_sheet.json') + errors_yaml = os.path.join(self.resource_dir, 'EVA_Submission_test_errors_without_data_acknowledgement_sheet.yml') + self.parser.json(output_json) + self.parser.save_errors(errors_yaml) + + # confirm the error - expected sheet not found + with open(errors_yaml) as open_file: + errors_data = yaml.safe_load(open_file) + assert errors_data == [{'column': '', 'description': 'Could not find expected worksheet Statements. Download the latest unmodified ' + f'template from https://raw.githubusercontent.com/EBIvariation/eva-sub-cli/main/eva_sub_cli/' + f'etc/EVA_Submission_template.xlsx', 'row': '', 'sheet': 'Statements'}] + + with open(output_json) as open_file: + json_data = json.load(open_file) + # assert json file is created with expected data + assert sorted(json_data.keys()) == ['analysis', 'files', 'project', 'sample', 'statements', 'submitterDetails'] + expected_json_data = self.get_expected_json() + expected_json_data['statements'] = [] + self.assertEqual(expected_json_data, json_data) + + def test_template_statement_matches_data_access_statement_constant(self): + template_xlsx = os.path.join(ETC_DIR, 'EVA_Submission_template.xlsx') + conf_filename = os.path.join(ETC_DIR, 'spreadsheet2json_conf.yaml') + parser = XlsxParser(template_xlsx, conf_filename) + parser.active_worksheet = STATEMENTS + acknowledgements = parser.get_data_acknowledgement_json_data()[STATEMENTS_KEY] + template_statement = acknowledgements[0]['statement'] + self.assertEqual( + SemanticMetadataChecker._normalize(template_statement), + SemanticMetadataChecker._normalize(DATA_ACCESS_STATEMENT) + ) + def test_conversion_2_json_with_project_accession(self) -> None: xls_filename = os.path.join(self.resource_dir, 'EVA_Submission_test_with_project_accession.xlsx') self.parser = XlsxParser(xls_filename, self.conf_filename) @@ -139,7 +177,7 @@ def test_conversion_2_json_with_project_accession(self) -> None: with open(output_json) as open_file: json_data = json.load(open_file) # assert json file is created with expected data - assert sorted(json_data.keys()) == ['analysis', 'files', 'project', 'sample', 'submitterDetails'] + assert sorted(json_data.keys()) == ['analysis', 'files', 'project', 'sample', 'statements', 'submitterDetails'] # get expected json and remove other fields apart from project accession for comparison expected_json = self.get_expected_json() expected_json['project'] = {'projectAccession': 'PRJEB12345'} @@ -167,7 +205,7 @@ def test_conversion_2_json_with_links(self) -> None: with open(output_json) as open_file: json_data = json.load(open_file) # assert json file is created with expected data - assert sorted(json_data.keys()) == ['analysis', 'files', 'project', 'sample', 'submitterDetails'] + assert sorted(json_data.keys()) == ['analysis', 'files', 'project', 'sample', 'statements', 'submitterDetails'] expected_json = self.get_expected_json() expected_json['analysis'][0]['links'] = ['BioProject:PRJNA1435562'] self.assertEqual(expected_json, json_data) @@ -201,7 +239,7 @@ def test_json_conversion_succeeds_with_invalid_metadata(self): assert os.path.exists(output_json) with open(output_json) as open_file: json_data = json.load(open_file) - assert sorted(json_data.keys()) == ['analysis', 'files', 'project', 'sample', 'submitterDetails'] + assert sorted(json_data.keys()) == ['analysis', 'files', 'project', 'sample', 'statements', 'submitterDetails'] # required field taxId is missing assert 'taxId' not in json_data['project'] # novel sample is missing scientific name in characteristics and sample name @@ -233,6 +271,16 @@ def build_large_string_of_length(self, length): return "A" * length def get_expected_json(self): + json_data = self.get_expected_json_v2() + json_data["statements"] = [ + { + "acknowledgement": True, + "statement": "To the best of my knowledge, the data being submitted are not subject to any restrictions that prohibit their open sharing and are submitted in compliance with applicable national and international access and benefit-sharing obligations, in accordance with the EMBL-EBI Terms of Use (https://www.ebi.ac.uk/about/terms-of-use/)." + } + ] + return json_data + + def get_expected_json_v2(self): return { "submitterDetails": [ {