Add checkbox for ABS statement

.github/workflows/tests.yml

@@ -2,9 +2,9 @@ name: Python tests
 
 on:
   push:
-    branches: [ main ]
+    branches: [ main, add-checkbox ]
   pull_request:
-    branches: [ main ]
+    branches: [ main, add-checkbox ]
 
 permissions:
   contents: read

docs/input_file_overview.md

@@ -87,6 +87,17 @@ One important column to note is the Hold Date, which is the date until which the
 release the data automatically after this date. If it is missing, the default value is three days after the date of
 submission.
 
+### Statements
+
+This sheet contains statements that the submitter must agree to before submitting data. Please read each
+statement carefully and confirm your agreement before submitting.
+
+Additional information about the Access and Benefit-Sharing statement is available in our
+[submission FAQs](https://www.ebi.ac.uk/eva/?Help#submissionPanel&link=confirm-data-shared-openly).
+
+NOTE: To use your Webin account, you must also log in to your account and do a one time confirmation of the Access and
+Benefit-Sharing acknowledgement there. [Webin login](https://www.ebi.ac.uk/ena/submit/webin/login) => Manage Account => Tick the box => Save.
+
 ### Analysis
 
 For EVA, an analysis is a grouping of samples and data files. This sheet allows us to link VCF files to a project and

eva_sub_cli/etc/eva_schema.json

@@ -63,6 +63,25 @@
         }
       }
     },
+    "statements": {
+      "type": "array",
+      "description": "Submitter acknowledgements of various EVA data statements",
+      "items": {
+        "type": "object",
+        "required": ["acknowledgement", "statement"],
+        "additionalProperties": false,
+        "properties": {
+          "acknowledgement": {
+            "type": "boolean",
+            "description": "Whether the submitter confirmed this statement"
+          },
+          "statement": {
+            "type": "string",
+            "description": "Full text of the statement as presented to the submitter"
+          }
+        }
+      }
+    },
     "project": {
       "oneOf": [
         {

eva_sub_cli/etc/spreadsheet2json_conf.yaml

@@ -11,6 +11,7 @@
 # List of worksheets to be parsed.
 worksheets:
   Submitter Details: submitterDetails
+  Statements: statements
   Project: project
   Analysis: analysis
   Sample: sample
@@ -26,6 +27,14 @@ Submitter Details:
   optional:
     Address: address
 
+Statements:
+  header_row: 2
+  required:
+    Acknowledgement: acknowledgement
+    Statement: statement
+  cast:
+    Acknowledgement: boolean
+
 # List of field for Projects
 Project:
   header_row: 3

eva_sub_cli/executables/xlsx2json.py

@@ -22,6 +22,8 @@
 SAMPLE_NAME_KEY = 'BioSample Name'
 SCIENTIFIC_NAME_KEY = 'Scientific Name'
 SPECIES = 'species'
+STATEMENTS = 'Statements'
+STATEMENTS_ACK_HEADER = 'Acknowledgement'
 
 logger = logging_config.get_logger(__name__)
 
@@ -72,7 +74,9 @@ def active_worksheet(self, worksheet):
         if self.worksheets is None:
             self.valid_worksheets()
         if worksheet not in self.worksheets:
-            self.add_error(f'Tried to access an invalid worksheet {worksheet}', sheet=worksheet)
+            self.add_error(f'Could not find expected worksheet {worksheet}. Download the latest unmodified '
+                           f'template from https://raw.githubusercontent.com/EBIvariation/eva-sub-cli/main/eva_sub_cli/'
+                           f'etc/EVA_Submission_template.xlsx', sheet=worksheet)
             return
 
         self._active_worksheet = worksheet
@@ -178,6 +182,21 @@ def get_rows(self):
 
         return rows
 
+    def get_data_acknowledgement_json_data(self):
+        json_key = self.xlsx_conf[WORKSHEETS_KEY_NAME][STATEMENTS]
+        ack_json_key = self.xlsx_conf[STATEMENTS][REQUIRED_HEADERS_KEY_NAME][STATEMENTS_ACK_HEADER]
+
+        acknowledgements = []
+        for row in self.get_rows():
+            row.pop('row_num')
+            json_value = {self.translate_header(STATEMENTS, k): v
+                          for k, v in row.items() if v is not None}
+            # An unchecked/blank checkbox should record as an explicit False
+            json_value.setdefault(ack_json_key, False)
+            acknowledgements.append(json_value)
+
+        return {json_key: acknowledgements}
+
     def get_project_json_data(self):
         json_key = self.xlsx_conf[WORKSHEETS_KEY_NAME][PROJECT]
 
@@ -265,6 +284,8 @@ def json(self, output_json_file):
             elif title == SAMPLE:
                 sample_data = self.get_sample_json_data()
                 json_data.update(sample_data)
+            elif title == STATEMENTS:
+                json_data.update(self.get_data_acknowledgement_json_data())
             else:
                 json_data[self.xlsx_conf[WORKSHEETS_KEY_NAME][title]] = []
                 for row in self.get_rows():

eva_sub_cli/semantic_metadata.py

@@ -1,4 +1,5 @@
 import json
+import re
 from collections import Counter
 from copy import deepcopy
 from datetime import datetime, timedelta
@@ -13,6 +14,7 @@
 from eva_sub_cli.date_utils import check_date
 
 PROJECT_KEY = 'project'
+STATEMENTS_KEY = 'statements'
 ANALYSIS_KEY = 'analysis'
 SAMPLE_KEY = 'sample'
 FILES_KEY = 'files'
@@ -33,6 +35,11 @@
 # Samples created before this date are not required to have collection date or geographic location
 threshold_2023 = datetime(2023, 1, 1)
 
+DATA_ACCESS_STATEMENT = """To the best of my knowledge, the data being submitted are not subject to any 
+       restrictions that prohibit their open sharing and are submitted in compliance with applicable national and 
+       international access and benefit-sharing obligations, in accordance with the EMBL-EBI Terms of Use
+       (https://www.ebi.ac.uk/about/terms-of-use/)."""
+
 
 def cast_list(l, type_to_cast=str):
     for e in l:
@@ -64,6 +71,20 @@ def check_all(self):
         self.check_all_analysis_contain_samples()
         self.check_all_samples_have_sample_in_vcf()
         self.check_hold_date()
+        self.check_statements()
+
+    @staticmethod
+    def _normalize(text):
+        return re.sub(r'\s+', ' ', text).strip().lower()
+
+    def check_statements(self):
+        if STATEMENTS_KEY in self.metadata:
+            for entry in self.metadata[STATEMENTS_KEY]:
+                if (self._normalize(entry.get('statement') or '') == self._normalize(DATA_ACCESS_STATEMENT)
+                        and entry.get('acknowledgement')):
+                    return
+
+        self.add_error(f'/{STATEMENTS_KEY}', 'Data Access & Sharing statement not acknowledged')
 
     def check_hold_date(self):
         """Check that holdDate is no later than 2 years from today."""
@@ -329,8 +350,8 @@ def check_all_samples_have_sample_in_vcf(self):
             json_path = f'/{SAMPLE_KEY}/{idx}/{SAMPLE_IN_VCF_KEY}'
             analysis_aliases = sample.get(ANALYSIS_ALIAS_KEY, [])
             if any([self.evidence_type_results.get(analysis_alias, {}).get('evidence_type') != 'allele_frequency' for
-                        analysis_alias in analysis_aliases]):
+                    analysis_alias in analysis_aliases]):
                 # SampleInVCF is required
                 if sample.get(SAMPLE_IN_VCF_KEY) is None or sample.get(SAMPLE_IN_VCF_KEY) == '':
-                    self.add_error(json_path, f'{SAMPLE_IN_VCF_KEY} must be provided when Genotypes are present in the VCF file')
-
+                    self.add_error(json_path,
+                                   f'{SAMPLE_IN_VCF_KEY} must be provided when Genotypes are present in the VCF file')

tests/test_orchestrator.py

@@ -33,7 +33,7 @@ class TestOrchestrator(unittest.TestCase):
     mapping_file = os.path.join(test_sub_dir, 'vcf_mapping_file.csv')
     metadata_json = os.path.join(test_sub_dir, 'sub_metadata.json')
     metadata_xlsx = os.path.join(test_sub_dir, 'sub_metadata.xlsx')
-    metadata_xlsx_version = '3.0.1'
+    metadata_xlsx_version = '3.1.0'
     metadata_xlsx_with_project_accession = os.path.join(test_sub_dir, 'EVA_Submission_test_with_project_accession.xlsx')
     metadata_xlsx_version_v2 = '2.0.1'
     metadata_xlsx_v2 = os.path.join(test_sub_dir, 'EVA_Submission_test_V2.xlsx')

tests/test_semantic_metadata.py

@@ -1,12 +1,12 @@
 from datetime import datetime, timedelta
 from unittest import TestCase
 from unittest.mock import patch
-from requests import HTTPError, Response
 
 import pytest
 from ebi_eva_common_pyutils.biosamples_communicators import NoAuthHALCommunicator
+from requests import HTTPError, Response
 
-from eva_sub_cli.semantic_metadata import SemanticMetadataChecker
+from eva_sub_cli.semantic_metadata import SemanticMetadataChecker, DATA_ACCESS_STATEMENT
 
 metadata = {
     "sample": [
@@ -105,7 +105,8 @@ def test_check_all_project_accessions(self):
             m_ena_download.side_effect = [True, True, HTTPError('problem downloading', response=Response())]
             checker.check_all_project_accessions()
             self.assertEqual(checker.errors, [
-                {'property': '/project/childProjects/1', 'description': 'Project PRJEBNA does not exist in ENA or is private'}
+                {'property': '/project/childProjects/1',
+                 'description': 'Project PRJEBNA does not exist in ENA or is private'}
             ])
 
     def test_check_all_taxonomy_codes(self):
@@ -174,7 +175,7 @@ def test_check_uniqueness_analysis_alias(self):
                 'description': 'Analysis alias alias1 is present 2 times in the Analysis Sheet'
             }
         ]
-)
+                         )
 
     def test_check_all_scientific_names(self):
         metadata = {
@@ -220,7 +221,8 @@ def test_check_all_scientific_names(self):
     def test_check_existing_biosamples_with_checklist(self):
         checker = SemanticMetadataChecker(metadata, {})
         with patch.object(SemanticMetadataChecker, '_get_biosample',
-                          side_effect=[valid_sample, ValueError, invalid_sample1, invalid_sample2, old_invalid_sample, old_invalid_sample2]) as m_get_sample:
+                          side_effect=[valid_sample, ValueError, invalid_sample1, invalid_sample2, old_invalid_sample,
+                                       old_invalid_sample2]) as m_get_sample:
             checker.check_existing_biosamples()
             self.assertEqual(
                 checker.errors[0],
@@ -236,7 +238,7 @@ def test_check_existing_biosamples_with_checklist(self):
                 'Error validating existing sample SAME00003: geographic location (country and~1or sea) must be equal to one of the allowed values:'))
 
             self.assertEqual(checker.errors[3]['description'],
-                "Error validating existing sample SAME00004: must have required property 'geographic location (country and/or sea)'")
+                             "Error validating existing sample SAME00004: must have required property 'geographic location (country and/or sea)'")
 
             # error message lists long regex for collection date
             self.assertTrue(checker.errors[4]['description'].startswith(
@@ -246,13 +248,17 @@ def test_check_existing_biosamples_with_checklist(self):
     def test_check_existing_biosamples(self):
         checker = SemanticMetadataChecker(metadata, {}, sample_checklist=None)
         with patch.object(NoAuthHALCommunicator, 'follows_link',
-                          side_effect=[valid_sample, ValueError, invalid_sample1, invalid_sample2, old_invalid_sample, old_invalid_sample2]) as m_follows_link:
+                          side_effect=[valid_sample, ValueError, invalid_sample1, invalid_sample2, old_invalid_sample,
+                                       old_invalid_sample2]) as m_follows_link:
             checker.check_existing_biosamples()
             self.assertEqual(checker.errors, [
-                {'description': 'SAME00002 does not exist or is private','property': '/sample/1/bioSampleAccession'},
-                {'description': 'Existing sample SAME00003 does not have a valid collection date', 'property': '/sample/2/bioSampleAccession'},
-                {'description': 'Existing sample SAME00004 does not have a valid collection date', 'property': '/sample/3/bioSampleAccession'},
-                {'description': 'Existing sample SAME00004 does not have a valid geographic location', 'property': '/sample/3/bioSampleAccession'}])
+                {'description': 'SAME00002 does not exist or is private', 'property': '/sample/1/bioSampleAccession'},
+                {'description': 'Existing sample SAME00003 does not have a valid collection date',
+                 'property': '/sample/2/bioSampleAccession'},
+                {'description': 'Existing sample SAME00004 does not have a valid collection date',
+                 'property': '/sample/3/bioSampleAccession'},
+                {'description': 'Existing sample SAME00004 does not have a valid geographic location',
+                 'property': '/sample/3/bioSampleAccession'}])
 
     @pytest.mark.skip(reason='Contact BioSample API')
     def test_check_existing_real_biosamples(self):
@@ -313,7 +319,8 @@ def test_check_all_analysis_run_accessions(self):
 
         checker.check_all_analysis_run_accessions()
         assert checker.errors == [
-            {'property': '/analysis/1/runAccessions', 'description': 'Run SRR00000000001 does not exist in ENA or is private'}]
+            {'property': '/analysis/1/runAccessions',
+             'description': 'Run SRR00000000001 does not exist in ENA or is private'}]
 
     def test_check_all_analysis_contain_samples(self):
         # all analysis contain samples
@@ -414,4 +421,31 @@ def test_check_hold_date(self):
         metadata = {"project": {}, "sample": [], "analysis": [], "files": []}
         checker = SemanticMetadataChecker(metadata, {})
         checker.check_hold_date()
-        self.assertEqual(checker.errors, [])
+        self.assertEqual(checker.errors, [])
+
+    def test_check_statementss(self):
+        # statement present and acknowledged
+        metadata = {"statements": [{
+            "acknowledgement": True,
+            "statement": DATA_ACCESS_STATEMENT
+        }]}
+        checker = SemanticMetadataChecker(metadata, {})
+        checker.check_statements()
+        self.assertEqual(checker.errors, [])
+
+        # statement present but not acknowledged
+        metadata = {"statements": [{
+            "acknowledgement": False,
+            "statement": DATA_ACCESS_STATEMENT
+        }]}
+        checker = SemanticMetadataChecker(metadata, {})
+        checker.check_statements()
+        self.assertEqual(checker.errors, [{'description': 'Data Access & Sharing statement not acknowledged',
+                                           'property': '/statements'}])
+
+        # no data present for acknowledgement
+        metadata = {"statements": []}
+        checker = SemanticMetadataChecker(metadata, {})
+        checker.check_statements()
+        self.assertEqual(checker.errors, [{'description': 'Data Access & Sharing statement not acknowledged',
+                                           'property': '/statements'}])