diff --git a/docker-compose.yml b/docker-compose.yml index f45227ea7..7e1b2fa24 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -176,6 +176,9 @@ services: - localstack environment: - LOCALSTACK_HOST=localstack + - PERSISTENCE=1 + volumes: + - localstack_data:/var/lib/localstack networks: backend: @@ -187,3 +190,5 @@ volumes: name: ${GIGWA_CONTAINER_NAME:-gigwa}_data gigwa_mongo_data: name: ${GIGWA_CONTAINER_NAME:-gigwa}_mongo_data + localstack_data: + name: localstack_data diff --git a/pom.xml b/pom.xml index 1788b1604..e37301b29 100644 --- a/pom.xml +++ b/pom.xml @@ -89,7 +89,7 @@ 31.0.1-jre 4.9.3 4.3.1 - 2.2.0 + 2.2.1 2.11.0 2.2.1 @@ -464,6 +464,11 @@ micronaut-amazon-awssdk-s3 2.0.5-micronaut-2.0 + + com.github.samtools + htsjdk + 2.24.1 + diff --git a/settings.xml b/settings.xml index 9b5fb7aa5..54c730caf 100644 --- a/settings.xml +++ b/settings.xml @@ -49,6 +49,12 @@ true true + + central-snapshots + https://central.sonatype.com/repository/maven-snapshots/ + false + true + github-fannypack FannyPack github repository diff --git a/src/main/java/org/breedinginsight/api/model/v1/request/query/GenotypeImportQuery.java b/src/main/java/org/breedinginsight/api/model/v1/request/query/GenotypeImportQuery.java new file mode 100644 index 000000000..a9d596bcf --- /dev/null +++ b/src/main/java/org/breedinginsight/api/model/v1/request/query/GenotypeImportQuery.java @@ -0,0 +1,69 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.api.model.v1.request.query; + +import io.micronaut.core.annotation.Introspected; +import lombok.Getter; +import org.apache.commons.lang3.StringUtils; + +import java.util.ArrayList; +import java.util.List; + +@Getter +@Introspected +public class GenotypeImportQuery extends QueryParams { + + private String sampleSubmissionId; + private String projectNameForSampleSubmission; + private String sampleSubmissionCreatedBy; + private String genotypingFileName; + private String genotypingImportDate; + private String genotypingImportBy; + + public SearchRequest constructSearchRequest() { + List filters = new ArrayList<>(); + + if (!StringUtils.isBlank(getSampleSubmissionId())) { + filters.add(constructFilterRequest("sampleSubmissionId", getSampleSubmissionId())); + } + if (!StringUtils.isBlank(getProjectNameForSampleSubmission())) { + filters.add(constructFilterRequest("projectNameForSampleSubmission", getProjectNameForSampleSubmission())); + } + if (!StringUtils.isBlank(getSampleSubmissionCreatedBy())) { + filters.add(constructFilterRequest("sampleSubmissionCreatedBy", getSampleSubmissionCreatedBy())); + } + if (!StringUtils.isBlank(getGenotypingFileName())) { + filters.add(constructFilterRequest("genotypingFileName", getGenotypingFileName())); + } + if (!StringUtils.isBlank(getGenotypingImportDate())) { + filters.add(constructFilterRequest("genotypingImportDate", getGenotypingImportDate())); + } + if (!StringUtils.isBlank(getGenotypingImportBy())) { + filters.add(constructFilterRequest("genotypingImportBy", getGenotypingImportBy())); + } + + return new SearchRequest(filters); + } + + private FilterRequest constructFilterRequest(String field, String value) { + return FilterRequest.builder() + .field(field) + .value(value) + .build(); + } +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/api/v1/controller/geno/GenotypeDataUploadController.java b/src/main/java/org/breedinginsight/api/v1/controller/geno/GenotypeDataUploadController.java index df6615458..85f3d3311 100644 --- a/src/main/java/org/breedinginsight/api/v1/controller/geno/GenotypeDataUploadController.java +++ b/src/main/java/org/breedinginsight/api/v1/controller/geno/GenotypeDataUploadController.java @@ -1,23 +1,52 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.breedinginsight.api.v1.controller.geno; +import io.micronaut.http.HttpHeaders; import io.micronaut.http.HttpResponse; +import io.micronaut.http.HttpStatus; import io.micronaut.http.MediaType; import io.micronaut.http.annotation.*; import io.micronaut.http.multipart.CompletedFileUpload; +import io.micronaut.http.server.types.files.StreamedFile; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; -import org.breedinginsight.api.auth.AuthenticatedUser; -import org.breedinginsight.api.auth.ProgramSecured; -import org.breedinginsight.api.auth.ProgramSecuredRole; -import org.breedinginsight.api.auth.SecurityService; +import org.breedinginsight.api.auth.*; +import org.breedinginsight.api.model.v1.request.query.GenotypeImportQuery; +import org.breedinginsight.api.model.v1.request.query.SearchRequest; +import org.breedinginsight.api.model.v1.response.DataResponse; import org.breedinginsight.api.model.v1.response.Response; +import org.breedinginsight.api.model.v1.validators.QueryValid; import org.breedinginsight.api.v1.controller.metadata.AddMetadata; import org.breedinginsight.brapps.importer.model.response.ImportResponse; +import org.breedinginsight.model.DownloadFile; +import org.breedinginsight.model.GenotypeImportDetails; +import org.breedinginsight.model.Program; +import org.breedinginsight.services.ProgramService; import org.breedinginsight.services.exceptions.AuthorizationException; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.services.geno.GenotypeService; +import org.breedinginsight.utilities.response.ResponseUtils; +import org.breedinginsight.utilities.response.mappers.GenotypeImportQueryMapper; import javax.inject.Inject; +import javax.validation.Valid; +import java.io.IOException; +import java.util.Optional; import java.util.UUID; @Slf4j @@ -25,22 +54,73 @@ public class GenotypeDataUploadController { private final GenotypeService genoService; private final SecurityService securityService; + private final ProgramService programService; + private final GenotypeImportQueryMapper genotypeImportQueryMapper; @Inject - public GenotypeDataUploadController(GenotypeService genoService, SecurityService securityService) { + public GenotypeDataUploadController(GenotypeService genoService, SecurityService securityService, + ProgramService programService, GenotypeImportQueryMapper genotypeImportQueryMapper) { this.genoService = genoService; this.securityService = securityService; + this.programService = programService; + this.genotypeImportQueryMapper = genotypeImportQueryMapper; } - @Post("programs/{programId}/experiments/{experimentId}/geno/import") + @Get("programs/{programId}/geno/imports{?genotypeImportQuery*}") + @Produces(MediaType.APPLICATION_JSON) + @ProgramSecured(roleGroups = ProgramSecuredRoleGroup.PROGRAM_SCOPED_ROLES) + public HttpResponse>> getGenotypeImports( + @PathVariable UUID programId, + @QueryValue @QueryValid(using = GenotypeImportQueryMapper.class) @Valid GenotypeImportQuery genotypeImportQuery) { + Optional program = programService.getById(programId); + if (program.isEmpty()) { + log.info("programId not found: {}", programId.toString()); + return HttpResponse.notFound(); + } + + SearchRequest searchRequest = genotypeImportQuery.constructSearchRequest(); + + return ResponseUtils.getQueryResponse( + genoService.getGenotypeImports(programId), + genotypeImportQueryMapper, + searchRequest, + genotypeImportQuery + ); + } + + @Get("programs/{programId}/geno/imports/{genotypeImportId}/download") + @ProgramSecured(roleGroups = ProgramSecuredRoleGroup.PROGRAM_SCOPED_ROLES) + @Produces(value = {"application/octet-stream"}) + public HttpResponse downloadGenotypeImport(@PathVariable UUID programId, @PathVariable UUID genotypeImportId) { + Optional program = programService.getById(programId); + if (program.isEmpty()) { + log.info("programId not found: {}", programId.toString()); + return HttpResponse.notFound(); + } + + try { + Optional downloadFile = genoService.downloadGenotypeImport(programId, genotypeImportId); + if (downloadFile.isEmpty()) { + return HttpResponse.notFound(); + } + + return HttpResponse.ok(downloadFile.get().getStreamedFile()) + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + downloadFile.get().getFileName()); + } catch (IOException e) { + log.error("Error downloading genotype import", e); + return HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, "Error downloading genotype import"); + } + } + + @Post("programs/{programId}/submissions/{submissionId}/geno/import") @Consumes(MediaType.MULTIPART_FORM_DATA) @Produces(MediaType.APPLICATION_JSON) @AddMetadata @ProgramSecured(roles = {ProgramSecuredRole.PROGRAM_ADMIN}) - public HttpResponse> uploadData(@PathVariable UUID programId, @PathVariable UUID experimentId, @Part("file") CompletedFileUpload upload) { + public HttpResponse> uploadData(@PathVariable UUID programId, @PathVariable UUID submissionId, @Part("file") CompletedFileUpload upload) { AuthenticatedUser actingUser = securityService.getUser(); try { - ImportResponse result = genoService.submitGenotypeData(actingUser.getId(), programId, experimentId, upload); + ImportResponse result = genoService.submitGenotypeData(actingUser.getId(), programId, submissionId, upload); Response response = new Response<>(result); return HttpResponse.ok(response); } catch (DoesNotExistException e) { diff --git a/src/main/java/org/breedinginsight/api/v1/controller/geno/SampleSubmissionController.java b/src/main/java/org/breedinginsight/api/v1/controller/geno/SampleSubmissionController.java index e01ab2c08..741d533a5 100644 --- a/src/main/java/org/breedinginsight/api/v1/controller/geno/SampleSubmissionController.java +++ b/src/main/java/org/breedinginsight/api/v1/controller/geno/SampleSubmissionController.java @@ -54,6 +54,11 @@ @Secured(SecurityRule.IS_AUTHENTICATED) public class SampleSubmissionController { + public static final String DELETE_STATUS_NOT_ALLOWED_ERROR_MESSAGE = + "Sample submission cannot be deleted because status is submitted or completed"; + public static final String DELETE_GENOTYPE_DATA_NOT_ALLOWED_ERROR_MESSAGE = + "Sample submission cannot be deleted because associated genotype data exists"; + private final boolean brapiSubmissionEnabled; private final SampleSubmissionService sampleSubmissionService; private final ProgramService programService; @@ -303,20 +308,20 @@ public HttpResponse deleteSubmissionById(@PathVariable UUID programId, @PathVari return HttpResponse.notFound(); } - // sample status validation - Optional submissionOpt = sampleSubmissionService.getSampleSubmission(program.get(), submissionId, false); - - if(submissionOpt.isEmpty()) { - return HttpResponse.notFound(); - } - SampleSubmission submission = submissionOpt.get(); - if (!submission.isDeletable()) { - return HttpResponse.notAllowed(); + SampleSubmissionService.DeleteResult result = sampleSubmissionService.deleteSampleSubmission(program.get(), submissionId); + switch (result) { + case NOT_FOUND: + return HttpResponse.notFound(); + case STATUS_NOT_ALLOWED: + return HttpResponse.notAllowed() + .body(DELETE_STATUS_NOT_ALLOWED_ERROR_MESSAGE); + case GENOTYPE_DATA_NOT_ALLOWED: + return HttpResponse.notAllowed() + .body(DELETE_GENOTYPE_DATA_NOT_ALLOWED_ERROR_MESSAGE); + case DELETED: + default: + return HttpResponse.ok(); } - - sampleSubmissionService.deleteSampleSubmission(program.get(), submissionId); - - return HttpResponse.ok(); } } diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIGermplasmController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIGermplasmController.java index 850ed8138..b26abf7ea 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIGermplasmController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIGermplasmController.java @@ -439,7 +439,8 @@ public HttpResponse> getGermplasmGenotype(@PathVaria try { BrAPIGermplasm germplasm = germplasmDAO.getGermplasmByUUID(germplasmId, programId); - GermplasmGenotype germplasmGenotype = genoService.retrieveGenotypeData(programId, germplasm); + + GermplasmGenotype germplasmGenotype = genoService.retrieveGenotypeData(programId, UUID.fromString(germplasmId)); Response response = new Response(germplasmGenotype); return HttpResponse.ok(response); diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java index 478c99bcf..58910396b 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java @@ -204,8 +204,7 @@ private Map processGermplasmForDisplay(List processGermplasmForDisplay(List - private String processBreedbasePedigree(String pedigree) { - - if (pedigree != null) { - if (pedigree.equals("NA/NA")) { - return ""; - } - - // Technically processGermplasmForDisplay should handle ok without stripping these NAs but will strip anyways - // for consistency. - // We only allow the /NA case for single parent as we require a female parent in the pedigree - // keep the leading slash, will be handled by processGermplasmForDisplay - if (pedigree.endsWith("/NA")) { - return pedigree.substring(0, pedigree.length()-2); - } - - // shouldn't have this case in our data but just in case - if (pedigree.startsWith("NA/")) { - return pedigree.substring(2); - } - } - return pedigree; - } - public List createBrAPIGermplasm(List postBrAPIGermplasmList, UUID programId, ImportUpload upload) { GermplasmApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), GermplasmApi.class); var program = programDAO.fetchOneById(programId); diff --git a/src/main/java/org/breedinginsight/brapi/v2/model/request/query/GermplasmQuery.java b/src/main/java/org/breedinginsight/brapi/v2/model/request/query/GermplasmQuery.java index 81039b7ef..464634d54 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/model/request/query/GermplasmQuery.java +++ b/src/main/java/org/breedinginsight/brapi/v2/model/request/query/GermplasmQuery.java @@ -22,6 +22,7 @@ public class GermplasmQuery extends BrapiQuery { private String femaleParentGID; private String maleParentGID; private String createdDate; + private String externalUID; private String createdByUserName; private String synonym; // This is a meta-parameter, it describes the display format of any date fields. @@ -56,9 +57,15 @@ public SearchRequest constructSearchRequest() { if (!StringUtils.isBlank(getMaleParentGID())) { filters.add(constructFilterRequest("maleParentGID", getMaleParentGID())); } + if (!StringUtils.isBlank(getCreatedDate())) { filters.add(constructFilterRequest("createdDate", getCreatedDate())); } + + if (!StringUtils.isBlank(getExternalUID())) { + filters.add(constructFilterRequest("externalUID", getExternalUID())); + } + if (!StringUtils.isBlank(getCreatedByUserName())) { filters.add(constructFilterRequest("createdByUserName", getCreatedByUserName())); } diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java index 7e4a01e67..b7ba4a5eb 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java @@ -119,24 +119,31 @@ public List> processListData(List germplasm, row.put("GID", Integer.valueOf(germplasmEntry.getAccessionNumber())); // Strip programKey and accessionNumber from germplasmName for the file output. row.put("Germplasm Name", Utilities.removeProgramKeyAnyAccession(germplasmEntry.getGermplasmName(), program.getKey())); - row.put("Breeding Method", germplasmEntry.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_BREEDING_METHOD).getAsString()); - String source = germplasmEntry.getSeedSource(); - row.put("Source", source); + if (germplasmEntry.getAdditionalInfo() != null && + germplasmEntry.getAdditionalInfo().has(BrAPIAdditionalInfoFields.GERMPLASM_BREEDING_METHOD) && + !germplasmEntry.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_BREEDING_METHOD).isJsonNull()) { + row.put("Breeding Method", germplasmEntry.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_BREEDING_METHOD).getAsString()); + } // Use the entry number in the list map if generated - if(listData == null) { + if (listData == null) { // Not downloading a real list, use GID (https://breedinginsight.atlassian.net/browse/BI-2266). row.put("Entry No", Integer.valueOf(germplasmEntry.getAccessionNumber())); } else { row.put("Entry No", entryNumber); } - //If germplasm was imported with an external UID, it will be stored in external reference with same source as seed source + String source = germplasmEntry.getSeedSource(); + if (source != null) { + row.put("Source", source);} + List externalReferences = germplasmEntry.getExternalReferences(); - for (BrAPIExternalReference reference: externalReferences){ - if (reference.getReferenceSource().equals(source)) { - row.put("External UID", reference.getReferenceID()); - break; + if (externalReferences != null) { + for (BrAPIExternalReference reference : externalReferences) { + if ("External UID".equals(reference.getReferenceSource())) { + row.put("External UID", reference.getReferenceID()); + break; + } } } @@ -171,6 +178,12 @@ public List> processListData(List germplasm, row.put("Synonyms", joinedSynonyms); } + // Pedigrees + if (germplasmEntry.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_PEDIGREE_BY_NAME) != null) { + String pedigreeString = germplasmEntry.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_PEDIGREE_BY_NAME).getAsString(); + row.put("Pedigree", pedigreeString); + } + processedData.add(row); } return processedData; diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPISampleDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPISampleDAO.java index 372bb63be..85dc19138 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPISampleDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPISampleDAO.java @@ -40,7 +40,6 @@ import javax.inject.Inject; import javax.inject.Singleton; -import java.io.IOException; import java.util.Collections; import java.util.List; @@ -75,25 +74,15 @@ public List createSamples(Program program, List sample return brAPIDAOUtil.post(samplesToSave, upload, samplesApi::samplesPost, importDAO::update); } - public List readSamplesByIds(Program program, List sampleExternalIds) throws ApiException { - if(sampleExternalIds.isEmpty()) { - return Collections.emptyList(); - } - - BrAPISampleSearchRequest searchRequest = new BrAPISampleSearchRequest().externalReferenceIDs(sampleExternalIds) - .externalReferenceSources(List.of(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.SAMPLES))); - SamplesApi samplesApi = brAPIEndpointProvider.get(programDAO.getSampleClient(program.getId()), SamplesApi.class); - return brAPIDAOUtil.search(samplesApi::searchSamplesPost, samplesApi::searchSamplesSearchResultsDbIdGet, searchRequest); - } - public List readSamplesByPlateIds(Program program, List plateExternalIds) throws ApiException { - if(plateExternalIds.isEmpty()) { + public List readSamplesByGermplasmIds(Program program, List germplasmExternalIds) throws ApiException { + if(germplasmExternalIds.isEmpty()) { return Collections.emptyList(); } - BrAPISampleSearchRequest searchRequest = new BrAPISampleSearchRequest().externalReferenceIDs(plateExternalIds) - .externalReferenceSources(List.of(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PLATES))); + BrAPISampleSearchRequest searchRequest = new BrAPISampleSearchRequest().externalReferenceIDs(germplasmExternalIds) + .externalReferenceSources(List.of(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.GERMPLASM))); SamplesApi samplesApi = brAPIEndpointProvider.get(programDAO.getSampleClient(program.getId()), SamplesApi.class); return brAPIDAOUtil.search(samplesApi::searchSamplesPost, samplesApi::searchSamplesSearchResultsDbIdGet, searchRequest); diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/base/Germplasm.java b/src/main/java/org/breedinginsight/brapps/importer/model/base/Germplasm.java index de1cf19f1..775ecc860 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/base/Germplasm.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/base/Germplasm.java @@ -57,12 +57,10 @@ public class Germplasm implements BrAPIObject { private String germplasmName; @ImportFieldType(type= ImportFieldTypeEnum.TEXT) - @ImportMappingRequired @ImportFieldMetadata(id="breedingMethod", name="Breeding Method", description = "The breeding method name or code") private String breedingMethod; @ImportFieldType(type= ImportFieldTypeEnum.TEXT) - @ImportMappingRequired @ImportFieldMetadata(id="germplasmSource", name="Source", description = "The germplasm origin. If External UID present, assumed to be the source associated with the External UID.") private String germplasmSource; @@ -160,12 +158,12 @@ public static String constructGermplasmListName(String listName, Program program * * @param germplasm germplasm object * @param program program - * @param listId list id * @param commit flag indicating if commit changes should be made * @param updatePedigree flag indicating if pedigree should be updated * @return mutated indicator */ - public boolean updateBrAPIGermplasm(BrAPIGermplasm germplasm, Program program, UUID listId, boolean commit, boolean updatePedigree) { + public boolean updateBrAPIGermplasm(BrAPIGermplasm germplasm, Program program, boolean commit, + boolean updatePedigree, ProgramBreedingMethodEntity breedingMethod) { boolean mutated = false; @@ -185,6 +183,27 @@ public boolean updateBrAPIGermplasm(BrAPIGermplasm germplasm, Program program, U mutated = true; } + // Append Source only when DB value is blank + if (StringUtils.isBlank(germplasm.getSeedSource()) && StringUtils.isNotBlank(getGermplasmSource())) { + germplasm.setSeedSource(getGermplasmSource()); + mutated = true; + } + + // Append Breeding Method only when DB field is blank + if (breedingMethod != null) { + boolean hasBreedingMethodId = hasDateInAdditionalInfo(germplasm, BrAPIAdditionalInfoFields.GERMPLASM_BREEDING_METHOD_ID); + boolean hasBreedingMethodName = hasDateInAdditionalInfo(germplasm, BrAPIAdditionalInfoFields.GERMPLASM_BREEDING_METHOD); + + if (!hasBreedingMethodId) { + germplasm.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GERMPLASM_BREEDING_METHOD_ID, breedingMethod.getId().toString()); + mutated = true; + } + if (!hasBreedingMethodName) { + germplasm.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GERMPLASM_BREEDING_METHOD, breedingMethod.getName()); + mutated = true; + } + } + // Append synonyms to germplasm that don't already exist // Synonym comparison is based on name and type if (synonyms != null) { @@ -211,6 +230,14 @@ public boolean updateBrAPIGermplasm(BrAPIGermplasm germplasm, Program program, U return mutated; } + //Method to check whether breeding method already exists in DB additionalInfo + private boolean hasDateInAdditionalInfo(BrAPIGermplasm germplasm, String key) { + JsonObject additionalInfo = germplasm.getAdditionalInfo(); + return additionalInfo != null + && additionalInfo.has(key) + && !additionalInfo.get(key).isJsonNull() + && StringUtils.isNotBlank(additionalInfo.get(key).getAsString()); + } public void setUpdateCommitFields(BrAPIGermplasm germplasm, String programKey) { @@ -262,13 +289,13 @@ public BrAPIGermplasm constructBrAPIGermplasm(ProgramBreedingMethodEntity breedi // Seed Source //If there is an external uid, source is associated with it as an additional external reference BrAPIExternalReference uidExternalReference = null; - if (germplasmSource != null) { + if (StringUtils.isNotBlank(getGermplasmSource())) { germplasm.setSeedSource(getGermplasmSource()); - if (externalUID != null) { - uidExternalReference = new BrAPIExternalReference(); - uidExternalReference.setReferenceID(getExternalUID()); - uidExternalReference.setReferenceSource(getGermplasmSource()); - } + } + if (StringUtils.isNotBlank(getExternalUID())) { + uidExternalReference = new BrAPIExternalReference(); + uidExternalReference.setReferenceSource("External UID"); + uidExternalReference.setReferenceID(getExternalUID()); } // External references diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/SampleSubmissionProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/SampleSubmissionProcessor.java index 396bf8658..8ded93d89 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/SampleSubmissionProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/SampleSubmissionProcessor.java @@ -67,7 +67,7 @@ public class SampleSubmissionProcessor implements Processor { private static final String UNKNOWN_GID = "Unknown germplasm GID"; private static final String INVALID_COLUMN = "Column must be a number between 1 and 12"; private static final String INVALID_ROW = "Row must be a letter between A and H"; - private static final String MULTIPLE_SAMPLES_SINGLE_WELL = "The sample in row %d is already in row: %s, column: %d"; + private static final String MULTIPLE_SAMPLES_SINGLE_WELL = "Plate position not unique, duplicate in row %d"; private final String referenceSource; private final BrAPIGermplasmDAO germplasmDAO; private final BrAPIObservationUnitDAO observationUnitDAO; @@ -230,7 +230,7 @@ private boolean validRow(SampleSubmissionImport row, int rowNum, ValidationError if (plateLayout[plateRow][plateCol] > 0) { validationErrors.addError(rowNum, new ValidationError(SampleSubmissionImport.Columns.ROW + "/" + SampleSubmissionImport.Columns.COLUMN, - String.format(MULTIPLE_SAMPLES_SINGLE_WELL, plateLayout[plateRow][plateCol], Character.toString('A' + plateRow), plateCol), + String.format(MULTIPLE_SAMPLES_SINGLE_WELL, plateLayout[plateRow][plateCol]), HttpStatus.UNPROCESSABLE_ENTITY)); } else { plateLayout[plateRow][plateCol] = rowNum; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java index 5defe7d10..899924f4b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java @@ -17,7 +17,6 @@ package org.breedinginsight.brapps.importer.services.processors.germplasm; import com.google.gson.Gson; -import com.google.gson.JsonElement; import io.micronaut.context.annotation.Property; import io.micronaut.context.annotation.Prototype; import io.micronaut.http.HttpStatus; @@ -94,6 +93,7 @@ public class GermplasmProcessor implements Processor { public static String missingParentalEntryNoMsg = "The following parental entry numbers were not found in the file: %s"; public static String badBreedMethodsMsg = "Invalid breeding method"; public static String badGermplasmNameMsg = "Germplasm name cannot contain /"; + public static String missingBreedingMethodForParentalGIDorEntryNo = "Required field \"Breeding Method\" cannot contain empty values when parent(s) are specified"; public static String missingEntryNumbersMsg = "Either all or none of the germplasm must have entry numbers"; public static String duplicateEntryNoMsg = "Entry numbers must be unique. Duplicated entry numbers found: %s"; public static String circularDependency = "Circular dependency in the pedigree tree"; @@ -160,17 +160,16 @@ public void getExistingBrapiData(List importRows, Program program) }); // Get parental accession nos in file - for (int i = 0; i < importRows.size(); i++) { - BrAPIImport germplasmImport = importRows.get(i); + for (BrAPIImport germplasmImport : importRows) { Germplasm germplasm = germplasmImport.getGermplasm(); if (germplasm != null) { // Retrieve parent accession numbers to assess if already in db - if (germplasm.getFemaleParentAccessionNumber() != null) { - germplasmAccessionNumbers.put(germplasm.getFemaleParentAccessionNumber(), true); - } - if (germplasm.getMaleParentAccessionNumber() != null) { - germplasmAccessionNumbers.put(germplasm.getMaleParentAccessionNumber(), true); - } + if (germplasm.getFemaleParentAccessionNumber() != null) { + germplasmAccessionNumbers.put(germplasm.getFemaleParentAccessionNumber(), true); + } + if (germplasm.getMaleParentAccessionNumber() != null) { + germplasmAccessionNumbers.put(germplasm.getMaleParentAccessionNumber(), true); + } } } @@ -269,7 +268,7 @@ public Map process(ImportUpload upload, List
nextVal = () -> dsl.nextval(germplasmSequenceName.toLowerCase()); @@ -285,14 +284,13 @@ public Map process(ImportUpload upload, List
(); updatedGermplasmList = new ArrayList<>(); Map breedingMethods = new HashMap<>(); - Boolean nullEntryNotFound = false; List badBreedingMethods = new ArrayList<>(); Map entryNumberCounts = new HashMap<>(); List userProvidedEntryNumbers = new ArrayList<>(); ValidationErrors validationErrors = new ValidationErrors(); for (int i = 0; i < importRows.size(); i++) { - log.debug("processing germplasm row: " + (i+1)); + log.debug("processing germplasm row: {}", i + 1); BrAPIImport brapiImport = importRows.get(i); PendingImport mappedImportRow = mappedBrAPIImport.getOrDefault(i, new PendingImport()); @@ -315,28 +313,28 @@ public Map process(ImportUpload upload, List
0 && userProvidedEntryNumbers.size() < importRows.size()) { + if (!userProvidedEntryNumbers.isEmpty() && userProvidedEntryNumbers.size() < importRows.size()) { throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, missingEntryNumbersMsg); } // Check for duplicate entry numbers if (entryNumberCounts.size() < importRows.size()) { List dups = entryNumberCounts.keySet().stream() - .filter(key -> entryNumberCounts.get(key) > 1) - .collect(Collectors.toList()); + .filter(key -> entryNumberCounts.get(key) > 1) + .collect(Collectors.toList()); throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, - String.format(duplicateEntryNoMsg, arrayOfStringFormatter.apply(dups))); + String.format(duplicateEntryNoMsg, arrayOfStringFormatter.apply(dups))); } // Construct pedigree @@ -351,25 +349,12 @@ private void processNewGermplasm(Germplasm germplasm, ValidationErrors validatio List badBreedingMethods, Program program, UUID importListId, boolean commit, PendingImport mappedImportRow, int i, User user, Supplier nextVal) { germplasm = removeBreedingMethodBlanks(germplasm); + //Validating if Breeding Method exists for valid parent entries + validateGermplasmBreedingMethod(germplasm, i + 2, validationErrors); // Get the breeding method database object - ProgramBreedingMethodEntity breedingMethod = null; - if (germplasm.getBreedingMethod() != null) { - if (breedingMethods.containsKey(germplasm.getBreedingMethod())) { - breedingMethod = breedingMethods.get(germplasm.getBreedingMethod()); - } else { - List breedingMethodResults = breedingMethodDAO.findByNameOrAbbreviation(germplasm.getBreedingMethod(), program.getId()); - if (breedingMethodResults.size() > 0) { - breedingMethods.put(germplasm.getBreedingMethod(), breedingMethodResults.get(0)); - breedingMethod = breedingMethods.get(germplasm.getBreedingMethod()); - } else { - ValidationError ve = new ValidationError("Breeding Method", badBreedMethodsMsg, HttpStatus.UNPROCESSABLE_ENTITY); - validationErrors.addError(i + 2, ve); // +2 instead of +1 to account for the column header row. - badBreedingMethods.add(germplasm.getBreedingMethod()); - } - } - } + ProgramBreedingMethodEntity breedingMethod = resolveBreedingMethod(germplasm, validationErrors, breedingMethods, badBreedingMethods, program, i + 2); - validateGermplasmName(germplasm, i+2, validationErrors); + validateGermplasmName(germplasm, i + 2, validationErrors); validatePedigree(germplasm, i + 2, validationErrors); if (germplasm.pedigreeExists()) { @@ -391,16 +376,17 @@ private void processNewGermplasm(Germplasm germplasm, ValidationErrors validatio // Removes leading and trailing blanks from the germplasm breedingMethod private Germplasm removeBreedingMethodBlanks(Germplasm germplasm) { - if(germplasm.getBreedingMethod() != null ) { + if (germplasm.getBreedingMethod() != null) { germplasm.setBreedingMethod(germplasm.getBreedingMethod().strip()); } return germplasm; } - private boolean processExistingGermplasm(Germplasm germplasm, ValidationErrors validationErrors, List importRows, Program program, UUID importListId, boolean commit, PendingImport mappedImportRow, int rowIndex) { + private void processExistingGermplasm(Germplasm germplasm, ValidationErrors validationErrors, List importRows, Map breedingMethods, + List badBreedingMethods, Program program, boolean commit, PendingImport mappedImportRow, int rowIndex) { BrAPIGermplasm existingGermplasm; String gid = germplasm.getAccessionNumber(); - boolean mutated = false; + boolean mutated; boolean updatePedigree = false; if (germplasmByAccessionNumber.containsKey(gid)) { @@ -410,31 +396,37 @@ private boolean processExistingGermplasm(Germplasm germplasm, ValidationErrors v } else { //should be caught in getExistingBrapiData ValidationError ve = new ValidationError("GID", String.format(missingGID, gid), HttpStatus.NOT_FOUND); - validationErrors.addError(rowIndex+2, ve ); // +2 instead of +1 to account for the column header row. - return false; + validationErrors.addError(rowIndex + 2, ve); // +2 instead of +1 to account for the column header row. + return; } + germplasm = removeBreedingMethodBlanks(germplasm); + //Validating if Breeding Method exists for valid parent entries + validateGermplasmBreedingMethod(germplasm, rowIndex + 2, validationErrors); + // Get the breeding method database object + ProgramBreedingMethodEntity breedingMethod = resolveBreedingMethod(germplasm, validationErrors, breedingMethods, badBreedingMethods, program, rowIndex + 2); + // Error conditions: // has existing pedigree and file pedigree is different and not empty // Valid conditions: // no existing pedigree and file different pedigree // existing pedigree and file pedigree same // existing pedigree and file pedigree empty - if(hasPedigree(existingGermplasm) && germplasm.pedigreeExists()) { - if(!arePedigreesEqual(existingGermplasm, germplasm, importRows)) { + if (hasPedigree(existingGermplasm) && germplasm.pedigreeExists()) { + if (!arePedigreesEqual(existingGermplasm, germplasm, importRows)) { ValidationError ve = new ValidationError("Pedigree", pedigreeAlreadyExists, HttpStatus.UNPROCESSABLE_ENTITY); validationErrors.addError(rowIndex + 2, ve); // +2 instead of +1 to account for the column header row. - return false; + return; } } // if no existing pedigree and file has pedigree then validate and update - if(germplasm.pedigreeExists() && !hasPedigree(existingGermplasm)) { + if (germplasm.pedigreeExists() && !hasPedigree(existingGermplasm)) { validatePedigree(germplasm, rowIndex + 2, validationErrors); updatePedigree = true; } - mutated = germplasm.updateBrAPIGermplasm(existingGermplasm, program, importListId, commit, updatePedigree); + mutated = germplasm.updateBrAPIGermplasm(existingGermplasm, program, commit, updatePedigree, breedingMethod); if (mutated) { updatedGermplasmList.add(existingGermplasm); @@ -448,7 +440,6 @@ private boolean processExistingGermplasm(Germplasm germplasm, ValidationErrors v // add to list regardless of mutated or not importList.addDataItem(existingGermplasm.getGermplasmName()); - return true; } private boolean canUpdatePedigree(BrAPIGermplasm existingGermplasm, Germplasm germplasm) { @@ -604,6 +595,35 @@ private void validateGermplasmName(Germplasm germplasm, Integer rowNumber, Valid } } + private void validateGermplasmBreedingMethod(Germplasm germplasm, Integer rowNumber, ValidationErrors validationErrors) { + if (germplasm.pedigreeExists() && StringUtils.isBlank(germplasm.getBreedingMethod())) { + ValidationError error = new ValidationError("Breeding Method", missingBreedingMethodForParentalGIDorEntryNo, HttpStatus.UNPROCESSABLE_ENTITY); + validationErrors.addError(rowNumber, error); + } + } + + //shared Breeding Method lookup for create and update processes + private ProgramBreedingMethodEntity resolveBreedingMethod(Germplasm germplasm, ValidationErrors validationErrors, + Map breedingMethods, + List badBreedingMethods, Program program, int rowNumber) { + if (germplasm.getBreedingMethod() == null) { + return null; + } + if (breedingMethods.containsKey(germplasm.getBreedingMethod())) { + return breedingMethods.get(germplasm.getBreedingMethod()); + } + List breedingMethodResults = breedingMethodDAO.findByNameOrAbbreviation(germplasm.getBreedingMethod(), program.getId()); + + if (breedingMethodResults.isEmpty()) { + ValidationError ve = new ValidationError("Breeding Method", badBreedMethodsMsg, HttpStatus.UNPROCESSABLE_ENTITY); + validationErrors.addError(rowNumber, ve); + badBreedingMethods.add(germplasm.getBreedingMethod()); + return null; + } + breedingMethods.put(germplasm.getBreedingMethod(), breedingMethodResults.get(0)); + return breedingMethods.get(germplasm.getBreedingMethod()); + } + private void validatePedigree(Germplasm germplasm, Integer rowNumber, ValidationErrors validationErrors) { String femaleParentEntryNo = germplasm.getFemaleParentEntryNo(); String maleParentEntryNo = germplasm.getMaleParentEntryNo(); @@ -657,7 +677,7 @@ private void createPostOrder() { } totalRecorded += createList.size(); - if (createList.size() > 0) { + if (!createList.isEmpty()) { created.addAll(createList.stream().map(GermplasmImportIdUtils::getImportId).collect(Collectors.toList())); postOrder.add(createList); } else if (totalRecorded < newGermplasmList.size()) { diff --git a/src/main/java/org/breedinginsight/daos/GenotypeImportDAO.java b/src/main/java/org/breedinginsight/daos/GenotypeImportDAO.java new file mode 100644 index 000000000..2d5d7c7f7 --- /dev/null +++ b/src/main/java/org/breedinginsight/daos/GenotypeImportDAO.java @@ -0,0 +1,111 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.breedinginsight.daos; + +import io.micronaut.http.HttpStatus; +import org.breedinginsight.dao.db.tables.BiUserTable; +import org.breedinginsight.dao.db.tables.daos.GenotypeImportDao; +import org.breedinginsight.dao.db.tables.pojos.GenotypeImportEntity; +import org.breedinginsight.model.GenotypeImportDetails; +import org.breedinginsight.model.GenotypeImportDownloadDetails; +import org.jooq.Configuration; +import org.jooq.DSLContext; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.time.OffsetDateTime; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +import static org.breedinginsight.dao.db.Tables.*; + +@Singleton +public class GenotypeImportDAO extends GenotypeImportDao { + + private final DSLContext dsl; + + @Inject + public GenotypeImportDAO(Configuration config, DSLContext dsl) { + super(config); + this.dsl = dsl; + } + + public void createGenotypeImportLink(UUID submissionId, UUID importerImportId, UUID userId) { + OffsetDateTime now = OffsetDateTime.now(); + + insert(GenotypeImportEntity.builder() + .id(UUID.randomUUID()) + .sampleSubmissionId(submissionId) + .importerImportId(importerImportId) + .createdAt(now) + .updatedAt(now) + .createdBy(userId) + .updatedBy(userId) + .build()); + } + + public boolean existsBySampleSubmissionId(UUID submissionId) { + return dsl.fetchExists( + dsl.selectOne() + .from(GENOTYPE_IMPORT) + .where(GENOTYPE_IMPORT.SAMPLE_SUBMISSION_ID.eq(submissionId)) + ); + } + + public List getGenotypeImportsByProgramId(UUID programId) { + BiUserTable sampleSubmissionCreatedByUser = BI_USER.as("sampleSubmissionCreatedByUser"); + BiUserTable genotypingImportByUser = BI_USER.as("genotypingImportByUser"); + + return dsl.select( + GENOTYPE_IMPORT.ID, + SAMPLE_SUBMISSION.ID, + SAMPLE_SUBMISSION.NAME, + sampleSubmissionCreatedByUser.NAME, + IMPORTER_IMPORT.UPLOAD_FILE_NAME, + IMPORTER_IMPORT.CREATED_AT, + genotypingImportByUser.NAME) + .from(GENOTYPE_IMPORT) + .join(SAMPLE_SUBMISSION).on(GENOTYPE_IMPORT.SAMPLE_SUBMISSION_ID.eq(SAMPLE_SUBMISSION.ID)) + .join(sampleSubmissionCreatedByUser).on(SAMPLE_SUBMISSION.CREATED_BY.eq(sampleSubmissionCreatedByUser.ID)) + .join(IMPORTER_IMPORT).on(GENOTYPE_IMPORT.IMPORTER_IMPORT_ID.eq(IMPORTER_IMPORT.ID)) + .join(IMPORTER_PROGRESS).on(IMPORTER_IMPORT.IMPORTER_PROGRESS_ID.eq(IMPORTER_PROGRESS.ID)) + .join(genotypingImportByUser).on(IMPORTER_IMPORT.USER_ID.eq(genotypingImportByUser.ID)) + .where(SAMPLE_SUBMISSION.PROGRAM_ID.eq(programId)) + .and(IMPORTER_IMPORT.PROGRAM_ID.eq(programId)) + .and(IMPORTER_PROGRESS.STATUSCODE.eq((short) HttpStatus.OK.getCode())) + .orderBy(IMPORTER_IMPORT.CREATED_AT.desc()) + .fetch(record -> GenotypeImportDetails + .parseSqlRecord(record, sampleSubmissionCreatedByUser, genotypingImportByUser)); + } + + public Optional getDownloadableGenotypeImportById(UUID programId, UUID genotypeImportId) { + return dsl.select( + GENOTYPE_IMPORT.SAMPLE_SUBMISSION_ID, + GENOTYPE_IMPORT.IMPORTER_IMPORT_ID, + IMPORTER_IMPORT.UPLOAD_FILE_NAME) + .from(GENOTYPE_IMPORT) + .join(SAMPLE_SUBMISSION).on(GENOTYPE_IMPORT.SAMPLE_SUBMISSION_ID.eq(SAMPLE_SUBMISSION.ID)) + .join(IMPORTER_IMPORT).on(GENOTYPE_IMPORT.IMPORTER_IMPORT_ID.eq(IMPORTER_IMPORT.ID)) + .join(IMPORTER_PROGRESS).on(IMPORTER_IMPORT.IMPORTER_PROGRESS_ID.eq(IMPORTER_PROGRESS.ID)) + .where(GENOTYPE_IMPORT.ID.eq(genotypeImportId)) + .and(SAMPLE_SUBMISSION.PROGRAM_ID.eq(programId)) + .and(IMPORTER_IMPORT.PROGRAM_ID.eq(programId)) + .and(IMPORTER_PROGRESS.STATUSCODE.eq((short) HttpStatus.OK.getCode())) + .fetchOptional(GenotypeImportDownloadDetails::parseSqlRecord); + } +} diff --git a/src/main/java/org/breedinginsight/model/GenotypeImportDetails.java b/src/main/java/org/breedinginsight/model/GenotypeImportDetails.java new file mode 100644 index 000000000..f9ad900f3 --- /dev/null +++ b/src/main/java/org/breedinginsight/model/GenotypeImportDetails.java @@ -0,0 +1,68 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.model; + +import io.micronaut.core.annotation.Introspected; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; +import lombok.experimental.Accessors; +import lombok.experimental.SuperBuilder; +import lombok.extern.jackson.Jacksonized; +import org.breedinginsight.dao.db.tables.BiUserTable; +import org.jooq.Record; + +import java.time.OffsetDateTime; +import java.util.UUID; + +import static org.breedinginsight.dao.db.Tables.IMPORTER_IMPORT; +import static org.breedinginsight.dao.db.Tables.SAMPLE_SUBMISSION; +import static org.breedinginsight.dao.db.Tables.GENOTYPE_IMPORT; + +@Getter +@Setter +@Accessors(chain = true) +@ToString +@SuperBuilder +@NoArgsConstructor +@Introspected +@Jacksonized +public class GenotypeImportDetails { + private UUID genotypeImportId; + private UUID sampleSubmissionId; + private String projectNameForSampleSubmission; + private String sampleSubmissionCreatedBy; + private String genotypingFileName; + private OffsetDateTime genotypingImportDate; + private String genotypingImportBy; + + public static GenotypeImportDetails parseSqlRecord(Record record, + BiUserTable sampleSubmissionCreatedByUser, + BiUserTable genotypingImportByUser) { + return GenotypeImportDetails.builder() + .genotypeImportId(record.get(GENOTYPE_IMPORT.ID)) + .sampleSubmissionId(record.get(SAMPLE_SUBMISSION.ID)) + .projectNameForSampleSubmission(record.get(SAMPLE_SUBMISSION.NAME)) + .sampleSubmissionCreatedBy(record.get(sampleSubmissionCreatedByUser.NAME)) + .genotypingFileName(record.get(IMPORTER_IMPORT.UPLOAD_FILE_NAME)) + .genotypingImportDate(record.get(IMPORTER_IMPORT.CREATED_AT)) + .genotypingImportBy(record.get(genotypingImportByUser.NAME)) + .build(); + } +} diff --git a/src/main/java/org/breedinginsight/model/GenotypeImportDownloadDetails.java b/src/main/java/org/breedinginsight/model/GenotypeImportDownloadDetails.java new file mode 100644 index 000000000..7fe874be0 --- /dev/null +++ b/src/main/java/org/breedinginsight/model/GenotypeImportDownloadDetails.java @@ -0,0 +1,54 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.breedinginsight.model; + +import io.micronaut.core.annotation.Introspected; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; +import lombok.experimental.Accessors; +import lombok.experimental.SuperBuilder; +import lombok.extern.jackson.Jacksonized; +import org.jooq.Record; + +import java.util.UUID; + +import static org.breedinginsight.dao.db.Tables.GENOTYPE_IMPORT; +import static org.breedinginsight.dao.db.Tables.IMPORTER_IMPORT; + +@Getter +@Setter +@Accessors(chain = true) +@ToString +@SuperBuilder +@NoArgsConstructor +@Introspected +@Jacksonized +public class GenotypeImportDownloadDetails { + private UUID sampleSubmissionId; + private UUID importerImportId; + private String genotypeFileName; + + public static GenotypeImportDownloadDetails parseSqlRecord(Record record) { + return GenotypeImportDownloadDetails.builder() + .sampleSubmissionId(record.get(GENOTYPE_IMPORT.SAMPLE_SUBMISSION_ID)) + .importerImportId(record.get(GENOTYPE_IMPORT.IMPORTER_IMPORT_ID)) + .genotypeFileName(record.get(IMPORTER_IMPORT.UPLOAD_FILE_NAME)) + .build(); + } +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/services/SampleSubmissionService.java b/src/main/java/org/breedinginsight/services/SampleSubmissionService.java index 3309dd0dc..7f364f619 100644 --- a/src/main/java/org/breedinginsight/services/SampleSubmissionService.java +++ b/src/main/java/org/breedinginsight/services/SampleSubmissionService.java @@ -41,6 +41,7 @@ import org.breedinginsight.brapps.importer.model.exports.FileType; import org.breedinginsight.brapps.importer.model.imports.sample.SampleSubmissionImport; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.daos.GenotypeImportDAO; import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.daos.SampleSubmissionDAO; import org.breedinginsight.model.*; @@ -78,6 +79,7 @@ public class SampleSubmissionService { private final BrAPISampleDAO sampleDAO; private final BrAPIEndpointProvider brAPIEndpointProvider; private final ProgramDAO programDAO; + private final GenotypeImportDAO genotypeImportDAO; private final DSLContext dsl; @Inject @@ -92,6 +94,7 @@ public SampleSubmissionService(@Property(name = "brapi.server.reference-source") BrAPISampleDAO sampleDAO, BrAPIEndpointProvider brAPIEndpointProvider, ProgramDAO programDAO, + GenotypeImportDAO genotypeImportDAO, DSLContext dsl) { this.referenceSource = referenceSource; this.dartBrapiUrl = dartBrapiUrl; @@ -104,6 +107,7 @@ public SampleSubmissionService(@Property(name = "brapi.server.reference-source") this.sampleDAO = sampleDAO; this.brAPIEndpointProvider = brAPIEndpointProvider; this.programDAO = programDAO; + this.genotypeImportDAO = genotypeImportDAO; this.dsl = dsl; } @@ -432,9 +436,21 @@ public Optional updateSubmissionStatus(Program program, UUID s * Deletes BrAPI plates and submission objects as well as sample submission record in bidb * We do not currently cache plates or samples so don't need to worry about that * @param submissionId sample submission UUID to delete + * @return result describing whether the submission was deleted or why deletion was rejected * @exception ApiException if a BrAPI call fails */ - public void deleteSampleSubmission(Program program, UUID submissionId) throws ApiException { + public DeleteResult deleteSampleSubmission(Program program, UUID submissionId) throws ApiException { + Optional submission = getSampleSubmission(program, submissionId, false); + if (submission.isEmpty()) { + return DeleteResult.NOT_FOUND; + } + if (!submission.get().isDeletable()) { + return DeleteResult.STATUS_NOT_ALLOWED; + } + if (genotypeImportDAO.existsBySampleSubmissionId(submissionId)) { + return DeleteResult.GENOTYPE_DATA_NOT_ALLOWED; + } + // create a batch of sampleIds and plateIds to delete // get samples with the sample submission xref List samples = sampleDAO.readSamplesBySubmissionIds(program, List.of(submissionId.toString())); @@ -449,6 +465,15 @@ public void deleteSampleSubmission(Program program, UUID submissionId) throws Ap // delete sample submission record from bidb submissionDAO.deleteById(submissionId); + + return DeleteResult.DELETED; + } + + public enum DeleteResult { + DELETED, + NOT_FOUND, + STATUS_NOT_ALLOWED, + GENOTYPE_DATA_NOT_ALLOWED } } diff --git a/src/main/java/org/breedinginsight/services/geno/GenotypeService.java b/src/main/java/org/breedinginsight/services/geno/GenotypeService.java index bceb33503..f54e1be93 100644 --- a/src/main/java/org/breedinginsight/services/geno/GenotypeService.java +++ b/src/main/java/org/breedinginsight/services/geno/GenotypeService.java @@ -1,17 +1,41 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.breedinginsight.services.geno; import io.micronaut.http.multipart.CompletedFileUpload; import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.germ.BrAPIGermplasm; import org.breedinginsight.brapps.importer.model.response.ImportResponse; +import org.breedinginsight.model.DownloadFile; import org.breedinginsight.model.GermplasmGenotype; import org.breedinginsight.services.exceptions.AuthorizationException; import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.model.GenotypeImportDetails; +import java.io.IOException; +import java.util.List; +import java.util.Optional; import java.util.UUID; public interface GenotypeService { - ImportResponse submitGenotypeData(UUID userId, UUID programId, UUID experimentId, CompletedFileUpload uploadedFile) throws DoesNotExistException, AuthorizationException, ApiException; + ImportResponse submitGenotypeData(UUID userId, UUID programId, UUID submissionId, CompletedFileUpload uploadedFile) throws DoesNotExistException, AuthorizationException, ApiException; - GermplasmGenotype retrieveGenotypeData(UUID programId, BrAPIGermplasm germplasm) throws DoesNotExistException, AuthorizationException, ApiException; + GermplasmGenotype retrieveGenotypeData(UUID programId, UUID germplasmId) throws DoesNotExistException, AuthorizationException, ApiException; + + List getGenotypeImports(UUID programId); + + Optional downloadGenotypeImport(UUID programId, UUID genotypeImportId)throws IOException; } diff --git a/src/main/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImpl.java b/src/main/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImpl.java index 97c78cf05..a6dc67a4b 100644 --- a/src/main/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImpl.java +++ b/src/main/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImpl.java @@ -1,13 +1,35 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.services.geno.impl; import com.agorapulse.micronaut.amazon.awssdk.s3.SimpleStorageService; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonObject; +import htsjdk.samtools.util.CloseableIterator; +import htsjdk.tribble.TribbleException; +import htsjdk.variant.variantcontext.VariantContext; +import htsjdk.variant.vcf.VCFFileReader; import io.micronaut.context.annotation.Property; import io.micronaut.http.HttpStatus; import io.micronaut.http.multipart.CompletedFileUpload; import io.micronaut.http.server.exceptions.InternalServerException; +import io.micronaut.http.server.types.files.StreamedFile; import lombok.extern.slf4j.Slf4j; import okhttp3.*; import org.apache.commons.lang3.tuple.Pair; @@ -18,18 +40,12 @@ import org.brapi.client.v2.auth.OAuth; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.client.v2.model.queryParams.core.ProgramQueryParams; -import org.brapi.client.v2.model.queryParams.core.TrialQueryParams; import org.brapi.client.v2.modules.core.ProgramsApi; -import org.brapi.client.v2.modules.core.TrialsApi; import org.brapi.client.v2.modules.genotype.CallSetsApi; import org.brapi.client.v2.modules.genotype.CallsApi; import org.brapi.client.v2.modules.genotype.SamplesApi; import org.brapi.client.v2.modules.genotype.VariantsApi; -import org.brapi.client.v2.modules.phenotype.ObservationUnitsApi; -import org.brapi.v2.model.BrAPIExternalReference; -import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.core.response.BrAPIProgramListResponse; -import org.brapi.v2.model.core.response.BrAPITrialListResponse; import org.brapi.v2.model.geno.BrAPICall; import org.brapi.v2.model.geno.BrAPICallSet; import org.brapi.v2.model.geno.BrAPISample; @@ -39,28 +55,27 @@ import org.brapi.v2.model.geno.request.BrAPISampleSearchRequest; import org.brapi.v2.model.geno.request.BrAPIVariantsSearchRequest; import org.brapi.v2.model.germ.BrAPIGermplasm; -import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.brapi.v2.model.pheno.request.BrAPIObservationUnitSearchRequest; import org.breedinginsight.brapi.v1.controller.BrapiVersion; +import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; +import org.breedinginsight.brapps.importer.daos.BrAPISampleDAO; import org.breedinginsight.brapps.importer.daos.ImportDAO; import org.breedinginsight.brapps.importer.daos.ImportMappingDAO; import org.breedinginsight.brapps.importer.model.ImportProgress; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.mapping.ImportMapping; import org.breedinginsight.brapps.importer.model.response.ImportResponse; -import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.daos.GenotypeImportDAO; import org.breedinginsight.daos.ProgramDAO; +import org.breedinginsight.daos.SampleSubmissionDAO; import org.breedinginsight.daos.UserDAO; -import org.breedinginsight.model.GermplasmGenotype; -import org.breedinginsight.model.Program; -import org.breedinginsight.model.User; +import org.breedinginsight.model.*; import org.breedinginsight.services.brapi.BrAPIEndpointProvider; import org.breedinginsight.services.exceptions.AuthorizationException; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.services.geno.GenotypeService; import org.breedinginsight.services.parsers.MimeTypeParser; import org.breedinginsight.utilities.BrAPIDAOUtil; -import org.breedinginsight.utilities.Utilities; +import org.jetbrains.annotations.NotNull; import org.jooq.DSLContext; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.*; @@ -70,6 +85,9 @@ import javax.inject.Singleton; import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; import java.time.LocalDateTime; import java.time.OffsetDateTime; import java.util.*; @@ -86,12 +104,14 @@ public class GigwaGenotypeServiceImpl implements GenotypeService { private static final String BEARER = "Bearer "; private static final String GIGWA_REST_BASE_PATH = "gigwa/rest"; private static final String GIGWA_BRAPI_BASE_PATH = GIGWA_REST_BASE_PATH + BrapiVersion.BRAPI_V2; + private static final String INVALID_REF_ALT_MESSAGE = "The file is not a valid VCF or contains unsupported REF/ALT allele values."; + private static final String DUPLICATE_POSITIONAL_KEY_MESSAGE = "Duplicate chromosomal position(s) detected. CHROM:POS key must be unique for variant type."; private static final MediaType MEDIA_TYPE_JSON = MediaType.parse("application/json"); private final Executor executor = Executors.newCachedThreadPool(); - private String referenceSource; + private final String referenceSource; private final String gigwaHost; private final String username; private final String password; @@ -100,7 +120,9 @@ public class GigwaGenotypeServiceImpl implements GenotypeService { private final ProgramDAO programDAO; private final UserDAO userDAO; private final ImportDAO importDAO; - + private final SampleSubmissionDAO sampleSubmissionDAO; + private final BrAPISampleDAO sampleDAO; + private final GenotypeImportDAO genotypeImportDAO; private final ImportMappingDAO importMappingDAO; private final SimpleStorageService storageService; @@ -114,6 +136,8 @@ public class GigwaGenotypeServiceImpl implements GenotypeService { private final BrAPIEndpointProvider brAPIEndpointProvider; + private final BrAPIGermplasmDAO germplasmDAO; + @Inject public GigwaGenotypeServiceImpl(@Property(name = "gigwa.host") String gigwaHost, @Property(name = "gigwa.username") String username, @@ -122,21 +146,28 @@ public GigwaGenotypeServiceImpl(@Property(name = "gigwa.host") String gigwaHost, ProgramDAO programDAO, UserDAO userDAO, ImportDAO importDAO, + SampleSubmissionDAO sampleSubmissionDAO, + BrAPISampleDAO sampleDAO, + GenotypeImportDAO genotypeImportDAO, ImportMappingDAO importMappingDAO, @Named("genotype") SimpleStorageService storageService, S3Client s3Client, DSLContext dsl, MimeTypeParser mimeTypeParser, BrAPIDAOUtil brAPIDAOUtil, - BrAPIEndpointProvider brAPIEndpointProvider) { + BrAPIEndpointProvider brAPIEndpointProvider, + BrAPIGermplasmDAO germplasmDAO) { this.gigwaHost = gigwaHost.endsWith("/") ? gigwaHost : gigwaHost + "/"; this.username = username; this.password = password; this.referenceSource = referenceSource; + this.genotypeImportDAO = genotypeImportDAO; this.gson = new GsonBuilder().create(); this.programDAO = programDAO; this.userDAO = userDAO; this.importDAO = importDAO; + this.sampleSubmissionDAO = sampleSubmissionDAO; + this.sampleDAO = sampleDAO; this.importMappingDAO = importMappingDAO; this.storageService = storageService; this.s3Client = s3Client; @@ -144,10 +175,11 @@ public GigwaGenotypeServiceImpl(@Property(name = "gigwa.host") String gigwaHost, this.mimeTypeParser = mimeTypeParser; this.brAPIDAOUtil = brAPIDAOUtil; this.brAPIEndpointProvider = brAPIEndpointProvider; + this.germplasmDAO = germplasmDAO; } @Override - public ImportResponse submitGenotypeData(UUID userId, UUID programId, UUID experimentId, CompletedFileUpload uploadedFile) throws DoesNotExistException, AuthorizationException, ApiException { + public ImportResponse submitGenotypeData(UUID userId, UUID programId, UUID submissionId, CompletedFileUpload uploadedFile) throws DoesNotExistException, AuthorizationException, ApiException { Program program = getProgram(programId); User user = userDAO.getUser(userId) @@ -204,10 +236,11 @@ public ImportResponse submitGenotypeData(UUID userId, UUID programId, UUID exper try { byte[] fileContents = uploadedFile.getBytes(); - if(validateSamples(program, experimentId, fileContents, upload, gigwaAuthToken)) { + if (validateSamples(program, submissionId, fileContents, upload) + && validateVariantRecords(fileContents, uploadedFile.getFilename(), upload)) { executor.execute(() -> { try { - processSubmission(gigwaAuthToken, program, experimentId, fileContents, uploadedFile.getFilename(), upload, progress); + processSubmission(gigwaAuthToken, program, submissionId, fileContents, uploadedFile.getFilename(), upload, progress); } catch (Exception e) { log.error(e.getMessage(), e); } @@ -227,7 +260,9 @@ public ImportResponse submitGenotypeData(UUID userId, UUID programId, UUID exper } @Override - public GermplasmGenotype retrieveGenotypeData(UUID programId, BrAPIGermplasm germplasm) throws DoesNotExistException, AuthorizationException, ApiException { + public GermplasmGenotype retrieveGenotypeData(UUID programId, UUID germplasmId) throws DoesNotExistException, AuthorizationException, ApiException { + BrAPIGermplasm germplasm = germplasmDAO.getGermplasmByUUID(germplasmId.toString(), programId); + log.debug("fetching genotypes for " + germplasm.getGermplasmName()); Program program = getProgram(programId); BrAPIClient brAPIClient = programDAO.getCoreClient(programId); @@ -237,18 +272,19 @@ public GermplasmGenotype retrieveGenotypeData(UUID programId, BrAPIGermplasm ger ((OAuth)authorizationToken).setAccessToken(getAuthToken()); } - BrAPIClient brapiPhenoClient = programDAO.getPhenoClient(programId); - if(verifyProgramExists(brAPIClient, program)) { - List germplasmOUs = fetchObservationUnits(brapiPhenoClient, germplasm); - List germplasmSamples = fetchSamples(brAPIClient, program, germplasmOUs); + // get sample names from brapi server + List samples = fetchSamples(program, germplasmId); + List sampleNames = samples.stream().map(BrAPISample::getSampleName).collect(Collectors.toList()); - List callSets = fetchCallsets(brAPIClient, germplasmSamples); - - List calls = fetchCalls(brAPIClient, callSets); - - List variants = fetchVariants(brAPIClient, calls); + // get samples from gigwa given sample names + List gigwaSamples = fetchGigwaSamples(brAPIClient, program, sampleNames); + List callSets = fetchCallsets(brAPIClient, gigwaSamples); + //Deprecated to avoid running out of memory destabilizing the system + /* + List calls = fetchCalls(brAPIClient, callSets) + List variants = fetchVariants(brAPIClient, calls); return GermplasmGenotype.builder() .germplasm(germplasm) @@ -256,23 +292,78 @@ public GermplasmGenotype retrieveGenotypeData(UUID programId, BrAPIGermplasm ger .callSets(callSets.stream().collect(Collectors.toMap(BrAPICallSet::getCallSetDbId, callset -> callset))) .variants(variants.stream().collect(Collectors.toMap(BrAPIVariant::getVariantDbId, variant -> variant))) .build(); + */ + + return GermplasmGenotype.builder() + .germplasm(germplasm) + .calls(null) + .callSets(callSets.stream().collect(Collectors.toMap(BrAPICallSet::getCallSetDbId, callset -> callset))) + .variants(null) + .build(); } else { return new GermplasmGenotype(); } } - private boolean validateSamples(Program program, UUID experimentId, byte[] fileContents, ImportUpload upload, String gigwaAuthToken) throws DoesNotExistException, ApiException { - log.debug("Validating samples in submitted VCF file for experiment: " + experimentId); + @Override + public List getGenotypeImports(UUID programId) { - BrAPIClient brAPIClient = programDAO.getCoreClient(program.getId()); - brAPIClient.setBasePath(gigwaHost + GIGWA_BRAPI_BASE_PATH); - Authentication authorizationToken = brAPIClient.getAuthentication("AuthorizationToken"); - if(authorizationToken instanceof OAuth) { - ((OAuth)authorizationToken).setAccessToken(gigwaAuthToken); + return genotypeImportDAO.getGenotypeImportsByProgramId(programId); + } + + @Override + public Optional downloadGenotypeImport(UUID programId, UUID genotypeImportId) { + Optional genotypeImportDownloadDetails = genotypeImportDAO + .getDownloadableGenotypeImportById(programId, genotypeImportId); + + if (genotypeImportDownloadDetails.isEmpty()) { + return Optional.empty(); } - BrAPIClient brapiPhenoClient = programDAO.getPhenoClient(program.getId()); - Set obsUnitNames = fetchObservationUnits(brapiPhenoClient, experimentId).stream().map(ou -> Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())).collect(Collectors.toSet()); + UUID submissionId = genotypeImportDownloadDetails.get().getSampleSubmissionId(); + UUID importerImportId = genotypeImportDownloadDetails.get().getImporterImportId(); + String originalFileName = genotypeImportDownloadDetails.get().getGenotypeFileName(); + + Optional storedKey = findStoredGenotypeImportKey(programId, submissionId, importerImportId); + if (storedKey.isEmpty()) { + return Optional.empty(); + } + + InputStream inputStream = s3Client.getObject(GetObjectRequest.builder() + .bucket(storageService.getDefaultBucketName()) + .key(storedKey.get()) + .build()); + + return Optional.of(new DownloadFile( + originalFileName, + new StreamedFile( + inputStream, + new io.micronaut.http.MediaType(io.micronaut.http.MediaType.APPLICATION_OCTET_STREAM) + ) + )); + } + + private Optional findStoredGenotypeImportKey(UUID programId, UUID submissionId, UUID importerImportId) { + String prefix = programId + "/" + submissionId + "/" + importerImportId; + + ListObjectsV2Response response = s3Client.listObjectsV2(ListObjectsV2Request.builder() + .bucket(storageService.getDefaultBucketName()) + .prefix(prefix) + .maxKeys(1) + .build()); + + return response.contents().stream() + .map(S3Object::key) + .findFirst(); + } + + private boolean validateSamples(Program program, UUID submissionId, byte[] fileContents, ImportUpload upload) throws DoesNotExistException, ApiException { + log.debug("Validating samples in submitted VCF file for submission: " + submissionId); + + Set submissionSampleNames = fetchSubmissionSamples(program, submissionId).stream() + .map(BrAPISample::getSampleName) + .filter(Objects::nonNull) + .collect(Collectors.toSet()); log.debug("searching for the VCF header row"); String[] headerParts = null; @@ -317,17 +408,17 @@ private boolean validateSamples(Program program, UUID experimentId, byte[] fileC return false; } - log.debug("pulled all the samples from the VCF, now checking each one has an OU record"); - List samplesMissingOu = new ArrayList<>(); + log.debug("pulled all the samples from the VCF, now checking each one belongs to the submission"); + List samplesMissingSubmission = new ArrayList<>(); samples.forEach(s -> { - if(!obsUnitNames.contains(s)) { - samplesMissingOu.add(s); + if(!submissionSampleNames.contains(s)) { + samplesMissingSubmission.add(s); } }); - if(!samplesMissingOu.isEmpty()) { + if(!samplesMissingSubmission.isEmpty()) { upload.getProgress().setStatuscode((short)HttpStatus.BAD_REQUEST.getCode()); - upload.getProgress().setMessage("There are samples that do not have an existing observation unit"); + upload.getProgress().setMessage("There are samples that are not linked to the selected submission"); importDAO.updateProgress(upload.getProgress()); return false; } @@ -336,6 +427,70 @@ private boolean validateSamples(Program program, UUID experimentId, byte[] fileC return true; } + private boolean validateVariantRecords(byte[] fileContents, String filename, ImportUpload upload) { + Set positionalKeys = new HashSet<>(); + Path tempVcfFile = null; + int parsedVariantCount = 0; + + try { + tempVcfFile = Files.createTempFile("bi-vcf-validation-", ".vcf"); + Files.write(tempVcfFile, fileContents); + + try (VCFFileReader reader = new VCFFileReader(tempVcfFile.toFile(), false); + CloseableIterator variants = reader.iterator()) { + + while (variants.hasNext()) { + VariantContext variant = variants.next(); + parsedVariantCount++; + + String positionalKey = + variant.getType() + ":" + + variant.getContig() + ":" + + variant.getStart(); + + if (!positionalKeys.add(positionalKey)) { + log.error("Duplicate Gigwa positional key detected during VCF validation for file '{}'. Parsed records: {}. Key: {}", + filename, parsedVariantCount, positionalKey); + + upload.getProgress().setStatuscode((short) HttpStatus.BAD_REQUEST.getCode()); + upload.getProgress().setMessage(DUPLICATE_POSITIONAL_KEY_MESSAGE); + importDAO.updateProgress(upload.getProgress()); + return false; + } + } + } + + log.info("Completed HTSJDK VCF validation for file '{}'. Parsed {} variant record(s) with no validation errors", + filename, parsedVariantCount); + } catch (TribbleException | IllegalArgumentException e) { + log.error("HTSJDK VCF validation failed for file '{}'. Parsed {} variant record(s) before failure. Error: {}", + filename, parsedVariantCount, e.getMessage(), e); + + upload.getProgress().setStatuscode((short) HttpStatus.BAD_REQUEST.getCode()); + upload.getProgress().setMessage(INVALID_REF_ALT_MESSAGE); + importDAO.updateProgress(upload.getProgress()); + return false; + } catch (IOException e) { + log.error("I/O failure during VCF validation setup for file '{}'. Parsed {} variant record(s) before failure. Error: {}", + filename, parsedVariantCount, e.getMessage(), e); + + upload.getProgress().setStatuscode((short) HttpStatus.INTERNAL_SERVER_ERROR.getCode()); + upload.getProgress().setMessage("An error occurred while trying to validate VCF variant information"); + importDAO.updateProgress(upload.getProgress()); + return false; + } finally { + if (tempVcfFile != null) { + try { + Files.deleteIfExists(tempVcfFile); + } catch (IOException e) { + log.warn("Unable to delete temporary VCF validation file {}", tempVcfFile, e); + } + } + } + + return true; + } + private boolean validateVcfHeader(String[] headerParts) { if(headerParts.length < 8) { return false; @@ -369,11 +524,7 @@ private boolean validateVcfHeader(String[] headerParts) { return false; } - if(!headerParts[7].equals("INFO")) { - return false; - } - - return true; + return headerParts[7].equals("INFO"); } private boolean verifyProgramExists(BrAPIClient genoBrAPIClient, Program program) throws ApiException { @@ -383,68 +534,35 @@ private boolean verifyProgramExists(BrAPIClient genoBrAPIClient, Program program return brAPIProgramListResponseApiResponse.getBody().getResult().getData().size() == 1; } - private List fetchSamples(BrAPIClient genoBrAPIClient, Program program, List observationUnits) throws ApiException { - log.debug("fetching samples for OUs"); - if(observationUnits.isEmpty()) { - log.debug("No OUs were supplied, returning"); + private List fetchGigwaSamples(BrAPIClient genoBrAPIClient, Program program, List sampleNames) throws ApiException { + log.debug("fetching gigwa samples"); + if(sampleNames.isEmpty()) { + log.debug("No samples were supplied, returning"); return Collections.emptyList(); } SamplesApi samplesApi = brAPIEndpointProvider.get(genoBrAPIClient, SamplesApi.class); BrAPISampleSearchRequest sampleSearchRequest = new BrAPISampleSearchRequest(); - - sampleSearchRequest.setGermplasmDbIds(observationUnits.stream().map(ou -> program.getKey() + "§" + Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())).collect(Collectors.toList())); + sampleSearchRequest.setGermplasmDbIds(sampleNames.stream().map(sampleName -> program.getKey() + "§" +sampleName).collect(Collectors.toList())); return brAPIDAOUtil.search(samplesApi::searchSamplesPost, samplesApi::searchSamplesSearchResultsDbIdGet, sampleSearchRequest); } - private List fetchObservationUnits(BrAPIClient phenoBrAPIClient, BrAPIGermplasm germplasm) throws ApiException { - ObservationUnitsApi observationUnitsApi = brAPIEndpointProvider.get(phenoBrAPIClient, ObservationUnitsApi.class); - BrAPIObservationUnitSearchRequest searchRequest = new BrAPIObservationUnitSearchRequest(); - searchRequest.addGermplasmDbIdsItem(germplasm.getGermplasmDbId()); - - return brAPIDAOUtil.search(observationUnitsApi::searchObservationunitsPost, observationUnitsApi::searchObservationunitsSearchResultsDbIdGet, searchRequest); + private List fetchSamples(Program program, @NotNull UUID germplasmId) throws ApiException { + String germplasmIdString = germplasmId.toString(); + java.util.List germplasmIdList = List.of(germplasmIdString); + List sampleNames = sampleDAO.readSamplesByGermplasmIds(program, germplasmIdList); + return sampleNames; } - private List fetchObservationUnits(BrAPIClient phenoBrAPIClient, UUID experimentId) throws ApiException, DoesNotExistException { - log.debug("fetching observationUnits for experiment: " + experimentId); - TrialsApi trialsApi = brAPIEndpointProvider.get(phenoBrAPIClient, TrialsApi.class); - ApiResponse brAPITrialListResponseApiResponse = trialsApi.trialsGet(new TrialQueryParams().externalReferenceID(experimentId.toString()) - .externalReferenceSource(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.TRIALS))); - - BrAPITrial brAPITrial = null; - if(brAPITrialListResponseApiResponse.getBody().getResult().getData() != null) { - if (brAPITrialListResponseApiResponse.getBody().getResult().getData().size() == 1) { - brAPITrial = brAPITrialListResponseApiResponse.getBody().getResult().getData().get(0); - } else { - String trialReferenceSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.TRIALS); - for (BrAPITrial trial : brAPITrialListResponseApiResponse.getBody().getResult().getData()) { - if (trial.getExternalReferences() != null) { - Optional xref = trial.getExternalReferences() - .stream() - .filter(externalReference -> externalReference.getReferenceSource().equals(trialReferenceSource)) - .findFirst(); - if (xref.isPresent() && xref.get().getReferenceID().equals(experimentId.toString())) { - brAPITrial = trial; - break; - } - } - } - } + private List fetchSubmissionSamples(Program program, UUID submissionId) throws ApiException, DoesNotExistException { + if(sampleSubmissionDAO.getBySubmissionId(program, submissionId).isEmpty()) { + throw new DoesNotExistException("Could not find sample submission in database"); } - if(brAPITrial != null) { - ObservationUnitsApi observationUnitsApi = brAPIEndpointProvider.get(phenoBrAPIClient, ObservationUnitsApi.class); - - BrAPIObservationUnitSearchRequest searchRequest = new BrAPIObservationUnitSearchRequest(); - searchRequest.addTrialDbIdsItem(brAPITrial.getTrialDbId()); - - return brAPIDAOUtil.search(observationUnitsApi::searchObservationunitsPost, observationUnitsApi::searchObservationunitsSearchResultsDbIdGet, searchRequest); - } else { - throw new DoesNotExistException("Could not find experiment in database"); - } + return sampleDAO.readSamplesBySubmissionIds(program, List.of(submissionId.toString())); } private List fetchCallsets(BrAPIClient genoBrAPIClient, List germplasmSamples) throws ApiException { @@ -500,12 +618,12 @@ private List fetchVariants(BrAPIClient genoBrAPIClient, List uploadedFileResult; try { progress.setMessage("Uploading file"); importDAO.updateProgress(progress); - uploadedFileResult = uploadGenotypeData(program.getId(), experimentId, upload.getId(), fileContents, filename); + uploadedFileResult = uploadGenotypeData(program.getId(), submissionId, upload.getId(), fileContents, filename); log.debug("file saved to: " + uploadedFileResult.getLeft()); } catch (Exception e) { progress.setStatuscode((short) HttpStatus.INTERNAL_SERVER_ERROR.getCode()); @@ -518,10 +636,12 @@ protected void processSubmission(String gigwaAuthToken, Program program, UUID ex importDAO.updateProgress(progress); OkHttpClient client = new OkHttpClient(); - String gigwaProgressToken = submitRequestToGigwa(client, program, experimentId, uploadedFileResult.getLeft(), gigwaAuthToken, progress); + String gigwaProgressToken = submitRequestToGigwa(client, program, submissionId, uploadedFileResult.getLeft(), gigwaAuthToken, progress); - if(checkGigwaProgress(client, gigwaAuthToken, gigwaProgressToken, progress)) { + if (checkGigwaProgress(client, gigwaAuthToken, gigwaProgressToken, progress)) { log.debug("Gigwa import was successful!"); + //logic to add record to the new JOIN table + genotypeImportDAO.createGenotypeImportLink(submissionId, upload.getId(), upload.getCreatedBy()); progress.setMessage("Import successful"); progress.setStatuscode((short) HttpStatus.OK.getCode()); importDAO.updateProgress(progress); @@ -593,19 +713,19 @@ private boolean checkGigwaProgress(OkHttpClient client, String gigwaAuthToken, S * Submits the upload request to Gigwa, and returns the progress token * @param client * @param program - * @param experimentId + * @param submissionId * @param fileUrl * @param gigwaAuthToken * @param progress * @return the progress token to check on the import's progress * @throws IOException */ - private String submitRequestToGigwa(OkHttpClient client, Program program, UUID experimentId, String fileUrl, String gigwaAuthToken, ImportProgress progress) throws IOException { + private String submitRequestToGigwa(OkHttpClient client, Program program, UUID submissionId, String fileUrl, String gigwaAuthToken, ImportProgress progress) throws IOException { Request request = new Request.Builder() .url(HttpUrl.parse(buildPath("gigwa/genotypeImport")) .newBuilder() .addQueryParameter("module", program.getKey()) - .addQueryParameter("project", experimentId.toString()) + .addQueryParameter("project", submissionId.toString()) .addQueryParameter("run", LocalDateTime.now().toString()) .addQueryParameter("dataFile1", fileUrl) @@ -631,7 +751,7 @@ private String submitRequestToGigwa(OkHttpClient client, Program program, UUID e } } - private Pair uploadGenotypeData(UUID programId, UUID experimentId, UUID uploadId, byte[] fileContents, String filename) throws IOException, MimeTypeException { + private Pair uploadGenotypeData(UUID programId, UUID submissionId, UUID uploadId, byte[] fileContents, String filename) throws IOException, MimeTypeException { log.debug("saving genotype data to S3"); if(!storageService.listBucketNames().contains(storageService.getDefaultBucketName())) { @@ -641,7 +761,7 @@ private Pair uploadGenotypeData(UUID programId, UUID experimentId, var mimeType = mimeTypeParser.getMimeType(fileContents, filename); - var key = programId.toString() + "/" + experimentId.toString() + "/" + uploadId + mimeType.getExtension(); + var key = programId.toString() + "/" + submissionId.toString() + "/" + uploadId + mimeType.getExtension(); var path = storeMultipartFile(key, fileContents, Map.of("originalFileName", filename)); Long fileSize = Long.valueOf(fileContents.length); diff --git a/src/main/java/org/breedinginsight/services/parsers/germplasm/GermplasmFileColumns.java b/src/main/java/org/breedinginsight/services/parsers/germplasm/GermplasmFileColumns.java index 613cfdf6a..6a87ccd1b 100644 --- a/src/main/java/org/breedinginsight/services/parsers/germplasm/GermplasmFileColumns.java +++ b/src/main/java/org/breedinginsight/services/parsers/germplasm/GermplasmFileColumns.java @@ -28,6 +28,7 @@ public enum GermplasmFileColumns { NAME("Germplasm Name", Column.ColumnDataType.STRING), BREEDING_METHOD("Breeding Method", Column.ColumnDataType.STRING), SOURCE("Source", Column.ColumnDataType.STRING), + PEDIGREE("Pedigree", Column.ColumnDataType.STRING), FEMALE_PARENT_GID("Female Parent GID", Column.ColumnDataType.INTEGER), MALE_PARENT_GID("Male Parent GID", Column.ColumnDataType.INTEGER), ENTRY_NO("Entry No", Column.ColumnDataType.INTEGER), diff --git a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java index cab44d951..f48d3bf80 100644 --- a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java +++ b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java @@ -33,7 +33,6 @@ import org.brapi.client.v2.ApiResponse; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.*; -import org.breedinginsight.api.model.v1.response.DataResponse; import org.breedinginsight.brapi.v1.controller.BrapiVersion; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.model.ProgramBrAPIEndpoints; diff --git a/src/main/java/org/breedinginsight/utilities/response/mappers/GenotypeImportQueryMapper.java b/src/main/java/org/breedinginsight/utilities/response/mappers/GenotypeImportQueryMapper.java new file mode 100644 index 000000000..3193a91a6 --- /dev/null +++ b/src/main/java/org/breedinginsight/utilities/response/mappers/GenotypeImportQueryMapper.java @@ -0,0 +1,54 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.utilities.response.mappers; + +import lombok.Getter; +import org.breedinginsight.model.GenotypeImportDetails; + +import javax.inject.Singleton; +import java.util.Map; +import java.util.function.Function; + +@Getter +@Singleton +public class GenotypeImportQueryMapper extends AbstractQueryMapper { + + private final Map> fields; + + public GenotypeImportQueryMapper() { + fields = Map.ofEntries( + Map.entry("sampleSubmissionId", GenotypeImportDetails::getSampleSubmissionId), + Map.entry("projectNameForSampleSubmission", GenotypeImportDetails::getProjectNameForSampleSubmission), + Map.entry("sampleSubmissionCreatedBy", GenotypeImportDetails::getSampleSubmissionCreatedBy), + Map.entry("genotypingFileName", GenotypeImportDetails::getGenotypingFileName), + Map.entry("genotypingImportDate", GenotypeImportDetails::getGenotypingImportDate), + Map.entry("genotypingImportBy", GenotypeImportDetails::getGenotypingImportBy) + ); + } + + @Override + public boolean exists(String fieldName) { + return getFields().containsKey(fieldName); + } + + @Override + public Function getField(String fieldName) throws NullPointerException { + if (fields.containsKey(fieldName)) return fields.get(fieldName); + else throw new NullPointerException(); + } +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/utilities/response/mappers/GermplasmQueryMapper.java b/src/main/java/org/breedinginsight/utilities/response/mappers/GermplasmQueryMapper.java index e75c7c8aa..322eacf7f 100644 --- a/src/main/java/org/breedinginsight/utilities/response/mappers/GermplasmQueryMapper.java +++ b/src/main/java/org/breedinginsight/utilities/response/mappers/GermplasmQueryMapper.java @@ -2,6 +2,7 @@ import lombok.Getter; import lombok.Setter; +import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.germ.BrAPIGermplasm; import org.breedinginsight.api.v1.controller.metadata.SortOrder; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; @@ -11,6 +12,7 @@ import javax.inject.Singleton; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; +import java.util.List; import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; @@ -60,6 +62,20 @@ public GermplasmQueryMapper() { germplasm.getAdditionalInfo() != null && germplasm.getAdditionalInfo().has(BrAPIAdditionalInfoFields.GERMPLASM_MALE_PARENT_GID) ? germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_MALE_PARENT_GID).getAsString() : null), + Map.entry("externalUID", (germplasm) ->{ + String externalUID = null; + if (germplasm.getExternalReferences() != null) { + List externalReferences = germplasm.getExternalReferences(); + for (BrAPIExternalReference reference : externalReferences) { + if ("External UID".equals(reference.getReferenceSource())) { + externalUID = reference.getReferenceID(); + break; + } + } + } + + return externalUID; + }), Map.entry("createdDate", (germplasm) ->{ String createdDate = null; if (germplasm.getAdditionalInfo() != null && germplasm.getAdditionalInfo().has(BrAPIAdditionalInfoFields.CREATED_DATE)) { diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index d58a8d8d2..e236ded35 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -27,10 +27,10 @@ micronaut: thread-selection: AUTO multipart: enabled: true - max-file-size: '100MB' + max-file-size: '800MB' mixed: true threshold: '10MB' - max-request-size: '100MB' + max-request-size: '800MB' bi: api: version: v1 diff --git a/src/main/resources/db/migration/V1.36.0__create_genotype_import_table.sql b/src/main/resources/db/migration/V1.36.0__create_genotype_import_table.sql new file mode 100644 index 000000000..9d123fd90 --- /dev/null +++ b/src/main/resources/db/migration/V1.36.0__create_genotype_import_table.sql @@ -0,0 +1,55 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Join table linking sample submissions to genotype imports +create table genotype_import +( + like base_entity including defaults including constraints including indexes, + sample_submission_id uuid not null, + importer_import_id uuid not null, + like base_edit_track_entity including all +); + +alter table genotype_import + add constraint fk_gi_sample_submission + foreign key (sample_submission_id) + references sample_submission (id); + +alter table genotype_import + add constraint fk_gi_importer_import + foreign key (importer_import_id) + references importer_import (id); + +alter table genotype_import + add constraint fk_gi_created_by + foreign key (created_by) + references bi_user (id); + +alter table genotype_import + add constraint fk_gi_updated_by + foreign key (updated_by) + references bi_user (id); + +alter table genotype_import + add constraint uq_gi_importer_import_id + unique (importer_import_id); + +create index idx_gi_sample_submission_id + on genotype_import (sample_submission_id); + +create index idx_gi_importer_import_id + on genotype_import (importer_import_id); \ No newline at end of file diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 3c769284c..9dba17ca3 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.3.0 -versionInfo=https://github.com/Breeding-Insight/bi-api/releases/tag/v1.3.0 +version=v1.4.0+1214 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/444c882ef19073a80cd12933910981c3bce7db74 \ No newline at end of file diff --git a/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java index 79c8095cb..3c7cb0051 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java @@ -25,6 +25,7 @@ import io.micronaut.http.HttpStatus; import io.micronaut.http.client.RxHttpClient; import io.micronaut.http.client.annotation.Client; +import io.micronaut.http.client.exceptions.HttpClientResponseException; import io.micronaut.http.netty.cookies.NettyCookie; import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import io.reactivex.Flowable; @@ -43,6 +44,7 @@ import org.breedinginsight.brapps.importer.model.imports.sample.SampleSubmissionImport.Columns; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.dao.db.tables.pojos.SpeciesEntity; +import org.breedinginsight.daos.GenotypeImportDAO; import org.breedinginsight.daos.SpeciesDAO; import org.breedinginsight.daos.UserDAO; import org.breedinginsight.model.*; @@ -61,7 +63,11 @@ import java.util.*; import static io.micronaut.http.HttpRequest.*; +import static org.breedinginsight.api.v1.controller.geno.SampleSubmissionController.DELETE_GENOTYPE_DATA_NOT_ALLOWED_ERROR_MESSAGE; +import static org.breedinginsight.api.v1.controller.geno.SampleSubmissionController.DELETE_STATUS_NOT_ALLOWED_ERROR_MESSAGE; import static org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields.SUBMISSION_NAME; +import static org.breedinginsight.dao.db.Tables.GENOTYPE_IMPORT; +import static org.breedinginsight.dao.db.Tables.IMPORTER_IMPORT; import static org.junit.jupiter.api.Assertions.*; @MicronautTest @@ -70,6 +76,7 @@ public class SampleSubmissionControllerIntegrationTest extends BrAPITest { private Program program; + private User testUser; private ImportTestUtils importTestUtils; @Property(name = "brapi.server.reference-source") @@ -84,6 +91,8 @@ public class SampleSubmissionControllerIntegrationTest extends BrAPITest { private OntologyService ontologyService; @Inject private BrAPIGermplasmDAO germplasmDAO; + @Inject + private GenotypeImportDAO genotypeImportDAO; @Inject @Client("/${micronaut.bi.api.version}") @@ -101,7 +110,7 @@ void setup() throws Exception { FannyPack brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); // Test User - User testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(Exception::new); + testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(Exception::new); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); // Species @@ -340,6 +349,73 @@ public void testGenerateLookupFile() throws IOException, InterruptedException, P assertEquals("GID", lookupTable.column(2).name()); } + @Test + public void testDeleteSubmissionEnforcesStatusAndGenotypeImportRules() throws IOException, InterruptedException { + SampleSubmission submission = createSubmission(program).getLeft(); + String submissionUrl = String.format("/programs/%s/submissions/%s", program.getId(), submission.getId()); + NettyCookie authCookie = new NettyCookie("phylo-token", "test-registered-user"); + + client.exchange( + PUT(submissionUrl + "/status", "{status:\"SUBMITTED\"}").cookie(authCookie), + String.class + ).blockingFirst(); + + HttpClientResponseException statusError = assertThrows(HttpClientResponseException.class, () -> client.exchange( + DELETE(submissionUrl).cookie(authCookie), String.class + ).blockingFirst()); + assertEquals(HttpStatus.METHOD_NOT_ALLOWED, statusError.getStatus()); + assertEquals( + DELETE_STATUS_NOT_ALLOWED_ERROR_MESSAGE, + statusError.getResponse().getBody(String.class).orElse(null) + ); + + client.exchange( + PUT(submissionUrl + "/status", "{status:\"NOT SUBMITTED\"}").cookie(authCookie), + String.class + ).blockingFirst(); + + UUID importerImportId = dsl.select(IMPORTER_IMPORT.ID) + .from(IMPORTER_IMPORT) + .where(IMPORTER_IMPORT.PROGRAM_ID.eq(program.getId())) + .orderBy(IMPORTER_IMPORT.CREATED_AT.desc()) + .limit(1) + .fetchOne(IMPORTER_IMPORT.ID); + genotypeImportDAO.createGenotypeImportLink(submission.getId(), importerImportId, testUser.getId()); + + HttpClientResponseException genotypeError = assertThrows(HttpClientResponseException.class, () -> client.exchange( + DELETE(submissionUrl).cookie(authCookie), String.class + ).blockingFirst()); + assertEquals(HttpStatus.METHOD_NOT_ALLOWED, genotypeError.getStatus()); + assertEquals( + DELETE_GENOTYPE_DATA_NOT_ALLOWED_ERROR_MESSAGE, + genotypeError.getResponse().getBody(String.class).orElse(null) + ); + + HttpResponse preservedSubmission = client.exchange( + GET(submissionUrl + "?details=true").cookie(authCookie), String.class + ).blockingFirst(); + SampleSubmission preserved = gson.fromJson( + JsonParser.parseString(preservedSubmission.body()).getAsJsonObject().getAsJsonObject("result"), + SampleSubmission.class + ); + assertEquals(96, preserved.getSamples().size()); + assertEquals(1, preserved.getPlates().size()); + + dsl.deleteFrom(GENOTYPE_IMPORT) + .where(GENOTYPE_IMPORT.SAMPLE_SUBMISSION_ID.eq(submission.getId())) + .execute(); + + HttpResponse deleteResponse = client.exchange( + DELETE(submissionUrl).cookie(authCookie), String.class + ).blockingFirst(); + assertEquals(HttpStatus.OK, deleteResponse.getStatus()); + + HttpClientResponseException notFound = assertThrows(HttpClientResponseException.class, () -> client.exchange( + GET(submissionUrl + "?details=true").cookie(authCookie), String.class + ).blockingFirst()); + assertEquals(HttpStatus.NOT_FOUND, notFound.getStatus()); + } + private Pair>> createSubmission(Program program) throws IOException, InterruptedException { Flowable> call = client.exchange( diff --git a/src/test/java/org/breedinginsight/api/v1/controller/geno/GenotypeDataUploadControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/geno/GenotypeDataUploadControllerIntegrationTest.java new file mode 100644 index 000000000..eebb2191e --- /dev/null +++ b/src/test/java/org/breedinginsight/api/v1/controller/geno/GenotypeDataUploadControllerIntegrationTest.java @@ -0,0 +1,443 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.api.v1.controller.geno; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import io.kowalski.fannypack.FannyPack; +import io.micronaut.http.HttpHeaders; +import io.micronaut.http.HttpResponse; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.MediaType; +import io.micronaut.http.client.RxHttpClient; +import io.micronaut.http.client.annotation.Client; +import io.micronaut.http.client.exceptions.HttpClientResponseException; +import io.micronaut.http.client.multipart.MultipartBody; +import io.micronaut.http.multipart.CompletedFileUpload; +import io.micronaut.http.netty.cookies.NettyCookie; +import io.micronaut.http.server.types.files.StreamedFile; +import io.micronaut.test.annotation.MockBean; +import io.micronaut.test.extensions.junit5.annotation.MicronautTest; +import org.breedinginsight.DatabaseTest; +import org.breedinginsight.api.v1.controller.TestTokenValidator; +import org.breedinginsight.brapps.importer.model.ImportProgress; +import org.breedinginsight.brapps.importer.model.response.ImportResponse; +import org.breedinginsight.daos.ProgramDAO; +import org.breedinginsight.daos.UserDAO; +import org.breedinginsight.model.*; +import org.breedinginsight.services.ProgramService; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.geno.GenotypeService; +import org.jooq.DSLContext; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; + +import javax.inject.Inject; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Optional; +import java.util.UUID; + +import static io.micronaut.http.HttpRequest.GET; +import static io.micronaut.http.HttpRequest.POST; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.*; + +@MicronautTest +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class GenotypeDataUploadControllerIntegrationTest extends DatabaseTest { + + @Inject + @Client("/${micronaut.bi.api.version}") + private RxHttpClient client; + + @Inject + private GenotypeService genotypeService; + + @Inject + private DSLContext dsl; + + @Inject + private ProgramDAO programDAO; + + @Inject + private UserDAO userDAO; + + @Inject + private ProgramService programService; + + private Program program; + private User testUser; + + @MockBean(GenotypeService.class) + GenotypeService genotypeService() { + return mock(GenotypeService.class); + } + + @MockBean(ProgramService.class) + ProgramService programService() { + return mock(ProgramService.class); + } + + @BeforeAll + void setup() { + FannyPack fp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); + dsl.execute(fp.get("InsertPrograms")); + program = programDAO.getAll().get(0); + testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(); + dsl.execute(fp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), program.getId()); + } + + @BeforeEach + void resetMocks() { + reset(genotypeService); + reset(programService); + } + + @Test + void uploadDataUsesSubmissionScopedRouteAndServiceContract() throws Exception { + UUID submissionId = UUID.randomUUID(); + UUID importId = UUID.randomUUID(); + + ImportResponse importResponse = new ImportResponse(); + importResponse.setImportId(importId); + importResponse.setProgress(ImportProgress.builder() + .statuscode((short) HttpStatus.ACCEPTED.getCode()) + .build()); + + doReturn(getBrAPIEndpoints()).when(programService).getBrapiEndpoints(program.getId()); + doReturn(importResponse).when(genotypeService) + .submitGenotypeData(eq(testUser.getId()), eq(program.getId()), eq(submissionId), any(CompletedFileUpload.class)); + + HttpResponse response = client.exchange( + POST(String.format("/programs/%s/submissions/%s/geno/import", program.getId(), submissionId), multipartBody()) + .contentType(MediaType.MULTIPART_FORM_DATA_TYPE) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), + String.class + ).blockingFirst(); + + assertEquals(HttpStatus.OK, response.getStatus()); + verify(genotypeService).submitGenotypeData(eq(testUser.getId()), eq(program.getId()), eq(submissionId), any(CompletedFileUpload.class)); + } + + @Test + void experimentScopedUploadRouteIsRemoved() { + UUID experimentId = UUID.randomUUID(); + + HttpClientResponseException exception = assertThrows(HttpClientResponseException.class, () -> client.exchange( + POST(String.format("/programs/%s/experiments/%s/geno/import", program.getId(), experimentId), multipartBody()) + .contentType(MediaType.MULTIPART_FORM_DATA_TYPE) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), + String.class + ).blockingFirst()); + + assertEquals(HttpStatus.FORBIDDEN, exception.getStatus()); + verifyNoInteractions(genotypeService); + } + + @Test + void getGenotypeImportsReturnsPagedAndSortedResponse() throws DoesNotExistException { + doReturn(getBrAPIEndpoints()).when(programService).getBrapiEndpoints(program.getId()); + doReturn(Optional.of(program)).when(programService).getById(program.getId()); + + GenotypeImportDetails older = GenotypeImportDetails.builder() + .genotypeImportId(UUID.fromString("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")) + .sampleSubmissionId(UUID.fromString("11111111-1111-1111-1111-111111111111")) + .projectNameForSampleSubmission("Older Submission") + .sampleSubmissionCreatedBy("Test User") + .genotypingFileName("older.vcf") + .genotypingImportDate(OffsetDateTime.parse("2026-06-01T10:00:00Z")) + .genotypingImportBy("Importer A") + .build(); + + GenotypeImportDetails newer = GenotypeImportDetails.builder() + .genotypeImportId(UUID.fromString("bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb")) + .sampleSubmissionId(UUID.fromString("22222222-2222-2222-2222-222222222222")) + .projectNameForSampleSubmission("Newer Submission") + .sampleSubmissionCreatedBy("Test User") + .genotypingFileName("newer.vcf") + .genotypingImportDate(OffsetDateTime.parse("2026-06-02T10:00:00Z")) + .genotypingImportBy("Importer B") + .build(); + + doReturn(new ArrayList<>(Arrays.asList(older, newer))) // mutable list required because ResponseUtils sorts in place + .when(genotypeService) + .getGenotypeImports(program.getId()); + + HttpResponse response = client.exchange( + GET(String.format("/programs/%s/geno/imports?page=1&pageSize=1&sortField=genotypingImportDate&sortOrder=DESC", program.getId())) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), + String.class + ).blockingFirst(); + + assertEquals(HttpStatus.OK, response.getStatus()); + + JsonObject body = JsonParser.parseString(response.body()).getAsJsonObject(); + JsonObject pagination = body.getAsJsonObject("metadata").getAsJsonObject("pagination"); + JsonArray data = body.getAsJsonObject("result").getAsJsonArray("data"); + + assertEquals(2, pagination.get("totalCount").getAsInt()); + assertEquals(1, pagination.get("pageSize").getAsInt()); + assertEquals(2, pagination.get("totalPages").getAsInt()); + assertEquals(1, pagination.get("currentPage").getAsInt()); + assertEquals(1, data.size()); + + JsonObject firstRow = data.get(0).getAsJsonObject(); + assertEquals("bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb", firstRow.get("genotypeImportId").getAsString()); + assertEquals("22222222-2222-2222-2222-222222222222", firstRow.get("sampleSubmissionId").getAsString()); + assertEquals("Newer Submission", firstRow.get("projectNameForSampleSubmission").getAsString()); + assertEquals("Test User", firstRow.get("sampleSubmissionCreatedBy").getAsString()); + assertEquals("newer.vcf", firstRow.get("genotypingFileName").getAsString()); + assertEquals("Importer B", firstRow.get("genotypingImportBy").getAsString()); + + verify(programService).getBrapiEndpoints(program.getId()); + verify(programService).getById(program.getId()); + verify(genotypeService).getGenotypeImports(program.getId()); + } + + @Test + void getGenotypeImportsReturnsNotFoundWhenProgramLookupFails() throws DoesNotExistException { + doReturn(getBrAPIEndpoints()).when(programService).getBrapiEndpoints(program.getId()); + doReturn(Optional.empty()).when(programService).getById(program.getId()); + + HttpClientResponseException exception = assertThrows(HttpClientResponseException.class, () -> client.exchange( + GET(String.format("/programs/%s/geno/imports?page=1&pageSize=10", program.getId())) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), + String.class + ).blockingFirst()); + + assertEquals(HttpStatus.NOT_FOUND, exception.getStatus()); + + verify(programService).getBrapiEndpoints(program.getId()); + verify(programService).getById(program.getId()); + verifyNoInteractions(genotypeService); + } + + @Test + void getGenotypeImportsRejectsInvalidSortField() throws DoesNotExistException { + doReturn(getBrAPIEndpoints()).when(programService).getBrapiEndpoints(program.getId()); + + HttpClientResponseException exception = assertThrows(HttpClientResponseException.class, () -> client.exchange( + GET(String.format("/programs/%s/geno/imports?page=1&pageSize=10&sortField=badField&sortOrder=DESC", program.getId())) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), + String.class + ).blockingFirst()); + + assertEquals(HttpStatus.BAD_REQUEST, exception.getStatus()); + + verify(programService).getBrapiEndpoints(program.getId()); + verifyNoInteractions(genotypeService); + } + + @Test + void getGenotypeImportsFiltersByProjectNameForSampleSubmission() throws DoesNotExistException { + doReturn(getBrAPIEndpoints()).when(programService).getBrapiEndpoints(program.getId()); + doReturn(Optional.of(program)).when(programService).getById(program.getId()); + + GenotypeImportDetails older = GenotypeImportDetails.builder() + .sampleSubmissionId(UUID.fromString("11111111-1111-1111-1111-111111111111")) + .projectNameForSampleSubmission("Older Submission") + .sampleSubmissionCreatedBy("Test User") + .genotypingFileName("older.vcf") + .genotypingImportDate(OffsetDateTime.parse("2026-06-01T10:00:00Z")) + .genotypingImportBy("Importer A") + .build(); + + GenotypeImportDetails newer = GenotypeImportDetails.builder() + .sampleSubmissionId(UUID.fromString("22222222-2222-2222-2222-222222222222")) + .projectNameForSampleSubmission("Newer Submission") + .sampleSubmissionCreatedBy("Test User") + .genotypingFileName("newer.vcf") + .genotypingImportDate(OffsetDateTime.parse("2026-06-02T10:00:00Z")) + .genotypingImportBy("Importer B") + .build(); + + doReturn(new ArrayList<>(Arrays.asList(older, newer))) //safe mutable list + .when(genotypeService) + .getGenotypeImports(program.getId()); + String param = URLEncoder.encode("Newer Submission", StandardCharsets.UTF_8); + HttpResponse response = client.exchange( + GET(String.format( + "/programs/%s/geno/imports?page=1&pageSize=10&projectNameForSampleSubmission=" + param, + program.getId())) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), + String.class + ).blockingFirst(); + + assertEquals(HttpStatus.OK, response.getStatus()); + + JsonObject body = JsonParser.parseString(response.body()).getAsJsonObject(); + JsonObject pagination = body.getAsJsonObject("metadata").getAsJsonObject("pagination"); + JsonArray data = body.getAsJsonObject("result").getAsJsonArray("data"); + + assertEquals(1, pagination.get("totalCount").getAsInt()); + assertEquals(10, pagination.get("pageSize").getAsInt()); + assertEquals(1, pagination.get("totalPages").getAsInt()); + assertEquals(1, pagination.get("currentPage").getAsInt()); + assertEquals(1, data.size()); + + JsonObject firstRow = data.get(0).getAsJsonObject(); + assertEquals("22222222-2222-2222-2222-222222222222", firstRow.get("sampleSubmissionId").getAsString()); + assertEquals("Newer Submission", firstRow.get("projectNameForSampleSubmission").getAsString()); + assertEquals("newer.vcf", firstRow.get("genotypingFileName").getAsString()); + + verify(programService).getBrapiEndpoints(program.getId()); + verify(programService).getById(program.getId()); + verify(genotypeService).getGenotypeImports(program.getId()); + } + + @Test + void getGenotypeImportsReturnsEmptyDataWhenFiltersDoNotMatch() throws DoesNotExistException { + doReturn(getBrAPIEndpoints()).when(programService).getBrapiEndpoints(program.getId()); + doReturn(Optional.of(program)).when(programService).getById(program.getId()); + + GenotypeImportDetails row = GenotypeImportDetails.builder() + .sampleSubmissionId(UUID.fromString("11111111-1111-1111-1111-111111111111")) + .projectNameForSampleSubmission("Older Submission") + .sampleSubmissionCreatedBy("Test User") + .genotypingFileName("older.vcf") + .genotypingImportDate(OffsetDateTime.parse("2026-06-01T10:00:00Z")) + .genotypingImportBy("Importer A") + .build(); + + doReturn(new ArrayList<>(Arrays.asList(row))) //safe mutable list + .when(genotypeService) + .getGenotypeImports(program.getId()); + + HttpResponse response = client.exchange( + GET(String.format( + "/programs/%s/geno/imports?page=1&pageSize=10&projectNameForSampleSubmission=DoesNotMatch", + program.getId())) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), + String.class + ).blockingFirst(); + + assertEquals(HttpStatus.OK, response.getStatus()); + + JsonObject body = JsonParser.parseString(response.body()).getAsJsonObject(); + JsonObject pagination = body.getAsJsonObject("metadata").getAsJsonObject("pagination"); + JsonArray data = body.getAsJsonObject("result").getAsJsonArray("data"); + + assertEquals(0, pagination.get("totalCount").getAsInt()); + assertEquals(0, data.size()); + + verify(programService, times(1)).getBrapiEndpoints(program.getId()); + verify(programService, times(1)).getById(program.getId()); + verify(genotypeService, times(1)).getGenotypeImports(program.getId()); + } + + @Test + void downloadGenotypeImportReturnsAttachmentWhenServiceReturnsFile() throws Exception { + UUID genotypeImportId = UUID.fromString("bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"); + + doReturn(getBrAPIEndpoints()).when(programService).getBrapiEndpoints(program.getId()); + doReturn(Optional.of(program)).when(programService).getById(program.getId()); + + DownloadFile downloadFile = new DownloadFile( + "sample.vcf", + new StreamedFile( + new ByteArrayInputStream("vcf-data".getBytes(StandardCharsets.UTF_8)), + new io.micronaut.http.MediaType(io.micronaut.http.MediaType.APPLICATION_OCTET_STREAM) + ) + ); + + doReturn(Optional.of(downloadFile)).when(genotypeService) + .downloadGenotypeImport(program.getId(), genotypeImportId); + + HttpResponse response = client.exchange( + GET(String.format("/programs/%s/geno/imports/%s/download", program.getId(), genotypeImportId)) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), + byte[].class + ).blockingFirst(); + + assertEquals(HttpStatus.OK, response.getStatus()); + assertEquals("attachment;filename=sample.vcf", response.header(HttpHeaders.CONTENT_DISPOSITION)); + + verify(programService).getBrapiEndpoints(program.getId()); + verify(programService).getById(program.getId()); + verify(genotypeService, times(1)).downloadGenotypeImport(program.getId(), genotypeImportId); + } + + @Test + void downloadGenotypeImportReturnsNotFoundWhenServiceReturnsEmpty() throws IOException, DoesNotExistException { + UUID genotypeImportId = UUID.fromString("cccccccc-cccc-cccc-cccc-cccccccccccc"); + + doReturn(getBrAPIEndpoints()).when(programService).getBrapiEndpoints(program.getId()); + doReturn(Optional.of(program)).when(programService).getById(program.getId()); + doReturn(Optional.empty()).when(genotypeService) + .downloadGenotypeImport(program.getId(), genotypeImportId); + + HttpClientResponseException exception = assertThrows(HttpClientResponseException.class, () -> client.exchange( + GET(String.format("/programs/%s/geno/imports/%s/download", program.getId(), genotypeImportId)) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), + byte[].class + ).blockingFirst()); + + assertEquals(HttpStatus.NOT_FOUND, exception.getStatus()); + + verify(programService).getBrapiEndpoints(program.getId()); + verify(programService).getById(program.getId()); + verify(genotypeService, times(1)).downloadGenotypeImport(program.getId(), genotypeImportId); + } + + @Test + void downloadGenotypeImportReturnsNotFoundWhenProgramLookupFails() throws DoesNotExistException { + UUID genotypeImportId = UUID.fromString("dddddddd-dddd-dddd-dddd-dddddddddddd"); + + doReturn(getBrAPIEndpoints()).when(programService).getBrapiEndpoints(program.getId()); + doReturn(Optional.empty()).when(programService).getById(program.getId()); + + HttpClientResponseException exception = assertThrows(HttpClientResponseException.class, () -> client.exchange( + GET(String.format("/programs/%s/geno/imports/%s/download", program.getId(), genotypeImportId)) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), + byte[].class + ).blockingFirst()); + + assertEquals(HttpStatus.NOT_FOUND, exception.getStatus()); + + verify(programService).getBrapiEndpoints(program.getId()); + verify(programService).getById(program.getId()); + verifyNoInteractions(genotypeService); + } + + private MultipartBody multipartBody() { + return MultipartBody.builder() + .addPart("file", new File("src/test/resources/files/geno/sample.vcf")) + .build(); + } + + private ProgramBrAPIEndpoints getBrAPIEndpoints() { + return ProgramBrAPIEndpoints.builder() + .coreUrl(Optional.of("http://localhost:8081/")) + .phenoUrl(Optional.of("http://localhost:8081/")) + .genoUrl(Optional.of("http://localhost:8081/")) + .build(); + } +} diff --git a/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java index ac2860441..92ed87e57 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java @@ -13,9 +13,12 @@ import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import io.reactivex.Flowable; import lombok.SneakyThrows; +import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.BrAPIListTypes; +import org.brapi.v2.model.germ.BrAPIGermplasm; import org.breedinginsight.BrAPITest; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; import org.breedinginsight.brapps.importer.model.base.Germplasm; import org.breedinginsight.brapps.importer.model.imports.germplasm.GermplasmImportService; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; @@ -70,6 +73,8 @@ public class GermplasmFileImportTest extends BrAPITest { private BreedingMethodDAO breedingMethodDAO; @Inject private DSLContext dsl; + @Inject + private BrAPIGermplasmDAO germplasmDAO; private ImportTestUtils importTestUtils; @@ -644,7 +649,7 @@ public void emptyRequiredFieldsError() { assertEquals(HttpStatus.UNPROCESSABLE_ENTITY, e.getStatus()); JsonArray rowErrors = JsonParser.parseString((String) e.getResponse().getBody().get()).getAsJsonObject().getAsJsonArray("rowErrors"); - assertEquals(2, rowErrors.size(), "Wrong number of row errors returned"); + assertEquals(1, rowErrors.size(), "Wrong number of row errors returned"); JsonObject rowError1 = rowErrors.get(0).getAsJsonObject(); JsonArray errors = rowError1.getAsJsonArray("errors"); @@ -653,14 +658,145 @@ public void emptyRequiredFieldsError() { assertEquals(422, error.get("httpStatusCode").getAsInt(), "Incorrect http status code"); assertEquals("Germplasm Name", error.get("field").getAsString(), "Incorrect field name"); assertEquals(importService.getBlankRequiredFieldMsg("Germplasm Name"), error.get("errorMessage").getAsString(), "Incorrect error message"); + } + + @Test + @SneakyThrows + public void blankBreedingMethodAllowedWithoutParents() { + JsonObject result = importGermplasm( + "src/test/resources/files/germplasm_import/breeding_method_optional_without_parents.csv", + "BlankBreedingMethodNoParents", + "Blank breeding method without parents should succeed", + false + ); + + assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt()); + } + + @Test + @SneakyThrows + public void blankBreedingMethodRejectedWhenParentProvided() { + JsonObject result = importGermplasm( + "src/test/resources/files/germplasm_import/breeding_method_required_with_parent.csv", + "BlankBreedingMethodWithParent", + "Blank breeding method with parent should fail", + false + ); + + assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt()); + + JsonArray rowErrors = result.getAsJsonObject("progress").getAsJsonArray("rowErrors"); + assertEquals(1, rowErrors.size()); + + JsonObject error = rowErrors.get(0).getAsJsonObject() + .getAsJsonArray("errors").get(0).getAsJsonObject(); + + assertEquals("Breeding Method", error.get("field").getAsString()); + assertEquals(GermplasmProcessor.missingBreedingMethodForParentalGIDorEntryNo, error.get("errorMessage").getAsString()); + } + + @Test + @SneakyThrows + public void blankSourceAllowedWhenParentProvided() { + JsonObject result = importGermplasm( + "src/test/resources/files/germplasm_import/source_optional_with_parent.csv", + "BlankSourceWithParent", + "Blank source with parent should succeed", + false + ); + + assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt()); + } - JsonObject rowError2 = rowErrors.get(1).getAsJsonObject(); - JsonArray errors2 = rowError2.getAsJsonArray("errors"); - assertEquals(1, errors2.size(), "Not enough errors were returned"); - JsonObject error2 = errors2.get(0).getAsJsonObject(); - assertEquals(422, error2.get("httpStatusCode").getAsInt(), "Incorrect http status code"); - assertEquals("Source", error2.get("field").getAsString(), "Incorrect field name"); - assertEquals(importService.getBlankRequiredFieldMsg("Source"), error2.get("errorMessage").getAsString(), "Incorrect error message"); + @Test + @SneakyThrows + public void appendSourceToExistingRecordWhenBlank() { + seedExistingGermplasm("9001", "Existing Blank Source", null, null); + + JsonObject result = importGermplasm( + "src/test/resources/files/germplasm_import/append_existing_blank_source.csv", + "AppendBlankSource", + "Append source to existing record", + false + ); + + assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt()); + assertEquals(ImportObjectState.MUTATED.name(), getPreviewState(result, 0)); + + JsonObject germplasm = getPreviewGermplasm(result, 0); + assertEquals("Wild", germplasm.get("seedSource").getAsString()); + } + + @Test + @SneakyThrows + public void appendBreedingMethodToExistingRecordWhenBlank() { + seedExistingGermplasm("9002", "Existing Blank Method", "Wild", null); + + JsonObject result = importGermplasm( + "src/test/resources/files/germplasm_import/append_existing_blank_breeding_method.csv", + "AppendBlankMethod", + "Append breeding method to existing record", + false + ); + + assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt()); + assertEquals(ImportObjectState.MUTATED.name(), getPreviewState(result, 0)); + + ProgramBreedingMethodEntity breedingMethod = breedingMethodDAO + .findByNameOrAbbreviation("BCR", validProgram.getId()) + .get(0); + + JsonObject germplasm = getPreviewGermplasm(result, 0); + JsonObject additionalInfo = germplasm.getAsJsonObject("additionalInfo"); + + assertEquals(breedingMethod.getName(), additionalInfo.get(BrAPIAdditionalInfoFields.GERMPLASM_BREEDING_METHOD).getAsString()); + assertEquals(breedingMethod.getId().toString(), additionalInfo.get(BrAPIAdditionalInfoFields.GERMPLASM_BREEDING_METHOD_ID).getAsString()); + } + + @Test + @SneakyThrows + public void doNotOverwriteExistingSourceOrBreedingMethod() { + seedExistingGermplasm("9003", "Germplasm1", "Existing Source", "ANE"); + + JsonObject result = importGermplasm( + "src/test/resources/files/germplasm_import/do_not_overwrite_existing_source_or_breeding_method.csv", + "DoNotOverwrite", + "Existing source and breeding method should not be overwritten", + false + ); + + assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt()); + assertEquals(ImportObjectState.EXISTING.name(), getPreviewState(result, 0)); + + ProgramBreedingMethodEntity breedingMethod = breedingMethodDAO + .findByNameOrAbbreviation("ANE", validProgram.getId()) + .get(0); + + JsonObject germplasm = getPreviewGermplasm(result, 0); + JsonObject additionalInfo = germplasm.getAsJsonObject("additionalInfo"); + + assertEquals("Existing Source", germplasm.get("seedSource").getAsString()); + assertEquals(breedingMethod.getName(), additionalInfo.get(BrAPIAdditionalInfoFields.GERMPLASM_BREEDING_METHOD).getAsString()); + assertEquals(breedingMethod.getId().toString(), additionalInfo.get(BrAPIAdditionalInfoFields.GERMPLASM_BREEDING_METHOD_ID).getAsString()); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + @SneakyThrows + public void externalUidCanonicalReferenceSource(boolean commit) { + String pathname = "src/test/resources/files/germplasm_import/external_uid_reference_source.csv"; + Table fileData = Table.read().file(pathname); + String listName = "ExternalUidCanonicalReferenceSource"; + String listDescription = "External UID should use canonical reference source"; + + JsonObject result = importGermplasm(pathname, listName, listDescription, commit); + assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt()); + + JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + for (int i = 0; i < previewRows.size(); i++) { + JsonObject germplasm = previewRows.get(i).getAsJsonObject().getAsJsonObject("germplasm").getAsJsonObject("brAPIObject"); + checkBasicResponse(germplasm, fileData, i); + } } @Test @@ -1006,23 +1142,42 @@ public void checkBasicResponse(JsonObject germplasm, Table fileData, Integer i) assertEquals(breedingMethod.getId().toString(), additionalInfo.get(BrAPIAdditionalInfoFields.GERMPLASM_BREEDING_METHOD_ID).getAsString(), "Wrong Breeding Method ID"); assertEquals(breedingMethod.getName(), additionalInfo.get(BrAPIAdditionalInfoFields.GERMPLASM_BREEDING_METHOD).getAsString(), "Wrong Breeding Method name"); // Seed source - assertEquals(fileData.getString(i, "Source"), germplasm.get("seedSource").getAsString(), "Wrong seed source"); - // External Reference (user specified) - // External Reference program + String source = fileData.getString(i, "Source"); + if (isNotBlank(source)) { + assertEquals(source, germplasm.get("seedSource").getAsString(), "Wrong seed source"); + } else { + assertTrue(!germplasm.has("seedSource") + || germplasm.get("seedSource").isJsonNull() + || germplasm.get("seedSource").getAsString().isBlank(), "Wrong seed source"); + } + + // External Reference program (user specified) JsonArray externalReferences = germplasm.getAsJsonArray("externalReferences"); Map expectedReferences = new HashMap<>(); expectedReferences.put(Utilities.generateReferenceSource(BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.PROGRAMS), validProgram.getId().toString()); - expectedReferences.put(fileData.getString(i, "Source"), fileData.getString(i, "External UID")); + + String externalUid = fileData.getString(i, "External UID"); + if (isNotBlank(externalUid)) { + expectedReferences.put("External UID", externalUid); + } + Integer referencesFound = 0; - for (JsonElement reference: externalReferences) { + boolean foundExternalUidReference = false; + for (JsonElement reference : externalReferences) { String referenceSource = reference.getAsJsonObject().get("referenceSource").getAsString(); String referenceID = reference.getAsJsonObject().get("referenceID").getAsString(); + if ("External UID".equals(referenceSource)) { + foundExternalUidReference = true; + } if (expectedReferences.containsKey(referenceSource)) { assertEquals(expectedReferences.get(referenceSource), referenceID); referencesFound += 1; } } assertEquals(expectedReferences.size(), referencesFound, "Not all expected references were returned"); + if (!isNotBlank(externalUid)) { + assertFalse(foundExternalUidReference, "External UID reference should not be created"); + } } public void checkGermplasmList(String listName, String listDescription, List germplasmNames) { @@ -1083,4 +1238,60 @@ public void checkGermplasmList(String listName, String listDescription, List externalReferences = new ArrayList<>(); + + BrAPIExternalReference programReference = new BrAPIExternalReference(); + programReference.setReferenceSource(String.format("%s/programs", BRAPI_REFERENCE_SOURCE)); + programReference.setReferenceId(validProgram.getId().toString()); + externalReferences.add(programReference); + + BrAPIExternalReference germplasmReference = new BrAPIExternalReference(); + germplasmReference.setReferenceSource(BRAPI_REFERENCE_SOURCE); + germplasmReference.setReferenceId(UUID.randomUUID().toString()); + externalReferences.add(germplasmReference); + + germplasm.setExternalReferences(externalReferences); + + germplasmDAO.createBrAPIGermplasm(List.of(germplasm), validProgram.getId(), null); + + //Refresh cache so importer lookup sees the seeded germplasm immediately. + germplasmDAO.repopulateGermplasmCacheForProgram(validProgram.getId()); + } } diff --git a/src/test/java/org/breedinginsight/services/BrAPIGermplasmServiceUnitTest.java b/src/test/java/org/breedinginsight/services/BrAPIGermplasmServiceUnitTest.java index 9d5f7b792..b957a3f88 100644 --- a/src/test/java/org/breedinginsight/services/BrAPIGermplasmServiceUnitTest.java +++ b/src/test/java/org/breedinginsight/services/BrAPIGermplasmServiceUnitTest.java @@ -1,5 +1,6 @@ package org.breedinginsight.services; +import com.google.gson.JsonNull; import com.google.gson.JsonObject; import io.reactivex.functions.Function; import io.reactivex.functions.Function3; @@ -37,7 +38,7 @@ import java.util.stream.Collectors; import static org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields.*; -import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.*; import static org.mockito.Mockito.*; @TestInstance(TestInstance.Lifecycle.PER_CLASS) @@ -109,6 +110,10 @@ public void getGermplasmListExport() { testReference.setReferenceSource(referenceSource); testReference.setReferenceID(parentUuid); externalRef.add(testReference); + BrAPIExternalReference externalUidReference = new BrAPIExternalReference(); + externalUidReference.setReferenceSource("External UID"); + externalUidReference.setReferenceID("UID-1"); + externalRef.add(externalUidReference); testGermplasm.setExternalReferences(externalRef); germplasm.add(testGermplasm); @@ -128,6 +133,10 @@ public void getGermplasmListExport() { testReference.setReferenceID(UUID.randomUUID().toString()); externalRef = new ArrayList<>(); externalRef.add(testReference); + BrAPIExternalReference externalUidReference2 = new BrAPIExternalReference(); + externalUidReference2.setReferenceSource("External UID"); + externalUidReference2.setReferenceID("UID-2"); + externalRef.add(externalUidReference2); testGermplasm.setExternalReferences(externalRef); germplasm.add(testGermplasm); @@ -165,12 +174,99 @@ public void getGermplasmListExport() { List expectedColumnNames = GermplasmFileColumns.getOrderedColumns().stream().map(c -> c.getValue()).collect(Collectors.toList()); //Check file values - assertEquals(listName+"_"+timestamp, downloadFile.getFileName(), "Incorrect export file name"); + assertEquals(listName + "_" + timestamp, downloadFile.getFileName(), "Incorrect export file name"); assertEquals(expectedColumnNames, resultTable.columnNames(), "Incorrect columns were exported"); assertEquals(2, resultTable.rowCount(), "Wrong number of rows were exported"); assertEquals("Germplasm A", resultTable.get(0, 1), "Incorrect data exported"); // Check that "GID" column matches "Entry No" for both (https://breedinginsight.atlassian.net/browse/BI-2266). - assertEquals(resultTable.get(0,0), resultTable.get(0, 6), "Incorrect data exported"); - assertEquals(resultTable.get(1,0), resultTable.get(1, 6), "Incorrect data exported"); + assertEquals(resultTable.get(0, 0), resultTable.get(0, 7), "Incorrect data exported"); + assertEquals(resultTable.get(1, 0), resultTable.get(1, 7), "Incorrect data exported"); + //Assert "Pedigree" column contains properly formatted data + assertEquals(resultTable.get(0, 4), "", "Incorrect data exported"); + assertEquals(resultTable.get(1, 4), "Germplasm A", "Incorrect data exported"); + //Asserting new External UID data + assertEquals("UID-1", resultTable.get(0, 10), "Incorrect data exported"); + assertEquals("UID-2", resultTable.get(1, 10), "Incorrect data exported"); + } + + @Test + public void processListDataAllowsBlankOptionalFieldsForProgramExport() { // NEW: covers full germplasm export path + Program testProgram = new Program(); + testProgram.setKey("TEST"); + + BrAPIGermplasm testGermplasm = new BrAPIGermplasm(); + testGermplasm.setGermplasmName("Germplasm A"); + testGermplasm.setAccessionNumber("1"); + + JsonObject additionalInfo = new JsonObject(); + additionalInfo.addProperty(MALE_PARENT_UNKNOWN, false); + additionalInfo.addProperty(FEMALE_PARENT_UNKNOWN, false); + + testGermplasm.setAdditionalInfo(additionalInfo); + testGermplasm.setExternalReferences(new ArrayList<>()); + + germplasmService = new BrAPIGermplasmService(listDAO, programService, germplasmDAO); + + List> processedData = germplasmService.processListData( + Collections.singletonList(testGermplasm), + null, + testProgram + ); + + assertEquals(1, processedData.size()); + assertEquals("Germplasm A", processedData.get(0).get("Germplasm Name")); + assertEquals(1, processedData.get(0).get("Entry No")); + assertFalse(processedData.get(0).containsKey("Breeding Method")); + assertFalse(processedData.get(0).containsKey("Source")); + assertFalse(processedData.get(0).containsKey("External UID")); + } + + @Test + public void processListDataAllowsBlankOptionalFieldsForListExport() { + Program testProgram = new Program(); + testProgram.setKey("TEST"); + + BrAPIGermplasm blankOptionalFields = new BrAPIGermplasm(); + blankOptionalFields.setGermplasmName("Germplasm A"); + blankOptionalFields.setAccessionNumber("1"); + blankOptionalFields.setSeedSource(""); + + JsonObject blankInfo = new JsonObject(); + blankInfo.add(GERMPLASM_BREEDING_METHOD, JsonNull.INSTANCE); + blankInfo.addProperty(MALE_PARENT_UNKNOWN, false); + blankInfo.addProperty(FEMALE_PARENT_UNKNOWN, false); + + blankOptionalFields.setAdditionalInfo(blankInfo); + blankOptionalFields.setExternalReferences(new ArrayList<>()); + + BrAPIGermplasm orderedFirst = new BrAPIGermplasm(); + orderedFirst.setGermplasmName("Germplasm B"); + orderedFirst.setAccessionNumber("2"); + orderedFirst.setSeedSource("Cultivated"); + + JsonObject orderedFirstInfo = new JsonObject(); + orderedFirstInfo.addProperty(GERMPLASM_BREEDING_METHOD, "Autopolyploid"); + orderedFirstInfo.addProperty(MALE_PARENT_UNKNOWN, false); + orderedFirstInfo.addProperty(FEMALE_PARENT_UNKNOWN, false); + + orderedFirst.setAdditionalInfo(orderedFirstInfo); + orderedFirst.setExternalReferences(new ArrayList<>()); + + germplasmService = new BrAPIGermplasmService(listDAO, programService, germplasmDAO); + + List> processedData = germplasmService.processListData( + Arrays.asList(blankOptionalFields, orderedFirst), + Arrays.asList("Germplasm B [TEST-2]", "Germplasm A [TEST-1]"), + testProgram + ); + + assertEquals(2, processedData.size()); + assertEquals("Germplasm B", processedData.get(0).get("Germplasm Name")); + assertEquals(1, processedData.get(0).get("Entry No")); + assertEquals("Germplasm A", processedData.get(1).get("Germplasm Name")); + assertEquals(2, processedData.get(1).get("Entry No")); + assertEquals("", processedData.get(1).get("Source")); + assertFalse(processedData.get(1).containsKey("Breeding Method")); + assertFalse(processedData.get(1).containsKey("External UID")); } } diff --git a/src/test/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImplIntegrationTest.java b/src/test/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImplIntegrationTest.java index 94770d912..e0cf269ac 100644 --- a/src/test/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImplIntegrationTest.java +++ b/src/test/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImplIntegrationTest.java @@ -5,7 +5,7 @@ import com.agorapulse.micronaut.amazon.awssdk.s3.SimpleStorageServiceConfiguration; import com.fasterxml.jackson.databind.ObjectMapper; import io.micronaut.context.ApplicationContext; -import io.micronaut.context.annotation.Property; +import io.micronaut.context.annotation.*; import io.micronaut.context.event.BeanCreatedEventListener; import io.micronaut.http.HttpStatus; import io.micronaut.http.MediaType; @@ -13,8 +13,6 @@ import io.micronaut.inject.qualifiers.Qualifiers; import io.micronaut.test.annotation.MockBean; import io.micronaut.test.extensions.junit5.annotation.MicronautTest; -import io.reactivex.functions.Function; -import io.reactivex.functions.Function3; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.tika.mime.MimeTypeException; @@ -25,50 +23,40 @@ import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.client.v2.model.queryParams.core.ProgramQueryParams; import org.brapi.client.v2.model.queryParams.core.StudyQueryParams; -import org.brapi.client.v2.model.queryParams.core.TrialQueryParams; import org.brapi.client.v2.modules.core.ProgramsApi; import org.brapi.client.v2.modules.core.StudiesApi; -import org.brapi.client.v2.modules.core.TrialsApi; +import org.brapi.client.v2.modules.genotype.SamplesApi; import org.brapi.client.v2.modules.phenotype.ObservationUnitsApi; -import org.brapi.v2.model.BrAPIExternalReference; -import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.core.response.BrAPIProgramListResponse; import org.brapi.v2.model.core.response.BrAPIStudyListResponse; -import org.brapi.v2.model.core.response.BrAPITrialListResponse; -import org.brapi.v2.model.core.response.BrAPITrialListResponseResult; +import org.brapi.v2.model.geno.BrAPISample; +import org.brapi.v2.model.geno.request.BrAPISampleSearchRequest; +import org.brapi.v2.model.geno.response.BrAPISampleListResponse; +import org.brapi.v2.model.geno.response.BrAPISampleListResponseResult; import org.brapi.v2.model.germ.BrAPIGermplasm; -import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.brapi.v2.model.pheno.request.BrAPIObservationUnitSearchRequest; -import org.brapi.v2.model.pheno.response.BrAPIObservationUnitListResponse; -import org.brapi.v2.model.pheno.response.BrAPIObservationUnitListResponseResult; import org.breedinginsight.DatabaseTest; -import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; +import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; +import org.breedinginsight.brapi.v2.dao.impl.ImportMappingDAOImpl; +import org.breedinginsight.brapps.importer.daos.BrAPISampleDAO; import org.breedinginsight.brapps.importer.daos.ImportDAO; import org.breedinginsight.brapps.importer.daos.ImportMappingDAO; -import org.breedinginsight.brapi.v2.dao.impl.BrAPITrialDAOImpl; -import org.breedinginsight.brapi.v2.dao.impl.ImportMappingDAOImpl; import org.breedinginsight.brapps.importer.model.ImportProgress; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.mapping.ImportMapping; import org.breedinginsight.brapps.importer.model.response.ImportResponse; -import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.dao.db.tables.pojos.ImporterImportEntity; +import org.breedinginsight.daos.GenotypeImportDAO; import org.breedinginsight.daos.ProgramDAO; +import org.breedinginsight.daos.SampleSubmissionDAO; import org.breedinginsight.daos.UserDAO; import org.breedinginsight.daos.impl.ProgramDAOImpl; import org.breedinginsight.daos.impl.UserDAOImpl; -import org.breedinginsight.model.BrAPIConstants; -import org.breedinginsight.model.GermplasmGenotype; -import org.breedinginsight.model.Program; -import org.breedinginsight.model.User; -import org.breedinginsight.services.ProgramService; +import org.breedinginsight.model.*; import org.breedinginsight.services.brapi.BrAPIClientProvider; import org.breedinginsight.services.brapi.BrAPIEndpointProvider; import org.breedinginsight.services.brapi.BrAPIProvider; import org.breedinginsight.services.exceptions.AuthorizationException; import org.breedinginsight.services.exceptions.DoesNotExistException; -import org.breedinginsight.utilities.BrAPIDAOUtil; -import org.breedinginsight.utilities.Utilities; import org.jetbrains.annotations.NotNull; import org.jooq.Configuration; import org.jooq.DSLContext; @@ -83,6 +71,7 @@ import javax.inject.Inject; import javax.inject.Named; +import javax.inject.Singleton; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; @@ -129,7 +118,16 @@ public class GigwaGenotypeServiceImplIntegrationTest extends DatabaseTest { private ImportDAO importDAO; @Inject - private BrAPITrialDAO trialDAO; + private SampleSubmissionDAO sampleSubmissionDAO; + + @Inject + private BrAPISampleDAO sampleDAO; + + @Inject + private BrAPIGermplasmDAO germplasmDAO; + + @Inject + private GenotypeImportDAO genotypeImportDAO; @Inject private ObjectMapper objectMapper; @@ -147,9 +145,6 @@ public class GigwaGenotypeServiceImplIntegrationTest extends DatabaseTest { @Inject private S3Presigner presigner; - @Inject - private BrAPIDAOUtil brAPIDAOUtil; - @Inject private BrAPIEndpointProvider brAPIEndpointProvider; @@ -188,31 +183,54 @@ ImportDAO importDAO() { return mock(ImportDAO.class); } + @MockBean(BrAPIEndpointProvider.class) + BrAPIEndpointProvider brAPIEndpointProvider() { + return spy(new BrAPIEndpointProvider()); + } - @MockBean(BrAPITrialDAOImpl.class) - BrAPITrialDAO trialDAO() { - return mock(BrAPITrialDAOImpl.class); + @MockBean(BrAPISampleDAO.class) + BrAPISampleDAO sampleDAO() { + return mock(BrAPISampleDAO.class); } - @MockBean(ProgramService.class) - ProgramService programService() { - return mock(ProgramService.class); + @MockBean(value = SimpleStorageService.class, named = "genotype") + SimpleStorageService simpleStorageService() { + return spy(new DefaultSimpleStorageService(bucketName, s3Client, presigner)); } - @MockBean(BrAPIDAOUtil.class) - BrAPIDAOUtil brAPIDAOUtil() { - return spy(new BrAPIDAOUtil(1000, Duration.of(10, ChronoUnit.MINUTES), 1000, 100, 65000, programService())); + // @MockBean cannot replace SampleSubmissionDAO here because it extends generated jOOQ DAO code; + // Scope this replacement to this spec so it does not affect other test contexts. + @Factory + @Requires(property = "micronaut.test.active.spec", value = "org.breedinginsight.services.geno.impl.GigwaGenotypeServiceImplIntegrationTest") + static class SampleSubmissionTestFactory { + + @Singleton + @Replaces(SampleSubmissionDAO.class) + SampleSubmissionDAO sampleSubmissionDAO() { + return mock(SampleSubmissionDAO.class); + } } - @MockBean(SimpleStorageService.class) - @Named("genotype") - SimpleStorageService simpleStorageService() { - return spy(new DefaultSimpleStorageService(bucketName, s3Client, presigner)); + @Factory + @Requires(property = "micronaut.test.active.spec", value = "org.breedinginsight.services.geno.impl.GigwaGenotypeServiceImplIntegrationTest") + static class GermplasmDaoTestFactory { + + @Context + @Replaces(BrAPIGermplasmDAO.class) + BrAPIGermplasmDAO germplasmDAO() { + return mock(BrAPIGermplasmDAO.class); + } } - @MockBean(BrAPIEndpointProvider.class) - BrAPIEndpointProvider brAPIEndpointProvider() { - return spy(new BrAPIEndpointProvider()); + @Factory + @Requires(property = "micronaut.test.active.spec", value = "org.breedinginsight.services.geno.impl.GigwaGenotypeServiceImplIntegrationTest") + static class GenotypeImportDaoTestFactory { + + @Singleton + @Replaces(GenotypeImportDAO.class) + GenotypeImportDAO genotypeImportDAO() { + return mock(GenotypeImportDAO.class); + } } private GenericContainer gigwa; @@ -247,12 +265,12 @@ public GigwaGenotypeServiceImplIntegrationTest() { .withEnv("GIGWA.serversAllowedToImport", gigwaAllowedServer) .waitingFor( Wait.forHttp("/gigwa") - .forStatusCode(200) - .withStartupTimeout(Duration.of(2, ChronoUnit.MINUTES))); + .forStatusCode(200) + .withStartupTimeout(Duration.of(2, ChronoUnit.MINUTES))); gigwa.start(); localStackContainer = new LocalStackContainer(DockerImageName.parse("localstack/localstack") - .withTag("3.0.2")) + .withTag("3.0.2")) .withServices(LocalStackContainer.Service.S3) .withNetwork(super.getNetwork()) .withNetworkAliases("localstack") @@ -265,7 +283,7 @@ public GigwaGenotypeServiceImplIntegrationTest() { public Map getProperties() { Map properties = super.getProperties(); - properties.put("gigwa.host", "http://"+gigwa.getContainerIpAddress()+":"+gigwa.getMappedPort(8080)+"/"); + properties.put("gigwa.host", "http://" + gigwa.getContainerIpAddress() + ":" + gigwa.getMappedPort(8080) + "/"); properties.put("gigwa.username", "gigwadmin"); properties.put("gigwa.password", "nimda"); @@ -293,7 +311,12 @@ public void setup() throws IllegalAccessException, NoSuchFieldException { storageService = applicationContext.getBean(SimpleStorageService.class, Qualifiers.byName("genotype")); storageService.createBucket(); - } + } + + @BeforeEach + public void resetGenotypeImportDaoMock() { + reset(genotypeImportDAO); + } @AfterAll public void teardown() { @@ -306,41 +329,42 @@ public void teardown() { public void testUpload() throws ApiException, AuthorizationException { UUID programId = UUID.fromString("360766b8-480b-4b0a-862c-7eaa651dda28"); String programKey = "TEST"; - UUID expId = UUID.randomUUID(); + UUID submissionId = UUID.randomUUID(); UUID importId = UUID.randomUUID(); - assertTimeout(Duration.of(2, ChronoUnit.MINUTES), () -> uploadGenoData(programId, programKey, expId, importId), "Upload did not complete within the time period"); + assertTimeout(Duration.of(2, ChronoUnit.MINUTES), () -> uploadGenoData(programId, programKey, submissionId, importId), "Upload did not complete within the time period"); - assertTrue(storageService.exists(storageService.getDefaultBucketName(), programId + "/" + expId + "/" + importId + ".vcf"), "File was not uploaded to s3"); + assertTrue(storageService.exists(storageService.getDefaultBucketName(), programId + "/" + submissionId + "/" + importId + ".vcf"), "File was not uploaded to s3"); + verify(genotypeImportDAO).createGenotypeImportLink(eq(submissionId), eq(importId), any(UUID.class)); BrAPIClient brAPIClient = new BrAPIClient(gigwaHost + "gigwa/rest/brapi/v2"); Authentication authorizationToken = brAPIClient.getAuthentication("AuthorizationToken"); - if(authorizationToken instanceof OAuth) { - ((OAuth)authorizationToken).setAccessToken(gigwaGenoStorageService.getAuthToken()); + if (authorizationToken instanceof OAuth) { + ((OAuth) authorizationToken).setAccessToken(gigwaGenoStorageService.getAuthToken()); } ProgramsApi programsApi = new ProgramsApi(brAPIClient); try { ApiResponse brAPIProgramListResponseApiResponse = programsApi.programsGet(ProgramQueryParams.builder() - .programDbId(programKey) - .build()); + .programDbId(programKey) + .build()); assertEquals(1, - brAPIProgramListResponseApiResponse.getBody() - .getResult() - .getData() - .size()); + brAPIProgramListResponseApiResponse.getBody() + .getResult() + .getData() + .size()); StudiesApi studiesApi = new StudiesApi(brAPIClient); ApiResponse brAPIStudyListResponseApiResponse = studiesApi.studiesGet(StudyQueryParams.builder() - .build()); + .build()); assertEquals(1, - brAPIStudyListResponseApiResponse.getBody() - .getResult() - .getData() - .stream() - .filter(brAPIStudy -> brAPIStudy.getStudyName() - .equals(expId.toString())) - .count()); + brAPIStudyListResponseApiResponse.getBody() + .getResult() + .getData() + .stream() + .filter(brAPIStudy -> brAPIStudy.getStudyName() + .equals(submissionId.toString())) + .count()); } catch (ApiException e) { System.err.println(e.getMessage()); System.err.println(e.getResponseBody()); @@ -352,34 +376,40 @@ public void testUpload() throws ApiException, AuthorizationException { public void testFetchGermplasmGenotype() throws AuthorizationException, ApiException, DoesNotExistException { UUID programId = UUID.fromString("8b667063-480b-4b0a-862c-7eaa651dda28"); String programKey = "TESTFETCH"; - UUID expId = UUID.randomUUID(); + UUID submissionId = UUID.randomUUID(); UUID importId = UUID.randomUUID(); - assertTimeout(Duration.of(2, ChronoUnit.MINUTES), () -> uploadGenoData(programId, programKey, expId, importId), "Upload did not complete within the time period"); - - BrAPIGermplasm germplasm = new BrAPIGermplasm().germplasmDbId(UUID.randomUUID().toString()).germplasmName("Test Germ"); + assertTimeout(Duration.of(2, ChronoUnit.MINUTES), () -> uploadGenoData(programId, programKey, submissionId, importId), "Upload did not complete within the time period"); - BrAPITrial trial = new BrAPITrial().externalReferences(List.of(new BrAPIExternalReference().referenceSource(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.TRIALS)) - .referenceID(UUID.randomUUID() - .toString()))); + String germplasmName = "USDAMSP1"; + String sampleName = germplasmName + "_A01"; + BrAPIGermplasm germplasm = new BrAPIGermplasm().germplasmDbId(UUID.randomUUID().toString()).germplasmName(germplasmName); + SamplesApi mockSamplesApi = spy(new SamplesApi()); + BrAPISample sample = new BrAPISample().sampleName(sampleName) + .germplasmDbId(programKey + "§" + sampleName); + doReturn(List.of(sample)).when(sampleDAO) + .readSamplesByGermplasmIds(any(Program.class), eq(List.of(germplasm.getGermplasmDbId()))); + doReturn(new ApiResponse<>(200, + new HashMap<>(), + Pair.of(Optional.of(new BrAPISampleListResponse().result(new BrAPISampleListResponseResult().data(List.of(sample)))), + Optional.empty()))) + .when(mockSamplesApi).searchSamplesPost(any(BrAPISampleSearchRequest.class)); - doReturn(List.of(trial)).when(trialDAO).getTrials(any(UUID.class)); - - doAnswer(invocation -> { - Object searchObject = invocation.getArgument(2); - if(searchObject instanceof BrAPIObservationUnitSearchRequest) { - return List.of(new BrAPIObservationUnit().observationUnitName("USDAMSP1_A01")); - } else { - return invocation.callRealMethod(); - } - }).when(brAPIDAOUtil) - .search(any(Function.class), any(Function3.class), any()); + doReturn(mockSamplesApi).when(brAPIEndpointProvider).get(any(BrAPIClient.class), eq(SamplesApi.class)); doReturn(new BrAPIClient("", 300000)).when(programDAO).getCoreClient(any(UUID.class)); doReturn(new BrAPIClient("", 300000)).when(programDAO).getPhenoClient(any(UUID.class)); - GermplasmGenotype germplasmGenotype = gigwaGenoStorageService.retrieveGenotypeData(programId, germplasm); + doReturn(germplasm).when(germplasmDAO) + .getGermplasmByUUID(any(String.class), any(UUID.class)); + GermplasmGenotype germplasmGenotype = gigwaGenoStorageService.retrieveGenotypeData(programId, UUID.fromString(germplasm.getGermplasmDbId())); + + verify(sampleDAO).readSamplesByGermplasmIds(any(Program.class), eq(List.of(germplasm.getGermplasmDbId()))); + verify(brAPIEndpointProvider, never()).get(any(BrAPIClient.class), eq(ObservationUnitsApi.class)); + verify(mockSamplesApi).searchSamplesPost(argThat(searchRequest -> searchRequest.getGermplasmDbIds() != null && + searchRequest.getGermplasmDbIds().contains(programKey + "§" + sample.getSampleName()) && + (searchRequest.getObservationUnitDbIds() == null || searchRequest.getObservationUnitDbIds().isEmpty()))); assertNotNull(germplasmGenotype); assertFalse(germplasmGenotype.getCalls().isEmpty()); assertFalse(germplasmGenotype.getCallSets().isEmpty()); @@ -390,180 +420,409 @@ public void testFetchGermplasmGenotype() throws AuthorizationException, ApiExcep public void testSubmitValidFile() throws IOException, ApiException { UUID programId = UUID.fromString("29162e85-e739-4f19-9fd0-0c377ed59956"); String programKey = "TESTSUBMITVALID"; - UUID expId = UUID.randomUUID(); - - Scanner sc = new Scanner(new FileInputStream("src/test/resources/files/geno/sample.vcf"), "UTF-8"); - String[] headerParts = null; - boolean foundHeader = false; - while (sc.hasNextLine() && !foundHeader) { - String line = sc.nextLine(); - if(line.startsWith("#CHROM")) { - foundHeader = true; - headerParts = line.split("\t"); - } - } - assertTrue(foundHeader, "Could not find sample.vcf header file"); + UUID submissionId = UUID.randomUUID(); - List ous = new ArrayList<>(); - for(int i = 9; i < headerParts.length; i++) { - ous.add(new BrAPIObservationUnit().observationUnitName(headerParts[i] + " ["+programKey+"-"+(i-7)+"]")); - } - setupMocksForSubmitGenoData(expId, ous); - - doAnswer(invocation -> { - if(invocation.getArgument(2) instanceof BrAPIObservationUnitSearchRequest) { - return invocation.callRealMethod(); - } - return invocation.getMock(); - }).when(brAPIDAOUtil).search(any(Function.class), any(Function3.class), any()); + List samples = buildSamplesFromValidVcf(); + setupMocksForSubmitGenoData(programId, submissionId, samples); AtomicReference importResponse = new AtomicReference<>(); - assertTimeout(Duration.of(2, ChronoUnit.MINUTES), () -> importResponse.set(submitGenoData(programId, programKey, expId, "sample.vcf")), "Upload did not complete within the time period"); + assertTimeout(Duration.of(2, ChronoUnit.MINUTES), () -> + importResponse.set(submitGenoData(programId, programKey, submissionId, "sample.vcf")), + "Upload did not complete within the time period"); ImportResponse response = importResponse.get(); assertNotNull(response); assertNotNull(response.getProgress()); - assertEquals((short)HttpStatus.ACCEPTED.getCode(), response.getProgress().getStatuscode(), "Error importing geno file: " + response.getProgress().getMessage()); + assertEquals((short) HttpStatus.ACCEPTED.getCode(), + response.getProgress().getStatuscode(), + "Error importing geno file: " + response.getProgress().getMessage()); + } + + @Test + public void testGetGenotypeImportsReturnsRowsIncludingSampleSubmissionId() { + UUID programId = UUID.randomUUID(); + UUID olderSubmissionId = UUID.randomUUID(); + UUID newerSubmissionId = UUID.randomUUID(); + UUID genotypeImportId1 = UUID.randomUUID(); + UUID genotypeImportId2 = UUID.randomUUID(); + + GenotypeImportDetails older = GenotypeImportDetails.builder() + .genotypeImportId(genotypeImportId1) + .sampleSubmissionId(olderSubmissionId) + .projectNameForSampleSubmission("Submission " + olderSubmissionId) + .sampleSubmissionCreatedBy("system") + .genotypingFileName("sample.vcf") + .genotypingImportDate(OffsetDateTime.parse("2026-06-01T10:00:00Z")) + .genotypingImportBy("system") + .build(); + + GenotypeImportDetails newer = GenotypeImportDetails.builder() + .genotypeImportId(genotypeImportId2) + .sampleSubmissionId(newerSubmissionId) + .projectNameForSampleSubmission("Submission " + newerSubmissionId) + .sampleSubmissionCreatedBy("system") + .genotypingFileName("sample.vcf") + .genotypingImportDate(OffsetDateTime.parse("2026-06-02T10:00:00Z")) + .genotypingImportBy("system") + .build(); + + doReturn(List.of(newer, older)).when(genotypeImportDAO).getGenotypeImportsByProgramId(programId); + + List rows = gigwaGenoStorageService.getGenotypeImports(programId); + + assertNotNull(rows); + assertEquals(2, rows.size()); + + assertEquals(genotypeImportId2, rows.get(0).getGenotypeImportId()); + assertEquals(newerSubmissionId, rows.get(0).getSampleSubmissionId()); + assertEquals("Submission " + newerSubmissionId, rows.get(0).getProjectNameForSampleSubmission()); + assertEquals("sample.vcf", rows.get(0).getGenotypingFileName()); + assertNotNull(rows.get(0).getGenotypingImportDate()); + assertEquals("system", rows.get(0).getSampleSubmissionCreatedBy()); + assertEquals("system", rows.get(0).getGenotypingImportBy()); + + assertEquals(genotypeImportId1, rows.get(1).getGenotypeImportId()); + assertEquals(olderSubmissionId, rows.get(1).getSampleSubmissionId()); + assertEquals("Submission " + olderSubmissionId, rows.get(1).getProjectNameForSampleSubmission()); + assertEquals("sample.vcf", rows.get(1).getGenotypingFileName()); + assertNotNull(rows.get(1).getGenotypingImportDate()); + assertEquals("system", rows.get(1).getSampleSubmissionCreatedBy()); + assertEquals("system", rows.get(1).getGenotypingImportBy()); + + verify(genotypeImportDAO).getGenotypeImportsByProgramId(programId); + } + + @Test + public void testGetGenotypeImportsReturnsEmptyListWhenNoImportsExist() { + UUID programId = UUID.randomUUID(); + doReturn(Collections.emptyList()).when(genotypeImportDAO).getGenotypeImportsByProgramId(programId); + + List rows = gigwaGenoStorageService.getGenotypeImports(programId); + + assertNotNull(rows); + assertTrue(rows.isEmpty()); + verify(genotypeImportDAO).getGenotypeImportsByProgramId(programId); + } + + @Test + public void testDownloadGenotypeImportReturnsOriginalUploadedFile() throws Exception { + UUID programId = UUID.randomUUID(); + String programKey = "TESTDOWNLOADGENOIMPORT"; + UUID submissionId = UUID.randomUUID(); + UUID importerImportId = UUID.randomUUID(); + UUID genotypeImportId = UUID.randomUUID(); + + uploadGenoData(programId, programKey, submissionId, importerImportId); + doReturn(Optional.of(GenotypeImportDownloadDetails.builder() + .sampleSubmissionId(submissionId) + .importerImportId(importerImportId) + .genotypeFileName("sample.vcf") + .build())) + .when(genotypeImportDAO).getDownloadableGenotypeImportById(programId, genotypeImportId); + + Optional downloadFile = gigwaGenoStorageService.downloadGenotypeImport(programId, genotypeImportId); + + verify(genotypeImportDAO, times(1)).getDownloadableGenotypeImportById(programId, genotypeImportId); + assertTrue(downloadFile.isPresent()); + assertEquals("sample.vcf", downloadFile.get().getFileName()); + assertArrayEquals( + new TestFileUpload("src/test/resources/files/geno/sample.vcf", MediaType.of("application/vcard")).getBytes(), + downloadFile.get().getStreamedFile().getInputStream().readAllBytes() + ); + } + + @Test + public void testDownloadGenotypeImportReturnsEmptyWhenDaoReturnsEmpty() { + UUID programId = UUID.randomUUID(); + UUID genotypeImportId = UUID.randomUUID(); + + doReturn(Optional.empty()).when(genotypeImportDAO) + .getDownloadableGenotypeImportById(programId, genotypeImportId); + + Optional downloadFile = gigwaGenoStorageService.downloadGenotypeImport(programId, genotypeImportId); + + assertTrue(downloadFile.isEmpty()); + verify(genotypeImportDAO, times(1)).getDownloadableGenotypeImportById(programId, genotypeImportId); } @Test public void testSubmitInvalidHeader() throws ApiException { UUID programId = UUID.fromString("29162e85-e739-4f19-9fd0-0c377ed59956"); String programKey = "TESTSUBMITINVALID"; - UUID expId = UUID.randomUUID(); - setupMocksForSubmitGenoData(expId, Collections.emptyList()); + UUID submissionId = UUID.randomUUID(); + setupMocksForSubmitGenoData(programId, submissionId, Collections.emptyList()); AtomicReference importResponse = new AtomicReference<>(); - assertTimeout(Duration.of(2, ChronoUnit.MINUTES), () -> importResponse.set(submitGenoData(programId, programKey, expId, "sample_invalid.vcf")), "Upload did not complete within the time period"); + assertTimeout(Duration.of(2, ChronoUnit.MINUTES), () -> importResponse.set(submitGenoData(programId, programKey, submissionId, "sample_invalid.vcf")), "Upload did not complete within the time period"); ImportResponse response = importResponse.get(); assertNotNull(response); assertNotNull(response.getProgress()); - assertEquals((short)HttpStatus.BAD_REQUEST.getCode(), response.getProgress().getStatuscode()); + assertEquals((short) HttpStatus.BAD_REQUEST.getCode(), response.getProgress().getStatuscode()); assertEquals("Header row is not valid VCF format", response.getProgress().getMessage()); } @Test - public void testSubmitMissingOUs() throws ApiException { + public void testSubmitMissingSubmissionSamples() throws ApiException { UUID programId = UUID.fromString("29162e85-e739-4f19-9fd0-0c377ed59956"); String programKey = "TESTSUBMITMISSINGOU"; - UUID expId = UUID.randomUUID(); + UUID submissionId = UUID.randomUUID(); - setupMocksForSubmitGenoData(expId, Collections.emptyList()); + setupMocksForSubmitGenoData(programId, submissionId, Collections.emptyList()); AtomicReference importResponse = new AtomicReference<>(); - assertTimeout(Duration.of(2, ChronoUnit.MINUTES), () -> importResponse.set(submitGenoData(programId, programKey, expId, "sample.vcf")), "Upload did not complete within the time period"); + assertTimeout(Duration.of(2, ChronoUnit.MINUTES), () -> importResponse.set(submitGenoData(programId, programKey, submissionId, "sample.vcf")), "Upload did not complete within the time period"); ImportResponse response = importResponse.get(); assertNotNull(response); assertNotNull(response.getProgress()); - assertEquals((short)HttpStatus.BAD_REQUEST.getCode(), response.getProgress().getStatuscode()); - assertEquals("There are samples that do not have an existing observation unit", response.getProgress().getMessage()); + assertEquals((short) HttpStatus.BAD_REQUEST.getCode(), response.getProgress().getStatuscode()); + assertEquals("There are samples that are not linked to the selected submission", response.getProgress().getMessage()); } - private void setupMocksForSubmitGenoData(UUID expId, List ous) throws ApiException { - TrialsApi mockTrialsApi = spy(new TrialsApi()); - doReturn(new ApiResponse(200, - new HashMap<>(), - new BrAPITrialListResponse().result(new BrAPITrialListResponseResult().addDataItem(new BrAPITrial().trialDbId(expId.toString()))))) - .when(mockTrialsApi).trialsGet(any(TrialQueryParams.class)); + @Test + public void testSubmitDuplicatePositionalKeysShowsSingleMessage() throws Exception { + UUID programId = UUID.fromString("29162e85-e739-4f19-9fd0-0c377ed59956"); + String programKey = "TESTDUPKEY"; + UUID submissionId = UUID.randomUUID(); - doReturn(mockTrialsApi).when(brAPIEndpointProvider).get(any(BrAPIClient.class), eq(TrialsApi.class)); + setupMocksForSubmitGenoData(programId, submissionId, buildSamplesFromVcf("sample_duplicate_positional_key.vcf")); - ObservationUnitsApi mockOUsApi = spy(new ObservationUnitsApi()); - doReturn(new ApiResponse<>(200, - new HashMap<>(), - Pair.of(Optional.of(new BrAPIObservationUnitListResponse().result(new BrAPIObservationUnitListResponseResult().data(ous))), - Optional.empty()))) - .when(mockOUsApi).searchObservationunitsPost(any(BrAPIObservationUnitSearchRequest.class)); + AtomicReference importResponse = new AtomicReference<>(); + assertTimeout(Duration.of(2, ChronoUnit.MINUTES), () -> importResponse.set(submitGenoData(programId, programKey, submissionId, "sample_duplicate_positional_key.vcf")), "Upload did not complete within the time period"); + + ImportResponse response = importResponse.get(); + assertNotNull(response); + assertNotNull(response.getProgress()); + assertEquals((short) HttpStatus.BAD_REQUEST.getCode(), response.getProgress().getStatuscode()); + assertEquals("Duplicate chromosomal position(s) detected. CHROM:POS key must be unique for variant type.", response.getProgress().getMessage()); + } + + @Test + public void testSubmitSameChromPosDifferentVariantTypesAccepted() throws Exception { + UUID programId = UUID.fromString("29162e85-e739-4f19-9fd0-0c377ed59956"); + String programKey = "TESTSAMEPOSDIFFTYPE"; + UUID submissionId = UUID.randomUUID(); - doReturn(mockOUsApi).when(brAPIEndpointProvider).get(any(BrAPIClient.class), eq(ObservationUnitsApi.class)); + setupMocksForSubmitGenoData(programId, submissionId, buildSamplesFromVcf("sample_same_pos_different_variant_type.vcf")); + + AtomicReference importResponse = new AtomicReference<>(); + assertTimeout( + Duration.of(2, ChronoUnit.MINUTES), + () -> importResponse.set(submitGenoData(programId, programKey, submissionId, "sample_same_pos_different_variant_type.vcf")), + "Upload did not complete within the time period" + ); + + ImportResponse response = importResponse.get(); + assertNotNull(response); + assertNotNull(response.getProgress()); + assertEquals( + (short) HttpStatus.ACCEPTED.getCode(), + response.getProgress().getStatuscode(), + "Error importing geno file: " + response.getProgress().getMessage() + ); + } + + @Test + public void testSubmitInvalidRefShowsSingleMessage() throws Exception { + UUID programId = UUID.fromString("29162e85-e739-4f19-9fd0-0c377ed59956"); + String programKey = "TESTBADREF"; + UUID submissionId = UUID.randomUUID(); + + setupMocksForSubmitGenoData(programId, submissionId, buildSamplesFromVcf("sample_invalid_ref.vcf")); + + AtomicReference importResponse = new AtomicReference<>(); + assertTimeout(Duration.of(2, ChronoUnit.MINUTES), () -> importResponse.set(submitGenoData(programId, programKey, submissionId, "sample_invalid_ref.vcf")), "Upload did not complete within the time period"); + + ImportResponse response = importResponse.get(); + assertNotNull(response); + assertNotNull(response.getProgress()); + assertEquals((short) HttpStatus.BAD_REQUEST.getCode(), response.getProgress().getStatuscode()); + assertEquals("The file is not a valid VCF or contains unsupported REF/ALT allele values.", response.getProgress().getMessage()); + } + + @Test + public void testSubmitInvalidAltShowsSingleMessage() throws Exception { + UUID programId = UUID.fromString("29162e85-e739-4f19-9fd0-0c377ed59956"); + String programKey = "TESTBADALT"; + UUID submissionId = UUID.randomUUID(); + + setupMocksForSubmitGenoData(programId, submissionId, buildSamplesFromVcf("sample_invalid_alt.vcf")); + + AtomicReference importResponse = new AtomicReference<>(); + assertTimeout(Duration.of(2, ChronoUnit.MINUTES), () -> importResponse.set(submitGenoData(programId, programKey, submissionId, "sample_invalid_alt.vcf")), "Upload did not complete within the time period"); + + ImportResponse response = importResponse.get(); + assertNotNull(response); + assertNotNull(response.getProgress()); + assertEquals((short) HttpStatus.BAD_REQUEST.getCode(), response.getProgress().getStatuscode()); + assertEquals("The file is not a valid VCF or contains unsupported REF/ALT allele values.", response.getProgress().getMessage()); + } + + @Test + public void testSubmitRefDotShowsSingleMessage() throws Exception { + UUID programId = UUID.fromString("29162e85-e739-4f19-9fd0-0c377ed59956"); + String programKey = "TESTDOTREF"; + UUID submissionId = UUID.randomUUID(); + + setupMocksForSubmitGenoData(programId, submissionId, buildSamplesFromVcf("sample_invalid_ref_dot.vcf")); + + AtomicReference importResponse = new AtomicReference<>(); + assertTimeout(Duration.of(2, ChronoUnit.MINUTES), () -> importResponse.set(submitGenoData(programId, programKey, submissionId, "sample_invalid_ref_dot.vcf")), "Upload did not complete within the time period"); + + ImportResponse response = importResponse.get(); + assertNotNull(response); + assertNotNull(response.getProgress()); + assertEquals((short) HttpStatus.BAD_REQUEST.getCode(), response.getProgress().getStatuscode()); + assertEquals("The file is not a valid VCF or contains unsupported REF/ALT allele values.", response.getProgress().getMessage()); + } + + @Test + public void testSubmitMultiAllelicAltAccepted() throws Exception { + UUID programId = UUID.fromString("29162e85-e739-4f19-9fd0-0c377ed59956"); + String programKey = "TESTMULTIALT"; + UUID submissionId = UUID.randomUUID(); + + setupMocksForSubmitGenoData(programId, submissionId, buildSamplesFromVcf("sample_valid_multi_alt.vcf")); + + AtomicReference importResponse = new AtomicReference<>(); + assertTimeout(Duration.of(2, ChronoUnit.MINUTES), () -> importResponse.set(submitGenoData(programId, programKey, submissionId, "sample_valid_multi_alt.vcf")), "Upload did not complete within the time period"); + + ImportResponse response = importResponse.get(); + assertNotNull(response); + assertNotNull(response.getProgress()); + assertEquals((short) HttpStatus.ACCEPTED.getCode(), response.getProgress().getStatuscode(), "Error importing geno file: " + response.getProgress().getMessage()); } - private void uploadGenoData(UUID programId, String programKey, UUID expId, UUID importId) throws AuthorizationException, MimeTypeException, IOException, ApiException { + private void setupMocksForSubmitGenoData(UUID programId, UUID submissionId, List samples) throws ApiException { + SampleSubmission submission = new SampleSubmission(); + submission.setId(submissionId); + submission.setProgramId(programId); + + doReturn(List.of(submission)).when(sampleSubmissionDAO) + .getBySubmissionId(any(Program.class), eq(submissionId)); + doReturn(samples).when(sampleDAO) + .readSamplesBySubmissionIds(any(Program.class), eq(List.of(submissionId.toString()))); + } + + private void uploadGenoData(UUID programId, String programKey, UUID submissionId, UUID importId) throws AuthorizationException, MimeTypeException, IOException, ApiException { Program program = Program.builder() - .id(programId) - .key(programKey) - .brapiUrl(BrAPIConstants.SYSTEM_DEFAULT.getValue()) - .build(); + .id(programId) + .key(programKey) + .brapiUrl(BrAPIConstants.SYSTEM_DEFAULT.getValue()) + .build(); doReturn(List.of(program)).when(programDAO) - .get(any(UUID.class)); + .get(any(UUID.class)); User user = User.builder() - .id(UUID.randomUUID()) - .build(); + .id(UUID.randomUUID()) + .build(); doReturn(Optional.of(user)).when(userDAO) - .getUser(any(UUID.class)); + .getUser(any(UUID.class)); ImportMapping mapping = ImportMapping.builder() - .build(); + .build(); doReturn(List.of(mapping)).when(importMappingDAO) - .getSystemMappingByName(any(String.class)); + .getSystemMappingByName(any(String.class)); doAnswer(invocation -> { var importEntity = invocation.getArgument(0, ImporterImportEntity.class); importEntity.setId(importId); return importEntity; }).when(importDAO) - .insert(any(ImporterImportEntity.class)); + .insert(any(ImporterImportEntity.class)); ImportProgress progress = ImportProgress.builder() - .createdBy(user.getId()) - .createdAt(OffsetDateTime.now()) - .updatedAt(OffsetDateTime.now()) - .updatedBy(user.getId()) - .statuscode((short) HttpStatus.ACCEPTED.getCode()) - .message("Uploading file") - .build(); + .createdBy(user.getId()) + .createdAt(OffsetDateTime.now()) + .updatedAt(OffsetDateTime.now()) + .updatedBy(user.getId()) + .statuscode((short) HttpStatus.ACCEPTED.getCode()) + .message("Uploading file") + .build(); ImportUpload importUpload = ImportUpload.uploadBuilder() - .createdBy(user.getId()) - .createdAt(OffsetDateTime.now()) - .updatedBy(user.getId()) - .updatedAt(OffsetDateTime.now()) - .programId(program.getId()) - .importerProgressId(progress.getId()) - .importerMappingId(mapping.getId()) - .id(importId) - .build(); + .createdBy(user.getId()) + .createdAt(OffsetDateTime.now()) + .updatedBy(user.getId()) + .updatedAt(OffsetDateTime.now()) + .programId(program.getId()) + .importerProgressId(progress.getId()) + .importerMappingId(mapping.getId()) + .id(importId) + .build(); System.out.println("====================== program ID: " + program.getId() + " ==============="); - System.out.println("=================== experiment ID: " + expId + " ==============="); - gigwaGenoStorageService.processSubmission(gigwaGenoStorageService.getAuthToken(), program, expId, new TestFileUpload("src/test/resources/files/geno/sample.vcf", MediaType.of("application/vcard")).getBytes(), "sample.vcf", importUpload, progress); + System.out.println("=================== submission ID: " + submissionId + " ==============="); + gigwaGenoStorageService.processSubmission(gigwaGenoStorageService.getAuthToken(), program, submissionId, new TestFileUpload("src/test/resources/files/geno/sample.vcf", MediaType.of("application/vcard")).getBytes(), "sample.vcf", importUpload, progress); } - private ImportResponse submitGenoData(UUID programId, String programKey, UUID expId, String file) throws AuthorizationException, IOException, ApiException, DoesNotExistException { + private ImportResponse submitGenoData(UUID programId, String programKey, UUID submissionId, String file) + throws AuthorizationException, IOException, ApiException, DoesNotExistException { Program program = Program.builder() - .id(programId) - .key(programKey) - .brapiUrl(BrAPIConstants.SYSTEM_DEFAULT.getValue()) - .build(); - doReturn(List.of(program)).when(programDAO) - .get(any(UUID.class)); + .id(programId) + .key(programKey) + .brapiUrl(BrAPIConstants.SYSTEM_DEFAULT.getValue()) + .build(); + doReturn(List.of(program)).when(programDAO).get(any(UUID.class)); User user = User.builder() - .id(UUID.randomUUID()) - .build(); - doReturn(Optional.of(user)).when(userDAO) - .getUser(any(UUID.class)); + .id(UUID.randomUUID()) + .build(); + doReturn(Optional.of(user)).when(userDAO).getUser(any(UUID.class)); ImportMapping mapping = ImportMapping.builder() - .build(); - doReturn(List.of(mapping)).when(importMappingDAO) - .getSystemMappingByName(any(String.class)); + .id(UUID.randomUUID()) + .build(); + doReturn(List.of(mapping)).when(importMappingDAO).getSystemMappingByName(any(String.class)); + + doAnswer(invocation -> { + ImportProgress progress = invocation.getArgument(0, ImportProgress.class); + progress.setId(UUID.randomUUID()); + return null; + }).when(importDAO).createProgress(any(ImportProgress.class)); UUID importId = UUID.randomUUID(); doAnswer(invocation -> { - var importEntity = invocation.getArgument(0, ImporterImportEntity.class); + ImporterImportEntity importEntity = invocation.getArgument(0, ImporterImportEntity.class); importEntity.setId(importId); - return importEntity; - }).when(importDAO) - .insert(any(ImporterImportEntity.class)); + return null; + }).when(importDAO).insert(any(ImporterImportEntity.class)); doReturn(new BrAPIClient("", 300000)).when(programDAO).getCoreClient(any(UUID.class)); doReturn(new BrAPIClient("", 300000)).when(programDAO).getPhenoClient(any(UUID.class)); - System.out.println("====================== program ID: " + program.getId() + " ==============="); - System.out.println("=================== experiment ID: " + expId + " ==============="); - return gigwaGenoStorageService.submitGenotypeData(user.getId(), programId, expId, new TestFileUpload("src/test/resources/files/geno/"+file, MediaType.of("application/vcard"))); + return gigwaGenoStorageService.submitGenotypeData( + user.getId(), + programId, + submissionId, + new TestFileUpload("src/test/resources/files/geno/" + file, MediaType.of("application/vcard")) + ); + } + + private List buildSamplesFromValidVcf() throws IOException { + return buildSamplesFromVcf("sample.vcf"); + } + + private List buildSamplesFromVcf(String fileName) throws IOException { + try (Scanner sc = new Scanner(new FileInputStream("src/test/resources/files/geno/" + fileName), "UTF-8")) { + String[] headerParts = null; + boolean foundHeader = false; + while (sc.hasNextLine() && !foundHeader) { + String line = sc.nextLine(); + if (line.startsWith("#CHROM")) { + foundHeader = true; + headerParts = line.split("\t"); + } + } + + assertTrue(foundHeader, "Could not find " + fileName + " header file"); + + List samples = new ArrayList<>(); + for (int i = 9; i < headerParts.length; i++) { + samples.add(new BrAPISample().sampleName(headerParts[i])); + } + + return samples; + } } private class TestFileUpload implements CompletedFileUpload { diff --git a/src/test/java/org/breedinginsight/utilities/response/mappers/GermplasmQueryMapperUnitTest.java b/src/test/java/org/breedinginsight/utilities/response/mappers/GermplasmQueryMapperUnitTest.java new file mode 100644 index 000000000..3ec74d25a --- /dev/null +++ b/src/test/java/org/breedinginsight/utilities/response/mappers/GermplasmQueryMapperUnitTest.java @@ -0,0 +1,62 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.breedinginsight.utilities.response.mappers; + +import lombok.SneakyThrows; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.junit.jupiter.api.*; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class GermplasmQueryMapperUnitTest { + + GermplasmQueryMapper germplasmQueryMapper; + + @BeforeAll + @SneakyThrows + public void setup() { + germplasmQueryMapper = new GermplasmQueryMapper(); + } + + @Test + public void testExternalUidMappingUsesCanonicalReferenceSource() { + BrAPIGermplasm germplasm = new BrAPIGermplasm(); + germplasm.setSeedSource("USDA"); + + List externalReferences = new ArrayList<>(); + + BrAPIExternalReference sourceReference = new BrAPIExternalReference(); + sourceReference.setReferenceSource("USDA"); + sourceReference.setReferenceID("OLD-UID"); + externalReferences.add(sourceReference); + + BrAPIExternalReference externalUidReference = new BrAPIExternalReference(); + externalUidReference.setReferenceSource("External UID"); + externalUidReference.setReferenceID("ABC-123"); + externalReferences.add(externalUidReference); + + germplasm.setExternalReferences(externalReferences); + + assertEquals("ABC-123", germplasmQueryMapper.getField("externalUID").apply(germplasm), "Wrong getter"); + } +} \ No newline at end of file diff --git a/src/test/resources/files/geno/sample_duplicate_positional_key.vcf b/src/test/resources/files/geno/sample_duplicate_positional_key.vcf new file mode 100644 index 000000000..a538b8624 --- /dev/null +++ b/src/test/resources/files/geno/sample_duplicate_positional_key.vcf @@ -0,0 +1,4 @@ +##fileformat=VCFv4.2 +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT USDAMSP1_A01 +1 100 var1 A T . PASS . GT 0/1 +1 100 var2 G C . PASS . GT 0/1 diff --git a/src/test/resources/files/geno/sample_invalid_alt.vcf b/src/test/resources/files/geno/sample_invalid_alt.vcf new file mode 100644 index 000000000..3563d66c5 --- /dev/null +++ b/src/test/resources/files/geno/sample_invalid_alt.vcf @@ -0,0 +1,3 @@ +##fileformat=VCFv4.2 +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT USDAMSP1_A01 +1 100 var1 A - . PASS . GT 0/1 diff --git a/src/test/resources/files/geno/sample_invalid_ref.vcf b/src/test/resources/files/geno/sample_invalid_ref.vcf new file mode 100644 index 000000000..22e7fb0e1 --- /dev/null +++ b/src/test/resources/files/geno/sample_invalid_ref.vcf @@ -0,0 +1,3 @@ +##fileformat=VCFv4.2 +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT USDAMSP1_A01 +1 100 var1 X T . PASS . GT 0/1 diff --git a/src/test/resources/files/geno/sample_invalid_ref_dot.vcf b/src/test/resources/files/geno/sample_invalid_ref_dot.vcf new file mode 100644 index 000000000..97d4db7dc --- /dev/null +++ b/src/test/resources/files/geno/sample_invalid_ref_dot.vcf @@ -0,0 +1,3 @@ +##fileformat=VCFv4.2 +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT USDAMSP1_A01 +1 100 var1 . T . PASS . GT 0/1 diff --git a/src/test/resources/files/geno/sample_same_pos_different_variant_type.vcf b/src/test/resources/files/geno/sample_same_pos_different_variant_type.vcf new file mode 100644 index 000000000..a3743e676 --- /dev/null +++ b/src/test/resources/files/geno/sample_same_pos_different_variant_type.vcf @@ -0,0 +1,4 @@ +##fileformat=VCFv4.2 +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT USDAMSP1_A01 +1 100 var1 A T . PASS . GT 0/1 +1 100 var2 A AG . PASS . GT 0/1 \ No newline at end of file diff --git a/src/test/resources/files/geno/sample_valid_missing_ref_alt.vcf b/src/test/resources/files/geno/sample_valid_missing_ref_alt.vcf new file mode 100644 index 000000000..df0ebb00f --- /dev/null +++ b/src/test/resources/files/geno/sample_valid_missing_ref_alt.vcf @@ -0,0 +1,3 @@ +##fileformat=VCFv4.2 +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT USDAMSP1_A01 +1 100 var1 . . . PASS . GT 0/0 diff --git a/src/test/resources/files/geno/sample_valid_multi_alt.vcf b/src/test/resources/files/geno/sample_valid_multi_alt.vcf new file mode 100644 index 000000000..2f0b66b04 --- /dev/null +++ b/src/test/resources/files/geno/sample_valid_multi_alt.vcf @@ -0,0 +1,3 @@ +##fileformat=VCFv4.2 +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT USDAMSP1_A01 +1 100 var1 C CAG,T,CGG . PASS . GT 1/2 diff --git a/src/test/resources/files/germplasm_import/append_existing_blank_breeding_method.csv b/src/test/resources/files/germplasm_import/append_existing_blank_breeding_method.csv new file mode 100644 index 000000000..146bc0424 --- /dev/null +++ b/src/test/resources/files/germplasm_import/append_existing_blank_breeding_method.csv @@ -0,0 +1,2 @@ +GID,Germplasm Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +9002,Existing Blank Method,BCR,,,,,,,, diff --git a/src/test/resources/files/germplasm_import/append_existing_blank_source.csv b/src/test/resources/files/germplasm_import/append_existing_blank_source.csv new file mode 100644 index 000000000..7966e6d08 --- /dev/null +++ b/src/test/resources/files/germplasm_import/append_existing_blank_source.csv @@ -0,0 +1,2 @@ +GID,Germplasm Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +9001,Existing Blank Source,,Wild,,,,,,, \ No newline at end of file diff --git a/src/test/resources/files/germplasm_import/breeding_method_optional_without_parents.csv b/src/test/resources/files/germplasm_import/breeding_method_optional_without_parents.csv new file mode 100644 index 000000000..4921c9aab --- /dev/null +++ b/src/test/resources/files/germplasm_import/breeding_method_optional_without_parents.csv @@ -0,0 +1,3 @@ +GID,Germplasm Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +,No Parent 1,,Wild,,,,,,, +,No Parent 2,,,,,,,,, \ No newline at end of file diff --git a/src/test/resources/files/germplasm_import/breeding_method_required_with_parent.csv b/src/test/resources/files/germplasm_import/breeding_method_required_with_parent.csv new file mode 100644 index 000000000..7fe32e128 --- /dev/null +++ b/src/test/resources/files/germplasm_import/breeding_method_required_with_parent.csv @@ -0,0 +1,2 @@ +GID,Germplasm Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +,Name1,,Wild,0,0,,,,, \ No newline at end of file diff --git a/src/test/resources/files/germplasm_import/do_not_overwrite_existing_source_or_breeding_method.csv b/src/test/resources/files/germplasm_import/do_not_overwrite_existing_source_or_breeding_method.csv new file mode 100644 index 000000000..09804522a --- /dev/null +++ b/src/test/resources/files/germplasm_import/do_not_overwrite_existing_source_or_breeding_method.csv @@ -0,0 +1,2 @@ +GID,Germplasm Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +9003,Germplasm1,BCR,New Source,,,,,,, \ No newline at end of file diff --git a/src/test/resources/files/germplasm_import/external_uid_reference_source.csv b/src/test/resources/files/germplasm_import/external_uid_reference_source.csv new file mode 100644 index 000000000..3e56677c0 --- /dev/null +++ b/src/test/resources/files/germplasm_import/external_uid_reference_source.csv @@ -0,0 +1,5 @@ +GID,Germplasm Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +,With Source And UID,BCR,USDA,,,1,,,ABC-123, +,With UID Only,BCR,,,,2,,,ABC-456, +,With Source Only,BCR,USDA,,,3,,,, +,With Neither,BCR,,,,4,,,, \ No newline at end of file diff --git a/src/test/resources/files/germplasm_import/source_optional_with_parent.csv b/src/test/resources/files/germplasm_import/source_optional_with_parent.csv new file mode 100644 index 000000000..1db1272be --- /dev/null +++ b/src/test/resources/files/germplasm_import/source_optional_with_parent.csv @@ -0,0 +1,2 @@ +GID,Germplasm Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +,Name1,BCR,,0,,,,,, \ No newline at end of file